From b37a478cacf0a3d92ec5a63a38b10dc24c22cc33 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Thu, 5 Dec 2024 14:00:12 -0800 Subject: [PATCH 001/223] go.mod: bump x/net and dependencies Pulling in upstream fix for #14201. Updates #14201 Signed-off-by: James Tucker --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 1924e93ed5d32..e57573f188933 100644 --- a/go.mod +++ b/go.mod @@ -95,14 +95,14 @@ require ( go.uber.org/zap v1.27.0 go4.org/mem v0.0.0-20220726221520-4f986261bf13 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba - golang.org/x/crypto v0.25.0 + golang.org/x/crypto v0.30.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/mod v0.19.0 - golang.org/x/net v0.27.0 + golang.org/x/net v0.32.0 golang.org/x/oauth2 v0.16.0 - golang.org/x/sync v0.9.0 - golang.org/x/sys v0.27.0 - golang.org/x/term v0.22.0 + golang.org/x/sync v0.10.0 + golang.org/x/sys v0.28.0 + golang.org/x/term v0.27.0 golang.org/x/time v0.5.0 golang.org/x/tools v0.23.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 @@ -386,7 +386,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/image v0.18.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index fadfb22b1a0c8..1cbb440fa7d58 100644 --- a/go.sum +++ b/go.sum @@ -1062,8 +1062,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1153,8 +1153,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1176,8 +1176,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1239,8 +1239,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1248,8 +1248,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1262,8 +1262,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 66aa77416744037baec93206ae212012a2314f83 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Thu, 5 Dec 2024 17:00:54 -0600 Subject: [PATCH 002/223] cmd/gitops-pusher: default previousEtag to controlEtag (#14296) If previousEtag is empty, then we assume control ACLs were not modified manually and push the local ACLs. Instead, we defaulted to localEtag which would be different if local ACLs were different from control. AFAIK this was always buggy, but never reported? Fixes #14295 Signed-off-by: Andrew Lytvynov --- cmd/gitops-pusher/gitops-pusher.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/gitops-pusher/gitops-pusher.go b/cmd/gitops-pusher/gitops-pusher.go index c33937ef24959..e7a0aeee10260 100644 --- a/cmd/gitops-pusher/gitops-pusher.go +++ b/cmd/gitops-pusher/gitops-pusher.go @@ -58,8 +58,8 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte } if cache.PrevETag == "" { - log.Println("no previous etag found, assuming local file is correct and recording that") - cache.PrevETag = localEtag + log.Println("no previous etag found, assuming the latest control etag") + cache.PrevETag = controlEtag } log.Printf("control: %s", controlEtag) @@ -105,8 +105,8 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex } if cache.PrevETag == "" { - log.Println("no previous etag found, assuming local file is correct and recording that") - cache.PrevETag = localEtag + log.Println("no previous etag found, assuming the latest control etag") + cache.PrevETag = controlEtag } log.Printf("control: %s", controlEtag) @@ -148,8 +148,8 @@ func getChecksums(cache *Cache, client *http.Client, tailnet, apiKey string) fun } if cache.PrevETag == "" { - log.Println("no previous etag found, assuming local file is correct and recording that") - cache.PrevETag = Shuck(localEtag) + log.Println("no previous etag found, assuming control etag") + cache.PrevETag = Shuck(controlEtag) } log.Printf("control: %s", controlEtag) From a482dc037bf6d22624e8750ef889f8e025da8a6e Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Thu, 5 Dec 2024 15:50:24 -0800 Subject: [PATCH 003/223] logpolicy: cleanup options API and allow setting http.Client (#11503) This package grew organically over time and is an awful mix of explicitly declared options and globally set parameters via environment variables and other subtle effects. Add a new Options and TransportOptions type to allow for the creation of a Policy or http.RoundTripper with some set of options. The options struct avoids the need to add yet more NewXXX functions for every possible combination of ordered arguments. The goal of this refactor is to allow specifying the http.Client to use with the Policy. Updates tailscale/corp#18177 Signed-off-by: Joe Tsai --- logpolicy/logpolicy.go | 187 ++++++++++++++++++++++++++++------------- 1 file changed, 130 insertions(+), 57 deletions(-) diff --git a/logpolicy/logpolicy.go b/logpolicy/logpolicy.go index d657c4e9352f3..fa882ad3a2afa 100644 --- a/logpolicy/logpolicy.go +++ b/logpolicy/logpolicy.go @@ -9,6 +9,7 @@ package logpolicy import ( "bufio" "bytes" + "cmp" "context" "crypto/tls" "encoding/json" @@ -449,25 +450,63 @@ func tryFixLogStateLocation(dir, cmdname string, logf logger.Logf) { } } -// New returns a new log policy (a logger and its instance ID) for a given -// collection name. -// -// The netMon parameter is optional. It should be specified in environments where -// Tailscaled is manipulating the routing table. -// -// The logf parameter is optional; if non-nil, information logs (e.g. when -// migrating state) are sent to that logger, and global changes to the log -// package are avoided. If nil, logs will be printed using log.Printf. +// Deprecated: Use [Options.New] instead. func New(collection string, netMon *netmon.Monitor, health *health.Tracker, logf logger.Logf) *Policy { - return NewWithConfigPath(collection, "", "", netMon, health, logf) + return Options{ + Collection: collection, + NetMon: netMon, + Health: health, + Logf: logf, + }.New() } -// NewWithConfigPath is identical to New, but uses the specified directory and -// command name. If either is empty, it derives them automatically. -// -// The netMon parameter is optional. It should be specified in environments where -// Tailscaled is manipulating the routing table. +// Deprecated: Use [Options.New] instead. func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, health *health.Tracker, logf logger.Logf) *Policy { + return Options{ + Collection: collection, + Dir: dir, + CmdName: cmdName, + NetMon: netMon, + Health: health, + Logf: logf, + }.New() +} + +// Options is used to construct a [Policy]. +type Options struct { + // Collection is a required collection to upload logs under. + // Collection is a namespace for the type logs. + // For example, logs for a node use "tailnode.log.tailscale.io". + Collection string + + // Dir is an optional directory to store the log configuration. + // If empty, [LogsDir] is used. + Dir string + + // CmdName is an optional name of the current binary. + // If empty, [version.CmdName] is used. + CmdName string + + // NetMon is an optional parameter for monitoring. + // If non-nil, it's used to do faster interface lookups. + NetMon *netmon.Monitor + + // Health is an optional parameter for health status. + // If non-nil, it's used to construct the default HTTP client. + Health *health.Tracker + + // Logf is an optional logger to use. + // If nil, [log.Printf] will be used instead. + Logf logger.Logf + + // HTTPC is an optional client to use upload logs. + // If nil, [TransportOptions.New] is used to construct a new client + // with that particular transport sending logs to the default logs server. + HTTPC *http.Client +} + +// New returns a new log policy (a logger and its instance ID). +func (opts Options) New() *Policy { if hostinfo.IsNATLabGuestVM() { // In NATLab Gokrazy instances, tailscaled comes up concurently with // DHCP and the doesn't have DNS for a while. Wait for DHCP first. @@ -495,23 +534,23 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, earlyErrBuf.WriteByte('\n') } - if dir == "" { - dir = LogsDir(earlyLogf) + if opts.Dir == "" { + opts.Dir = LogsDir(earlyLogf) } - if cmdName == "" { - cmdName = version.CmdName() + if opts.CmdName == "" { + opts.CmdName = version.CmdName() } - useStdLogger := logf == nil + useStdLogger := opts.Logf == nil if useStdLogger { - logf = log.Printf + opts.Logf = log.Printf } - tryFixLogStateLocation(dir, cmdName, logf) + tryFixLogStateLocation(opts.Dir, opts.CmdName, opts.Logf) - cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", cmdName)) + cfgPath := filepath.Join(opts.Dir, fmt.Sprintf("%s.log.conf", opts.CmdName)) if runtime.GOOS == "windows" { - switch cmdName { + switch opts.CmdName { case "tailscaled": // Tailscale 1.14 and before stored state under %LocalAppData% // (usually "C:\WINDOWS\system32\config\systemprofile\AppData\Local" @@ -542,7 +581,7 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, cfgPath = paths.TryConfigFileMigration(earlyLogf, oldPath, cfgPath) case "tailscale-ipn": for _, oldBase := range []string{"wg64.log.conf", "wg32.log.conf"} { - oldConf := filepath.Join(dir, oldBase) + oldConf := filepath.Join(opts.Dir, oldBase) if fi, err := os.Stat(oldConf); err == nil && fi.Mode().IsRegular() { cfgPath = paths.TryConfigFileMigration(earlyLogf, oldConf, cfgPath) break @@ -555,9 +594,9 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, if err != nil { earlyLogf("logpolicy.ConfigFromFile %v: %v", cfgPath, err) } - if err := newc.Validate(collection); err != nil { + if err := newc.Validate(opts.Collection); err != nil { earlyLogf("logpolicy.Config.Validate for %v: %v", cfgPath, err) - newc = NewConfig(collection) + newc = NewConfig(opts.Collection) if err := newc.Save(cfgPath); err != nil { earlyLogf("logpolicy.Config.Save for %v: %v", cfgPath, err) } @@ -568,31 +607,39 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, PrivateID: newc.PrivateID, Stderr: logWriter{console}, CompressLogs: true, - HTTPC: &http.Client{Transport: NewLogtailTransport(logtail.DefaultHost, netMon, health, logf)}, } - if collection == logtail.CollectionNode { + if opts.Collection == logtail.CollectionNode { conf.MetricsDelta = clientmetric.EncodeLogTailMetricsDelta conf.IncludeProcID = true conf.IncludeProcSequence = true } if envknob.NoLogsNoSupport() || testenv.InTest() { - logf("You have disabled logging. Tailscale will not be able to provide support.") + opts.Logf("You have disabled logging. Tailscale will not be able to provide support.") conf.HTTPC = &http.Client{Transport: noopPretendSuccessTransport{}} } else { // Only attach an on-disk filch buffer if we are going to be sending logs. // No reason to persist them locally just to drop them later. - attachFilchBuffer(&conf, dir, cmdName, logf) - - if val := getLogTarget(); val != "" { - logf("You have enabled a non-default log target. Doing without being told to by Tailscale staff or your network administrator will make getting support difficult.") - conf.BaseURL = val - u, _ := url.Parse(val) - conf.HTTPC = &http.Client{Transport: NewLogtailTransport(u.Host, netMon, health, logf)} + attachFilchBuffer(&conf, opts.Dir, opts.CmdName, opts.Logf) + conf.HTTPC = opts.HTTPC + + if conf.HTTPC == nil { + logHost := logtail.DefaultHost + if val := getLogTarget(); val != "" { + opts.Logf("You have enabled a non-default log target. Doing without being told to by Tailscale staff or your network administrator will make getting support difficult.") + conf.BaseURL = val + u, _ := url.Parse(val) + logHost = u.Host + } + conf.HTTPC = &http.Client{Transport: TransportOptions{ + Host: logHost, + NetMon: opts.NetMon, + Health: opts.Health, + Logf: opts.Logf, + }.New()} } - } - lw := logtail.NewLogger(conf, logf) + lw := logtail.NewLogger(conf, opts.Logf) var logOutput io.Writer = lw @@ -610,19 +657,19 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, log.SetOutput(logOutput) } - logf("Program starting: v%v, Go %v: %#v", + opts.Logf("Program starting: v%v, Go %v: %#v", version.Long(), goVersion(), os.Args) - logf("LogID: %v", newc.PublicID) + opts.Logf("LogID: %v", newc.PublicID) if earlyErrBuf.Len() != 0 { - logf("%s", earlyErrBuf.Bytes()) + opts.Logf("%s", earlyErrBuf.Bytes()) } return &Policy{ Logtail: lw, PublicID: newc.PublicID, - Logf: logf, + Logf: opts.Logf, } } @@ -763,23 +810,48 @@ func dialContext(ctx context.Context, netw, addr string, netMon *netmon.Monitor, return c, err } -// NewLogtailTransport returns an HTTP Transport particularly suited to uploading -// logs to the given host name. See DialContext for details on how it works. -// -// The netMon parameter is optional. It should be specified in environments where -// Tailscaled is manipulating the routing table. -// -// The logf parameter is optional; if non-nil, logs are printed using the -// provided function; if nil, log.Printf will be used instead. +// Deprecated: Use [TransportOptions.New] instead. func NewLogtailTransport(host string, netMon *netmon.Monitor, health *health.Tracker, logf logger.Logf) http.RoundTripper { + return TransportOptions{Host: host, NetMon: netMon, Health: health, Logf: logf}.New() +} + +// TransportOptions is used to construct an [http.RoundTripper]. +type TransportOptions struct { + // Host is the optional hostname of the logs server. + // If empty, then [logtail.DefaultHost] is used. + Host string + + // NetMon is an optional parameter for monitoring. + // If non-nil, it's used to do faster interface lookups. + NetMon *netmon.Monitor + + // Health is an optional parameter for health status. + // If non-nil, it's used to construct the default HTTP client. + Health *health.Tracker + + // Logf is an optional logger to use. + // If nil, [log.Printf] will be used instead. + Logf logger.Logf + + // TLSClientConfig is an optional TLS configuration to use. + // If non-nil, the configuration will be cloned. + TLSClientConfig *tls.Config +} + +// New returns an HTTP Transport particularly suited to uploading logs +// to the given host name. See [DialContext] for details on how it works. +func (opts TransportOptions) New() http.RoundTripper { if testenv.InTest() { return noopPretendSuccessTransport{} } - if netMon == nil { - netMon = netmon.NewStatic() + if opts.NetMon == nil { + opts.NetMon = netmon.NewStatic() } // Start with a copy of http.DefaultTransport and tweak it a bit. tr := http.DefaultTransport.(*http.Transport).Clone() + if opts.TLSClientConfig != nil { + tr.TLSClientConfig = opts.TLSClientConfig.Clone() + } tr.Proxy = tshttpproxy.ProxyFromEnvironment tshttpproxy.SetTransportGetProxyConnectHeader(tr) @@ -790,10 +862,10 @@ func NewLogtailTransport(host string, netMon *netmon.Monitor, health *health.Tra tr.DisableCompression = true // Log whenever we dial: - if logf == nil { - logf = log.Printf + if opts.Logf == nil { + opts.Logf = log.Printf } - tr.DialContext = MakeDialFunc(netMon, logf) + tr.DialContext = MakeDialFunc(opts.NetMon, opts.Logf) // We're uploading logs ideally infrequently, with specific timing that will // change over time. Try to keep the connection open, to avoid repeatedly @@ -815,7 +887,8 @@ func NewLogtailTransport(host string, netMon *netmon.Monitor, health *health.Tra tr.TLSNextProto = map[string]func(authority string, c *tls.Conn) http.RoundTripper{} } - tr.TLSClientConfig = tlsdial.Config(host, health, tr.TLSClientConfig) + host := cmp.Or(opts.Host, logtail.DefaultHost) + tr.TLSClientConfig = tlsdial.Config(host, opts.Health, tr.TLSClientConfig) // Force TLS 1.3 since we know log.tailscale.io supports it. tr.TLSClientConfig.MinVersion = tls.VersionTLS13 From dc6728729e903e83d7bc91de51dc38e115d79624 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 5 Dec 2024 15:45:48 -0800 Subject: [PATCH 004/223] health: fix TestHealthMetric to pass on release branch Fixes #14302 Change-Id: I9fd893a97711c72b713fe5535f2ccb93fadf7452 Signed-off-by: Brad Fitzpatrick --- health/health_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/health/health_test.go b/health/health_test.go index 69e586066cdd6..ebdddc988edc7 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -14,6 +14,7 @@ import ( "tailscale.com/tailcfg" "tailscale.com/types/opt" "tailscale.com/util/usermetric" + "tailscale.com/version" ) func TestAppendWarnableDebugFlags(t *testing.T) { @@ -352,6 +353,11 @@ func TestShowUpdateWarnable(t *testing.T) { } func TestHealthMetric(t *testing.T) { + unstableBuildWarning := 0 + if version.IsUnstableBuild() { + unstableBuildWarning = 1 + } + tests := []struct { desc string check bool @@ -361,20 +367,20 @@ func TestHealthMetric(t *testing.T) { }{ // When running in dev, and not initialising the client, there will be two warnings // by default: - // - is-using-unstable-version + // - is-using-unstable-version (except on the release branch) // - wantrunning-false { desc: "base-warnings", check: true, cv: nil, - wantMetricCount: 2, + wantMetricCount: unstableBuildWarning + 1, }, // with: update-available { desc: "update-warning", check: true, cv: &tailcfg.ClientVersion{RunningLatest: false, LatestVersion: "1.2.3"}, - wantMetricCount: 3, + wantMetricCount: unstableBuildWarning + 2, }, } for _, tt := range tests { From 06a82f416f2339e3309eec32ab98b4858d045697 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Wed, 4 Dec 2024 14:43:43 -0600 Subject: [PATCH 005/223] cmd,{get-authkey,tailscale}: remove unnecessary scope qualifier from OAuth clients OAuth clients that were used to generate an auth_key previously specified the scope 'device'. 'device' is not an actual scope, the real scope is 'devices'. The resulting OAuth token ended up including all scopes from the specified OAuth client, so the code was able to successfully create auth_keys. It's better not to hardcode a scope here anyway, so that we have the flexibility of changing which scope(s) are used in the future without having to update old clients. Since the qualifier never actually did anything, this commit simply removes it. Updates tailscale/corp#24934 Signed-off-by: Percy Wegmann --- cmd/get-authkey/main.go | 1 - cmd/tailscale/cli/up.go | 1 - 2 files changed, 2 deletions(-) diff --git a/cmd/get-authkey/main.go b/cmd/get-authkey/main.go index 777258d64b21b..95c9307565948 100644 --- a/cmd/get-authkey/main.go +++ b/cmd/get-authkey/main.go @@ -46,7 +46,6 @@ func main() { ClientID: clientID, ClientSecret: clientSecret, TokenURL: baseURL + "/api/v2/oauth/token", - Scopes: []string{"device"}, } ctx := context.Background() diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index 6c5c6f337f909..e86687527e5e0 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -1157,7 +1157,6 @@ func resolveAuthKey(ctx context.Context, v, tags string) (string, error) { ClientID: "some-client-id", // ignored ClientSecret: clientSecret, TokenURL: baseURL + "/api/v2/oauth/token", - Scopes: []string{"device"}, } tsClient := tailscale.NewClient("-", nil) From f81786007989c5d3e37253f269a561f9937dccfc Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Fri, 6 Dec 2024 11:17:11 -0600 Subject: [PATCH 006/223] VERSION.txt: this is v1.79.0 Signed-off-by: Nick Khyl --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 79e15fd49370a..b3a8c61e6a864 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.77.0 +1.79.0 From c2761162a002e65e2305f7570b2c54c561ac151f Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Fri, 6 Dec 2024 14:27:52 -0500 Subject: [PATCH 007/223] cmd/stunc: enforce read timeout deadline (#14309) Make argparsing use flag for adding a new parameter that requires parsing. Enforce a read timeout deadline waiting for response from the stun server provided in the args. Otherwise the program will never exit. Fixes #14267 Signed-off-by: Mike O'Driscoll --- cmd/stunc/stunc.go | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/cmd/stunc/stunc.go b/cmd/stunc/stunc.go index 9743a33007265..c4b2eedd39f90 100644 --- a/cmd/stunc/stunc.go +++ b/cmd/stunc/stunc.go @@ -5,24 +5,40 @@ package main import ( + "flag" "log" "net" "os" "strconv" + "time" "tailscale.com/net/stun" ) func main() { log.SetFlags(0) - - if len(os.Args) < 2 || len(os.Args) > 3 { - log.Fatalf("usage: %s [port]", os.Args[0]) - } - host := os.Args[1] + var host string port := "3478" - if len(os.Args) == 3 { - port = os.Args[2] + + var readTimeout time.Duration + flag.DurationVar(&readTimeout, "timeout", 3*time.Second, "response wait timeout") + + flag.Parse() + + values := flag.Args() + if len(values) < 1 || len(values) > 2 { + log.Printf("usage: %s [port]", os.Args[0]) + flag.PrintDefaults() + os.Exit(1) + } else { + for i, value := range values { + switch i { + case 0: + host = value + case 1: + port = value + } + } } _, err := strconv.ParseUint(port, 10, 16) if err != nil { @@ -46,6 +62,10 @@ func main() { log.Fatal(err) } + err = c.SetReadDeadline(time.Now().Add(readTimeout)) + if err != nil { + log.Fatal(err) + } var buf [1024]byte n, raddr, err := c.ReadFromUDPAddrPort(buf[:]) if err != nil { From 06c5e83c204b29496e67a8184d9ed7791c05b23c Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 9 Dec 2024 20:42:10 +0000 Subject: [PATCH 008/223] hostinfo: fix testing in container (#14330) Previously this unit test failed if it was run in a container. Update the assert to focus on exactly the condition we are trying to assert: the package type should only be 'container' if we use the build tag. Updates #14317 Signed-off-by: Tom Proctor --- hostinfo/hostinfo_linux_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hostinfo/hostinfo_linux_test.go b/hostinfo/hostinfo_linux_test.go index c8bd2abbeb230..0286fadf329ab 100644 --- a/hostinfo/hostinfo_linux_test.go +++ b/hostinfo/hostinfo_linux_test.go @@ -35,8 +35,12 @@ remotes/origin/QTSFW_5.0.0` } } -func TestInContainer(t *testing.T) { - if got := inContainer(); !got.EqualBool(false) { - t.Errorf("inContainer = %v; want false due to absence of ts_package_container build tag", got) +func TestPackageTypeNotContainer(t *testing.T) { + var got string + if packageType != nil { + got = packageType() + } + if got == "container" { + t.Fatal("packageType = container; should only happen if build tag ts_package_container is set") } } From 24b243c19490be0d8d133659901be07281f4b745 Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Tue, 10 Dec 2024 08:58:27 -0500 Subject: [PATCH 009/223] derp: add env var setting server send queue depth (#14334) Use envknob to configure the per client send queue depth for the derp server. Fixes tailscale/corp#24978 Signed-off-by: Mike O'Driscoll --- derp/derp_server.go | 23 ++++++++++++++++++----- derp/derp_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/derp/derp_server.go b/derp/derp_server.go index ab0ab0a908a07..8066b7f19ef43 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -84,11 +84,19 @@ func init() { } const ( - perClientSendQueueDepth = 32 // packets buffered for sending - writeTimeout = 2 * time.Second - privilegedWriteTimeout = 30 * time.Second // for clients with the mesh key + defaultPerClientSendQueueDepth = 32 // default packets buffered for sending + writeTimeout = 2 * time.Second + privilegedWriteTimeout = 30 * time.Second // for clients with the mesh key ) +func getPerClientSendQueueDepth() int { + if v, ok := envknob.LookupInt("TS_DEBUG_DERP_PER_CLIENT_SEND_QUEUE_DEPTH"); ok { + return v + } + + return defaultPerClientSendQueueDepth +} + // dupPolicy is a temporary (2021-08-30) mechanism to change the policy // of how duplicate connection for the same key are handled. type dupPolicy int8 @@ -190,6 +198,9 @@ type Server struct { // maps from netip.AddrPort to a client's public key keyOfAddr map[netip.AddrPort]key.NodePublic + // Sets the client send queue depth for the server. + perClientSendQueueDepth int + clock tstime.Clock } @@ -377,6 +388,8 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server { s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco") s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other") + + s.perClientSendQueueDepth = getPerClientSendQueueDepth() return s } @@ -849,8 +862,8 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem done: ctx.Done(), remoteIPPort: remoteIPPort, connectedAt: s.clock.Now(), - sendQueue: make(chan pkt, perClientSendQueueDepth), - discoSendQueue: make(chan pkt, perClientSendQueueDepth), + sendQueue: make(chan pkt, s.perClientSendQueueDepth), + discoSendQueue: make(chan pkt, s.perClientSendQueueDepth), sendPongCh: make(chan [8]byte, 1), peerGone: make(chan peerGoneMsg), canMesh: s.isMeshPeer(clientInfo), diff --git a/derp/derp_test.go b/derp/derp_test.go index 9185194dd79cf..f0fc52fe7c71b 100644 --- a/derp/derp_test.go +++ b/derp/derp_test.go @@ -6,6 +6,7 @@ package derp import ( "bufio" "bytes" + "cmp" "context" "crypto/x509" "encoding/asn1" @@ -23,6 +24,7 @@ import ( "testing" "time" + qt "github.com/frankban/quicktest" "go4.org/mem" "golang.org/x/time/rate" "tailscale.com/disco" @@ -1598,3 +1600,29 @@ func TestServerRepliesToPing(t *testing.T) { } } } + +func TestGetPerClientSendQueueDepth(t *testing.T) { + c := qt.New(t) + envKey := "TS_DEBUG_DERP_PER_CLIENT_SEND_QUEUE_DEPTH" + + testCases := []struct { + envVal string + want int + }{ + // Empty case, envknob treats empty as missing also. + { + "", defaultPerClientSendQueueDepth, + }, + { + "64", 64, + }, + } + + for _, tc := range testCases { + t.Run(cmp.Or(tc.envVal, "empty"), func(t *testing.T) { + t.Setenv(envKey, tc.envVal) + val := getPerClientSendQueueDepth() + c.Assert(val, qt.Equals, tc.want) + }) + } +} From ea3d0bcfd4452697b966a5f5842fd812855a8828 Mon Sep 17 00:00:00 2001 From: Mario Minardi Date: Tue, 10 Dec 2024 10:51:03 -0700 Subject: [PATCH 010/223] prober,derp/derphttp: make dev-mode DERP probes work without TLS (#14347) Make dev-mode DERP probes work without TLS. Properly dial port `3340` when not using HTTPS when dialing nodes in `derphttp_client`. Skip verifying TLS state in `newConn` if we are not running a prober. Updates tailscale/corp#24635 Signed-off-by: Percy Wegmann Co-authored-by: Percy Wegmann --- derp/derphttp/derphttp_client.go | 3 +++ prober/derp.go | 28 ++++++++++++++++------------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/derp/derphttp/derphttp_client.go b/derp/derphttp/derphttp_client.go index c95d072b1a572..7387b60b4a04d 100644 --- a/derp/derphttp/derphttp_client.go +++ b/derp/derphttp/derphttp_client.go @@ -757,6 +757,9 @@ func (c *Client) dialNode(ctx context.Context, n *tailcfg.DERPNode) (net.Conn, e } dst := cmp.Or(dstPrimary, n.HostName) port := "443" + if !c.useHTTPS() { + port = "3340" + } if n.DERPPort != 0 { port = fmt.Sprint(n.DERPPort) } diff --git a/prober/derp.go b/prober/derp.go index b1ebc590d4f98..bce40e34c8519 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -597,18 +597,22 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr if err != nil { return nil, err } - cs, ok := dc.TLSConnectionState() - if !ok { - dc.Close() - return nil, errors.New("no TLS state") - } - if len(cs.PeerCertificates) == 0 { - dc.Close() - return nil, errors.New("no peer certificates") - } - if cs.ServerName != n.HostName { - dc.Close() - return nil, fmt.Errorf("TLS server name %q != derp hostname %q", cs.ServerName, n.HostName) + + // Only verify TLS state if this is a prober. + if isProber { + cs, ok := dc.TLSConnectionState() + if !ok { + dc.Close() + return nil, errors.New("no TLS state") + } + if len(cs.PeerCertificates) == 0 { + dc.Close() + return nil, errors.New("no peer certificates") + } + if cs.ServerName != n.HostName { + dc.Close() + return nil, fmt.Errorf("TLS server name %q != derp hostname %q", cs.ServerName, n.HostName) + } } errc := make(chan error, 1) From fa28b024d6f9b9174a9e00ae2d798a7ed8d43a99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:32:04 -0700 Subject: [PATCH 011/223] .github: Bump actions/cache from 4.1.2 to 4.2.0 (#14331) Bumps [actions/cache](https://github.com/actions/cache) from 4.1.2 to 4.2.0. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/6849a6489940f00c2f30c0fb92c6274307ccb58a...1bd1e32a3bdc45362d1e726936510720a7c30a57) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f9bb5cae2235f..a4dccd103d237 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,7 +80,7 @@ jobs: - name: checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Restore Cache - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: # Note: unlike the other setups, this is only grabbing the mod download # cache, rather than the whole mod directory, as the download cache @@ -159,7 +159,7 @@ jobs: cache: false - name: Restore Cache - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: # Note: unlike the other setups, this is only grabbing the mod download # cache, rather than the whole mod directory, as the download cache @@ -260,7 +260,7 @@ jobs: - name: checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Restore Cache - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: # Note: unlike the other setups, this is only grabbing the mod download # cache, rather than the whole mod directory, as the download cache @@ -319,7 +319,7 @@ jobs: - name: checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Restore Cache - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: # Note: unlike the other setups, this is only grabbing the mod download # cache, rather than the whole mod directory, as the download cache @@ -367,7 +367,7 @@ jobs: - name: checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Restore Cache - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: # Note: unlike the other setups, this is only grabbing the mod download # cache, rather than the whole mod directory, as the download cache From d54cd593905fc0bc7cc13009e1db2741bf6960e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:15:11 -0700 Subject: [PATCH 012/223] .github: Bump github/codeql-action from 3.27.1 to 3.27.6 (#14332) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.27.1 to 3.27.6. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/4f3212b61783c3c68e8309a0f18a699764811cda...aa578102511db1f4524ed59b8cc2bae4f6e88195) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d9a287be32d8d..ba21e8fe94a5b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 + uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 + uses: github/codeql-action/autobuild@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 + uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 From 8b1d01161bbca8a26c2a50208444087c9fa2b3f1 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Wed, 11 Dec 2024 02:52:56 -0700 Subject: [PATCH 013/223] cmd/containerboot: guard kubeClient against nil dereference (#14357) A method on kc was called unconditionally, even if was not initialized, leading to a nil pointer dereference when TS_SERVE_CONFIG was set outside Kubernetes. Add a guard symmetric with other uses of the kubeClient. Fixes #14354. Signed-off-by: Bjorn Neergaard --- cmd/containerboot/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index ad1c0db201aa5..7411ea9496cfd 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -331,8 +331,10 @@ authLoop: if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil { log.Fatalf("failed to unset serve config: %v", err) } - if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil { - log.Fatalf("failed to update HTTPS endpoint in tailscale state: %v", err) + if hasKubeStateStore(cfg) { + if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil { + log.Fatalf("failed to update HTTPS endpoint in tailscale state: %v", err) + } } } From 0cc071f15409071f2649c3e142eceaf7cabff560 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Wed, 11 Dec 2024 10:56:12 +0000 Subject: [PATCH 014/223] cmd/containerboot: don't attempt to write kube Secret in non-kube environments (#14358) Updates tailscale/tailscale#14354 Signed-off-by: Irbe Krumina --- cmd/containerboot/serve.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 29ee7347f0c14..c8b9e098d8394 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -72,8 +72,10 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil { log.Fatalf("serve proxy: error updating serve config: %v", err) } - if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil { - log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err) + if kc != nil { + if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil { + log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err) + } } prevServeConfig = sc } From fa655e6ed366af5bdf2284449e1eb29dd784303a Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Wed, 11 Dec 2024 12:59:42 +0000 Subject: [PATCH 015/223] cmd/containerboot: add more tests, check that egress service config only set on kube (#14360) Updates tailscale/tailscale#14357 Signed-off-by: Irbe Krumina --- cmd/containerboot/main_test.go | 124 ++++++++++++++++++++++++++++++--- cmd/containerboot/settings.go | 3 + 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index 83e001b62c09e..dacfb5bc687b1 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -31,6 +31,7 @@ import ( "github.com/google/go-cmp/cmp" "golang.org/x/sys/unix" "tailscale.com/ipn" + "tailscale.com/kube/egressservices" "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/types/netmap" @@ -57,6 +58,16 @@ func TestContainerBoot(t *testing.T) { if err != nil { t.Fatalf("error unmarshaling tailscaled config: %v", err) } + serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} + serveConfBytes, err := json.Marshal(serveConf) + if err != nil { + t.Fatalf("error unmarshaling serve config: %v", err) + } + egressSvcsCfg := egressservices.Configs{"foo": {TailnetTarget: egressservices.TailnetTarget{FQDN: "foo.tailnetxyx.ts.net"}}} + egressSvcsCfgBytes, err := json.Marshal(egressSvcsCfg) + if err != nil { + t.Fatalf("error unmarshaling egress services config: %v", err) + } dirs := []string{ "var/lib", @@ -73,14 +84,16 @@ func TestContainerBoot(t *testing.T) { } } files := map[string][]byte{ - "usr/bin/tailscaled": fakeTailscaled, - "usr/bin/tailscale": fakeTailscale, - "usr/bin/iptables": fakeTailscale, - "usr/bin/ip6tables": fakeTailscale, - "dev/net/tun": []byte(""), - "proc/sys/net/ipv4/ip_forward": []byte("0"), - "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), - "etc/tailscaled/cap-95.hujson": tailscaledConfBytes, + "usr/bin/tailscaled": fakeTailscaled, + "usr/bin/tailscale": fakeTailscale, + "usr/bin/iptables": fakeTailscale, + "usr/bin/ip6tables": fakeTailscale, + "dev/net/tun": []byte(""), + "proc/sys/net/ipv4/ip_forward": []byte("0"), + "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), + "etc/tailscaled/cap-95.hujson": tailscaledConfBytes, + "etc/tailscaled/serve-config.json": serveConfBytes, + "etc/tailscaled/egress-services-config.json": egressSvcsCfgBytes, } resetFiles := func() { for path, content := range files { @@ -829,6 +842,101 @@ func TestContainerBoot(t *testing.T) { }, }, }, + { + Name: "serve_config_no_kube", + Env: map[string]string{ + "TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"), + "TS_AUTHKEY": "tskey-key", + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", + }, + }, + { + Notify: runningNotify, + }, + }, + }, + { + Name: "serve_config_kube", + Env: map[string]string{ + "KUBERNETES_SERVICE_HOST": kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, + "TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"), + }, + KubeSecret: map[string]string{ + "authkey": "tskey-key", + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", + }, + WantKubeSecret: map[string]string{ + "authkey": "tskey-key", + }, + }, + { + Notify: runningNotify, + WantKubeSecret: map[string]string{ + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + "https_endpoint": "no-https", + "tailscale_capver": capver, + }, + }, + }, + }, + { + Name: "egress_svcs_config_kube", + Env: map[string]string{ + "KUBERNETES_SERVICE_HOST": kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, + "TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"), + }, + KubeSecret: map[string]string{ + "authkey": "tskey-key", + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", + }, + WantKubeSecret: map[string]string{ + "authkey": "tskey-key", + }, + }, + { + Notify: runningNotify, + WantKubeSecret: map[string]string{ + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + "tailscale_capver": capver, + }, + }, + }, + }, + { + Name: "egress_svcs_config_no_kube", + Env: map[string]string{ + "TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"), + "TS_AUTHKEY": "tskey-key", + }, + Phases: []phase{ + { + WantFatalLog: "TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes", + }, + }, + }, } for _, test := range tests { diff --git a/cmd/containerboot/settings.go b/cmd/containerboot/settings.go index 4fae58584cec7..e80dbee57a82a 100644 --- a/cmd/containerboot/settings.go +++ b/cmd/containerboot/settings.go @@ -199,6 +199,9 @@ func (s *settings) validate() error { if s.HealthCheckEnabled && s.HealthCheckAddrPort != "" { return errors.New("TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0, use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT") } + if s.EgressSvcsCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") { + return errors.New("TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes") + } return nil } From f1ccdcc713bfebc1500ea666d523f36301a9f782 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 11 Dec 2024 14:48:57 +0000 Subject: [PATCH 016/223] cmd/k8s-operator,k8s-operator: operator integration tests (#12792) This is the start of an integration/e2e test suite for the tailscale operator. It currently only tests two major features, ingress proxy and API server proxy, but we intend to expand it to cover more features over time. It also only supports manual runs for now. We intend to integrate it into CI checks in a separate update when we have planned how to securely provide CI with the secrets required for connecting to a test tailnet. Updates #12622 Change-Id: I31e464bb49719348b62a563790f2bc2ba165a11b Co-authored-by: Irbe Krumina Signed-off-by: Tom Proctor --- cmd/k8s-operator/e2e/ingress_test.go | 108 +++++++++++++++ cmd/k8s-operator/e2e/main_test.go | 194 +++++++++++++++++++++++++++ cmd/k8s-operator/e2e/proxy_test.go | 156 +++++++++++++++++++++ k8s-operator/conditions.go | 11 ++ 4 files changed, 469 insertions(+) create mode 100644 cmd/k8s-operator/e2e/ingress_test.go create mode 100644 cmd/k8s-operator/e2e/main_test.go create mode 100644 cmd/k8s-operator/e2e/proxy_test.go diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go new file mode 100644 index 0000000000000..373dd2c7dc88f --- /dev/null +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + kube "tailscale.com/k8s-operator" + "tailscale.com/tstest" +) + +// See [TestMain] for test requirements. +func TestIngress(t *testing.T) { + if tsClient == nil { + t.Skip("TestIngress requires credentials for a tailscale client") + } + + ctx := context.Background() + cfg := config.GetConfigOrDie() + cl, err := client.New(cfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + // Apply nginx + createAndCleanup(t, ctx, cl, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }) + // Apply service to expose it as ingress + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + Annotations: map[string]string{ + "tailscale.com/expose": "true", + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: "TCP", + Port: 80, + }, + }, + }, + } + createAndCleanup(t, ctx, cl, svc) + + // TODO: instead of timing out only when test times out, cancel context after 60s or so. + if err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")} + if err := get(ctx, cl, maybeReadySvc); err != nil { + return false, err + } + isReady := kube.SvcIsReady(maybeReadySvc) + if isReady { + t.Log("Service is ready") + } + return isReady, nil + }); err != nil { + t.Fatalf("error waiting for the Service to become Ready: %v", err) + } + + var resp *http.Response + if err := tstest.WaitFor(time.Second*60, func() error { + // TODO(tomhjp): Get the tailnet DNS name from the associated secret instead. + // If we are not the first tailnet node with the requested name, we'll get + // a -N suffix. + resp, err = tsClient.HTTPClient.Get(fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name)) + if err != nil { + return err + } + return nil + }); err != nil { + t.Fatalf("error trying to reach service: %v", err) + } + + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %v; response body s", resp.StatusCode) + } +} diff --git a/cmd/k8s-operator/e2e/main_test.go b/cmd/k8s-operator/e2e/main_test.go new file mode 100644 index 0000000000000..ae23c939c8925 --- /dev/null +++ b/cmd/k8s-operator/e2e/main_test.go @@ -0,0 +1,194 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "slices" + "strings" + "testing" + + "github.com/go-logr/zapr" + "github.com/tailscale/hujson" + "go.uber.org/zap/zapcore" + "golang.org/x/oauth2/clientcredentials" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" + "tailscale.com/client/tailscale" +) + +const ( + e2eManagedComment = "// This is managed by the k8s-operator e2e tests" +) + +var ( + tsClient *tailscale.Client + testGrants = map[string]string{ + "test-proxy": `{ + "src": ["tag:e2e-test-proxy"], + "dst": ["tag:k8s-operator"], + "app": { + "tailscale.com/cap/kubernetes": [{ + "impersonate": { + "groups": ["ts:e2e-test-proxy"], + }, + }], + }, + }`, + } +) + +// This test suite is currently not run in CI. +// It requires some setup not handled by this code: +// - Kubernetes cluster with tailscale operator installed +// - Current kubeconfig context set to connect to that cluster (directly, no operator proxy) +// - Operator installed with --set apiServerProxyConfig.mode="true" +// - ACLs that define tag:e2e-test-proxy tag. TODO(tomhjp): Can maybe replace this prereq onwards with an API key +// - OAuth client ID and secret in TS_API_CLIENT_ID and TS_API_CLIENT_SECRET env +// - OAuth client must have auth_keys and policy_file write for tag:e2e-test-proxy tag +func TestMain(m *testing.M) { + code, err := runTests(m) + if err != nil { + log.Fatal(err) + } + os.Exit(code) +} + +func runTests(m *testing.M) (int, error) { + zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar() + logf.SetLogger(zapr.NewLogger(zlog.Desugar())) + tailscale.I_Acknowledge_This_API_Is_Unstable = true + + if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" { + cleanup, err := setupClientAndACLs() + if err != nil { + return 0, err + } + defer func() { + err = errors.Join(err, cleanup()) + }() + } + + return m.Run(), nil +} + +func setupClientAndACLs() (cleanup func() error, _ error) { + ctx := context.Background() + credentials := clientcredentials.Config{ + ClientID: os.Getenv("TS_API_CLIENT_ID"), + ClientSecret: os.Getenv("TS_API_CLIENT_SECRET"), + TokenURL: "https://login.tailscale.com/api/v2/oauth/token", + Scopes: []string{"auth_keys", "policy_file"}, + } + tsClient = tailscale.NewClient("-", nil) + tsClient.HTTPClient = credentials.Client(ctx) + + if err := patchACLs(ctx, tsClient, func(acls *hujson.Value) { + for test, grant := range testGrants { + deleteTestGrants(test, acls) + addTestGrant(test, grant, acls) + } + }); err != nil { + return nil, err + } + + return func() error { + return patchACLs(ctx, tsClient, func(acls *hujson.Value) { + for test := range testGrants { + deleteTestGrants(test, acls) + } + }) + }, nil +} + +func patchACLs(ctx context.Context, tsClient *tailscale.Client, patchFn func(*hujson.Value)) error { + acls, err := tsClient.ACLHuJSON(ctx) + if err != nil { + return err + } + hj, err := hujson.Parse([]byte(acls.ACL)) + if err != nil { + return err + } + + patchFn(&hj) + + hj.Format() + acls.ACL = hj.String() + if _, err := tsClient.SetACLHuJSON(ctx, *acls, true); err != nil { + return err + } + + return nil +} + +func addTestGrant(test, grant string, acls *hujson.Value) error { + v, err := hujson.Parse([]byte(grant)) + if err != nil { + return err + } + + // Add the managed comment to the first line of the grant object contents. + v.Value.(*hujson.Object).Members[0].Name.BeforeExtra = hujson.Extra(fmt.Sprintf("%s: %s\n", e2eManagedComment, test)) + + if err := acls.Patch([]byte(fmt.Sprintf(`[{"op": "add", "path": "/grants/-", "value": %s}]`, v.String()))); err != nil { + return err + } + + return nil +} + +func deleteTestGrants(test string, acls *hujson.Value) error { + grants := acls.Find("/grants") + + var patches []string + for i, g := range grants.Value.(*hujson.Array).Elements { + members := g.Value.(*hujson.Object).Members + if len(members) == 0 { + continue + } + comment := strings.TrimSpace(string(members[0].Name.BeforeExtra)) + if name, found := strings.CutPrefix(comment, e2eManagedComment+": "); found && name == test { + patches = append(patches, fmt.Sprintf(`{"op": "remove", "path": "/grants/%d"}`, i)) + } + } + + // Remove in reverse order so we don't affect the found indices as we mutate. + slices.Reverse(patches) + + if err := acls.Patch([]byte(fmt.Sprintf("[%s]", strings.Join(patches, ",")))); err != nil { + return err + } + + return nil +} + +func objectMeta(namespace, name string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + } +} + +func createAndCleanup(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) { + t.Helper() + if err := cl.Create(ctx, obj); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := cl.Delete(ctx, obj); err != nil { + t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err) + } + }) +} + +func get(ctx context.Context, cl client.Client, obj client.Object) error { + return cl.Get(ctx, client.ObjectKeyFromObject(obj), obj) +} diff --git a/cmd/k8s-operator/e2e/proxy_test.go b/cmd/k8s-operator/e2e/proxy_test.go new file mode 100644 index 0000000000000..eac983e88d613 --- /dev/null +++ b/cmd/k8s-operator/e2e/proxy_test.go @@ -0,0 +1,156 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "tailscale.com/client/tailscale" + "tailscale.com/tsnet" + "tailscale.com/tstest" +) + +// See [TestMain] for test requirements. +func TestProxy(t *testing.T) { + if tsClient == nil { + t.Skip("TestProxy requires credentials for a tailscale client") + } + + ctx := context.Background() + cfg := config.GetConfigOrDie() + cl, err := client.New(cfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + + // Create role and role binding to allow a group we'll impersonate to do stuff. + createAndCleanup(t, ctx, cl, &rbacv1.Role{ + ObjectMeta: objectMeta("tailscale", "read-secrets"), + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Verbs: []string{"get"}, + Resources: []string{"secrets"}, + }}, + }) + createAndCleanup(t, ctx, cl, &rbacv1.RoleBinding{ + ObjectMeta: objectMeta("tailscale", "read-secrets"), + Subjects: []rbacv1.Subject{{ + Kind: "Group", + Name: "ts:e2e-test-proxy", + }}, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: "read-secrets", + }, + }) + + // Get operator host name from kube secret. + operatorSecret := corev1.Secret{ + ObjectMeta: objectMeta("tailscale", "operator"), + } + if err := get(ctx, cl, &operatorSecret); err != nil { + t.Fatal(err) + } + + // Connect to tailnet with test-specific tag so we can use the + // [testGrants] ACLs when connecting to the API server proxy + ts := tsnetServerWithTag(t, ctx, "tag:e2e-test-proxy") + proxyCfg := &rest.Config{ + Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)), + Dial: ts.Dial, + } + proxyCl, err := client.New(proxyCfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + + // Expect success. + allowedSecret := corev1.Secret{ + ObjectMeta: objectMeta("tailscale", "operator"), + } + // Wait for up to a minute the first time we use the proxy, to give it time + // to provision the TLS certs. + if err := tstest.WaitFor(time.Second*60, func() error { + return get(ctx, proxyCl, &allowedSecret) + }); err != nil { + t.Fatal(err) + } + + // Expect forbidden. + forbiddenSecret := corev1.Secret{ + ObjectMeta: objectMeta("default", "operator"), + } + if err := get(ctx, proxyCl, &forbiddenSecret); err == nil || !apierrors.IsForbidden(err) { + t.Fatalf("expected forbidden error fetching secret from default namespace: %s", err) + } +} + +func tsnetServerWithTag(t *testing.T, ctx context.Context, tag string) *tsnet.Server { + caps := tailscale.KeyCapabilities{ + Devices: tailscale.KeyDeviceCapabilities{ + Create: tailscale.KeyDeviceCreateCapabilities{ + Reusable: false, + Preauthorized: true, + Ephemeral: true, + Tags: []string{tag}, + }, + }, + } + + authKey, authKeyMeta, err := tsClient.CreateKey(ctx, caps) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := tsClient.DeleteKey(ctx, authKeyMeta.ID); err != nil { + t.Errorf("error deleting auth key: %s", err) + } + }) + + ts := &tsnet.Server{ + Hostname: "test-proxy", + Ephemeral: true, + Dir: t.TempDir(), + AuthKey: authKey, + } + _, err = ts.Up(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := ts.Close(); err != nil { + t.Errorf("error shutting down tsnet.Server: %s", err) + } + }) + + return ts +} + +func hostNameFromOperatorSecret(t *testing.T, s corev1.Secret) string { + profiles := map[string]any{} + if err := json.Unmarshal(s.Data["_profiles"], &profiles); err != nil { + t.Fatal(err) + } + key, ok := strings.CutPrefix(string(s.Data["_current-profile"]), "profile-") + if !ok { + t.Fatal(string(s.Data["_current-profile"])) + } + profile, ok := profiles[key] + if !ok { + t.Fatal(profiles) + } + + return ((profile.(map[string]any))["Name"]).(string) +} diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index ace0fb7e33a75..1ecedfc0751aa 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -167,3 +167,14 @@ func DNSCfgIsReady(cfg *tsapi.DNSConfig) bool { cond := cfg.Status.Conditions[idx] return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == cfg.Generation } + +func SvcIsReady(svc *corev1.Service) bool { + idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool { + return cond.Type == string(tsapi.ProxyReady) + }) + if idx == -1 { + return false + } + cond := svc.Status.Conditions[idx] + return cond.Status == metav1.ConditionTrue +} From 6e552f66a0289f6309477fb024019b62a251da16 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Wed, 11 Dec 2024 14:58:44 +0000 Subject: [PATCH 017/223] cmd/containerboot: don't attempt to patch a Secret field without permissions (#14365) Signed-off-by: Irbe Krumina --- cmd/containerboot/kube.go | 1 + cmd/containerboot/serve.go | 2 +- cmd/containerboot/settings.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 643eef385ee0c..4d00687ee4566 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -24,6 +24,7 @@ import ( type kubeClient struct { kubeclient.Client stateSecret string + canPatch bool // whether the client has permissions to patch Kubernetes Secrets } func newKubeClient(root string, stateSecret string) (*kubeClient, error) { diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index c8b9e098d8394..14c7f00d7450f 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -72,7 +72,7 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil { log.Fatalf("serve proxy: error updating serve config: %v", err) } - if kc != nil { + if kc != nil && kc.canPatch { if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil { log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err) } diff --git a/cmd/containerboot/settings.go b/cmd/containerboot/settings.go index e80dbee57a82a..5fc6cc3f06d15 100644 --- a/cmd/containerboot/settings.go +++ b/cmd/containerboot/settings.go @@ -217,6 +217,7 @@ func (cfg *settings) setupKube(ctx context.Context, kc *kubeClient) error { return fmt.Errorf("some Kubernetes permissions are missing, please check your RBAC configuration: %v", err) } cfg.KubernetesCanPatch = canPatch + kc.canPatch = canPatch s, err := kc.GetSecret(ctx, cfg.KubeSecret) if err != nil { From 00458600605e8db25655f9abb95cac5bb78b3c55 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Wed, 11 Dec 2024 10:55:21 -0800 Subject: [PATCH 018/223] types/iox: add function types for Reader and Writer (#14366) Throughout our codebase we have types that only exist only to implement an io.Reader or io.Writer, when it would have been simpler, cleaner, and more readable to use an inlined function literal that closes over the relevant types. This is arguably more readable since it keeps the semantic logic in place rather than have it be isolated elsewhere. Note that a function literal that closes over some variables is semantic equivalent to declaring a struct with fields and having the Read or Write method mutate those fields. Updates #cleanup Signed-off-by: Joe Tsai --- types/iox/io.go | 23 +++++++++++++++++++++++ types/iox/io_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 types/iox/io.go create mode 100644 types/iox/io_test.go diff --git a/types/iox/io.go b/types/iox/io.go new file mode 100644 index 0000000000000..a5ca1be43f737 --- /dev/null +++ b/types/iox/io.go @@ -0,0 +1,23 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package iox provides types to implement [io] functionality. +package iox + +// TODO(https://go.dev/issue/21670): Deprecate or remove this functionality +// once the Go language supports implementing an 1-method interface directly +// using a function value of a matching signature. + +// ReaderFunc implements [io.Reader] using the underlying function value. +type ReaderFunc func([]byte) (int, error) + +func (f ReaderFunc) Read(b []byte) (int, error) { + return f(b) +} + +// WriterFunc implements [io.Writer] using the underlying function value. +type WriterFunc func([]byte) (int, error) + +func (f WriterFunc) Write(b []byte) (int, error) { + return f(b) +} diff --git a/types/iox/io_test.go b/types/iox/io_test.go new file mode 100644 index 0000000000000..9fba39605d28d --- /dev/null +++ b/types/iox/io_test.go @@ -0,0 +1,39 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package iox + +import ( + "bytes" + "io" + "testing" + "testing/iotest" + + "tailscale.com/util/must" +) + +func TestCopy(t *testing.T) { + const testdata = "the quick brown fox jumped over the lazy dog" + src := testdata + bb := new(bytes.Buffer) + if got := must.Get(io.Copy(bb, ReaderFunc(func(b []byte) (n int, err error) { + n = copy(b[:min(len(b), 7)], src) + src = src[n:] + if len(src) == 0 { + err = io.EOF + } + return n, err + }))); int(got) != len(testdata) { + t.Errorf("copy = %d, want %d", got, len(testdata)) + } + var dst []byte + if got := must.Get(io.Copy(WriterFunc(func(b []byte) (n int, err error) { + dst = append(dst, b...) + return len(b), nil + }), iotest.OneByteReader(bb))); int(got) != len(testdata) { + t.Errorf("copy = %d, want %d", got, len(testdata)) + } + if string(dst) != testdata { + t.Errorf("copy = %q, want %q", dst, testdata) + } +} From c9188d7760fb68a60e6791f0adf42f8dc1728251 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Wed, 11 Dec 2024 10:55:33 -0800 Subject: [PATCH 019/223] types/bools: add IfElse (#14272) The IfElse function is equivalent to the ternary (c ? a : b) operator in many other languages like C. Unfortunately, this function cannot perform short-circuit evaluation like in many other languages, but this is a restriction that's not much different than the pre-existing cmp.Or function. The argument against ternary operators in Go is that nested ternary operators become unreadable (e.g., (c1 ? (c2 ? a : b) : (c2 ? x : y))). But a single layer of ternary expressions can sometimes make code much more readable. Having the bools.IfElse function gives code authors the ability to decide whether use of this is more readable or not. Obviously, code authors will need to be judicious about their use of this helper function. Readability is more of an art than a science. Updates #cleanup Signed-off-by: Joe Tsai --- types/bools/bools.go | 28 +++++++++++++++++++ .../bools/{compare_test.go => bools_test.go} | 9 ++++++ types/bools/compare.go | 17 ----------- 3 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 types/bools/bools.go rename types/bools/{compare_test.go => bools_test.go} (70%) delete mode 100644 types/bools/compare.go diff --git a/types/bools/bools.go b/types/bools/bools.go new file mode 100644 index 0000000000000..962e39919e5ae --- /dev/null +++ b/types/bools/bools.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package bools contains the [Compare] and [Select] functions. +package bools + +// Compare compares two boolean values as if false is ordered before true. +func Compare[T ~bool](x, y T) int { + switch { + case x == false && y == true: + return -1 + case x == true && y == false: + return +1 + default: + return 0 + } +} + +// IfElse is a ternary operator that returns trueVal if condExpr is true +// otherwise it returns falseVal. +// IfElse(c, a, b) is roughly equivalent to (c ? a : b) in languages like C. +func IfElse[T any](condExpr bool, trueVal T, falseVal T) T { + if condExpr { + return trueVal + } else { + return falseVal + } +} diff --git a/types/bools/compare_test.go b/types/bools/bools_test.go similarity index 70% rename from types/bools/compare_test.go rename to types/bools/bools_test.go index 280294621e719..1b466db17b468 100644 --- a/types/bools/compare_test.go +++ b/types/bools/bools_test.go @@ -19,3 +19,12 @@ func TestCompare(t *testing.T) { t.Errorf("Compare(true, true) = %v, want 0", got) } } + +func TestIfElse(t *testing.T) { + if got := IfElse(true, 0, 1); got != 0 { + t.Errorf("IfElse(true, 0, 1) = %v, want 0", got) + } + if got := IfElse(false, 0, 1); got != 1 { + t.Errorf("IfElse(false, 0, 1) = %v, want 1", got) + } +} diff --git a/types/bools/compare.go b/types/bools/compare.go deleted file mode 100644 index ac433b240755a..0000000000000 --- a/types/bools/compare.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -// Package bools contains the bools.Compare function. -package bools - -// Compare compares two boolean values as if false is ordered before true. -func Compare[T ~bool](x, y T) int { - switch { - case x == false && y == true: - return -1 - case x == true && y == false: - return +1 - default: - return 0 - } -} From 716cb372563640d8a06deec218a03ea1982c1a15 Mon Sep 17 00:00:00 2001 From: Adrian Dewhurst Date: Wed, 11 Dec 2024 23:49:59 -0500 Subject: [PATCH 020/223] util/dnsname: use vizerror for all errors The errors emitted by util/dnsname are all written at least moderately friendly and none of them emit sensitive information. They should be safe to display to end users. Updates tailscale/corp#9025 Change-Id: Ic58705075bacf42f56378127532c5f28ff6bfc89 Signed-off-by: Adrian Dewhurst --- util/dnsname/dnsname.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/util/dnsname/dnsname.go b/util/dnsname/dnsname.go index dde0baaeda2f2..131bdd14b632e 100644 --- a/util/dnsname/dnsname.go +++ b/util/dnsname/dnsname.go @@ -5,9 +5,9 @@ package dnsname import ( - "errors" - "fmt" "strings" + + "tailscale.com/util/vizerror" ) const ( @@ -36,7 +36,7 @@ func ToFQDN(s string) (FQDN, error) { totalLen += 1 // account for missing dot } if totalLen > maxNameLength { - return "", fmt.Errorf("%q is too long to be a DNS name", s) + return "", vizerror.Errorf("%q is too long to be a DNS name", s) } st := 0 @@ -54,7 +54,7 @@ func ToFQDN(s string) (FQDN, error) { // // See https://github.com/tailscale/tailscale/issues/2024 for more. if len(label) == 0 || len(label) > maxLabelLength { - return "", fmt.Errorf("%q is not a valid DNS label", label) + return "", vizerror.Errorf("%q is not a valid DNS label", label) } st = i + 1 } @@ -97,23 +97,23 @@ func (f FQDN) Contains(other FQDN) bool { // ValidLabel reports whether label is a valid DNS label. func ValidLabel(label string) error { if len(label) == 0 { - return errors.New("empty DNS label") + return vizerror.New("empty DNS label") } if len(label) > maxLabelLength { - return fmt.Errorf("%q is too long, max length is %d bytes", label, maxLabelLength) + return vizerror.Errorf("%q is too long, max length is %d bytes", label, maxLabelLength) } if !isalphanum(label[0]) { - return fmt.Errorf("%q is not a valid DNS label: must start with a letter or number", label) + return vizerror.Errorf("%q is not a valid DNS label: must start with a letter or number", label) } if !isalphanum(label[len(label)-1]) { - return fmt.Errorf("%q is not a valid DNS label: must end with a letter or number", label) + return vizerror.Errorf("%q is not a valid DNS label: must end with a letter or number", label) } if len(label) < 2 { return nil } for i := 1; i < len(label)-1; i++ { if !isdnschar(label[i]) { - return fmt.Errorf("%q is not a valid DNS label: contains invalid character %q", label, label[i]) + return vizerror.Errorf("%q is not a valid DNS label: contains invalid character %q", label, label[i]) } } return nil From 73128e25230fda8c82696ed0ffef991bce68cecc Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 12 Dec 2024 09:38:07 -0800 Subject: [PATCH 021/223] ssh/tailssh: remove unused public key support When we first made Tailscale SSH, we assumed people would want public key support soon after. Turns out that hasn't been the case; people love the Tailscale identity authentication and check mode. In light of CVE-2024-45337, just remove all our public key code to not distract people, and to make the code smaller. We can always get it back from git if needed. Updates tailscale/corp#25131 Updates golang/go#70779 Co-authored-by: Percy Wegmann Change-Id: I87a6e79c2215158766a81942227a18b247333c22 Signed-off-by: Brad Fitzpatrick --- Makefile | 1 - ssh/tailssh/tailssh.go | 277 ++++-------------------------------- ssh/tailssh/tailssh_test.go | 88 +----------- tailcfg/tailcfg.go | 18 ++- tailcfg/tailcfg_clone.go | 12 +- tailcfg/tailcfg_view.go | 22 +-- 6 files changed, 54 insertions(+), 364 deletions(-) diff --git a/Makefile b/Makefile index 960f13885c11c..d3e50af0571b7 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,6 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \ echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \ - echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \ echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \ echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers diff --git a/ssh/tailssh/tailssh.go b/ssh/tailssh/tailssh.go index 7cb99c3813104..7f21ccd1182ee 100644 --- a/ssh/tailssh/tailssh.go +++ b/ssh/tailssh/tailssh.go @@ -10,7 +10,6 @@ import ( "bytes" "context" "crypto/rand" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -45,7 +44,6 @@ import ( "tailscale.com/util/clientmetric" "tailscale.com/util/httpm" "tailscale.com/util/mak" - "tailscale.com/util/slicesx" ) var ( @@ -80,16 +78,14 @@ type server struct { logf logger.Logf tailscaledPath string - pubKeyHTTPClient *http.Client // or nil for http.DefaultClient - timeNow func() time.Time // or nil for time.Now + timeNow func() time.Time // or nil for time.Now sessionWaitGroup sync.WaitGroup // mu protects the following - mu sync.Mutex - activeConns map[*conn]bool // set; value is always true - fetchPublicKeysCache map[string]pubKeyCacheEntry // by https URL - shutdownCalled bool + mu sync.Mutex + activeConns map[*conn]bool // set; value is always true + shutdownCalled bool } func (srv *server) now() time.Time { @@ -204,7 +200,6 @@ func (srv *server) OnPolicyChange() { // // Do the user auth // - NoClientAuthHandler -// - PublicKeyHandler (only if NoClientAuthHandler returns errPubKeyRequired) // // Once auth is done, the conn can be multiplexed with multiple sessions and // channels concurrently. At which point any of the following can be called @@ -234,10 +229,9 @@ type conn struct { finalAction *tailcfg.SSHAction // set by doPolicyAuth or resolveNextAction finalActionErr error // set by doPolicyAuth or resolveNextAction - info *sshConnInfo // set by setInfo - localUser *userMeta // set by doPolicyAuth - userGroupIDs []string // set by doPolicyAuth - pubKey gossh.PublicKey // set by doPolicyAuth + info *sshConnInfo // set by setInfo + localUser *userMeta // set by doPolicyAuth + userGroupIDs []string // set by doPolicyAuth acceptEnv []string // mu protects the following fields. @@ -268,9 +262,6 @@ func (c *conn) isAuthorized(ctx ssh.Context) error { action := c.currentAction for { if action.Accept { - if c.pubKey != nil { - metricPublicKeyAccepts.Add(1) - } return nil } if action.Reject || action.HoldAndDelegate == "" { @@ -293,10 +284,6 @@ func (c *conn) isAuthorized(ctx ssh.Context) error { // policy. var errDenied = errors.New("ssh: access denied") -// errPubKeyRequired is returned by NoClientAuthCallback to make the client -// resort to public-key auth; not user visible. -var errPubKeyRequired = errors.New("ssh publickey required") - // NoClientAuthCallback implements gossh.NoClientAuthCallback and is called by // the ssh.Server when the client first connects with the "none" // authentication method. @@ -305,13 +292,12 @@ var errPubKeyRequired = errors.New("ssh publickey required") // starting it afresh). It returns an error if the policy evaluation fails, or // if the decision is "reject" // -// It either returns nil (accept) or errPubKeyRequired or errDenied -// (reject). The errors may be wrapped. +// It either returns nil (accept) or errDenied (reject). The errors may be wrapped. func (c *conn) NoClientAuthCallback(ctx ssh.Context) error { if c.insecureSkipTailscaleAuth { return nil } - if err := c.doPolicyAuth(ctx, nil /* no pub key */); err != nil { + if err := c.doPolicyAuth(ctx); err != nil { return err } if err := c.isAuthorized(ctx); err != nil { @@ -332,8 +318,6 @@ func (c *conn) nextAuthMethodCallback(cm gossh.ConnMetadata, prevErrors []error) switch { case c.anyPasswordIsOkay: nextMethod = append(nextMethod, "password") - case slicesx.LastEqual(prevErrors, errPubKeyRequired): - nextMethod = append(nextMethod, "publickey") } // The fake "tailscale" method is always appended to next so OpenSSH renders @@ -353,41 +337,20 @@ func (c *conn) fakePasswordHandler(ctx ssh.Context, password string) bool { return c.anyPasswordIsOkay } -// PublicKeyHandler implements ssh.PublicKeyHandler is called by the -// ssh.Server when the client presents a public key. -func (c *conn) PublicKeyHandler(ctx ssh.Context, pubKey ssh.PublicKey) error { - if err := c.doPolicyAuth(ctx, pubKey); err != nil { - // TODO(maisem/bradfitz): surface the error here. - c.logf("rejecting SSH public key %s: %v", bytes.TrimSpace(gossh.MarshalAuthorizedKey(pubKey)), err) - return err - } - if err := c.isAuthorized(ctx); err != nil { - return err - } - c.logf("accepting SSH public key %s", bytes.TrimSpace(gossh.MarshalAuthorizedKey(pubKey))) - return nil -} - -// doPolicyAuth verifies that conn can proceed with the specified (optional) -// pubKey. It returns nil if the matching policy action is Accept or -// HoldAndDelegate. If pubKey is nil, there was no policy match but there is a -// policy that might match a public key it returns errPubKeyRequired. Otherwise, -// it returns errDenied. -func (c *conn) doPolicyAuth(ctx ssh.Context, pubKey ssh.PublicKey) error { +// doPolicyAuth verifies that conn can proceed. +// It returns nil if the matching policy action is Accept or +// HoldAndDelegate. Otherwise, it returns errDenied. +func (c *conn) doPolicyAuth(ctx ssh.Context) error { if err := c.setInfo(ctx); err != nil { c.logf("failed to get conninfo: %v", err) return errDenied } - a, localUser, acceptEnv, err := c.evaluatePolicy(pubKey) + a, localUser, acceptEnv, err := c.evaluatePolicy() if err != nil { - if pubKey == nil && c.havePubKeyPolicy() { - return errPubKeyRequired - } return fmt.Errorf("%w: %v", errDenied, err) } c.action0 = a c.currentAction = a - c.pubKey = pubKey c.acceptEnv = acceptEnv if a.Message != "" { if err := ctx.SendAuthBanner(a.Message); err != nil { @@ -448,7 +411,6 @@ func (srv *server) newConn() (*conn, error) { ServerConfigCallback: c.ServerConfig, NoClientAuthHandler: c.NoClientAuthCallback, - PublicKeyHandler: c.PublicKeyHandler, PasswordHandler: c.fakePasswordHandler, Handler: c.handleSessionPostSSHAuth, @@ -516,34 +478,6 @@ func (c *conn) mayForwardLocalPortTo(ctx ssh.Context, destinationHost string, de return false } -// havePubKeyPolicy reports whether any policy rule may provide access by means -// of a ssh.PublicKey. -func (c *conn) havePubKeyPolicy() bool { - if c.info == nil { - panic("havePubKeyPolicy called before setInfo") - } - // Is there any rule that looks like it'd require a public key for this - // sshUser? - pol, ok := c.sshPolicy() - if !ok { - return false - } - for _, r := range pol.Rules { - if c.ruleExpired(r) { - continue - } - if mapLocalUser(r.SSHUsers, c.info.sshUser) == "" { - continue - } - for _, p := range r.Principals { - if len(p.PubKeys) > 0 && c.principalMatchesTailscaleIdentity(p) { - return true - } - } - } - return false -} - // sshPolicy returns the SSHPolicy for current node. // If there is no SSHPolicy in the netmap, it returns a debugPolicy // if one is defined. @@ -620,117 +554,19 @@ func (c *conn) setInfo(ctx ssh.Context) error { } // evaluatePolicy returns the SSHAction and localUser after evaluating -// the SSHPolicy for this conn. The pubKey may be nil for "none" auth. -func (c *conn) evaluatePolicy(pubKey gossh.PublicKey) (_ *tailcfg.SSHAction, localUser string, acceptEnv []string, _ error) { +// the SSHPolicy for this conn. +func (c *conn) evaluatePolicy() (_ *tailcfg.SSHAction, localUser string, acceptEnv []string, _ error) { pol, ok := c.sshPolicy() if !ok { return nil, "", nil, fmt.Errorf("tailssh: rejecting connection; no SSH policy") } - a, localUser, acceptEnv, ok := c.evalSSHPolicy(pol, pubKey) + a, localUser, acceptEnv, ok := c.evalSSHPolicy(pol) if !ok { return nil, "", nil, fmt.Errorf("tailssh: rejecting connection; no matching policy") } return a, localUser, acceptEnv, nil } -// pubKeyCacheEntry is the cache value for an HTTPS URL of public keys (like -// "https://github.com/foo.keys") -type pubKeyCacheEntry struct { - lines []string - etag string // if sent by server - at time.Time -} - -const ( - pubKeyCacheDuration = time.Minute // how long to cache non-empty public keys - pubKeyCacheEmptyDuration = 15 * time.Second // how long to cache empty responses -) - -func (srv *server) fetchPublicKeysURLCached(url string) (ce pubKeyCacheEntry, ok bool) { - srv.mu.Lock() - defer srv.mu.Unlock() - // Mostly don't care about the size of this cache. Clean rarely. - if m := srv.fetchPublicKeysCache; len(m) > 50 { - tooOld := srv.now().Add(pubKeyCacheDuration * 10) - for k, ce := range m { - if ce.at.Before(tooOld) { - delete(m, k) - } - } - } - ce, ok = srv.fetchPublicKeysCache[url] - if !ok { - return ce, false - } - maxAge := pubKeyCacheDuration - if len(ce.lines) == 0 { - maxAge = pubKeyCacheEmptyDuration - } - return ce, srv.now().Sub(ce.at) < maxAge -} - -func (srv *server) pubKeyClient() *http.Client { - if srv.pubKeyHTTPClient != nil { - return srv.pubKeyHTTPClient - } - return http.DefaultClient -} - -// fetchPublicKeysURL fetches the public keys from a URL. The strings are in the -// the typical public key "type base64-string [comment]" format seen at e.g. -// https://github.com/USER.keys -func (srv *server) fetchPublicKeysURL(url string) ([]string, error) { - if !strings.HasPrefix(url, "https://") { - return nil, errors.New("invalid URL scheme") - } - - ce, ok := srv.fetchPublicKeysURLCached(url) - if ok { - return ce.lines, nil - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - if ce.etag != "" { - req.Header.Add("If-None-Match", ce.etag) - } - res, err := srv.pubKeyClient().Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - var lines []string - var etag string - switch res.StatusCode { - default: - err = fmt.Errorf("unexpected status %v", res.Status) - srv.logf("fetching public keys from %s: %v", url, err) - case http.StatusNotModified: - lines = ce.lines - etag = ce.etag - case http.StatusOK: - var all []byte - all, err = io.ReadAll(io.LimitReader(res.Body, 4<<10)) - if s := strings.TrimSpace(string(all)); s != "" { - lines = strings.Split(s, "\n") - } - etag = res.Header.Get("Etag") - } - - srv.mu.Lock() - defer srv.mu.Unlock() - mak.Set(&srv.fetchPublicKeysCache, url, pubKeyCacheEntry{ - at: srv.now(), - lines: lines, - etag: etag, - }) - return lines, err -} - // handleSessionPostSSHAuth runs an SSH session after the SSH-level authentication, // but not necessarily before all the Tailscale-level extra verification has // completed. It also handles SFTP requests. @@ -832,18 +668,6 @@ func (c *conn) expandDelegateURLLocked(actionURL string) string { ).Replace(actionURL) } -func (c *conn) expandPublicKeyURL(pubKeyURL string) string { - if !strings.Contains(pubKeyURL, "$") { - return pubKeyURL - } - loginName := c.info.uprof.LoginName - localPart, _, _ := strings.Cut(loginName, "@") - return strings.NewReplacer( - "$LOGINNAME_EMAIL", loginName, - "$LOGINNAME_LOCALPART", localPart, - ).Replace(pubKeyURL) -} - // sshSession is an accepted Tailscale SSH session. type sshSession struct { ssh.Session @@ -894,7 +718,7 @@ func (c *conn) newSSHSession(s ssh.Session) *sshSession { // isStillValid reports whether the conn is still valid. func (c *conn) isStillValid() bool { - a, localUser, _, err := c.evaluatePolicy(c.pubKey) + a, localUser, _, err := c.evaluatePolicy() c.vlogf("stillValid: %+v %v %v", a, localUser, err) if err != nil { return false @@ -1277,9 +1101,9 @@ func (c *conn) ruleExpired(r *tailcfg.SSHRule) bool { return r.RuleExpires.Before(c.srv.now()) } -func (c *conn) evalSSHPolicy(pol *tailcfg.SSHPolicy, pubKey gossh.PublicKey) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, ok bool) { +func (c *conn) evalSSHPolicy(pol *tailcfg.SSHPolicy) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, ok bool) { for _, r := range pol.Rules { - if a, localUser, acceptEnv, err := c.matchRule(r, pubKey); err == nil { + if a, localUser, acceptEnv, err := c.matchRule(r); err == nil { return a, localUser, acceptEnv, true } } @@ -1296,7 +1120,7 @@ var ( errInvalidConn = errors.New("invalid connection state") ) -func (c *conn) matchRule(r *tailcfg.SSHRule, pubKey gossh.PublicKey) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, err error) { +func (c *conn) matchRule(r *tailcfg.SSHRule) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, err error) { defer func() { c.vlogf("matchRule(%+v): %v", r, err) }() @@ -1326,9 +1150,7 @@ func (c *conn) matchRule(r *tailcfg.SSHRule, pubKey gossh.PublicKey) (a *tailcfg return nil, "", nil, errUserMatch } } - if ok, err := c.anyPrincipalMatches(r.Principals, pubKey); err != nil { - return nil, "", nil, err - } else if !ok { + if !c.anyPrincipalMatches(r.Principals) { return nil, "", nil, errPrincipalMatch } return r.Action, localUser, r.AcceptEnv, nil @@ -1345,30 +1167,20 @@ func mapLocalUser(ruleSSHUsers map[string]string, reqSSHUser string) (localUser return v } -func (c *conn) anyPrincipalMatches(ps []*tailcfg.SSHPrincipal, pubKey gossh.PublicKey) (bool, error) { +func (c *conn) anyPrincipalMatches(ps []*tailcfg.SSHPrincipal) bool { for _, p := range ps { if p == nil { continue } - if ok, err := c.principalMatches(p, pubKey); err != nil { - return false, err - } else if ok { - return true, nil + if c.principalMatchesTailscaleIdentity(p) { + return true } } - return false, nil -} - -func (c *conn) principalMatches(p *tailcfg.SSHPrincipal, pubKey gossh.PublicKey) (bool, error) { - if !c.principalMatchesTailscaleIdentity(p) { - return false, nil - } - return c.principalMatchesPubKey(p, pubKey) + return false } // principalMatchesTailscaleIdentity reports whether one of p's four fields // that match the Tailscale identity match (Node, NodeIP, UserLogin, Any). -// This function does not consider PubKeys. func (c *conn) principalMatchesTailscaleIdentity(p *tailcfg.SSHPrincipal) bool { ci := c.info if p.Any { @@ -1388,42 +1200,6 @@ func (c *conn) principalMatchesTailscaleIdentity(p *tailcfg.SSHPrincipal) bool { return false } -func (c *conn) principalMatchesPubKey(p *tailcfg.SSHPrincipal, clientPubKey gossh.PublicKey) (bool, error) { - if len(p.PubKeys) == 0 { - return true, nil - } - if clientPubKey == nil { - return false, nil - } - knownKeys := p.PubKeys - if len(knownKeys) == 1 && strings.HasPrefix(knownKeys[0], "https://") { - var err error - knownKeys, err = c.srv.fetchPublicKeysURL(c.expandPublicKeyURL(knownKeys[0])) - if err != nil { - return false, err - } - } - for _, knownKey := range knownKeys { - if pubKeyMatchesAuthorizedKey(clientPubKey, knownKey) { - return true, nil - } - } - return false, nil -} - -func pubKeyMatchesAuthorizedKey(pubKey ssh.PublicKey, wantKey string) bool { - wantKeyType, rest, ok := strings.Cut(wantKey, " ") - if !ok { - return false - } - if pubKey.Type() != wantKeyType { - return false - } - wantKeyB64, _, _ := strings.Cut(rest, " ") - wantKeyData, _ := base64.StdEncoding.DecodeString(wantKeyB64) - return len(wantKeyData) > 0 && bytes.Equal(pubKey.Marshal(), wantKeyData) -} - func randBytes(n int) []byte { b := make([]byte, n) if _, err := rand.Read(b); err != nil { @@ -1749,7 +1525,6 @@ func envEq(a, b string) bool { var ( metricActiveSessions = clientmetric.NewGauge("ssh_active_sessions") metricIncomingConnections = clientmetric.NewCounter("ssh_incoming_connections") - metricPublicKeyAccepts = clientmetric.NewCounter("ssh_publickey_accepts") // accepted subset of ssh_publickey_connections metricTerminalAccept = clientmetric.NewCounter("ssh_terminalaction_accept") metricTerminalReject = clientmetric.NewCounter("ssh_terminalaction_reject") metricTerminalMalformed = clientmetric.NewCounter("ssh_terminalaction_malformed") diff --git a/ssh/tailssh/tailssh_test.go b/ssh/tailssh/tailssh_test.go index ad9cb1e57b53d..9f3616d8ca8ab 100644 --- a/ssh/tailssh/tailssh_test.go +++ b/ssh/tailssh/tailssh_test.go @@ -10,7 +10,6 @@ import ( "context" "crypto/ed25519" "crypto/rand" - "crypto/sha256" "encoding/json" "errors" "fmt" @@ -229,7 +228,7 @@ func TestMatchRule(t *testing.T) { info: tt.ci, srv: &server{logf: t.Logf}, } - got, gotUser, gotAcceptEnv, err := c.matchRule(tt.rule, nil) + got, gotUser, gotAcceptEnv, err := c.matchRule(tt.rule) if err != tt.wantErr { t.Errorf("err = %v; want %v", err, tt.wantErr) } @@ -348,7 +347,7 @@ func TestEvalSSHPolicy(t *testing.T) { info: tt.ci, srv: &server{logf: t.Logf}, } - got, gotUser, gotAcceptEnv, match := c.evalSSHPolicy(tt.policy, nil) + got, gotUser, gotAcceptEnv, match := c.evalSSHPolicy(tt.policy) if match != tt.wantMatch { t.Errorf("match = %v; want %v", match, tt.wantMatch) } @@ -1129,89 +1128,6 @@ func parseEnv(out []byte) map[string]string { return e } -func TestPublicKeyFetching(t *testing.T) { - var reqsTotal, reqsIfNoneMatchHit, reqsIfNoneMatchMiss int32 - ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atomic.AddInt32((&reqsTotal), 1) - etag := fmt.Sprintf("W/%q", sha256.Sum256([]byte(r.URL.Path))) - w.Header().Set("Etag", etag) - if v := r.Header.Get("If-None-Match"); v != "" { - if v == etag { - atomic.AddInt32(&reqsIfNoneMatchHit, 1) - w.WriteHeader(304) - return - } - atomic.AddInt32(&reqsIfNoneMatchMiss, 1) - } - io.WriteString(w, "foo\nbar\n"+string(r.URL.Path)+"\n") - })) - ts.StartTLS() - defer ts.Close() - keys := ts.URL - - clock := &tstest.Clock{} - srv := &server{ - pubKeyHTTPClient: ts.Client(), - timeNow: clock.Now, - } - for range 2 { - got, err := srv.fetchPublicKeysURL(keys + "/alice.keys") - if err != nil { - t.Fatal(err) - } - if want := []string{"foo", "bar", "/alice.keys"}; !reflect.DeepEqual(got, want) { - t.Errorf("got %q; want %q", got, want) - } - } - if got, want := atomic.LoadInt32(&reqsTotal), int32(1); got != want { - t.Errorf("got %d requests; want %d", got, want) - } - if got, want := atomic.LoadInt32(&reqsIfNoneMatchHit), int32(0); got != want { - t.Errorf("got %d etag hits; want %d", got, want) - } - clock.Advance(5 * time.Minute) - got, err := srv.fetchPublicKeysURL(keys + "/alice.keys") - if err != nil { - t.Fatal(err) - } - if want := []string{"foo", "bar", "/alice.keys"}; !reflect.DeepEqual(got, want) { - t.Errorf("got %q; want %q", got, want) - } - if got, want := atomic.LoadInt32(&reqsTotal), int32(2); got != want { - t.Errorf("got %d requests; want %d", got, want) - } - if got, want := atomic.LoadInt32(&reqsIfNoneMatchHit), int32(1); got != want { - t.Errorf("got %d etag hits; want %d", got, want) - } - if got, want := atomic.LoadInt32(&reqsIfNoneMatchMiss), int32(0); got != want { - t.Errorf("got %d etag misses; want %d", got, want) - } - -} - -func TestExpandPublicKeyURL(t *testing.T) { - c := &conn{ - info: &sshConnInfo{ - uprof: tailcfg.UserProfile{ - LoginName: "bar@baz.tld", - }, - }, - } - if got, want := c.expandPublicKeyURL("foo"), "foo"; got != want { - t.Errorf("basic: got %q; want %q", got, want) - } - if got, want := c.expandPublicKeyURL("https://example.com/$LOGINNAME_LOCALPART.keys"), "https://example.com/bar.keys"; got != want { - t.Errorf("localpart: got %q; want %q", got, want) - } - if got, want := c.expandPublicKeyURL("https://example.com/keys?email=$LOGINNAME_EMAIL"), "https://example.com/keys?email=bar@baz.tld"; got != want { - t.Errorf("email: got %q; want %q", got, want) - } - c.info = new(sshConnInfo) - if got, want := c.expandPublicKeyURL("https://example.com/keys?email=$LOGINNAME_EMAIL"), "https://example.com/keys?email="; got != want { - t.Errorf("on empty: got %q; want %q", got, want) - } -} - func TestAcceptEnvPair(t *testing.T) { tests := []struct { in string diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 897e8d27f7f7b..be6c4f0be6b82 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -152,7 +152,8 @@ type CapabilityVersion int // - 107: 2024-10-30: add App Connector to conffile (PR #13942) // - 108: 2024-11-08: Client sends ServicesHash in Hostinfo, understands c2n GET /vip-services. // - 109: 2024-11-18: Client supports filtertype.Match.SrcCaps (issue #12542) -const CurrentCapabilityVersion CapabilityVersion = 109 +// - 110: 2024-12-12: removed never-before-used Tailscale SSH public key support (#14373) +const CurrentCapabilityVersion CapabilityVersion = 110 type StableID string @@ -2525,16 +2526,13 @@ type SSHPrincipal struct { Any bool `json:"any,omitempty"` // if true, match any connection // TODO(bradfitz): add StableUserID, once that exists - // PubKeys, if non-empty, means that this SSHPrincipal only - // matches if one of these public keys is presented by the user. + // UnusedPubKeys was public key support. It never became an official product + // feature and so as of 2024-12-12 is being removed. + // This stub exists to remind us not to re-use the JSON field name "pubKeys" + // in the future if we bring it back with different semantics. // - // As a special case, if len(PubKeys) == 1 and PubKeys[0] starts - // with "https://", then it's fetched (like https://github.com/username.keys). - // In that case, the following variable expansions are also supported - // in the URL: - // * $LOGINNAME_EMAIL ("foo@bar.com" or "foo@github") - // * $LOGINNAME_LOCALPART (the "foo" from either of the above) - PubKeys []string `json:"pubKeys,omitempty"` + // Deprecated: do not use. It does nothing. + UnusedPubKeys []string `json:"pubKeys,omitempty"` } // SSHAction is how to handle an incoming connection. diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index f4f02c01721dc..bf9bac2980df9 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -556,17 +556,17 @@ func (src *SSHPrincipal) Clone() *SSHPrincipal { } dst := new(SSHPrincipal) *dst = *src - dst.PubKeys = append(src.PubKeys[:0:0], src.PubKeys...) + dst.UnusedPubKeys = append(src.UnusedPubKeys[:0:0], src.UnusedPubKeys...) return dst } // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _SSHPrincipalCloneNeedsRegeneration = SSHPrincipal(struct { - Node StableNodeID - NodeIP string - UserLogin string - Any bool - PubKeys []string + Node StableNodeID + NodeIP string + UserLogin string + Any bool + UnusedPubKeys []string }{}) // Clone makes a deep copy of ControlDialPlan. diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index f275a6a9da5f2..6c21e5f450340 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -1260,19 +1260,21 @@ func (v *SSHPrincipalView) UnmarshalJSON(b []byte) error { return nil } -func (v SSHPrincipalView) Node() StableNodeID { return v.Đļ.Node } -func (v SSHPrincipalView) NodeIP() string { return v.Đļ.NodeIP } -func (v SSHPrincipalView) UserLogin() string { return v.Đļ.UserLogin } -func (v SSHPrincipalView) Any() bool { return v.Đļ.Any } -func (v SSHPrincipalView) PubKeys() views.Slice[string] { return views.SliceOf(v.Đļ.PubKeys) } +func (v SSHPrincipalView) Node() StableNodeID { return v.Đļ.Node } +func (v SSHPrincipalView) NodeIP() string { return v.Đļ.NodeIP } +func (v SSHPrincipalView) UserLogin() string { return v.Đļ.UserLogin } +func (v SSHPrincipalView) Any() bool { return v.Đļ.Any } +func (v SSHPrincipalView) UnusedPubKeys() views.Slice[string] { + return views.SliceOf(v.Đļ.UnusedPubKeys) +} // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _SSHPrincipalViewNeedsRegeneration = SSHPrincipal(struct { - Node StableNodeID - NodeIP string - UserLogin string - Any bool - PubKeys []string + Node StableNodeID + NodeIP string + UserLogin string + Any bool + UnusedPubKeys []string }{}) // View returns a readonly view of ControlDialPlan. From aa04f61d5ef74fcb11373490876740a0bd9b2bac Mon Sep 17 00:00:00 2001 From: James Tucker Date: Thu, 5 Dec 2024 12:42:45 -0800 Subject: [PATCH 022/223] net/netcheck: adjust HTTPS latency check to connection time and avoid data race The go-httpstat package has a data race when used with connections that are performing happy-eyeballs connection setups as we are in the DERP client. There is a long-stale PR upstream to address this, however revisiting the purpose of this code suggests we don't really need httpstat here. The code populates a latency table that may be used to compare to STUN latency, which is a lightweight RTT check. Switching out the reported timing here to simply the request HTTP request RTT avoids the problematic package. Fixes tailscale/corp#25095 Signed-off-by: James Tucker --- cmd/k8s-operator/depaware.txt | 1 - cmd/tailscale/depaware.txt | 3 +-- cmd/tailscaled/depaware.txt | 3 +-- net/netcheck/netcheck.go | 24 +++++++++++++++++------- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index d1d687432863b..0e42fe2b60dd0 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -225,7 +225,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/tailscale/wireguard-go/rwcancel from github.com/tailscale/wireguard-go/device+ github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device đŸ’Ŗ github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+ - github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4 L github.com/u-root/uio/uio from github.com/insomniacslk/dhcp/dhcpv4+ L github.com/vishvananda/netns from github.com/tailscale/netlink+ diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index d18d8887327fa..a8496c411b99a 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -58,7 +58,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep L đŸ’Ŗ github.com/tailscale/netlink from tailscale.com/util/linuxfw L đŸ’Ŗ github.com/tailscale/netlink/nl from github.com/tailscale/netlink github.com/tailscale/web-client-prebuilt from tailscale.com/client/web - github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck github.com/toqueteos/webbrowser from tailscale.com/cmd/tailscale/cli L github.com/vishvananda/netns from github.com/tailscale/netlink+ github.com/x448/float16 from github.com/fxamacker/cbor/v2 @@ -306,7 +305,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep net from crypto/tls+ net/http from expvar+ net/http/cgi from tailscale.com/cmd/tailscale/cli - net/http/httptrace from github.com/tcnksm/go-httpstat+ + net/http/httptrace from golang.org/x/net/http2+ net/http/httputil from tailscale.com/client/web+ net/http/internal from net/http+ net/netip from go4.org/netipx+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 81cd53271cf9e..264f8296f1e92 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -181,7 +181,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de đŸ’Ŗ github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+ github.com/tailscale/xnet/webdav from tailscale.com/drive/driveimpl+ github.com/tailscale/xnet/webdav/internal/xml from github.com/tailscale/xnet/webdav - github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck LD github.com/u-root/u-root/pkg/termios from tailscale.com/ssh/tailssh L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4 L github.com/u-root/uio/uio from github.com/insomniacslk/dhcp/dhcpv4+ @@ -553,7 +552,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de net from crypto/tls+ net/http from expvar+ net/http/httptest from tailscale.com/control/controlclient - net/http/httptrace from github.com/tcnksm/go-httpstat+ + net/http/httptrace from github.com/prometheus-community/pro-bing+ net/http/httputil from github.com/aws/smithy-go/transport/http+ net/http/internal from net/http+ net/http/pprof from tailscale.com/cmd/tailscaled+ diff --git a/net/netcheck/netcheck.go b/net/netcheck/netcheck.go index 7930f88f6dce6..c32eeee8b3ecd 100644 --- a/net/netcheck/netcheck.go +++ b/net/netcheck/netcheck.go @@ -23,7 +23,6 @@ import ( "syscall" "time" - "github.com/tcnksm/go-httpstat" "tailscale.com/derp/derphttp" "tailscale.com/envknob" "tailscale.com/net/captivedetection" @@ -1110,10 +1109,11 @@ func (c *Client) runHTTPOnlyChecks(ctx context.Context, last *Report, rs *report return nil } +// measureHTTPSLatency measures HTTP request latency to the DERP region, but +// only returns success if an HTTPS request to the region succeeds. func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegion) (time.Duration, netip.Addr, error) { metricHTTPSend.Add(1) - var result httpstat.Result - ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), httpsProbeTimeout) + ctx, cancel := context.WithTimeout(ctx, httpsProbeTimeout) defer cancel() var ip netip.Addr @@ -1121,6 +1121,8 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio dc := derphttp.NewNetcheckClient(c.logf, c.NetMon) defer dc.Close() + // DialRegionTLS may dial multiple times if a node is not available, as such + // it does not have stable timing to measure. tlsConn, tcpConn, node, err := dc.DialRegionTLS(ctx, reg) if err != nil { return 0, ip, err @@ -1138,6 +1140,8 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio connc := make(chan *tls.Conn, 1) connc <- tlsConn + // make an HTTP request to measure, as this enables us to account for MITM + // overhead in e.g. corp environments that have HTTP MITM in front of DERP. tr := &http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, errors.New("unexpected DialContext dial") @@ -1153,12 +1157,17 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio } hc := &http.Client{Transport: tr} + // This is the request that will be measured, the request and response + // should be small enough to fit into a single packet each way unless the + // connection has already become unstable. req, err := http.NewRequestWithContext(ctx, "GET", "https://"+node.HostName+"/derp/latency-check", nil) if err != nil { return 0, ip, err } + startTime := c.timeNow() resp, err := hc.Do(req) + reqDur := c.timeNow().Sub(startTime) if err != nil { return 0, ip, err } @@ -1175,11 +1184,12 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio if err != nil { return 0, ip, err } - result.End(c.timeNow()) - // TODO: decide best timing heuristic here. - // Maybe the server should return the tcpinfo_rtt? - return result.ServerProcessing, ip, nil + // return the connection duration, not the request duration, as this is the + // best approximation of the RTT latency to the node. Note that the + // connection setup performs happy-eyeballs and TLS so there are additional + // overheads. + return reqDur, ip, nil } func (c *Client) measureAllICMPLatency(ctx context.Context, rs *reportState, need []*tailcfg.DERPRegion) error { From 1ed9bd76d682299376f404521cf1958a7f9bea7a Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Tue, 10 Dec 2024 11:52:51 -0600 Subject: [PATCH 023/223] prober: perform DERP bandwidth probes over TUN device to mimic real client Updates tailscale/corp#24635 Co-authored-by: Mario Minardi Signed-off-by: Percy Wegmann --- cmd/derpprobe/derpprobe.go | 27 ++-- prober/derp.go | 323 +++++++++++++++++++++++++++++++++++-- prober/tun_darwin.go | 35 ++++ prober/tun_default.go | 18 +++ prober/tun_linux.go | 36 +++++ 5 files changed, 411 insertions(+), 28 deletions(-) create mode 100644 prober/tun_darwin.go create mode 100644 prober/tun_default.go create mode 100644 prober/tun_linux.go diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 8f04326b03980..620b966099176 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -18,18 +18,19 @@ import ( ) var ( - derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") - versionFlag = flag.Bool("version", false, "print version and exit") - listen = flag.String("listen", ":8030", "HTTP listen address") - probeOnce = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag") - spread = flag.Bool("spread", true, "whether to spread probing over time") - interval = flag.Duration("interval", 15*time.Second, "probe interval") - meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval") - stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval") - tlsInterval = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval") - bwInterval = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)") - bwSize = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size") - regionCode = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed") + derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") + versionFlag = flag.Bool("version", false, "print version and exit") + listen = flag.String("listen", ":8030", "HTTP listen address") + probeOnce = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag") + spread = flag.Bool("spread", true, "whether to spread probing over time") + interval = flag.Duration("interval", 15*time.Second, "probe interval") + meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval") + stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval") + tlsInterval = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval") + bwInterval = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)") + bwSize = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size") + bwTUNIPv4Address = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. We will use a /30 subnet including this IP address.") + regionCode = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed") ) func main() { @@ -46,7 +47,7 @@ func main() { prober.WithTLSProbing(*tlsInterval), } if *bwInterval > 0 { - opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize)) + opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address)) } if *regionCode != "" { opts = append(opts, prober.WithRegion(*regionCode)) diff --git a/prober/derp.go b/prober/derp.go index bce40e34c8519..8e8e6ac3dda97 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -12,20 +12,27 @@ import ( "errors" "expvar" "fmt" + "io" "log" "net" "net/http" + "net/netip" "strconv" "strings" "sync" "time" "github.com/prometheus/client_golang/prometheus" + wgconn "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" + "github.com/tailscale/wireguard-go/tun" + "go4.org/netipx" "tailscale.com/client/tailscale" "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/net/netmon" "tailscale.com/net/stun" + "tailscale.com/net/tstun" "tailscale.com/syncs" "tailscale.com/tailcfg" "tailscale.com/types/key" @@ -42,8 +49,9 @@ type derpProber struct { tlsInterval time.Duration // Optional bandwidth probing. - bwInterval time.Duration - bwProbeSize int64 + bwInterval time.Duration + bwProbeSize int64 + bwTUNIPv4Prefix *netip.Prefix // Optionally restrict probes to a single regionCode. regionCode string @@ -68,11 +76,18 @@ type DERPOpt func(*derpProber) // WithBandwidthProbing enables bandwidth probing. When enabled, a payload of // `size` bytes will be regularly transferred through each DERP server, and each -// pair of DERP servers in every region. -func WithBandwidthProbing(interval time.Duration, size int64) DERPOpt { +// pair of DERP servers in every region. If tunAddress is specified, probes will +// use a TCP connection over a TUN device at this address in order to exercise +// TCP-in-TCP in similar fashion to TCP over Tailscale via DERP +func WithBandwidthProbing(interval time.Duration, size int64, tunAddress string) DERPOpt { return func(d *derpProber) { d.bwInterval = interval d.bwProbeSize = size + prefix, err := netip.ParsePrefix(fmt.Sprintf("%s/30", tunAddress)) + if err != nil { + log.Fatalf("failed to parse IP prefix from bw-tun-ipv4-addr: %v", err) + } + d.bwTUNIPv4Prefix = &prefix } } @@ -200,7 +215,11 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { n := fmt.Sprintf("derp/%s/%s/%s/bw", region.RegionCode, server.Name, to.Name) wantProbes[n] = true if d.probes[n] == nil { - log.Printf("adding DERP bandwidth probe for %s->%s (%s) %v bytes every %v", server.Name, to.Name, region.RegionName, d.bwProbeSize, d.bwInterval) + tunString := "" + if d.bwTUNIPv4Prefix != nil { + tunString = " (TUN)" + } + log.Printf("adding%s DERP bandwidth probe for %s->%s (%s) %v bytes every %v", tunString, server.Name, to.Name, region.RegionName, d.bwProbeSize, d.bwInterval) d.probes[n] = d.p.Run(n, d.bwInterval, labels, d.bwProbeFn(server.Name, to.Name, d.bwProbeSize)) } } @@ -251,21 +270,24 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { if from == to { derpPath = "single" } - var transferTime expvar.Float + var transferTimeSeconds expvar.Float return ProbeClass{ Probe: func(ctx context.Context) error { fromN, toN, err := d.getNodePair(from, to) if err != nil { return err } - return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTime) + return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, d.bwTUNIPv4Prefix) + }, + Class: "derp_bw", + Labels: Labels{ + "derp_path": derpPath, + "tcp_in_tcp": strconv.FormatBool(d.bwTUNIPv4Prefix != nil), }, - Class: "derp_bw", - Labels: Labels{"derp_path": derpPath}, Metrics: func(l prometheus.Labels) []prometheus.Metric { return []prometheus.Metric{ prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_probe_size_bytes", "Payload size of the bandwidth prober", nil, l), prometheus.GaugeValue, float64(size)), - prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_transfer_time_seconds_total", "Time it took to transfer data", nil, l), prometheus.CounterValue, transferTime.Value()), + prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_transfer_time_seconds_total", "Time it took to transfer data", nil, l), prometheus.CounterValue, transferTimeSeconds.Value()), } }, } @@ -412,8 +434,10 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error { } // derpProbeBandwidth sends a payload of a given size between two local -// DERP clients connected to two DERP servers. -func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTime *expvar.Float) (err error) { +// DERP clients connected to two DERP servers.If tunIPv4Address is specified, +// probes will use a TCP connection over a TUN device at this address in order +// to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. +func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper logs with every packet // sent by the bandwidth probe. fromc, err := newConn(ctx, dm, from, false) @@ -434,10 +458,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail time.Sleep(100 * time.Millisecond) // pretty arbitrary } - start := time.Now() - defer func() { transferTime.Add(time.Since(start).Seconds()) }() + if tunIPv4Prefix != nil { + err = derpProbeBandwidthTUN(ctx, transferTimeSeconds, from, to, fromc, toc, size, tunIPv4Prefix) + } else { + err = derpProbeBandwidthDirect(ctx, transferTimeSeconds, from, to, fromc, toc, size) + } - if err := runDerpProbeNodePair(ctx, from, to, fromc, toc, size); err != nil { + if err != nil { // Record pubkeys on failed probes to aid investigation. return fmt.Errorf("%s -> %s: %w", fromc.SelfPublicKey().ShortString(), @@ -577,6 +604,272 @@ func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc return nil } +// derpProbeBandwidthDirect takes two DERP clients (fromc and toc) connected to two +// DERP servers (from and to) and sends a test payload of a given size from one +// to another using runDerpProbeNodePair. The time taken to finish the transfer is +// recorded in `transferTimeSeconds`. +func derpProbeBandwidthDirect(ctx context.Context, transferTimeSeconds *expvar.Float, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, size int64) error { + start := time.Now() + defer func() { transferTimeSeconds.Add(time.Since(start).Seconds()) }() + + return runDerpProbeNodePair(ctx, from, to, fromc, toc, size) +} + +// derpProbeBandwidthTUNMu ensures that TUN bandwidth probes don't run concurrently. +// This is necessary to avoid conflicts trying to create the TUN device, and +// it also has the nice benefit of preventing concurrent bandwidth probes from +// influencing each other's results. +// +// This guards derpProbeBandwidthTUN. +var derpProbeBandwidthTUNMu sync.Mutex + +// derpProbeBandwidthTUN takes two DERP clients (fromc and toc) connected to two +// DERP servers (from and to) and sends a test payload of a given size from one +// to another over a TUN device at an address at the start of the usable host IP +// range that the given tunAddress lives in. The time taken to finish the transfer +// is recorded in `transferTimeSeconds`. +func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds *expvar.Float, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, size int64, prefix *netip.Prefix) error { + // Make sure all goroutines have finished. + var wg sync.WaitGroup + defer wg.Wait() + + // Close the clients to make sure goroutines that are reading/writing from them terminate. + defer fromc.Close() + defer toc.Close() + + ipRange := netipx.RangeOfPrefix(*prefix) + // Start of the usable host IP range from the address we have been passed in. + ifAddr := ipRange.From().Next() + // Destination address to dial. This is the next address in the range from + // our ifAddr to ensure that the underlying networking stack is actually being + // utilized instead of being optimized away and treated as a loopback. Packets + // sent to this address will be routed over the TUN. + destinationAddr := ifAddr.Next() + + derpProbeBandwidthTUNMu.Lock() + defer derpProbeBandwidthTUNMu.Unlock() + + // Temporarily set up a TUN device with which to simulate a real client TCP connection + // tunneling over DERP. Use `tstun.DefaultTUNMTU()` (e.g., 1280) as our MTU as this is + // the minimum safe MTU used by Tailscale. + dev, err := tun.CreateTUN(tunName, int(tstun.DefaultTUNMTU())) + if err != nil { + return fmt.Errorf("failed to create TUN device: %w", err) + } + defer func() { + if err := dev.Close(); err != nil { + log.Printf("failed to close TUN device: %s", err) + } + }() + mtu, err := dev.MTU() + if err != nil { + return fmt.Errorf("failed to get TUN MTU: %w", err) + } + + name, err := dev.Name() + if err != nil { + return fmt.Errorf("failed to get device name: %w", err) + } + + // Perform platform specific configuration of the TUN device. + err = configureTUN(*prefix, name) + if err != nil { + return fmt.Errorf("failed to configure tun: %w", err) + } + + // Depending on platform, we need some space for headers at the front + // of TUN I/O op buffers. The below constant is more than enough space + // for any platform that this might run on. + tunStartOffset := device.MessageTransportHeaderSize + + // This goroutine reads packets from the TUN device and evaluates if they + // are IPv4 packets destined for loopback via DERP. If so, it performs L3 NAT + // (swap src/dst) and writes them towards DERP in order to loopback via the + // `toc` DERP client. It only reports errors to `tunReadErrC`. + wg.Add(1) + tunReadErrC := make(chan error, 1) + go func() { + defer wg.Done() + + numBufs := wgconn.IdealBatchSize + bufs := make([][]byte, 0, numBufs) + sizes := make([]int, numBufs) + for range numBufs { + bufs = append(bufs, make([]byte, mtu+tunStartOffset)) + } + + destinationAddrBytes := destinationAddr.AsSlice() + scratch := make([]byte, 4) + for { + n, err := dev.Read(bufs, sizes, tunStartOffset) + if err != nil { + tunReadErrC <- err + return + } + + for i := range n { + pkt := bufs[i][tunStartOffset : sizes[i]+tunStartOffset] + // Skip everything except valid IPv4 packets + if len(pkt) < 20 { + // Doesn't even have a full IPv4 header + continue + } + if pkt[0]>>4 != 4 { + // Not IPv4 + continue + } + + if !bytes.Equal(pkt[16:20], destinationAddrBytes) { + // Unexpected dst address + continue + } + + copy(scratch, pkt[12:16]) + copy(pkt[12:16], pkt[16:20]) + copy(pkt[16:20], scratch) + + if err := fromc.Send(toc.SelfPublicKey(), pkt); err != nil { + tunReadErrC <- err + return + } + } + } + }() + + // This goroutine reads packets from the `toc` DERP client and writes them towards the TUN. + // It only reports errors to `recvErrC` channel. + wg.Add(1) + recvErrC := make(chan error, 1) + go func() { + defer wg.Done() + + buf := make([]byte, mtu+tunStartOffset) + bufs := make([][]byte, 1) + + for { + m, err := toc.Recv() + if err != nil { + recvErrC <- fmt.Errorf("failed to receive: %w", err) + return + } + switch v := m.(type) { + case derp.ReceivedPacket: + if v.Source != fromc.SelfPublicKey() { + recvErrC <- fmt.Errorf("got data packet from unexpected source, %v", v.Source) + return + } + pkt := v.Data + copy(buf[tunStartOffset:], pkt) + bufs[0] = buf[:len(pkt)+tunStartOffset] + if _, err := dev.Write(bufs, tunStartOffset); err != nil { + recvErrC <- fmt.Errorf("failed to write to TUN device: %w", err) + return + } + case derp.KeepAliveMessage: + // Silently ignore. + default: + log.Printf("%v: ignoring Recv frame type %T", to.Name, v) + // Loop. + } + } + }() + + // Start a listener to receive the data + l, err := net.Listen("tcp", net.JoinHostPort(ifAddr.String(), "0")) + if err != nil { + return fmt.Errorf("failed to listen: %s", err) + } + defer l.Close() + + // 128KB by default + const writeChunkSize = 128 << 10 + + randData := make([]byte, writeChunkSize) + _, err = crand.Read(randData) + if err != nil { + return fmt.Errorf("failed to initialize random data: %w", err) + } + + // Dial ourselves + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return fmt.Errorf("failed to split address %q: %w", l.Addr().String(), err) + } + + connAddr := net.JoinHostPort(destinationAddr.String(), port) + conn, err := net.Dial("tcp", connAddr) + if err != nil { + return fmt.Errorf("failed to dial address %q: %w", connAddr, err) + } + defer conn.Close() + + // Timing only includes the actual sending and receiving of data. + start := time.Now() + + // This goroutine reads data from the TCP stream being looped back via DERP. + // It reports to `readFinishedC` when `size` bytes have been read, or if an + // error occurs. + wg.Add(1) + readFinishedC := make(chan error, 1) + go func() { + defer wg.Done() + + readConn, err := l.Accept() + if err != nil { + readFinishedC <- err + } + defer readConn.Close() + deadline, ok := ctx.Deadline() + if ok { + // Don't try reading past our context's deadline. + if err := readConn.SetReadDeadline(deadline); err != nil { + readFinishedC <- fmt.Errorf("unable to set read deadline: %w", err) + } + } + _, err = io.CopyN(io.Discard, readConn, size) + // Measure transfer time irrespective of whether it succeeded or failed. + transferTimeSeconds.Add(time.Since(start).Seconds()) + readFinishedC <- err + }() + + // This goroutine sends data to the TCP stream being looped back via DERP. + // It only reports errors to `sendErrC`. + wg.Add(1) + sendErrC := make(chan error, 1) + go func() { + defer wg.Done() + + for wrote := 0; wrote < int(size); wrote += len(randData) { + b := randData + if wrote+len(randData) > int(size) { + // This is the last chunk and we don't need the whole thing + b = b[0 : int(size)-wrote] + } + if _, err := conn.Write(b); err != nil { + sendErrC <- fmt.Errorf("failed to write to conn: %w", err) + return + } + } + }() + + select { + case <-ctx.Done(): + return fmt.Errorf("timeout: %w", ctx.Err()) + case err := <-tunReadErrC: + return fmt.Errorf("error reading from TUN via %q: %w", from.Name, err) + case err := <-sendErrC: + return fmt.Errorf("error sending via %q: %w", from.Name, err) + case err := <-recvErrC: + return fmt.Errorf("error receiving from %q: %w", to.Name, err) + case err := <-readFinishedC: + if err != nil { + return fmt.Errorf("error reading from %q to TUN: %w", to.Name, err) + } + } + + return nil +} + func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) { // To avoid spamming the log with regular connection messages. l := logger.Filtered(log.Printf, func(s string) bool { diff --git a/prober/tun_darwin.go b/prober/tun_darwin.go new file mode 100644 index 0000000000000..0ef22e41e4076 --- /dev/null +++ b/prober/tun_darwin.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin + +package prober + +import ( + "fmt" + "net/netip" + "os/exec" + + "go4.org/netipx" +) + +const tunName = "utun" + +func configureTUN(addr netip.Prefix, tunname string) error { + cmd := exec.Command("ifconfig", tunname, "inet", addr.String(), addr.Addr().String()) + res, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to add address: %w (%s)", err, string(res)) + } + + net := netipx.PrefixIPNet(addr) + nip := net.IP.Mask(net.Mask) + nstr := fmt.Sprintf("%v/%d", nip, addr.Bits()) + cmd = exec.Command("route", "-q", "-n", "add", "-inet", nstr, "-iface", addr.Addr().String()) + res, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to add route: %w (%s)", err, string(res)) + } + + return nil +} diff --git a/prober/tun_default.go b/prober/tun_default.go new file mode 100644 index 0000000000000..93a5b07fd442a --- /dev/null +++ b/prober/tun_default.go @@ -0,0 +1,18 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !linux && !darwin + +package prober + +import ( + "fmt" + "net/netip" + "runtime" +) + +const tunName = "unused" + +func configureTUN(addr netip.Prefix, tunname string) error { + return fmt.Errorf("not implemented on " + runtime.GOOS) +} diff --git a/prober/tun_linux.go b/prober/tun_linux.go new file mode 100644 index 0000000000000..52a31efbbf66a --- /dev/null +++ b/prober/tun_linux.go @@ -0,0 +1,36 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package prober + +import ( + "fmt" + "net/netip" + + "github.com/tailscale/netlink" + "go4.org/netipx" +) + +const tunName = "derpprobe" + +func configureTUN(addr netip.Prefix, tunname string) error { + link, err := netlink.LinkByName(tunname) + if err != nil { + return fmt.Errorf("failed to look up link %q: %w", tunname, err) + } + + // We need to bring the TUN device up before assigning an address. This + // allows the OS to automatically create a route for it. Otherwise, we'd + // have to manually create the route. + if err := netlink.LinkSetUp(link); err != nil { + return fmt.Errorf("failed to bring tun %q up: %w", tunname, err) + } + + if err := netlink.AddrReplace(link, &netlink.Addr{IPNet: netipx.PrefixIPNet(addr)}); err != nil { + return fmt.Errorf("failed to add address: %w", err) + } + + return nil +} From cc168d9f6bcd958872fd2d3b8999222a459f9d9a Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 16 Dec 2024 06:11:18 +0000 Subject: [PATCH 024/223] cmd/k8s-operator: fix ProxyGroup hostname (#14336) Updates tailscale/tailscale#14325 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/proxygroup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 39b7ccc01f6fb..60f470fc28bb5 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -479,7 +479,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32 } if pg.Spec.HostnamePrefix != "" { - conf.Hostname = ptr.To(fmt.Sprintf("%s%d", pg.Spec.HostnamePrefix, idx)) + conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) } if shouldAcceptRoutes(class) { From 5883ca72a7397df487a26ab64ed02b0086405b35 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Mon, 16 Dec 2024 10:56:55 -0800 Subject: [PATCH 025/223] types/opt: fix test to be agnostic to omitzero support (#14401) The omitzero tag option has been backported to v1 "encoding/json" from the "encoding/json/v2" prototype and will land in Go1.24. Until we fully upgrade to Go1.24, adjust the test to be agnostic to which version of Go someone is using. Updates tailscale/corp#25406 Signed-off-by: Joe Tsai --- types/opt/value_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/types/opt/value_test.go b/types/opt/value_test.go index 93d935e27581f..dbd8b255f5c7d 100644 --- a/types/opt/value_test.go +++ b/types/opt/value_test.go @@ -9,6 +9,8 @@ import ( "testing" jsonv2 "github.com/go-json-experiment/json" + "tailscale.com/types/bools" + "tailscale.com/util/must" ) type testStruct struct { @@ -87,7 +89,14 @@ func TestValue(t *testing.T) { False: ValueOf(false), ExplicitUnset: Value[bool]{}, }, - want: `{"True":true,"False":false,"Unset":null,"ExplicitUnset":null}`, + want: bools.IfElse( + // Detect whether v1 "encoding/json" supports `omitzero` or not. + // TODO(Go1.24): Remove this after `omitzero` is supported. + string(must.Get(json.Marshal(struct { + X int `json:",omitzero"` + }{}))) == `{}`, + `{"True":true,"False":false}`, // omitzero supported + `{"True":true,"False":false,"Unset":null,"ExplicitUnset":null}`), // omitzero not supported wantBack: struct { True Value[bool] `json:",omitzero"` False Value[bool] `json:",omitzero"` From 0cc2a8dc0d086f114f1031ef3cb621b8413ac946 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 16 Dec 2024 10:19:10 -0800 Subject: [PATCH 026/223] go.toolchain.rev: bump Go toolchain For https://github.com/tailscale/go/pull/108 so we can depend on it in other repos. (This repo can't yet use it; we permit building tailscale/tailscale with the latest stock Go release) But that will be in Go 1.24. We're just impatient elsewhere and would like it in the control plane code earlier. Updates tailscale/corp#25406 Change-Id: I53ff367318365c465cbd02cea387c8ff1eb49fab Signed-off-by: Brad Fitzpatrick --- go.toolchain.rev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.toolchain.rev b/go.toolchain.rev index 500d853e5e4bd..7be85deb6e796 100644 --- a/go.toolchain.rev +++ b/go.toolchain.rev @@ -1 +1 @@ -96578f73d04e1a231fa2a495ad3fa97747785bc6 +e005697288a8d2fadc87bb7c3e2c74778d08554a From 2506b81471914ad10fe40476e2f9aae25777cee6 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 16 Dec 2024 12:11:38 -0800 Subject: [PATCH 027/223] prober: fix WithBandwidthProbing behavior with optional tunAddress 1ed9bd76d682299376f404521cf1958a7f9bea7a meant to make tunAddress be optional. Updates tailscale/corp#24635 Change-Id: Idc4a8540b294e480df5bd291967024c04df751c0 Signed-off-by: Brad Fitzpatrick --- prober/derp.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/prober/derp.go b/prober/derp.go index 8e8e6ac3dda97..742e8a5f42e57 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -51,7 +51,7 @@ type derpProber struct { // Optional bandwidth probing. bwInterval time.Duration bwProbeSize int64 - bwTUNIPv4Prefix *netip.Prefix + bwTUNIPv4Prefix *netip.Prefix // or nil to not use TUN // Optionally restrict probes to a single regionCode. regionCode string @@ -78,16 +78,18 @@ type DERPOpt func(*derpProber) // `size` bytes will be regularly transferred through each DERP server, and each // pair of DERP servers in every region. If tunAddress is specified, probes will // use a TCP connection over a TUN device at this address in order to exercise -// TCP-in-TCP in similar fashion to TCP over Tailscale via DERP +// TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. func WithBandwidthProbing(interval time.Duration, size int64, tunAddress string) DERPOpt { return func(d *derpProber) { d.bwInterval = interval d.bwProbeSize = size - prefix, err := netip.ParsePrefix(fmt.Sprintf("%s/30", tunAddress)) - if err != nil { - log.Fatalf("failed to parse IP prefix from bw-tun-ipv4-addr: %v", err) + if tunAddress != "" { + prefix, err := netip.ParsePrefix(fmt.Sprintf("%s/30", tunAddress)) + if err != nil { + log.Fatalf("failed to parse IP prefix from bw-tun-ipv4-addr: %v", err) + } + d.bwTUNIPv4Prefix = &prefix } - d.bwTUNIPv4Prefix = &prefix } } From b62a013ecbaff241fda500e95365bc28b77595b0 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Mon, 16 Dec 2024 14:53:34 -0800 Subject: [PATCH 028/223] Switch logging service from log.tailscale.io to log.tailscale.com (#14398) Updates tailscale/corp#23617 Signed-off-by: Joe Tsai --- cmd/derper/bootstrap_dns_test.go | 12 ++++++------ docs/windows/policy/en-US/tailscale.adml | 2 +- ipn/ipnserver/proxyconnect.go | 2 +- logpolicy/logpolicy.go | 2 +- logpolicy/logpolicy_test.go | 4 +++- logtail/api.md | 4 ++-- logtail/example/logadopt/logadopt.go | 2 +- logtail/example/logreprocess/demo.sh | 2 +- logtail/example/logreprocess/logreprocess.go | 2 +- logtail/logtail.go | 8 ++++---- net/tlsdial/tlsdial.go | 4 ++-- tsnet/tsnet.go | 2 +- tstest/natlab/vnet/vip.go | 2 +- tstest/natlab/vnet/vnet.go | 2 +- 14 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cmd/derper/bootstrap_dns_test.go b/cmd/derper/bootstrap_dns_test.go index d151bc2b05fdf..9b99103abfe33 100644 --- a/cmd/derper/bootstrap_dns_test.go +++ b/cmd/derper/bootstrap_dns_test.go @@ -20,10 +20,10 @@ import ( ) func BenchmarkHandleBootstrapDNS(b *testing.B) { - tstest.Replace(b, bootstrapDNS, "log.tailscale.io,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com") + tstest.Replace(b, bootstrapDNS, "log.tailscale.com,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com") refreshBootstrapDNS() w := new(bitbucketResponseWriter) - req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.io"), nil) + req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.com"), nil) b.ReportAllocs() b.ResetTimer() b.RunParallel(func(b *testing.PB) { @@ -63,7 +63,7 @@ func TestUnpublishedDNS(t *testing.T) { nettest.SkipIfNoNetwork(t) const published = "login.tailscale.com" - const unpublished = "log.tailscale.io" + const unpublished = "log.tailscale.com" prev1, prev2 := *bootstrapDNS, *unpublishedDNS *bootstrapDNS = published @@ -119,18 +119,18 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { unpublishedDNSCache.Store(&dnsEntryMap{ IPs: map[string][]net.IP{ - "log.tailscale.io": {}, + "log.tailscale.com": {}, "controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}, }, Percent: map[string]float64{ - "log.tailscale.io": 1.0, + "log.tailscale.com": 1.0, "controlplane.tailscale.com": 1.0, }, }) t.Run("CacheMiss", func(t *testing.T) { // One domain in map but empty, one not in map at all - for _, q := range []string{"log.tailscale.io", "login.tailscale.com"} { + for _, q := range []string{"log.tailscale.com", "login.tailscale.com"} { resetMetrics() ips := getBootstrapDNS(t, q) diff --git a/docs/windows/policy/en-US/tailscale.adml b/docs/windows/policy/en-US/tailscale.adml index ebf1a5905f6e9..4d5893a32d5c7 100644 --- a/docs/windows/policy/en-US/tailscale.adml +++ b/docs/windows/policy/en-US/tailscale.adml @@ -31,7 +31,7 @@ See https://tailscale.com/kb/1315/mdm-keys#set-a-custom-control-server-url for m Specify which Tailnet should be used for Login diff --git a/ipn/ipnserver/proxyconnect.go b/ipn/ipnserver/proxyconnect.go index 1094a79f9daf9..030c4efe4a6b0 100644 --- a/ipn/ipnserver/proxyconnect.go +++ b/ipn/ipnserver/proxyconnect.go @@ -14,7 +14,7 @@ import ( ) // handleProxyConnectConn handles a CONNECT request to -// log.tailscale.io (or whatever the configured log server is). This +// log.tailscale.com (or whatever the configured log server is). This // is intended for use by the Windows GUI client to log via when an // exit node is in use, so the logs don't go out via the exit node and // instead go directly, like tailscaled's. The dialer tried to do that diff --git a/logpolicy/logpolicy.go b/logpolicy/logpolicy.go index fa882ad3a2afa..b9b813718cef7 100644 --- a/logpolicy/logpolicy.go +++ b/logpolicy/logpolicy.go @@ -889,7 +889,7 @@ func (opts TransportOptions) New() http.RoundTripper { host := cmp.Or(opts.Host, logtail.DefaultHost) tr.TLSClientConfig = tlsdial.Config(host, opts.Health, tr.TLSClientConfig) - // Force TLS 1.3 since we know log.tailscale.io supports it. + // Force TLS 1.3 since we know log.tailscale.com supports it. tr.TLSClientConfig.MinVersion = tls.VersionTLS13 return tr diff --git a/logpolicy/logpolicy_test.go b/logpolicy/logpolicy_test.go index fdbfe4506e038..fb5666f8686d4 100644 --- a/logpolicy/logpolicy_test.go +++ b/logpolicy/logpolicy_test.go @@ -7,6 +7,8 @@ import ( "os" "reflect" "testing" + + "tailscale.com/logtail" ) func TestLogHost(t *testing.T) { @@ -20,7 +22,7 @@ func TestLogHost(t *testing.T) { env string want string }{ - {"", "log.tailscale.io"}, + {"", logtail.DefaultHost}, {"http://foo.com", "foo.com"}, {"https://foo.com", "foo.com"}, {"https://foo.com/", "foo.com"}, diff --git a/logtail/api.md b/logtail/api.md index 8ec0b69c0f331..20726e2096704 100644 --- a/logtail/api.md +++ b/logtail/api.md @@ -6,14 +6,14 @@ retrieving, and processing log entries. # Overview HTTP requests are received at the service **base URL** -[https://log.tailscale.io](https://log.tailscale.io), and return JSON-encoded +[https://log.tailscale.com](https://log.tailscale.com), and return JSON-encoded responses using standard HTTP response codes. Authorization for the configuration and retrieval APIs is done with a secret API key passed as the HTTP basic auth username. Secret keys are generated via the web UI at base URL. An example of using basic auth with curl: - curl -u : https://log.tailscale.io/collections + curl -u : https://log.tailscale.com/collections In the future, an HTTP header will allow using MessagePack instead of JSON. diff --git a/logtail/example/logadopt/logadopt.go b/logtail/example/logadopt/logadopt.go index 984a8a35adc7a..eba3f93112d62 100644 --- a/logtail/example/logadopt/logadopt.go +++ b/logtail/example/logadopt/logadopt.go @@ -25,7 +25,7 @@ func main() { } log.SetFlags(0) - req, err := http.NewRequest("POST", "https://log.tailscale.io/instances", strings.NewReader(url.Values{ + req, err := http.NewRequest("POST", "https://log.tailscale.com/instances", strings.NewReader(url.Values{ "collection": []string{*collection}, "instances": []string{*publicID}, "adopt": []string{"true"}, diff --git a/logtail/example/logreprocess/demo.sh b/logtail/example/logreprocess/demo.sh index 4ec819a67450d..583929c12b4fe 100755 --- a/logtail/example/logreprocess/demo.sh +++ b/logtail/example/logreprocess/demo.sh @@ -13,7 +13,7 @@ # # Then generate a LOGTAIL_API_KEY and two test collections by visiting: # -# https://log.tailscale.io +# https://log.tailscale.com # # Then set the three variables below. trap 'rv=$?; [ "$rv" = 0 ] || echo "-- exiting with code $rv"; exit $rv' EXIT diff --git a/logtail/example/logreprocess/logreprocess.go b/logtail/example/logreprocess/logreprocess.go index 5dbf765788165..aae65df9f1321 100644 --- a/logtail/example/logreprocess/logreprocess.go +++ b/logtail/example/logreprocess/logreprocess.go @@ -37,7 +37,7 @@ func main() { }() } - req, err := http.NewRequest("GET", "https://log.tailscale.io/c/"+*collection+"?stream=true", nil) + req, err := http.NewRequest("GET", "https://log.tailscale.com/c/"+*collection+"?stream=true", nil) if err != nil { log.Fatal(err) } diff --git a/logtail/logtail.go b/logtail/logtail.go index 13e8e85fd40f7..0e9c4f28808b1 100644 --- a/logtail/logtail.go +++ b/logtail/logtail.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -// Package logtail sends logs to log.tailscale.io. +// Package logtail sends logs to log.tailscale.com. package logtail import ( @@ -55,7 +55,7 @@ const bufferSize = 4 << 10 // DefaultHost is the default host name to upload logs to when // Config.BaseURL isn't provided. -const DefaultHost = "log.tailscale.io" +const DefaultHost = "log.tailscale.com" const defaultFlushDelay = 2 * time.Second @@ -69,7 +69,7 @@ type Config struct { Collection string // collection name, a domain name PrivateID logid.PrivateID // private ID for the primary log stream CopyPrivateID logid.PrivateID // private ID for a log stream that is a superset of this log stream - BaseURL string // if empty defaults to "https://log.tailscale.io" + BaseURL string // if empty defaults to "https://log.tailscale.com" HTTPC *http.Client // if empty defaults to http.DefaultClient SkipClientTime bool // if true, client_time is not written to logs LowMemory bool // if true, logtail minimizes memory use @@ -507,7 +507,7 @@ func (l *Logger) upload(ctx context.Context, body []byte, origlen int) (retryAft } if runtime.GOOS == "js" { // We once advertised we'd accept optional client certs (for internal use) - // on log.tailscale.io but then Tailscale SSH js/wasm clients prompted + // on log.tailscale.com but then Tailscale SSH js/wasm clients prompted // users (on some browsers?) to pick a client cert. We'll fix the server's // TLS ServerHello, but we can also fix it client side for good measure. // diff --git a/net/tlsdial/tlsdial.go b/net/tlsdial/tlsdial.go index 7e847a8b6a656..2a109c790632d 100644 --- a/net/tlsdial/tlsdial.go +++ b/net/tlsdial/tlsdial.go @@ -89,8 +89,8 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // (with the baked-in fallback root) in the VerifyConnection hook. conf.InsecureSkipVerify = true conf.VerifyConnection = func(cs tls.ConnectionState) (retErr error) { - if host == "log.tailscale.io" && hostinfo.IsNATLabGuestVM() { - // Allow log.tailscale.io TLS MITM for integration tests when + if host == "log.tailscale.com" && hostinfo.IsNATLabGuestVM() { + // Allow log.tailscale.com TLS MITM for integration tests when // the client's running within a NATLab VM. return nil } diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 34cab7385558b..5f1d8073ae5b9 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -78,7 +78,7 @@ type Server struct { // If nil, a new FileStore is initialized at `Dir/tailscaled.state`. // See tailscale.com/ipn/store for supported stores. // - // Logs will automatically be uploaded to log.tailscale.io, + // Logs will automatically be uploaded to log.tailscale.com, // where the configuration file for logging will be saved at // `Dir/tailscaled.log.conf`. Store ipn.StateStore diff --git a/tstest/natlab/vnet/vip.go b/tstest/natlab/vnet/vip.go index c75f17cee5393..190c9e75f1a62 100644 --- a/tstest/natlab/vnet/vip.go +++ b/tstest/natlab/vnet/vip.go @@ -17,7 +17,7 @@ var ( fakeControl = newVIP("control.tailscale", 3) fakeDERP1 = newVIP("derp1.tailscale", "33.4.0.1") // 3340=DERP; 1=derp 1 fakeDERP2 = newVIP("derp2.tailscale", "33.4.0.2") // 3340=DERP; 2=derp 2 - fakeLogCatcher = newVIP("log.tailscale.io", 4) + fakeLogCatcher = newVIP("log.tailscale.com", 4) fakeSyslog = newVIP("syslog.tailscale", 9) ) diff --git a/tstest/natlab/vnet/vnet.go b/tstest/natlab/vnet/vnet.go index 92312c039bfc9..586fd28e02977 100644 --- a/tstest/natlab/vnet/vnet.go +++ b/tstest/natlab/vnet/vnet.go @@ -394,7 +394,7 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) { } } -// serveLogCatchConn serves a TCP connection to "log.tailscale.io", speaking the +// serveLogCatchConn serves a TCP connection to "log.tailscale.com", speaking the // logtail/logcatcher protocol. // // We terminate TLS with an arbitrary cert; the client is configured to not From b3d4ffe1688b4218feee06a5873fce75a839a022 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 17 Dec 2024 15:36:57 +0000 Subject: [PATCH 029/223] docs/k8s: add some high-level operator architecture diagrams (#13915) This is an experiment to see how useful we will find it to have some text-based diagrams to document how various components of the operator work. There are no plans to link to this from elsewhere yet, but hopefully it will be a useful reference internally. Updates #cleanup Change-Id: If5911ed39b09378fec0492e87738ec0cc3d8731e Signed-off-by: Tom Proctor --- docs/k8s/operator-architecture.md | 517 ++++++++++++++++++++++++++++++ 1 file changed, 517 insertions(+) create mode 100644 docs/k8s/operator-architecture.md diff --git a/docs/k8s/operator-architecture.md b/docs/k8s/operator-architecture.md new file mode 100644 index 0000000000000..26bfa854267c7 --- /dev/null +++ b/docs/k8s/operator-architecture.md @@ -0,0 +1,517 @@ +# Operator architecture diagrams + +The Tailscale [Kubernetes operator][kb-operator] has a collection of use-cases +that can be mixed and matched as required. The following diagrams illustrate +how the operator implements each use-case. + +In each diagram, the "tailscale" namespace is entirely managed by the operator +once the operator itself has been deployed. + +Tailscale devices are highlighted as black nodes. The salient devices for each +use-case are marked as "src" or "dst" to denote which node is a source or a +destination in the context of ACL rules that will apply to network traffic. + +Note, in some cases, the config and the state Secret may be the same Kubernetes +Secret. + +## API server proxy + +[Documentation][kb-operator-proxy] + +The operator runs the API server proxy in-process. If the proxy is running in +"noauth" mode, it forwards HTTP requests unmodified. If the proxy is running in +"auth" mode, it deletes any existing auth headers and adds +[impersonation headers][k8s-impersonation] to the request before forwarding to +the API server. A request with impersonation headers will look something like: + +``` +GET /api/v1/namespaces/default/pods HTTP/1.1 +Host: k8s-api.example.com +Authorization: Bearer +Impersonate-Group: tailnet-readers +Accept: application/json +``` + +```mermaid +%%{ init: { 'theme':'neutral' } }%% +flowchart LR + classDef tsnode color:#fff,fill:#000; + classDef pod fill:#fff; + + subgraph Key + ts[Tailscale device]:::tsnode + pod((Pod)):::pod + blank[" "]-->|WireGuard traffic| blank2[" "] + blank3[" "]-->|Other network traffic| blank4[" "] + end + + subgraph k8s[Kubernetes cluster] + subgraph tailscale-ns[namespace=tailscale] + operator(("operator (dst)")):::tsnode + end + + subgraph controlplane["Control plane"] + api[kube-apiserver] + end + end + + client["client (src)"]:::tsnode --> operator + operator -->|"proxy (maybe with impersonation headers)"| api + + linkStyle 0 stroke:red; + linkStyle 2 stroke:red; + + linkStyle 1 stroke:blue; + linkStyle 3 stroke:blue; + +``` + +## L3 ingress + +[Documentation][kb-operator-l3-ingress] + +The user deploys an app to the default namespace, and creates a normal Service +that selects the app's Pods. Either add the annotation +`tailscale.com/expose: "true"` or specify `.spec.type` as `Loadbalancer` and +`.spec.loadBalancerClass` as `tailscale`. The operator will create an ingress +proxy that allows devices anywhere on the tailnet to access the Service. + +The proxy Pod uses `iptables` or `nftables` rules to DNAT traffic bound for the +proxy's tailnet IP to the Service's internal Cluster IP instead. + +```mermaid +%%{ init: { 'theme':'neutral' } }%% +flowchart TD + classDef tsnode color:#fff,fill:#000; + classDef pod fill:#fff; + + subgraph Key + ts[Tailscale device]:::tsnode + pod((Pod)):::pod + blank[" "]-->|WireGuard traffic| blank2[" "] + blank3[" "]-->|Other network traffic| blank4[" "] + end + + subgraph k8s[Kubernetes cluster] + subgraph tailscale-ns[namespace=tailscale] + operator((operator)):::tsnode + ingress-sts["StatefulSet"] + ingress(("ingress proxy (dst)")):::tsnode + config-secret["config Secret"] + state-secret["state Secret"] + end + + subgraph defaultns[namespace=default] + svc[annotated Service] + svc --> pod1((pod1)) + svc --> pod2((pod2)) + end + end + + client["client (src)"]:::tsnode --> ingress + ingress -->|forwards traffic| svc + operator -.->|creates| ingress-sts + ingress-sts -.->|manages| ingress + operator -.->|reads| svc + operator -.->|creates| config-secret + config-secret -.->|mounted| ingress + ingress -.->|stores state| state-secret + + linkStyle 0 stroke:red; + linkStyle 4 stroke:red; + + linkStyle 1 stroke:blue; + linkStyle 2 stroke:blue; + linkStyle 3 stroke:blue; + linkStyle 5 stroke:blue; + +``` + +## L7 ingress + +[Documentation][kb-operator-l7-ingress] + +L7 ingress is relatively similar to L3 ingress. It is configured via an +`Ingress` object instead of a `Service`, and uses `tailscale serve` to accept +traffic instead of configuring `iptables` or `nftables` rules. Note that we use +tailscaled's local API (`SetServeConfig`) to set serve config, not the +`tailscale serve` command. + +```mermaid +%%{ init: { 'theme':'neutral' } }%% +flowchart TD + classDef tsnode color:#fff,fill:#000; + classDef pod fill:#fff; + + subgraph Key + ts[Tailscale device]:::tsnode + pod((Pod)):::pod + blank[" "]-->|WireGuard traffic| blank2[" "] + blank3[" "]-->|Other network traffic| blank4[" "] + end + + subgraph k8s[Kubernetes cluster] + subgraph tailscale-ns[namespace=tailscale] + operator((operator)):::tsnode + ingress-sts["StatefulSet"] + ingress-pod(("ingress proxy (dst)")):::tsnode + config-secret["config Secret"] + state-secret["state Secret"] + end + + subgraph defaultns[namespace=default] + ingress[tailscale Ingress] + svc["Service"] + svc --> pod1((pod1)) + svc --> pod2((pod2)) + end + end + + client["client (src)"]:::tsnode --> ingress-pod + ingress-pod -->|forwards /api prefix traffic| svc + operator -.->|creates| ingress-sts + ingress-sts -.->|manages| ingress-pod + operator -.->|reads| ingress + operator -.->|creates| config-secret + config-secret -.->|mounted| ingress-pod + ingress-pod -.->|stores state| state-secret + ingress -.->|/api prefix| svc + + linkStyle 0 stroke:red; + linkStyle 4 stroke:red; + + linkStyle 1 stroke:blue; + linkStyle 2 stroke:blue; + linkStyle 3 stroke:blue; + linkStyle 5 stroke:blue; + +``` + +## L3 egress + +[Documentation][kb-operator-l3-egress] + +1. The user deploys a Service with `type: ExternalName` and an annotation + `tailscale.com/tailnet-fqdn: db.tails-scales.ts.net`. +1. The operator creates a proxy Pod managed by a single replica StatefulSet, and a headless Service pointing at the proxy Pod. +1. The operator updates the `ExternalName` Service's `spec.externalName` field to point + at the headless Service it created in the previous step. + +(Optional) If the user also adds the `tailscale.com/proxy-group: egress-proxies` +annotation to their `ExternalName` Service, the operator will skip creating a +proxy Pod and instead point the headless Service at the existing ProxyGroup's +pods. In this case, ports are also required in the `ExternalName` Service spec. +See below for a more representative diagram. + +```mermaid +%%{ init: { 'theme':'neutral' } }%% + +flowchart TD + classDef tsnode color:#fff,fill:#000; + classDef pod fill:#fff; + + subgraph Key + ts[Tailscale device]:::tsnode + pod((Pod)):::pod + blank[" "]-->|WireGuard traffic| blank2[" "] + blank3[" "]-->|Other network traffic| blank4[" "] + end + + subgraph k8s[Kubernetes cluster] + subgraph tailscale-ns[namespace=tailscale] + operator((operator)):::tsnode + egress(("egress proxy (src)")):::tsnode + egress-sts["StatefulSet"] + headless-svc[headless Service] + cfg-secret["config Secret"] + state-secret["state Secret"] + end + + subgraph defaultns[namespace=default] + svc[ExternalName Service] + pod1((pod1)) --> svc + pod2((pod2)) --> svc + end + end + + node["db.tails-scales.ts.net (dst)"]:::tsnode + + svc -->|DNS points to| headless-svc + headless-svc -->|selects egress Pod| egress + egress -->|forwards traffic| node + operator -.->|creates| egress-sts + egress-sts -.->|manages| egress + operator -.->|creates| headless-svc + operator -.->|creates| cfg-secret + operator -.->|watches & updates| svc + cfg-secret -.->|mounted| egress + egress -.->|stores state| state-secret + + linkStyle 0 stroke:red; + linkStyle 6 stroke:red; + + linkStyle 1 stroke:blue; + linkStyle 2 stroke:blue; + linkStyle 3 stroke:blue; + linkStyle 4 stroke:blue; + linkStyle 5 stroke:blue; + +``` + +## `ProxyGroup` + +[Documentation][kb-operator-l3-egress-proxygroup] + +The `ProxyGroup` custom resource manages a collection of proxy Pods that +can be configured to egress traffic out of the cluster via ExternalName +Services. A `ProxyGroup` is both a high availability (HA) version of L3 +egress, and a mechanism to serve multiple ExternalName Services on a single +set of Tailscale devices (coalescing). + +In this diagram, the `ProxyGroup` is named `pg`. The Secrets associated with +the `ProxyGroup` Pods are omitted for simplicity. They are similar to the L3 +egress case above, but there is a pair of config + state Secrets _per Pod_. + +Each ExternalName Service defines which ports should be mapped to their defined +egress target. The operator maps from these ports to randomly chosen ephemeral +ports via the ClusterIP Service and its EndpointSlice. The operator then +generates the egress ConfigMap that tells the `ProxyGroup` Pods which incoming +ports map to which egress targets. + +`ProxyGroups` currently only support egress. + +```mermaid +%%{ init: { 'theme':'neutral' } }%% + +flowchart LR + classDef tsnode color:#fff,fill:#000; + classDef pod fill:#fff; + + subgraph Key + ts[Tailscale device]:::tsnode + pod((Pod)):::pod + blank[" "]-->|WireGuard traffic| blank2[" "] + blank3[" "]-->|Other network traffic| blank4[" "] + end + + subgraph k8s[Kubernetes cluster] + subgraph tailscale-ns[namespace=tailscale] + operator((operator)):::tsnode + pg-sts[StatefulSet] + pg-0(("pg-0 (src)")):::tsnode + pg-1(("pg-1 (src)")):::tsnode + db-cluster-ip[db ClusterIP Service] + api-cluster-ip[api ClusterIP Service] + egress-cm["egress ConfigMap"] + end + + subgraph cluster-scope["Cluster scoped resources"] + pg["ProxyGroup 'pg'"] + end + + subgraph defaultns[namespace=default] + db-svc[db ExternalName Service] + api-svc[api ExternalName Service] + pod1((pod1)) --> db-svc + pod2((pod2)) --> db-svc + pod1((pod1)) --> api-svc + pod2((pod2)) --> api-svc + end + end + + db["db.tails-scales.ts.net (dst)"]:::tsnode + api["api.tails-scales.ts.net (dst)"]:::tsnode + + db-svc -->|DNS points to| db-cluster-ip + api-svc -->|DNS points to| api-cluster-ip + db-cluster-ip -->|maps to ephemeral db ports| pg-0 + db-cluster-ip -->|maps to ephemeral db ports| pg-1 + api-cluster-ip -->|maps to ephemeral api ports| pg-0 + api-cluster-ip -->|maps to ephemeral api ports| pg-1 + pg-0 -->|forwards db port traffic| db + pg-0 -->|forwards api port traffic| api + pg-1 -->|forwards db port traffic| db + pg-1 -->|forwards api port traffic| api + operator -.->|creates & populates endpointslice| db-cluster-ip + operator -.->|creates & populates endpointslice| api-cluster-ip + operator -.->|stores port mapping| egress-cm + egress-cm -.->|mounted| pg-0 + egress-cm -.->|mounted| pg-1 + operator -.->|watches| pg + operator -.->|creates| pg-sts + pg-sts -.->|manages| pg-0 + pg-sts -.->|manages| pg-1 + operator -.->|watches| db-svc + operator -.->|watches| api-svc + + linkStyle 0 stroke:red; + linkStyle 12 stroke:red; + linkStyle 13 stroke:red; + linkStyle 14 stroke:red; + linkStyle 15 stroke:red; + + linkStyle 1 stroke:blue; + linkStyle 2 stroke:blue; + linkStyle 3 stroke:blue; + linkStyle 4 stroke:blue; + linkStyle 5 stroke:blue; + linkStyle 6 stroke:blue; + linkStyle 7 stroke:blue; + linkStyle 8 stroke:blue; + linkStyle 9 stroke:blue; + linkStyle 10 stroke:blue; + linkStyle 11 stroke:blue; + +``` + +## Connector + +[Subnet router and exit node documentation][kb-operator-connector] + +[App connector documentation][kb-operator-app-connector] + +The Connector Custom Resource can deploy either a subnet router, an exit node, +or an app connector. The following diagram shows all 3, but only one workflow +can be configured per Connector resource. + +```mermaid +%%{ init: { 'theme':'neutral' } }%% + +flowchart TD + classDef tsnode color:#fff,fill:#000; + classDef pod fill:#fff; + classDef hidden display:none; + + subgraph Key + ts[Tailscale device]:::tsnode + pod((Pod)):::pod + blank[" "]-->|WireGuard traffic| blank2[" "] + blank3[" "]-->|Other network traffic| blank4[" "] + end + + subgraph grouping[" "] + subgraph k8s[Kubernetes cluster] + subgraph tailscale-ns[namespace=tailscale] + operator((operator)):::tsnode + cn-sts[StatefulSet] + cn-pod(("tailscale (dst)")):::tsnode + cfg-secret["config Secret"] + state-secret["state Secret"] + end + + subgraph cluster-scope["Cluster scoped resources"] + cn["Connector"] + end + + subgraph defaultns["namespace=default"] + pod1 + end + end + + client["client (src)"]:::tsnode + Internet + end + + client --> cn-pod + cn-pod -->|app connector or exit node routes| Internet + cn-pod -->|subnet route| pod1 + operator -.->|watches| cn + operator -.->|creates| cn-sts + cn-sts -.->|manages| cn-pod + operator -.->|creates| cfg-secret + cfg-secret -.->|mounted| cn-pod + cn-pod -.->|stores state| state-secret + + class grouping hidden + + linkStyle 0 stroke:red; + linkStyle 2 stroke:red; + + linkStyle 1 stroke:blue; + linkStyle 3 stroke:blue; + linkStyle 4 stroke:blue; + +``` + +## Recorder nodes + +[Documentation][kb-operator-recorder] + +The `Recorder` custom resource makes it easier to deploy `tsrecorder` to a cluster. +It currently only supports a single replica. + +```mermaid +%%{ init: { 'theme':'neutral' } }%% + +flowchart TD + classDef tsnode color:#fff,fill:#000; + classDef pod fill:#fff; + classDef hidden display:none; + + subgraph Key + ts[Tailscale device]:::tsnode + pod((Pod)):::pod + blank[" "]-->|WireGuard traffic| blank2[" "] + blank3[" "]-->|Other network traffic| blank4[" "] + end + + subgraph grouping[" "] + subgraph k8s[Kubernetes cluster] + api["kube-apiserver"] + + subgraph tailscale-ns[namespace=tailscale] + operator(("operator (dst)")):::tsnode + rec-sts[StatefulSet] + rec-0(("tsrecorder")):::tsnode + cfg-secret-0["config Secret"] + state-secret-0["state Secret"] + end + + subgraph cluster-scope["Cluster scoped resources"] + rec["Recorder"] + end + end + + client["client (src)"]:::tsnode + kubectl-exec["kubectl exec (src)"]:::tsnode + server["server (dst)"]:::tsnode + s3["S3-compatible storage"] + end + + kubectl-exec -->|exec session| operator + operator -->|exec session recording| rec-0 + operator -->|exec session| api + client -->|ssh session| server + server -->|ssh session recording| rec-0 + rec-0 -->|session recordings| s3 + operator -.->|watches| rec + operator -.->|creates| rec-sts + rec-sts -.->|manages| rec-0 + operator -.->|creates| cfg-secret-0 + cfg-secret-0 -.->|mounted| rec-0 + rec-0 -.->|stores state| state-secret-0 + + class grouping hidden + + linkStyle 0 stroke:red; + linkStyle 2 stroke:red; + linkStyle 3 stroke:red; + linkStyle 5 stroke:red; + linkStyle 6 stroke:red; + + linkStyle 1 stroke:blue; + linkStyle 4 stroke:blue; + linkStyle 7 stroke:blue; + +``` + +[kb-operator]: https://tailscale.com/kb/1236/kubernetes-operator +[kb-operator-proxy]: https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy +[kb-operator-l3-ingress]: https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress#exposing-a-cluster-workload-using-a-kubernetes-service +[kb-operator-l7-ingress]: https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress#exposing-cluster-workloads-using-a-kubernetes-ingress +[kb-operator-l3-egress]: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +[kb-operator-l3-egress-proxygroup]: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress#configure-an-egress-service-using-proxygroup +[kb-operator-connector]: https://tailscale.com/kb/1441/kubernetes-operator-connector +[kb-operator-app-connector]: https://tailscale.com/kb/1517/kubernetes-operator-app-connector +[kb-operator-recorder]: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder +[k8s-impersonation]: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation From ff5b4bae99c7dc8bb57660bb579d5df4ab31b1ce Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Wed, 18 Dec 2024 17:11:22 -0800 Subject: [PATCH 030/223] syncs: add MutexValue (#14422) MutexValue is simply a value guarded by a mutex. For any type that is not pointer-sized, MutexValue will perform much better than AtomicValue since it will not incur an allocation boxing the value into an interface value (which is how Go's atomic.Value is implemented under-the-hood). Updates #cleanup Signed-off-by: Joe Tsai --- syncs/syncs.go | 62 +++++++++++++++++++++++++++++++++++++++++++++ syncs/syncs_test.go | 34 +++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/syncs/syncs.go b/syncs/syncs.go index acc0c88f2991e..337fca7557f34 100644 --- a/syncs/syncs.go +++ b/syncs/syncs.go @@ -25,6 +25,7 @@ func initClosedChan() <-chan struct{} { } // AtomicValue is the generic version of [atomic.Value]. +// See [MutexValue] for guidance on whether to use this type. type AtomicValue[T any] struct { v atomic.Value } @@ -74,6 +75,67 @@ func (v *AtomicValue[T]) CompareAndSwap(oldV, newV T) (swapped bool) { return v.v.CompareAndSwap(wrappedValue[T]{oldV}, wrappedValue[T]{newV}) } +// MutexValue is a value protected by a mutex. +// +// AtomicValue, [MutexValue], [atomic.Pointer] are similar and +// overlap in their use cases. +// +// - Use [atomic.Pointer] if the value being stored is a pointer and +// you only ever need load and store operations. +// An atomic pointer only occupies 1 word of memory. +// +// - Use [MutexValue] if the value being stored is not a pointer or +// you need the ability for a mutex to protect a set of operations +// performed on the value. +// A mutex-guarded value occupies 1 word of memory plus +// the memory representation of T. +// +// - AtomicValue is useful for non-pointer types that happen to +// have the memory layout of a single pointer. +// Examples include a map, channel, func, or a single field struct +// that contains any prior types. +// An atomic value occupies 2 words of memory. +// Consequently, Storing of non-pointer types always allocates. +// +// Note that [AtomicValue] has the ability to report whether it was set +// while [MutexValue] lacks the ability to detect if the value was set +// and it happens to be the zero value of T. If such a use case is +// necessary, then you could consider wrapping T in [opt.Value]. +type MutexValue[T any] struct { + mu sync.Mutex + v T +} + +// WithLock calls f with a pointer to the value while holding the lock. +// The provided pointer must not leak beyond the scope of the call. +func (m *MutexValue[T]) WithLock(f func(p *T)) { + m.mu.Lock() + defer m.mu.Unlock() + f(&m.v) +} + +// Load returns a shallow copy of the underlying value. +func (m *MutexValue[T]) Load() T { + m.mu.Lock() + defer m.mu.Unlock() + return m.v +} + +// Store stores a shallow copy of the provided value. +func (m *MutexValue[T]) Store(v T) { + m.mu.Lock() + defer m.mu.Unlock() + m.v = v +} + +// Swap stores new into m and returns the previous value. +func (m *MutexValue[T]) Swap(new T) (old T) { + m.mu.Lock() + defer m.mu.Unlock() + old, m.v = m.v, new + return old +} + // WaitGroupChan is like a sync.WaitGroup, but has a chan that closes // on completion that you can wait on. (This, you can only use the // value once) diff --git a/syncs/syncs_test.go b/syncs/syncs_test.go index ee3711e76587b..901d429486d13 100644 --- a/syncs/syncs_test.go +++ b/syncs/syncs_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "testing" + "time" "github.com/google/go-cmp/cmp" ) @@ -65,6 +66,39 @@ func TestAtomicValue(t *testing.T) { } } +func TestMutexValue(t *testing.T) { + var v MutexValue[time.Time] + if n := int(testing.AllocsPerRun(1000, func() { + v.Store(v.Load()) + v.WithLock(func(*time.Time) {}) + })); n != 0 { + t.Errorf("AllocsPerRun = %d, want 0", n) + } + + now := time.Now() + v.Store(now) + if !v.Load().Equal(now) { + t.Errorf("Load = %v, want %v", v.Load(), now) + } + + var group WaitGroup + var v2 MutexValue[int] + var sum int + for i := range 10 { + group.Go(func() { + old1 := v2.Load() + old2 := v2.Swap(old1 + i) + delta := old2 - old1 + v2.WithLock(func(p *int) { *p += delta }) + }) + sum += i + } + group.Wait() + if v2.Load() != sum { + t.Errorf("Load = %v, want %v", v2.Load(), sum) + } +} + func TestWaitGroupChan(t *testing.T) { wg := NewWaitGroupChan() From 6ae0287a5799f60eacdb1e7b1191fadb697fb75a Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Tue, 10 Dec 2024 13:54:31 -0800 Subject: [PATCH 031/223] cmd/systray: add account switcher Updates #1708 Signed-off-by: Andrew Lytvynov --- cmd/systray/systray.go | 91 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index aca38f627c65a..d175b55f309d7 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -49,6 +49,8 @@ type Menu struct { more *systray.MenuItem quit *systray.MenuItem + accountsCh chan ipn.ProfileID + eventCancel func() // cancel eventLoop } @@ -64,15 +66,32 @@ func onReady() { chState = make(chan ipn.State, 1) + menu := new(Menu) + menu.rebuild(fetchState(ctx)) + + go watchIPNBus(ctx) +} + +type state struct { + status *ipnstate.Status + curProfile ipn.LoginProfile + allProfiles []ipn.LoginProfile +} + +func fetchState(ctx context.Context) state { status, err := localClient.Status(ctx) if err != nil { log.Print(err) } - - menu := new(Menu) - menu.rebuild(status) - - go watchIPNBus(ctx) + curProfile, allProfiles, err := localClient.ProfileStatus(ctx) + if err != nil { + log.Print(err) + } + return state{ + status: status, + curProfile: curProfile, + allProfiles: allProfiles, + } } // rebuild the systray menu based on the current Tailscale state. @@ -80,14 +99,17 @@ func onReady() { // We currently rebuild the entire menu because it is not easy to update the existing menu. // You cannot iterate over the items in a menu, nor can you remove some items like separators. // So for now we rebuild the whole thing, and can optimize this later if needed. -func (menu *Menu) rebuild(status *ipnstate.Status) { +func (menu *Menu) rebuild(state state) { menu.mu.Lock() defer menu.mu.Unlock() if menu.eventCancel != nil { menu.eventCancel() } - menu.status = status + ctx := context.Background() + ctx, menu.eventCancel = context.WithCancel(ctx) + + menu.status = state.status systray.ResetMenu() menu.connect = systray.AddMenuItem("Connect", "") @@ -95,8 +117,46 @@ func (menu *Menu) rebuild(status *ipnstate.Status) { menu.disconnect.Hide() systray.AddSeparator() - if status != nil && status.Self != nil { - title := fmt.Sprintf("This Device: %s (%s)", status.Self.HostName, status.Self.TailscaleIPs[0]) + account := "Account" + if state.curProfile.Name != "" { + account += fmt.Sprintf(" (%s)", state.curProfile.Name) + } + accounts := systray.AddMenuItem(account, "") + // The dbus message about this menu item must propagate to the receiving + // end before we attach any submenu items. Otherwise the receiver may not + // yet record the parent menu item and error out. + // + // On waybar with libdbusmenu-gtk, this manifests as the following warning: + // (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu' + time.Sleep(100 * time.Millisecond) + // Aggregate all clicks into a shared channel. + menu.accountsCh = make(chan ipn.ProfileID) + for _, profile := range state.allProfiles { + title := fmt.Sprintf("%s (%s)", profile.Name, profile.NetworkProfile.DomainName) + // Note: we could use AddSubMenuItemCheckbox instead of this formatting + // hack, but checkboxes don't work across all desktops unfortunately. + if profile.ID == state.curProfile.ID { + title = "* " + title + } + item := accounts.AddSubMenuItem(title, "") + go func(profile ipn.LoginProfile) { + for { + select { + case <-ctx.Done(): + return + case <-item.ClickedCh: + select { + case <-ctx.Done(): + return + case menu.accountsCh <- profile.ID: + } + } + } + }(profile) + } + + if state.status != nil && state.status.Self != nil { + title := fmt.Sprintf("This Device: %s (%s)", state.status.Self.HostName, state.status.Self.TailscaleIPs[0]) menu.self = systray.AddMenuItem(title, "") } systray.AddSeparator() @@ -107,8 +167,6 @@ func (menu *Menu) rebuild(status *ipnstate.Status) { menu.quit = systray.AddMenuItem("Quit", "Quit the app") menu.quit.Enable() - ctx := context.Background() - ctx, menu.eventCancel = context.WithCancel(ctx) go menu.eventLoop(ctx) } @@ -124,11 +182,7 @@ func (menu *Menu) eventLoop(ctx context.Context) { switch state { case ipn.Running: setAppIcon(loading) - status, err := localClient.Status(ctx) - if err != nil { - log.Printf("error getting tailscale status: %v", err) - } - menu.rebuild(status) + menu.rebuild(fetchState(ctx)) setAppIcon(connected) menu.connect.SetTitle("Connected") menu.connect.Disable() @@ -172,6 +226,11 @@ func (menu *Menu) eventLoop(ctx context.Context) { case <-menu.more.ClickedCh: webbrowser.Open("http://100.100.100.100/") + case id := <-menu.accountsCh: + if err := localClient.SwitchProfile(ctx, id); err != nil { + log.Printf("failed switching to profile ID %v: %v", id, err) + } + case <-menu.quit.ClickedCh: systray.Quit() } From 00a4504cf1a1b150a8896283ccf5e01011112626 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Mon, 16 Dec 2024 23:05:46 -0600 Subject: [PATCH 032/223] cmd/derpprobe,prober: add ability to perform continuous queuing delay measurements against DERP servers This new type of probe sends DERP packets sized similarly to CallMeMaybe packets at a rate of 10 packets per second. It records the round-trip times in a Prometheus histogram. It also keeps track of how many packets are dropped. Packets that fail to arrive within 5 seconds are considered dropped. Updates tailscale/corp#24522 Signed-off-by: Percy Wegmann --- cmd/derpprobe/derpprobe.go | 31 +++--- prober/derp.go | 222 ++++++++++++++++++++++++++++++++++++- prober/histogram.go | 50 +++++++++ prober/histogram_test.go | 29 +++++ prober/prober.go | 64 +++++++++-- prober/prober_test.go | 17 +-- prober/status.go | 12 +- prober/status.html | 57 +++++++--- 8 files changed, 428 insertions(+), 54 deletions(-) create mode 100644 prober/histogram.go create mode 100644 prober/histogram_test.go diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 620b966099176..62b7d47a4e6bf 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -18,19 +18,21 @@ import ( ) var ( - derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") - versionFlag = flag.Bool("version", false, "print version and exit") - listen = flag.String("listen", ":8030", "HTTP listen address") - probeOnce = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag") - spread = flag.Bool("spread", true, "whether to spread probing over time") - interval = flag.Duration("interval", 15*time.Second, "probe interval") - meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval") - stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval") - tlsInterval = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval") - bwInterval = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)") - bwSize = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size") - bwTUNIPv4Address = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. We will use a /30 subnet including this IP address.") - regionCode = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed") + derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") + versionFlag = flag.Bool("version", false, "print version and exit") + listen = flag.String("listen", ":8030", "HTTP listen address") + probeOnce = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag") + spread = flag.Bool("spread", true, "whether to spread probing over time") + interval = flag.Duration("interval", 15*time.Second, "probe interval") + meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval") + stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval") + tlsInterval = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval") + bwInterval = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)") + bwSize = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size") + bwTUNIPv4Address = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP; we will use a /30 subnet including this IP address") + qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second") + qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings") + regionCode = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed") ) func main() { @@ -45,6 +47,7 @@ func main() { prober.WithMeshProbing(*meshInterval), prober.WithSTUNProbing(*stunInterval), prober.WithTLSProbing(*tlsInterval), + prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout), } if *bwInterval > 0 { opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address)) @@ -107,7 +110,7 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) { // Do not show probes that have not finished yet. continue } - if i.Result { + if i.Status == prober.ProbeStatusSucceeded { o.addGoodf("%s: %s", p, i.Latency) } else { o.addBadf("%s: %s", p, i.Error) diff --git a/prober/derp.go b/prober/derp.go index 742e8a5f42e57..5adc0c0b408d0 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -8,6 +8,7 @@ import ( "cmp" "context" crand "crypto/rand" + "encoding/binary" "encoding/json" "errors" "expvar" @@ -17,6 +18,7 @@ import ( "net" "net/http" "net/netip" + "slices" "strconv" "strings" "sync" @@ -53,6 +55,10 @@ type derpProber struct { bwProbeSize int64 bwTUNIPv4Prefix *netip.Prefix // or nil to not use TUN + // Optional queuing delay probing. + qdPacketsPerSecond int // in packets per second + qdPacketTimeout time.Duration + // Optionally restrict probes to a single regionCode. regionCode string @@ -64,6 +70,7 @@ type derpProber struct { udpProbeFn func(string, int) ProbeClass meshProbeFn func(string, string) ProbeClass bwProbeFn func(string, string, int64) ProbeClass + qdProbeFn func(string, string, int, time.Duration) ProbeClass sync.Mutex lastDERPMap *tailcfg.DERPMap @@ -93,6 +100,16 @@ func WithBandwidthProbing(interval time.Duration, size int64, tunAddress string) } } +// WithQueuingDelayProbing enables/disables queuing delay probing. qdSendRate +// is the number of packets sent per second. qdTimeout is the amount of time +// after which a sent packet is considered to have timed out. +func WithQueuingDelayProbing(qdPacketsPerSecond int, qdPacketTimeout time.Duration) DERPOpt { + return func(d *derpProber) { + d.qdPacketsPerSecond = qdPacketsPerSecond + d.qdPacketTimeout = qdPacketTimeout + } +} + // WithMeshProbing enables mesh probing. When enabled, a small message will be // transferred through each DERP server and each pair of DERP servers. func WithMeshProbing(interval time.Duration) DERPOpt { @@ -147,6 +164,7 @@ func DERP(p *Prober, derpMapURL string, opts ...DERPOpt) (*derpProber, error) { d.udpProbeFn = d.ProbeUDP d.meshProbeFn = d.probeMesh d.bwProbeFn = d.probeBandwidth + d.qdProbeFn = d.probeQueuingDelay return d, nil } @@ -213,7 +231,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { } } - if d.bwInterval > 0 && d.bwProbeSize > 0 { + if d.bwInterval != 0 && d.bwProbeSize > 0 { n := fmt.Sprintf("derp/%s/%s/%s/bw", region.RegionCode, server.Name, to.Name) wantProbes[n] = true if d.probes[n] == nil { @@ -225,6 +243,15 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { d.probes[n] = d.p.Run(n, d.bwInterval, labels, d.bwProbeFn(server.Name, to.Name, d.bwProbeSize)) } } + + if d.qdPacketsPerSecond > 0 { + n := fmt.Sprintf("derp/%s/%s/%s/qd", region.RegionCode, server.Name, to.Name) + wantProbes[n] = true + if d.probes[n] == nil { + log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName) + d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout)) + } + } } } } @@ -240,7 +267,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { return nil } -// probeMesh returs a probe class that sends a test packet through a pair of DERP +// probeMesh returns a probe class that sends a test packet through a pair of DERP // servers (or just one server, if 'from' and 'to' are the same). 'from' and 'to' // are expected to be names (DERPNode.Name) of two DERP servers in the same region. func (d *derpProber) probeMesh(from, to string) ProbeClass { @@ -263,7 +290,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass { } } -// probeBandwidth returs a probe class that sends a payload of a given size +// probeBandwidth returns a probe class that sends a payload of a given size // through a pair of DERP servers (or just one server, if 'from' and 'to' are // the same). 'from' and 'to' are expected to be names (DERPNode.Name) of two // DERP servers in the same region. @@ -295,6 +322,193 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { } } +// probeQueuingDelay returns a probe class that continuously sends packets +// through a pair of DERP servers (or just one server, if 'from' and 'to' are +// the same) at a rate of `packetsPerSecond` packets per second in order to +// measure queuing delays. Packets arriving after `packetTimeout` don't contribute +// to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are +// expected to be names (DERPNode.Name) of two DERP servers in the same region, +// and may refer to the same server. +func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass { + derpPath := "mesh" + if from == to { + derpPath = "single" + } + var packetsDropped expvar.Float + qdh := newHistogram([]float64{.005, .01, .025, .05, .1, .25, .5, 1}) + return ProbeClass{ + Probe: func(ctx context.Context) error { + fromN, toN, err := d.getNodePair(from, to) + if err != nil { + return err + } + return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh) + }, + Class: "derp_qd", + Labels: Labels{"derp_path": derpPath}, + Metrics: func(l prometheus.Labels) []prometheus.Metric { + qdh.mx.Lock() + result := []prometheus.Metric{ + prometheus.MustNewConstMetric(prometheus.NewDesc("derp_qd_probe_dropped_packets", "Total packets dropped", nil, l), prometheus.CounterValue, float64(packetsDropped.Value())), + prometheus.MustNewConstHistogram(prometheus.NewDesc("derp_qd_probe_delays_seconds", "Distribution of queuing delays", nil, l), qdh.count, qdh.sum, qdh.bucketedCounts), + } + qdh.mx.Unlock() + return result + }, + } +} + +// derpProbeQueuingDelay continuously sends data between two local DERP clients +// connected to two DERP servers in order to measure queuing delays. From and to +// can be the same server. +func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) { + // This probe uses clients with isProber=false to avoid spamming the derper + // logs with every packet sent by the queuing delay probe. + fromc, err := newConn(ctx, dm, from, false) + if err != nil { + return err + } + defer fromc.Close() + toc, err := newConn(ctx, dm, to, false) + if err != nil { + return err + } + defer toc.Close() + + // Wait a bit for from's node to hear about to existing on the + // other node in the region, in the case where the two nodes + // are different. + if from.Name != to.Name { + time.Sleep(100 * time.Millisecond) // pretty arbitrary + } + + if err := runDerpProbeQueuingDelayContinously(ctx, from, to, fromc, toc, packetsPerSecond, packetTimeout, packetsDropped, qdh); err != nil { + // Record pubkeys on failed probes to aid investigation. + return fmt.Errorf("%s -> %s: %w", + fromc.SelfPublicKey().ShortString(), + toc.SelfPublicKey().ShortString(), err) + } + return nil +} + +func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) error { + // Make sure all goroutines have finished. + var wg sync.WaitGroup + defer wg.Wait() + + // Close the clients to make sure goroutines that are reading/writing from them terminate. + defer fromc.Close() + defer toc.Close() + + type txRecord struct { + at time.Time + seq uint64 + } + // txRecords is sized to hold enough transmission records to keep timings + // for packets up to their timeout. As records age out of the front of this + // list, if the associated packet arrives, we won't have a txRecord for it + // and will consider it to have timed out. + txRecords := make([]txRecord, 0, packetsPerSecond*int(packetTimeout.Seconds())) + var txRecordsMu sync.Mutex + + // Send the packets. + sendErrC := make(chan error, 1) + // TODO: construct a disco CallMeMaybe in the same fashion as magicsock, e.g. magic bytes, src pub, seal payload. + // DERP server handling of disco may vary from non-disco, and we may want to measure queue delay of both. + pkt := make([]byte, 260) // the same size as a CallMeMaybe packet observed on a Tailscale client. + crand.Read(pkt) + + wg.Add(1) + go func() { + defer wg.Done() + t := time.NewTicker(time.Second / time.Duration(packetsPerSecond)) + defer t.Stop() + + seq := uint64(0) + for { + select { + case <-ctx.Done(): + return + case <-t.C: + txRecordsMu.Lock() + if len(txRecords) == cap(txRecords) { + txRecords = slices.Delete(txRecords, 0, 1) + packetsDropped.Add(1) + } + txRecords = append(txRecords, txRecord{time.Now(), seq}) + txRecordsMu.Unlock() + binary.BigEndian.PutUint64(pkt, seq) + seq++ + if err := fromc.Send(toc.SelfPublicKey(), pkt); err != nil { + sendErrC <- fmt.Errorf("sending packet %w", err) + return + } + } + } + }() + + // Receive the packets. + recvFinishedC := make(chan error, 1) + wg.Add(1) + go func() { + defer wg.Done() + defer close(recvFinishedC) // to break out of 'select' below. + for { + m, err := toc.Recv() + if err != nil { + recvFinishedC <- err + return + } + switch v := m.(type) { + case derp.ReceivedPacket: + now := time.Now() + if v.Source != fromc.SelfPublicKey() { + recvFinishedC <- fmt.Errorf("got data packet from unexpected source, %v", v.Source) + return + } + seq := binary.BigEndian.Uint64(v.Data) + txRecordsMu.Lock() + findTxRecord: + for i, record := range txRecords { + switch { + case record.seq == seq: + rtt := now.Sub(record.at) + qdh.add(rtt.Seconds()) + txRecords = slices.Delete(txRecords, i, i+1) + break findTxRecord + case record.seq > seq: + // No sent time found, probably a late arrival already + // recorded as drop by sender when deleted. + break findTxRecord + case record.seq < seq: + continue + } + } + txRecordsMu.Unlock() + + case derp.KeepAliveMessage: + // Silently ignore. + + default: + log.Printf("%v: ignoring Recv frame type %T", to.Name, v) + // Loop. + } + } + }() + + select { + case <-ctx.Done(): + return fmt.Errorf("timeout: %w", ctx.Err()) + case err := <-sendErrC: + return fmt.Errorf("error sending via %q: %w", from.Name, err) + case err := <-recvFinishedC: + if err != nil { + return fmt.Errorf("error receiving from %q: %w", to.Name, err) + } + } + return nil +} + // getNodePair returns DERPNode objects for two DERP servers based on their // short names. func (d *derpProber) getNodePair(n1, n2 string) (ret1, ret2 *tailcfg.DERPNode, _ error) { @@ -573,6 +787,8 @@ func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc recvc <- fmt.Errorf("got data packet %d from unexpected source, %v", idx, v.Source) return } + // This assumes that the packets are received reliably and in order. + // The DERP protocol does not guarantee this, but this probe assumes it. if got, want := v.Data, pkts[idx]; !bytes.Equal(got, want) { recvc <- fmt.Errorf("unexpected data packet %d (out of %d)", idx, len(pkts)) return diff --git a/prober/histogram.go b/prober/histogram.go new file mode 100644 index 0000000000000..e9005b452eb8a --- /dev/null +++ b/prober/histogram.go @@ -0,0 +1,50 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package prober + +import ( + "slices" + "sync" +) + +// histogram serves as an adapter to the Prometheus histogram datatype. +// The prober framework passes labels at custom metric collection time that +// it expects to be coupled with the returned metrics. See ProbeClass.Metrics +// and its call sites. Native prometheus histograms cannot be collected while +// injecting more labels. Instead we use this type and pass observations + +// collection labels to prometheus.MustNewConstHistogram() at prometheus +// metric collection time. +type histogram struct { + count uint64 + sum float64 + buckets []float64 + bucketedCounts map[float64]uint64 + mx sync.Mutex +} + +// newHistogram constructs a histogram that buckets data based on the given +// slice of upper bounds. +func newHistogram(buckets []float64) *histogram { + slices.Sort(buckets) + return &histogram{ + buckets: buckets, + bucketedCounts: make(map[float64]uint64, len(buckets)), + } +} + +func (h *histogram) add(v float64) { + h.mx.Lock() + defer h.mx.Unlock() + + h.count++ + h.sum += v + + for _, b := range h.buckets { + if v > b { + continue + } + h.bucketedCounts[b] += 1 + break + } +} diff --git a/prober/histogram_test.go b/prober/histogram_test.go new file mode 100644 index 0000000000000..a569167e63977 --- /dev/null +++ b/prober/histogram_test.go @@ -0,0 +1,29 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package prober + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestHistogram(t *testing.T) { + h := newHistogram([]float64{1, 2}) + h.add(0.5) + h.add(1) + h.add(1.5) + h.add(2) + h.add(2.5) + + if diff := cmp.Diff(h.count, uint64(5)); diff != "" { + t.Errorf("wrong count; (-got+want):%v", diff) + } + if diff := cmp.Diff(h.sum, 7.5); diff != "" { + t.Errorf("wrong sum; (-got+want):%v", diff) + } + if diff := cmp.Diff(h.bucketedCounts, map[float64]uint64{1: 2, 2: 2}); diff != "" { + t.Errorf("wrong bucketedCounts; (-got+want):%v", diff) + } +} diff --git a/prober/prober.go b/prober/prober.go index 2a43628bda908..e3860e7b9ba57 100644 --- a/prober/prober.go +++ b/prober/prober.go @@ -94,6 +94,9 @@ func newForTest(now func() time.Time, newTicker func(time.Duration) ticker) *Pro // Run executes probe class function every interval, and exports probe results under probeName. // +// If interval is negative, the probe will run continuously. If it encounters a failure while +// running continuously, it will pause for -1*interval and then retry. +// // Registering a probe under an already-registered name panics. func (p *Prober) Run(name string, interval time.Duration, labels Labels, pc ProbeClass) *Probe { p.mu.Lock() @@ -256,6 +259,11 @@ type Probe struct { latencyHist *ring.Ring } +// IsContinuous indicates that this is a continuous probe. +func (p *Probe) IsContinuous() bool { + return p.interval < 0 +} + // Close shuts down the Probe and unregisters it from its Prober. // It is safe to Run a new probe of the same name after Close returns. func (p *Probe) Close() error { @@ -288,6 +296,22 @@ func (p *Probe) loop() { return } + if p.IsContinuous() { + // Probe function is going to run continuously. + for { + p.run() + // Wait and then retry if probe fails. We use the inverse of the + // configured negative interval as our sleep period. + // TODO(percy):implement exponential backoff, possibly using logtail/backoff. + select { + case <-time.After(-1 * p.interval): + p.run() + case <-p.ctx.Done(): + return + } + } + } + p.tick = p.prober.newTicker(p.interval) defer p.tick.Stop() for { @@ -323,9 +347,13 @@ func (p *Probe) run() (pi ProbeInfo, err error) { p.recordEnd(err) } }() - timeout := time.Duration(float64(p.interval) * 0.8) - ctx, cancel := context.WithTimeout(p.ctx, timeout) - defer cancel() + ctx := p.ctx + if !p.IsContinuous() { + timeout := time.Duration(float64(p.interval) * 0.8) + var cancel func() + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } err = p.probeClass.Probe(ctx) p.recordEnd(err) @@ -365,6 +393,16 @@ func (p *Probe) recordEnd(err error) { p.successHist = p.successHist.Next() } +// ProbeStatus indicates the status of a probe. +type ProbeStatus string + +const ( + ProbeStatusUnknown = "unknown" + ProbeStatusRunning = "running" + ProbeStatusFailed = "failed" + ProbeStatusSucceeded = "succeeded" +) + // ProbeInfo is a snapshot of the configuration and state of a Probe. type ProbeInfo struct { Name string @@ -374,7 +412,7 @@ type ProbeInfo struct { Start time.Time End time.Time Latency time.Duration - Result bool + Status ProbeStatus Error string RecentResults []bool RecentLatencies []time.Duration @@ -402,6 +440,10 @@ func (pb ProbeInfo) RecentMedianLatency() time.Duration { return pb.RecentLatencies[len(pb.RecentLatencies)/2] } +func (pb ProbeInfo) Continuous() bool { + return pb.Interval < 0 +} + // ProbeInfo returns the state of all probes. func (p *Prober) ProbeInfo() map[string]ProbeInfo { out := map[string]ProbeInfo{} @@ -429,9 +471,14 @@ func (probe *Probe) probeInfoLocked() ProbeInfo { Labels: probe.metricLabels, Start: probe.start, End: probe.end, - Result: probe.succeeded, } - if probe.lastErr != nil { + inf.Status = ProbeStatusUnknown + if probe.end.Before(probe.start) { + inf.Status = ProbeStatusRunning + } else if probe.succeeded { + inf.Status = ProbeStatusSucceeded + } else if probe.lastErr != nil { + inf.Status = ProbeStatusFailed inf.Error = probe.lastErr.Error() } if probe.latency > 0 { @@ -467,7 +514,7 @@ func (p *Prober) RunHandler(w http.ResponseWriter, r *http.Request) error { p.mu.Lock() probe, ok := p.probes[name] p.mu.Unlock() - if !ok { + if !ok || probe.IsContinuous() { return tsweb.Error(http.StatusNotFound, fmt.Sprintf("unknown probe %q", name), nil) } @@ -531,7 +578,8 @@ func (p *Probe) Collect(ch chan<- prometheus.Metric) { if !p.start.IsZero() { ch <- prometheus.MustNewConstMetric(p.mStartTime, prometheus.GaugeValue, float64(p.start.Unix())) } - if p.end.IsZero() { + // For periodic probes that haven't ended, don't collect probe metrics yet. + if p.end.IsZero() && !p.IsContinuous() { return } ch <- prometheus.MustNewConstMetric(p.mEndTime, prometheus.GaugeValue, float64(p.end.Unix())) diff --git a/prober/prober_test.go b/prober/prober_test.go index 742a914b24661..3905bfbc91576 100644 --- a/prober/prober_test.go +++ b/prober/prober_test.go @@ -316,7 +316,7 @@ func TestProberProbeInfo(t *testing.T) { Interval: probeInterval, Labels: map[string]string{"class": "", "name": "probe1"}, Latency: 500 * time.Millisecond, - Result: true, + Status: ProbeStatusSucceeded, RecentResults: []bool{true}, RecentLatencies: []time.Duration{500 * time.Millisecond}, }, @@ -324,6 +324,7 @@ func TestProberProbeInfo(t *testing.T) { Name: "probe2", Interval: probeInterval, Labels: map[string]string{"class": "", "name": "probe2"}, + Status: ProbeStatusFailed, Error: "error2", RecentResults: []bool{false}, RecentLatencies: nil, // no latency for failed probes @@ -349,7 +350,7 @@ func TestProbeInfoRecent(t *testing.T) { }{ { name: "no_runs", - wantProbeInfo: ProbeInfo{}, + wantProbeInfo: ProbeInfo{Status: ProbeStatusUnknown}, wantRecentSuccessRatio: 0, wantRecentMedianLatency: 0, }, @@ -358,7 +359,7 @@ func TestProbeInfoRecent(t *testing.T) { results: []probeResult{{latency: 100 * time.Millisecond, err: nil}}, wantProbeInfo: ProbeInfo{ Latency: 100 * time.Millisecond, - Result: true, + Status: ProbeStatusSucceeded, RecentResults: []bool{true}, RecentLatencies: []time.Duration{100 * time.Millisecond}, }, @@ -369,7 +370,7 @@ func TestProbeInfoRecent(t *testing.T) { name: "single_failure", results: []probeResult{{latency: 100 * time.Millisecond, err: errors.New("error123")}}, wantProbeInfo: ProbeInfo{ - Result: false, + Status: ProbeStatusFailed, RecentResults: []bool{false}, RecentLatencies: nil, Error: "error123", @@ -390,7 +391,7 @@ func TestProbeInfoRecent(t *testing.T) { {latency: 80 * time.Millisecond, err: nil}, }, wantProbeInfo: ProbeInfo{ - Result: true, + Status: ProbeStatusSucceeded, Latency: 80 * time.Millisecond, RecentResults: []bool{false, true, true, false, true, true, false, true}, RecentLatencies: []time.Duration{ @@ -420,7 +421,7 @@ func TestProbeInfoRecent(t *testing.T) { {latency: 110 * time.Millisecond, err: nil}, }, wantProbeInfo: ProbeInfo{ - Result: true, + Status: ProbeStatusSucceeded, Latency: 110 * time.Millisecond, RecentResults: []bool{true, true, true, true, true, true, true, true, true, true}, RecentLatencies: []time.Duration{ @@ -483,7 +484,7 @@ func TestProberRunHandler(t *testing.T) { ProbeInfo: ProbeInfo{ Name: "success", Interval: probeInterval, - Result: true, + Status: ProbeStatusSucceeded, RecentResults: []bool{true, true}, }, PreviousSuccessRatio: 1, @@ -498,7 +499,7 @@ func TestProberRunHandler(t *testing.T) { ProbeInfo: ProbeInfo{ Name: "failure", Interval: probeInterval, - Result: false, + Status: ProbeStatusFailed, Error: "error123", RecentResults: []bool{false, false}, }, diff --git a/prober/status.go b/prober/status.go index aa9ef99d05d2c..20fbeec58a77e 100644 --- a/prober/status.go +++ b/prober/status.go @@ -62,8 +62,9 @@ func (p *Prober) StatusHandler(opts ...statusHandlerOpt) tsweb.ReturnHandlerFunc return func(w http.ResponseWriter, r *http.Request) error { type probeStatus struct { ProbeInfo - TimeSinceLast time.Duration - Links map[string]template.URL + TimeSinceLastStart time.Duration + TimeSinceLastEnd time.Duration + Links map[string]template.URL } vars := struct { Title string @@ -81,12 +82,15 @@ func (p *Prober) StatusHandler(opts ...statusHandlerOpt) tsweb.ReturnHandlerFunc for name, info := range p.ProbeInfo() { vars.TotalProbes++ - if !info.Result { + if info.Error != "" { vars.UnhealthyProbes++ } s := probeStatus{ProbeInfo: info} + if !info.Start.IsZero() { + s.TimeSinceLastStart = time.Since(info.Start).Truncate(time.Second) + } if !info.End.IsZero() { - s.TimeSinceLast = time.Since(info.End).Truncate(time.Second) + s.TimeSinceLastEnd = time.Since(info.End).Truncate(time.Second) } for textTpl, urlTpl := range params.probeLinks { text, err := renderTemplate(textTpl, info) diff --git a/prober/status.html b/prober/status.html index ff0f06c13fe62..d26588da19431 100644 --- a/prober/status.html +++ b/prober/status.html @@ -73,8 +73,9 @@

Probes:

Name Probe Class & Labels Interval - Last Attempt - Success + Last Finished + Last Started + Status Latency Last Error @@ -85,9 +86,11 @@

Probes:

{{$name}} {{range $text, $url := $probeInfo.Links}}
- + {{if not $probeInfo.Continuous}} + + {{end}} {{end}} {{$probeInfo.Class}}
@@ -97,28 +100,48 @@

Probes:

{{end}} - {{$probeInfo.Interval}} - - {{if $probeInfo.TimeSinceLast}} - {{$probeInfo.TimeSinceLast.String}} ago
+ + {{if $probeInfo.Continuous}} + Continuous + {{else}} + {{$probeInfo.Interval}} + {{end}} + + + {{if $probeInfo.TimeSinceLastEnd}} + {{$probeInfo.TimeSinceLastEnd.String}} ago
{{$probeInfo.End.Format "2006-01-02T15:04:05Z07:00"}} {{else}} Never {{end}} + + {{if $probeInfo.TimeSinceLastStart}} + {{$probeInfo.TimeSinceLastStart.String}} ago
+ {{$probeInfo.Start.Format "2006-01-02T15:04:05Z07:00"}} + {{else}} + Never + {{end}} + - {{if $probeInfo.Result}} - {{$probeInfo.Result}} + {{if $probeInfo.Error}} + {{$probeInfo.Status}} {{else}} - {{$probeInfo.Result}} + {{$probeInfo.Status}} {{end}}
-
Recent: {{$probeInfo.RecentResults}}
-
Mean: {{$probeInfo.RecentSuccessRatio}}
+ {{if not $probeInfo.Continuous}} +
Recent: {{$probeInfo.RecentResults}}
+
Mean: {{$probeInfo.RecentSuccessRatio}}
+ {{end}} - {{$probeInfo.Latency.String}} -
Recent: {{$probeInfo.RecentLatencies}}
-
Median: {{$probeInfo.RecentMedianLatency}}
+ {{if $probeInfo.Continuous}} + n/a + {{else}} + {{$probeInfo.Latency.String}} +
Recent: {{$probeInfo.RecentLatencies}}
+
Median: {{$probeInfo.RecentMedianLatency}}
+ {{end}} {{$probeInfo.Error}} From 2d4edd80f11b6263a4ef23c4e39032af00279b5a Mon Sep 17 00:00:00 2001 From: Will Norris Date: Thu, 19 Dec 2024 13:11:25 -0800 Subject: [PATCH 033/223] cmd/systray: add extra padding around notification icon Some notification managers crop the application icon to a circle, so ensure we have enough padding to account for that. Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/logo.go | 8 +++++++- cmd/systray/systray.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/systray/logo.go b/cmd/systray/logo.go index cd79c94a02ea4..ef8caca66d89e 100644 --- a/cmd/systray/logo.go +++ b/cmd/systray/logo.go @@ -128,8 +128,14 @@ var ( // render returns a PNG image of the logo. func (logo tsLogo) render() *bytes.Buffer { - const radius = 25 const borderUnits = 1 + return logo.renderWithBorder(borderUnits) +} + +// renderWithBorder returns a PNG image of the logo with the specified border width. +// One border unit is equal to the radius of a tailscale logo dot. +func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer { + const radius = 25 dim := radius * (8 + borderUnits*2) dc := gg.NewContext(dim, dim) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index d175b55f309d7..a3cd19c6454de 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -62,7 +62,7 @@ func onReady() { // dbus wants a file path for notification icons, so copy to a temp file. appIcon, _ = os.CreateTemp("", "tailscale-systray.png") - io.Copy(appIcon, connected.render()) + io.Copy(appIcon, connected.renderWithBorder(3)) chState = make(chan ipn.State, 1) From e8f1721147eb8d709232363e67cf96878adf8c9c Mon Sep 17 00:00:00 2001 From: James Tucker Date: Tue, 17 Dec 2024 12:22:44 -0800 Subject: [PATCH 034/223] syncs: add ShardedInt expvar.Var type ShardedInt provides an int type expvar.Var that supports more efficient writes at high frequencies (one order of magnigude on an M1 Max, much more on NUMA systems). There are two implementations of ShardValue, one that abuses sync.Pool that will work on current public Go versions, and one that takes a dependency on a runtime.TailscaleP function exposed in Tailscale's Go fork. The sync.Pool variant has about 10x the throughput of a single atomic integer on an M1 Max, and the runtime.TailscaleP variant is about 10x faster than the sync.Pool variant. Neither variant have perfect distribution, or perfectly always avoid cross-CPU sharing, as there is no locking or affinity to ensure that the time of yield is on the same core as the time of core biasing, but in the average case the distributions are enough to provide substantially better performance. See golang/go#18802 for a related upstream proposal. Updates tailscale/go#109 Updates tailscale/corp#25450 Signed-off-by: James Tucker --- go.toolchain.rev | 2 +- syncs/shardedint.go | 69 ++++++++++++++++++++ syncs/shardedint_test.go | 119 ++++++++++++++++++++++++++++++++++ syncs/shardvalue.go | 36 ++++++++++ syncs/shardvalue_go.go | 36 ++++++++++ syncs/shardvalue_tailscale.go | 24 +++++++ syncs/shardvalue_test.go | 119 ++++++++++++++++++++++++++++++++++ 7 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 syncs/shardedint.go create mode 100644 syncs/shardedint_test.go create mode 100644 syncs/shardvalue.go create mode 100644 syncs/shardvalue_go.go create mode 100644 syncs/shardvalue_tailscale.go create mode 100644 syncs/shardvalue_test.go diff --git a/go.toolchain.rev b/go.toolchain.rev index 7be85deb6e796..e90440d41701f 100644 --- a/go.toolchain.rev +++ b/go.toolchain.rev @@ -1 +1 @@ -e005697288a8d2fadc87bb7c3e2c74778d08554a +161c3b79ed91039e65eb148f2547dea6b91e2247 diff --git a/syncs/shardedint.go b/syncs/shardedint.go new file mode 100644 index 0000000000000..28c4168d54c79 --- /dev/null +++ b/syncs/shardedint.go @@ -0,0 +1,69 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package syncs + +import ( + "encoding/json" + "strconv" + "sync/atomic" + + "golang.org/x/sys/cpu" +) + +// ShardedInt provides a sharded atomic int64 value that optimizes high +// frequency (Mhz range and above) writes in highly parallel workloads. +// The zero value is not safe for use; use [NewShardedInt]. +// ShardedInt implements the expvar.Var interface. +type ShardedInt struct { + sv *ShardValue[intShard] +} + +// NewShardedInt returns a new [ShardedInt]. +func NewShardedInt() *ShardedInt { + return &ShardedInt{ + sv: NewShardValue[intShard](), + } +} + +// Add adds delta to the value. +func (m *ShardedInt) Add(delta int64) { + m.sv.One(func(v *intShard) { + v.Add(delta) + }) +} + +type intShard struct { + atomic.Int64 + _ cpu.CacheLinePad // avoid false sharing of neighboring shards +} + +// Value returns the current value. +func (m *ShardedInt) Value() int64 { + var v int64 + for s := range m.sv.All { + v += s.Load() + } + return v +} + +// GetDistribution returns the current value in each shard. +// This is intended for observability/debugging only. +func (m *ShardedInt) GetDistribution() []int64 { + v := make([]int64, 0, m.sv.Len()) + for s := range m.sv.All { + v = append(v, s.Load()) + } + return v +} + +// String implements the expvar.Var interface +func (m *ShardedInt) String() string { + v, _ := json.Marshal(m.Value()) + return string(v) +} + +// AppendText implements the encoding.TextAppender interface +func (m *ShardedInt) AppendText(b []byte) ([]byte, error) { + return strconv.AppendInt(b, m.Value(), 10), nil +} diff --git a/syncs/shardedint_test.go b/syncs/shardedint_test.go new file mode 100644 index 0000000000000..d355a15400a90 --- /dev/null +++ b/syncs/shardedint_test.go @@ -0,0 +1,119 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package syncs + +import ( + "expvar" + "sync" + "testing" + + "tailscale.com/tstest" +) + +var ( + _ expvar.Var = (*ShardedInt)(nil) + // TODO(raggi): future go version: + // _ encoding.TextAppender = (*ShardedInt)(nil) +) + +func BenchmarkShardedInt(b *testing.B) { + b.ReportAllocs() + + b.Run("expvar", func(b *testing.B) { + var m expvar.Int + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + m.Add(1) + } + }) + }) + + b.Run("sharded int", func(b *testing.B) { + m := NewShardedInt() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + m.Add(1) + } + }) + }) +} + +func TestShardedInt(t *testing.T) { + t.Run("basics", func(t *testing.T) { + m := NewShardedInt() + if got, want := m.Value(), int64(0); got != want { + t.Errorf("got %v, want %v", got, want) + } + m.Add(1) + if got, want := m.Value(), int64(1); got != want { + t.Errorf("got %v, want %v", got, want) + } + m.Add(2) + if got, want := m.Value(), int64(3); got != want { + t.Errorf("got %v, want %v", got, want) + } + m.Add(-1) + if got, want := m.Value(), int64(2); got != want { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("high concurrency", func(t *testing.T) { + m := NewShardedInt() + wg := sync.WaitGroup{} + numWorkers := 1000 + numIncrements := 1000 + wg.Add(numWorkers) + for i := 0; i < numWorkers; i++ { + go func() { + defer wg.Done() + for i := 0; i < numIncrements; i++ { + m.Add(1) + } + }() + } + wg.Wait() + if got, want := m.Value(), int64(numWorkers*numIncrements); got != want { + t.Errorf("got %v, want %v", got, want) + } + for i, shard := range m.GetDistribution() { + t.Logf("shard %d: %d", i, shard) + } + }) + + t.Run("encoding.TextAppender", func(t *testing.T) { + m := NewShardedInt() + m.Add(1) + b := make([]byte, 0, 10) + b, err := m.AppendText(b) + if err != nil { + t.Fatal(err) + } + if got, want := string(b), "1"; got != want { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("allocs", func(t *testing.T) { + m := NewShardedInt() + tstest.MinAllocsPerRun(t, 0, func() { + m.Add(1) + _ = m.Value() + }) + + // TODO(raggi): fix access to expvar's internal append based + // interface, unfortunately it's not currently closed for external + // use, this will alloc when it escapes. + tstest.MinAllocsPerRun(t, 0, func() { + m.Add(1) + _ = m.String() + }) + + b := make([]byte, 0, 10) + tstest.MinAllocsPerRun(t, 0, func() { + m.Add(1) + m.AppendText(b) + }) + }) +} diff --git a/syncs/shardvalue.go b/syncs/shardvalue.go new file mode 100644 index 0000000000000..b1474477c7082 --- /dev/null +++ b/syncs/shardvalue.go @@ -0,0 +1,36 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package syncs + +// TODO(raggi): this implementation is still imperfect as it will still result +// in cross CPU sharing periodically, we instead really want a per-CPU shard +// key, but the limitations of calling platform code make reaching for even the +// getcpu vdso very painful. See https://github.com/golang/go/issues/18802, and +// hopefully one day we can replace with a primitive that falls out of that +// work. + +// ShardValue contains a value sharded over a set of shards. +// In order to be useful, T should be aligned to cache lines. +// Users must organize that usage in One and All is concurrency safe. +// The zero value is not safe for use; use [NewShardValue]. +type ShardValue[T any] struct { + shards []T + + //lint:ignore U1000 unused under tailscale_go builds. + pool shardValuePool +} + +// Len returns the number of shards. +func (sp *ShardValue[T]) Len() int { + return len(sp.shards) +} + +// All yields a pointer to the value in each shard. +func (sp *ShardValue[T]) All(yield func(*T) bool) { + for i := range sp.shards { + if !yield(&sp.shards[i]) { + return + } + } +} diff --git a/syncs/shardvalue_go.go b/syncs/shardvalue_go.go new file mode 100644 index 0000000000000..9b9d252a796d4 --- /dev/null +++ b/syncs/shardvalue_go.go @@ -0,0 +1,36 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !tailscale_go + +package syncs + +import ( + "runtime" + "sync" + "sync/atomic" +) + +type shardValuePool struct { + atomic.Int64 + sync.Pool +} + +// NewShardValue constructs a new ShardValue[T] with a shard per CPU. +func NewShardValue[T any]() *ShardValue[T] { + sp := &ShardValue[T]{ + shards: make([]T, runtime.NumCPU()), + } + sp.pool.New = func() any { + i := sp.pool.Add(1) - 1 + return &sp.shards[i%int64(len(sp.shards))] + } + return sp +} + +// One yields a pointer to a single shard value with best-effort P-locality. +func (sp *ShardValue[T]) One(yield func(*T)) { + v := sp.pool.Get().(*T) + yield(v) + sp.pool.Put(v) +} diff --git a/syncs/shardvalue_tailscale.go b/syncs/shardvalue_tailscale.go new file mode 100644 index 0000000000000..8ef778ff3e669 --- /dev/null +++ b/syncs/shardvalue_tailscale.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// TODO(raggi): update build tag after toolchain update +//go:build tailscale_go + +package syncs + +import ( + "runtime" +) + +//lint:ignore U1000 unused under tailscale_go builds. +type shardValuePool struct{} + +// NewShardValue constructs a new ShardValue[T] with a shard per CPU. +func NewShardValue[T any]() *ShardValue[T] { + return &ShardValue[T]{shards: make([]T, runtime.NumCPU())} +} + +// One yields a pointer to a single shard value with best-effort P-locality. +func (sp *ShardValue[T]) One(f func(*T)) { + f(&sp.shards[runtime.TailscaleCurrentP()%len(sp.shards)]) +} diff --git a/syncs/shardvalue_test.go b/syncs/shardvalue_test.go new file mode 100644 index 0000000000000..8f6ac6414dee7 --- /dev/null +++ b/syncs/shardvalue_test.go @@ -0,0 +1,119 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package syncs + +import ( + "math" + "runtime" + "sync" + "sync/atomic" + "testing" + + "golang.org/x/sys/cpu" +) + +func TestShardValue(t *testing.T) { + type intVal struct { + atomic.Int64 + _ cpu.CacheLinePad + } + + t.Run("One", func(t *testing.T) { + sv := NewShardValue[intVal]() + sv.One(func(v *intVal) { + v.Store(10) + }) + + var v int64 + for i := range sv.shards { + v += sv.shards[i].Load() + } + if v != 10 { + t.Errorf("got %v, want 10", v) + } + }) + + t.Run("All", func(t *testing.T) { + sv := NewShardValue[intVal]() + for i := range sv.shards { + sv.shards[i].Store(int64(i)) + } + + var total int64 + sv.All(func(v *intVal) bool { + total += v.Load() + return true + }) + // triangle coefficient lower one order due to 0 index + want := int64(len(sv.shards) * (len(sv.shards) - 1) / 2) + if total != want { + t.Errorf("got %v, want %v", total, want) + } + }) + + t.Run("Len", func(t *testing.T) { + sv := NewShardValue[intVal]() + if got, want := sv.Len(), runtime.NumCPU(); got != want { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("distribution", func(t *testing.T) { + sv := NewShardValue[intVal]() + + goroutines := 1000 + iterations := 10000 + var wg sync.WaitGroup + wg.Add(goroutines) + for i := 0; i < goroutines; i++ { + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + sv.One(func(v *intVal) { + v.Add(1) + }) + } + }() + } + wg.Wait() + + var ( + total int64 + distribution []int64 + ) + t.Logf("distribution:") + sv.All(func(v *intVal) bool { + total += v.Load() + distribution = append(distribution, v.Load()) + t.Logf("%d", v.Load()) + return true + }) + + if got, want := total, int64(goroutines*iterations); got != want { + t.Errorf("got %v, want %v", got, want) + } + if got, want := len(distribution), runtime.NumCPU(); got != want { + t.Errorf("got %v, want %v", got, want) + } + + mean := total / int64(len(distribution)) + for _, v := range distribution { + if v < mean/10 || v > mean*10 { + t.Logf("distribution is very unbalanced: %v", distribution) + } + } + t.Logf("mean: %d", mean) + + var standardDev int64 + for _, v := range distribution { + standardDev += ((v - mean) * (v - mean)) + } + standardDev = int64(math.Sqrt(float64(standardDev / int64(len(distribution))))) + t.Logf("stdev: %d", standardDev) + + if standardDev > mean/3 { + t.Logf("standard deviation is too high: %v", standardDev) + } + }) +} From 89adcd853dab6463d437e3a023b90704b66f3a3f Mon Sep 17 00:00:00 2001 From: Will Norris Date: Thu, 19 Dec 2024 11:31:31 -0800 Subject: [PATCH 035/223] cmd/systray: improve profile menu Bring UI closer to macOS and windows: - split login and tailnet name over separate lines - render profile picture (with very simple caching) - use checkbox to indicate active profile. I've not found any desktops that can't render checkboxes, so I'd like to explore other options if needed. Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 55 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index a3cd19c6454de..504ca5b8c1e0e 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "log" + "net/http" "os" "strings" "sync" @@ -118,10 +119,11 @@ func (menu *Menu) rebuild(state state) { systray.AddSeparator() account := "Account" - if state.curProfile.Name != "" { - account += fmt.Sprintf(" (%s)", state.curProfile.Name) + if pt := profileTitle(state.curProfile); pt != "" { + account = pt } accounts := systray.AddMenuItem(account, "") + setRemoteIcon(accounts, state.curProfile.UserProfile.ProfilePicURL) // The dbus message about this menu item must propagate to the receiving // end before we attach any submenu items. Otherwise the receiver may not // yet record the parent menu item and error out. @@ -132,13 +134,14 @@ func (menu *Menu) rebuild(state state) { // Aggregate all clicks into a shared channel. menu.accountsCh = make(chan ipn.ProfileID) for _, profile := range state.allProfiles { - title := fmt.Sprintf("%s (%s)", profile.Name, profile.NetworkProfile.DomainName) - // Note: we could use AddSubMenuItemCheckbox instead of this formatting - // hack, but checkboxes don't work across all desktops unfortunately. + title := profileTitle(profile) + var item *systray.MenuItem if profile.ID == state.curProfile.ID { - title = "* " + title + item = accounts.AddSubMenuItemCheckbox(title, "", true) + } else { + item = accounts.AddSubMenuItem(title, "") } - item := accounts.AddSubMenuItem(title, "") + setRemoteIcon(item, profile.UserProfile.ProfilePicURL) go func(profile ipn.LoginProfile) { for { select { @@ -170,6 +173,44 @@ func (menu *Menu) rebuild(state state) { go menu.eventLoop(ctx) } +// profileTitle returns the title string for a profile menu item. +func profileTitle(profile ipn.LoginProfile) string { + title := profile.Name + if profile.NetworkProfile.DomainName != "" { + title += "\n" + profile.NetworkProfile.DomainName + } + return title +} + +var ( + cacheMu sync.Mutex + httpCache = map[string][]byte{} // URL => response body +) + +// setRemoteIcon sets the icon for menu to the specified remote image. +// Remote images are fetched as needed and cached. +func setRemoteIcon(menu *systray.MenuItem, urlStr string) { + if menu == nil || urlStr == "" { + return + } + + cacheMu.Lock() + b, ok := httpCache[urlStr] + if !ok { + resp, err := http.Get(urlStr) + if err == nil && resp.StatusCode == http.StatusOK { + b, _ = io.ReadAll(resp.Body) + httpCache[urlStr] = b + resp.Body.Close() + } + } + cacheMu.Unlock() + + if len(b) > 0 { + menu.SetIcon(b) + } +} + // eventLoop is the main event loop for handling click events on menu items // and responding to Tailscale state changes. // This method does not return until ctx.Done is closed. From 3adad364f137b072ef0342bf51aa23c4647908ba Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Fri, 20 Dec 2024 16:12:56 +0000 Subject: [PATCH 036/223] cmd/k8s-operator,k8s-operator: include top-level CRD descriptions (#14435) When reading https://doc.crds.dev/github.com/tailscale/tailscale/tailscale.com/ProxyGroup/v1alpha1@v1.78.3 I noticed there is no top-level description for ProxyGroup and Recorder. Add one to give some high-level direction. Updates #cleanup Change-Id: I3666c5445be272ea5a1d4d02b6d5ad4c23afb09f Signed-off-by: Tom Proctor --- .../deploy/crds/tailscale.com_proxygroups.yaml | 11 +++++++++++ .../deploy/crds/tailscale.com_recorders.yaml | 6 ++++++ cmd/k8s-operator/deploy/manifests/operator.yaml | 17 +++++++++++++++++ k8s-operator/api.md | 13 +++++++++++++ k8s-operator/apis/v1alpha1/types_proxygroup.go | 10 ++++++++++ k8s-operator/apis/v1alpha1/types_recorder.go | 5 +++++ 6 files changed, 62 insertions(+) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 66701bdf4afbd..5e6b537853794 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -23,6 +23,17 @@ spec: name: v1alpha1 schema: openAPIV3Schema: + description: |- + ProxyGroup defines a set of Tailscale devices that will act as proxies. + Currently only egress ProxyGroups are supported. + + Use the tailscale.com/proxy-group annotation on a Service to specify that + the egress proxy should be implemented by a ProxyGroup instead of a single + dedicated proxy. In addition to running a highly available set of proxies, + ProxyGroup also allows for serving many annotated Services from a single + set of proxies to minimise resource consumption. + + More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress type: object required: - spec diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml index fda8bcebdbe53..5b22297d8b774 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml @@ -27,6 +27,12 @@ spec: name: v1alpha1 schema: openAPIV3Schema: + description: |- + Recorder defines a tsrecorder device for recording SSH sessions. By default, + it will store recordings in a local ephemeral volume. If you want to persist + recordings, you can configure an S3-compatible API for storage. + + More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder type: object required: - spec diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 210a7b43463e5..dd34c2a1e5a5c 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2724,6 +2724,17 @@ spec: name: v1alpha1 schema: openAPIV3Schema: + description: |- + ProxyGroup defines a set of Tailscale devices that will act as proxies. + Currently only egress ProxyGroups are supported. + + Use the tailscale.com/proxy-group annotation on a Service to specify that + the egress proxy should be implemented by a ProxyGroup instead of a single + dedicated proxy. In addition to running a highly available set of proxies, + ProxyGroup also allows for serving many annotated Services from a single + set of proxies to minimise resource consumption. + + More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress properties: apiVersion: description: |- @@ -2916,6 +2927,12 @@ spec: name: v1alpha1 schema: openAPIV3Schema: + description: |- + Recorder defines a tsrecorder device for recording SSH sessions. By default, + it will store recordings in a local ephemeral volume. If you want to persist + recordings, you can configure an S3-compatible API for storage. + + More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder properties: apiVersion: description: |- diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 08e1284fe82e7..327f95ea9eb82 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -508,7 +508,16 @@ _Appears in:_ +ProxyGroup defines a set of Tailscale devices that will act as proxies. +Currently only egress ProxyGroups are supported. +Use the tailscale.com/proxy-group annotation on a Service to specify that +the egress proxy should be implemented by a ProxyGroup instead of a single +dedicated proxy. In addition to running a highly available set of proxies, +ProxyGroup also allows for serving many annotated Services from a single +set of proxies to minimise resource consumption. + +More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress @@ -602,7 +611,11 @@ _Appears in:_ +Recorder defines a tsrecorder device for recording SSH sessions. By default, +it will store recordings in a local ephemeral volume. If you want to persist +recordings, you can configure an S3-compatible API for storage. +More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index 7e5515ba9d66c..e7397f33ec5ba 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -14,6 +14,16 @@ import ( // +kubebuilder:resource:scope=Cluster,shortName=pg // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyGroupReady")].reason`,description="Status of the deployed ProxyGroup resources." +// ProxyGroup defines a set of Tailscale devices that will act as proxies. +// Currently only egress ProxyGroups are supported. +// +// Use the tailscale.com/proxy-group annotation on a Service to specify that +// the egress proxy should be implemented by a ProxyGroup instead of a single +// dedicated proxy. In addition to running a highly available set of proxies, +// ProxyGroup also allows for serving many annotated Services from a single +// set of proxies to minimise resource consumption. +// +// More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress type ProxyGroup struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/k8s-operator/apis/v1alpha1/types_recorder.go b/k8s-operator/apis/v1alpha1/types_recorder.go index 3728154b45170..a32b8eb93c1c9 100644 --- a/k8s-operator/apis/v1alpha1/types_recorder.go +++ b/k8s-operator/apis/v1alpha1/types_recorder.go @@ -16,6 +16,11 @@ import ( // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "RecorderReady")].reason`,description="Status of the deployed Recorder resources." // +kubebuilder:printcolumn:name="URL",type="string",JSONPath=`.status.devices[?(@.url != "")].url`,description="URL on which the UI is exposed if enabled." +// Recorder defines a tsrecorder device for recording SSH sessions. By default, +// it will store recordings in a local ephemeral volume. If you want to persist +// recordings, you can configure an S3-compatible API for storage. +// +// More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder type Recorder struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` From 5095efd62831110e65cad79740bf302492756f1e Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Fri, 20 Dec 2024 08:07:54 -0600 Subject: [PATCH 037/223] prober: make histogram buckets cumulative Histogram buckets should include counts for all values under the bucket ceiling, not just those between the ceiling and the next lower ceiling. See https://prometheus.io/docs/tutorials/understanding_metric_types/\#histogram Updates tailscale/corp#24522 Signed-off-by: Percy Wegmann --- prober/histogram.go | 1 - prober/histogram_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/prober/histogram.go b/prober/histogram.go index e9005b452eb8a..c544a5f79bb17 100644 --- a/prober/histogram.go +++ b/prober/histogram.go @@ -45,6 +45,5 @@ func (h *histogram) add(v float64) { continue } h.bucketedCounts[b] += 1 - break } } diff --git a/prober/histogram_test.go b/prober/histogram_test.go index a569167e63977..dbb5eda6741a5 100644 --- a/prober/histogram_test.go +++ b/prober/histogram_test.go @@ -23,7 +23,7 @@ func TestHistogram(t *testing.T) { if diff := cmp.Diff(h.sum, 7.5); diff != "" { t.Errorf("wrong sum; (-got+want):%v", diff) } - if diff := cmp.Diff(h.bucketedCounts, map[float64]uint64{1: 2, 2: 2}); diff != "" { + if diff := cmp.Diff(h.bucketedCounts, map[float64]uint64{1: 2, 2: 4}); diff != "" { t.Errorf("wrong bucketedCounts; (-got+want):%v", diff) } } From 256da8dfb5fc30ff8ac6405ef66bbc1880e01e30 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Fri, 20 Dec 2024 09:11:04 -0800 Subject: [PATCH 038/223] cmd/systray: remove new menu delay on KDE The new menu delay added to fix libdbusmenu systrays causes problems with KDE. Given the state of wildly varying systray implementations, I suspect we may need more desktop-specific hacks, so I'm setting this up to accommodate that. Updates #1708 Updates #14431 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 504ca5b8c1e0e..5b20ddde40d32 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -14,6 +14,7 @@ import ( "log" "net/http" "os" + "runtime" "strings" "sync" "time" @@ -32,6 +33,10 @@ var ( chState chan ipn.State // tailscale state changes appIcon *os.File + + // newMenuDelay is the amount of time to sleep after creating a new menu, + // but before adding items to it. This works around a bug in some dbus implementations. + newMenuDelay time.Duration ) func main() { @@ -55,6 +60,30 @@ type Menu struct { eventCancel func() // cancel eventLoop } +func init() { + if runtime.GOOS != "linux" { + // so far, these tweaks are only needed on Linux + return + } + + desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP")) + switch desktop { + case "kde": + // KDE doesn't need a delay, and actually won't render submenus + // if we delay for more than about 400Âĩs. + newMenuDelay = 0 + default: + // Add a slight delay to ensure the menu is created before adding items. + // + // Systray implementations that use libdbusmenu sometimes process messages out of order, + // resulting in errors such as: + // (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu' + // + // See also: https://github.com/fyne-io/systray/issues/12 + newMenuDelay = 100 * time.Millisecond + } +} + func onReady() { log.Printf("starting") ctx := context.Background() @@ -124,13 +153,7 @@ func (menu *Menu) rebuild(state state) { } accounts := systray.AddMenuItem(account, "") setRemoteIcon(accounts, state.curProfile.UserProfile.ProfilePicURL) - // The dbus message about this menu item must propagate to the receiving - // end before we attach any submenu items. Otherwise the receiver may not - // yet record the parent menu item and error out. - // - // On waybar with libdbusmenu-gtk, this manifests as the following warning: - // (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu' - time.Sleep(100 * time.Millisecond) + time.Sleep(newMenuDelay) // Aggregate all clicks into a shared channel. menu.accountsCh = make(chan ipn.ProfileID) for _, profile := range state.allProfiles { From 887472312d7b896e8d59d235b68257e2bb7ea317 Mon Sep 17 00:00:00 2001 From: Naman Sood Date: Fri, 20 Dec 2024 15:57:46 -0500 Subject: [PATCH 039/223] tailcfg: rename and retype ServiceHost capability (#14380) * tailcfg: rename and retype ServiceHost capability, add value type Updates tailscale/corp#22743. In #14046, this was accidentally made a PeerCapability when it should have been NodeCapability. Also, renaming it to use the nomenclature that we decided on after #14046 went up, and adding the type of the value that will be passed down in the RawMessage for this capability. This shouldn't break anything, since no one was using this string or variable yet. Signed-off-by: Naman Sood --- tailcfg/tailcfg.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index be6c4f0be6b82..ad07cff288a13 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -1453,11 +1453,6 @@ const ( // user groups as Kubernetes user groups. This capability is read by // peers that are Tailscale Kubernetes operator instances. PeerCapabilityKubernetes PeerCapability = "tailscale.com/cap/kubernetes" - - // PeerCapabilityServicesDestination grants a peer the ability to serve as - // a destination for a set of given VIP services, which is provided as the - // value of this key in NodeCapMap. - PeerCapabilityServicesDestination PeerCapability = "tailscale.com/cap/services-destination" ) // NodeCapMap is a map of capabilities to their optional values. It is valid for @@ -2401,6 +2396,15 @@ const ( // NodeAttrSSHEnvironmentVariables enables logic for handling environment variables sent // via SendEnv in the SSH server and applying them to the SSH session. NodeAttrSSHEnvironmentVariables NodeCapability = "ssh-env-vars" + + // NodeAttrServiceHost indicates the VIP Services for which the client is + // approved to act as a service host, and which IP addresses are assigned + // to those VIP Services. Any VIP Services that the client is not + // advertising can be ignored. + // Each value of this key in [NodeCapMap] is of type [ServiceIPMappings]. + // If multiple values of this key exist, they should be merged in sequence + // (replace conflicting keys). + NodeAttrServiceHost NodeCapability = "service-host" ) // SetDNSRequest is a request to add a DNS record. @@ -2883,3 +2887,21 @@ type EarlyNoise struct { // For some request types, the header may have multiple values. (e.g. OldNodeKey // vs NodeKey) const LBHeader = "Ts-Lb" + +// ServiceIPMappings maps service names (strings that conform to +// [CheckServiceName]) to lists of IP addresses. This is used as the value of +// the [NodeAttrServiceHost] capability, to inform service hosts what IP +// addresses they need to listen on for each service that they are advertising. +// +// This is of the form: +// +// { +// "svc:samba": ["100.65.32.1", "fd7a:115c:a1e0::1234"], +// "svc:web": ["100.102.42.3", "fd7a:115c:a1e0::abcd"], +// } +// +// where the IP addresses are the IPs of the VIP services. These IPs are also +// provided in AllowedIPs, but this lets the client know which services +// correspond to those IPs. Any services that don't correspond to a service +// this client is hosting can be ignored. +type ServiceIPMappings map[string][]netip.Addr From cb59943501b068911eecd490119d3ca8f98c7129 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Fri, 20 Dec 2024 15:37:00 -0800 Subject: [PATCH 040/223] cmd/systray: add exit nodes menu This commit builds the exit node menu including the recommended exit node, if available, as well as tailnet and mullvad exit nodes. This does not yet update the menu based on changes in exit node outside of the systray app, which will come later. This also does not include the ability to run as an exit node. Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 255 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 251 insertions(+), 4 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 5b20ddde40d32..1334a0351a8aa 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -7,14 +7,17 @@ package main import ( + "cmp" "context" "errors" "fmt" "io" "log" + "maps" "net/http" "os" "runtime" + "slices" "strings" "sync" "time" @@ -26,6 +29,7 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" ) var ( @@ -51,11 +55,13 @@ type Menu struct { connect *systray.MenuItem disconnect *systray.MenuItem - self *systray.MenuItem - more *systray.MenuItem - quit *systray.MenuItem + self *systray.MenuItem + more *systray.MenuItem + exitNodes *systray.MenuItem + quit *systray.MenuItem accountsCh chan ipn.ProfileID + exitNodeCh chan tailcfg.StableNodeID // ID of selected exit node eventCancel func() // cancel eventLoop } @@ -80,7 +86,7 @@ func init() { // (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu' // // See also: https://github.com/fyne-io/systray/issues/12 - newMenuDelay = 100 * time.Millisecond + newMenuDelay = 10 * time.Millisecond } } @@ -187,6 +193,9 @@ func (menu *Menu) rebuild(state state) { } systray.AddSeparator() + menu.exitNodeCh = make(chan tailcfg.StableNodeID) + menu.rebuildExitNodeMenu(ctx) + menu.more = systray.AddMenuItem("More settings", "") menu.more.Enable() @@ -295,6 +304,26 @@ func (menu *Menu) eventLoop(ctx context.Context) { log.Printf("failed switching to profile ID %v: %v", id, err) } + case exitNode := <-menu.exitNodeCh: + if exitNode.IsZero() { + log.Print("disable exit node") + if err := localClient.SetUseExitNode(ctx, false); err != nil { + log.Printf("failed disabling exit node: %v", err) + } + } else { + log.Printf("enable exit node: %v", exitNode) + mp := &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeID: exitNode, + }, + ExitNodeIDSet: true, + } + if _, err := localClient.EditPrefs(ctx, mp); err != nil { + log.Printf("failed setting exit node: %v", err) + } + } + menu.rebuild(fetchState(ctx)) + case <-menu.quit.ClickedCh: systray.Quit() } @@ -375,6 +404,224 @@ func sendNotification(title, content string) { } } +func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { + status := menu.status + menu.exitNodes = systray.AddMenuItem("Exit Nodes", "") + time.Sleep(newMenuDelay) + + // register a click handler for a menu item to set nodeID as the exit node. + onClick := func(item *systray.MenuItem, nodeID tailcfg.StableNodeID) { + go func() { + for { + select { + case <-ctx.Done(): + return + case <-item.ClickedCh: + select { + case <-ctx.Done(): + return + case menu.exitNodeCh <- nodeID: + } + } + } + }() + } + + noExitNodeMenu := menu.exitNodes.AddSubMenuItemCheckbox("None", "", status.ExitNodeStatus == nil) + onClick(noExitNodeMenu, "") + + // Show recommended exit node if available. + if status.Self.CapMap.Contains(tailcfg.NodeAttrSuggestExitNodeUI) { + sugg, err := localClient.SuggestExitNode(ctx) + if err == nil { + title := "Recommended: " + if loc := sugg.Location; loc.Valid() && loc.Country() != "" { + flag := countryFlag(loc.CountryCode()) + title += fmt.Sprintf("%s %s: %s", flag, loc.Country(), loc.City()) + } else { + title += strings.Split(sugg.Name, ".")[0] + } + menu.exitNodes.AddSeparator() + rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false) + onClick(rm, sugg.ID) + if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID { + rm.Check() + } + } + } + + // Add tailnet exit nodes if present. + var tailnetExitNodes []*ipnstate.PeerStatus + for _, ps := range status.Peer { + if ps.ExitNodeOption && ps.Location == nil { + tailnetExitNodes = append(tailnetExitNodes, ps) + } + } + if len(tailnetExitNodes) > 0 { + menu.exitNodes.AddSeparator() + menu.exitNodes.AddSubMenuItem("Tailnet Exit Nodes", "").Disable() + for _, ps := range status.Peer { + if !ps.ExitNodeOption || ps.Location != nil { + continue + } + name := strings.Split(ps.DNSName, ".")[0] + if !ps.Online { + name += " (offline)" + } + sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", false) + if !ps.Online { + sm.Disable() + } + if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID { + sm.Check() + } + onClick(sm, ps.ID) + } + } + + // Add mullvad exit nodes if present. + var mullvadExitNodes mullvadPeers + if status.Self.CapMap.Contains("mullvad") { + mullvadExitNodes = newMullvadPeers(status) + } + if len(mullvadExitNodes.countries) > 0 { + menu.exitNodes.AddSeparator() + menu.exitNodes.AddSubMenuItem("Location-based Exit Nodes", "").Disable() + mullvadMenu := menu.exitNodes.AddSubMenuItemCheckbox("Mullvad VPN", "", false) + + for _, country := range mullvadExitNodes.sortedCountries() { + flag := countryFlag(country.code) + countryMenu := mullvadMenu.AddSubMenuItemCheckbox(flag+" "+country.name, "", false) + + // single-city country, no submenu + if len(country.cities) == 1 { + onClick(countryMenu, country.best.ID) + if status.ExitNodeStatus != nil { + for _, city := range country.cities { + for _, ps := range city.peers { + if status.ExitNodeStatus.ID == ps.ID { + mullvadMenu.Check() + countryMenu.Check() + } + } + } + } + continue + } + + // multi-city country, build submenu with "best available" option and cities. + time.Sleep(newMenuDelay) + bm := countryMenu.AddSubMenuItemCheckbox("Best Available", "", false) + onClick(bm, country.best.ID) + countryMenu.AddSeparator() + + for _, city := range country.sortedCities() { + cityMenu := countryMenu.AddSubMenuItemCheckbox(city.name, "", false) + onClick(cityMenu, city.best.ID) + if status.ExitNodeStatus != nil { + for _, ps := range city.peers { + if status.ExitNodeStatus.ID == ps.ID { + mullvadMenu.Check() + countryMenu.Check() + cityMenu.Check() + } + } + } + } + } + } + + // TODO: "Allow Local Network Access" and "Run Exit Node" menu items +} + +// mullvadPeers contains all mullvad peer nodes, sorted by country and city. +type mullvadPeers struct { + countries map[string]*mvCountry // country code (uppercase) => country +} + +// sortedCountries returns countries containing mullvad nodes, sorted by name. +func (mp mullvadPeers) sortedCountries() []*mvCountry { + countries := slices.Collect(maps.Values(mp.countries)) + slices.SortFunc(countries, func(a, b *mvCountry) int { + return cmp.Compare(a.name, b.name) + }) + return countries +} + +type mvCountry struct { + code string + name string + best *ipnstate.PeerStatus // highest priority peer in the country + cities map[string]*mvCity // city code => city +} + +// sortedCities returns cities containing mullvad nodes, sorted by name. +func (mc *mvCountry) sortedCities() []*mvCity { + cities := slices.Collect(maps.Values(mc.cities)) + slices.SortFunc(cities, func(a, b *mvCity) int { + return cmp.Compare(a.name, b.name) + }) + return cities +} + +// countryFlag takes a 2-character ASCII string and returns the corresponding emoji flag. +// It returns the empty string on error. +func countryFlag(code string) string { + if len(code) != 2 { + return "" + } + runes := make([]rune, 0, 2) + for i := range 2 { + b := code[i] | 32 // lowercase + if b < 'a' || b > 'z' { + return "" + } + // https://en.wikipedia.org/wiki/Regional_indicator_symbol + runes = append(runes, 0x1F1E6+rune(b-'a')) + } + return string(runes) +} + +type mvCity struct { + name string + best *ipnstate.PeerStatus // highest priority peer in the city + peers []*ipnstate.PeerStatus +} + +func newMullvadPeers(status *ipnstate.Status) mullvadPeers { + countries := make(map[string]*mvCountry) + for _, ps := range status.Peer { + if !ps.ExitNodeOption || ps.Location == nil { + continue + } + loc := ps.Location + country, ok := countries[loc.CountryCode] + if !ok { + country = &mvCountry{ + code: loc.CountryCode, + name: loc.Country, + cities: make(map[string]*mvCity), + } + countries[loc.CountryCode] = country + } + city, ok := countries[loc.CountryCode].cities[loc.CityCode] + if !ok { + city = &mvCity{ + name: loc.City, + } + countries[loc.CountryCode].cities[loc.CityCode] = city + } + city.peers = append(city.peers, ps) + if city.best == nil || ps.Location.Priority > city.best.Location.Priority { + city.best = ps + } + if country.best == nil || ps.Location.Priority > country.best.Location.Priority { + country.best = ps + } + } + return mullvadPeers{countries} +} + func onExit() { log.Printf("exiting") os.Remove(appIcon.Name()) From 10d4057a64e5ad8c5e7f1bc786f61eeb76cc158f Mon Sep 17 00:00:00 2001 From: Will Norris Date: Fri, 20 Dec 2024 17:32:10 -0800 Subject: [PATCH 041/223] cmd/systray: add visual workarounds for gnome, mac, and windows Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 1334a0351a8aa..26316feebb6a4 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -41,6 +41,10 @@ var ( // newMenuDelay is the amount of time to sleep after creating a new menu, // but before adding items to it. This works around a bug in some dbus implementations. newMenuDelay time.Duration + + // if true, treat all mullvad exit node countries as single-city. + // Instead of rendering a submenu with cities, just select the highest-priority peer. + hideMullvadCities bool ) func main() { @@ -74,6 +78,12 @@ func init() { desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP")) switch desktop { + case "gnome": + // GNOME expands submenus downward in the main menu, rather than flyouts to the side. + // Either as a result of that or another limitation, there seems to be a maximum depth of submenus. + // Mullvad countries that have a city submenu are not being rendered, and so can't be selected. + // Handle this by simply treating all mullvad countries as single-city and select the best peer. + hideMullvadCities = true case "kde": // KDE doesn't need a delay, and actually won't render submenus // if we delay for more than about 400Âĩs. @@ -209,7 +219,12 @@ func (menu *Menu) rebuild(state state) { func profileTitle(profile ipn.LoginProfile) string { title := profile.Name if profile.NetworkProfile.DomainName != "" { - title += "\n" + profile.NetworkProfile.DomainName + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + // windows and mac don't support multi-line menu + title += " (" + profile.NetworkProfile.DomainName + ")" + } else { + title += "\n" + profile.NetworkProfile.DomainName + } } return title } @@ -494,7 +509,7 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { countryMenu := mullvadMenu.AddSubMenuItemCheckbox(flag+" "+country.name, "", false) // single-city country, no submenu - if len(country.cities) == 1 { + if len(country.cities) == 1 || hideMullvadCities { onClick(countryMenu, country.best.ID) if status.ExitNodeStatus != nil { for _, city := range country.cities { From 8d4ea4d90c76502bb6c15a6e6140f7d51de4c787 Mon Sep 17 00:00:00 2001 From: Jason Barnett Date: Thu, 11 Jan 2024 17:36:12 -0700 Subject: [PATCH 042/223] wgengine/router: add ip rules for unifi udm-pro Fixes: #4038 Signed-off-by: Jason Barnett --- version/distro/distro.go | 46 ++++++++++++++++++++++++ wgengine/router/router_linux.go | 52 ++++++++++++++++++++++------ wgengine/router/router_linux_test.go | 22 ++++++++++++ 3 files changed, 110 insertions(+), 10 deletions(-) diff --git a/version/distro/distro.go b/version/distro/distro.go index ce61137cf3280..8128ce395f555 100644 --- a/version/distro/distro.go +++ b/version/distro/distro.go @@ -9,6 +9,7 @@ import ( "os" "runtime" "strconv" + "strings" "tailscale.com/types/lazy" "tailscale.com/util/lineiter" @@ -30,6 +31,7 @@ const ( WDMyCloud = Distro("wdmycloud") Unraid = Distro("unraid") Alpine = Distro("alpine") + UDMPro = Distro("udmpro") ) var distro lazy.SyncValue[Distro] @@ -75,6 +77,9 @@ func linuxDistro() Distro { case have("/usr/local/bin/freenas-debug"): // TrueNAS Scale runs on debian return TrueNAS + case isUDMPro(): + // UDM-Pro runs on debian + return UDMPro case have("/etc/debian_version"): return Debian case have("/etc/arch-release"): @@ -147,3 +152,44 @@ func DSMVersion() int { return 0 }) } + +// isUDMPro checks a couple of files known to exist on a UDM-Pro and returns +// true if the expected content exists in the files. +func isUDMPro() bool { + // This is a performance guardrail against trying to load both + // /etc/board.info and /sys/firmware/devicetree/base/soc/board-cfg/id when + // not running on Debian so we don't make unnecessary calls in situations + // where we definitely are NOT on a UDM Pro. In other words, the have() call + // is much cheaper than the two os.ReadFile() in fileContainsAnyString(). + // That said, on Debian systems we will still be making the two + // os.ReadFile() in fileContainsAnyString(). + if !have("/etc/debian_version") { + return false + } + if exists, err := fileContainsAnyString("/etc/board.info", "UDMPRO", "Dream Machine PRO"); err == nil && exists { + return true + } + if exists, err := fileContainsAnyString("/sys/firmware/devicetree/base/soc/board-cfg/id", "udm pro"); err == nil && exists { + return true + } + return false +} + +// fileContainsAnyString is used to determine if one or more of the provided +// strings exists in a file. This is not efficient for larger files. If you want +// to use this function to parse large files, please refactor to use +// `io.LimitedReader`. +func fileContainsAnyString(filePath string, searchStrings ...string) (bool, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return false, err + } + + content := string(data) + for _, searchString := range searchStrings { + if strings.Contains(content, searchString) { + return true, nil + } + } + return false, nil +} diff --git a/wgengine/router/router_linux.go b/wgengine/router/router_linux.go index 2af73e26d2f28..e154a30fad91a 100644 --- a/wgengine/router/router_linux.go +++ b/wgengine/router/router_linux.go @@ -32,6 +32,8 @@ import ( "tailscale.com/version/distro" ) +var getDistroFunc = distro.Get + const ( netfilterOff = preftype.NetfilterOff netfilterNoDivert = preftype.NetfilterNoDivert @@ -222,7 +224,7 @@ func busyboxParseVersion(output string) (major, minor, patch int, err error) { } func useAmbientCaps() bool { - if distro.Get() != distro.Synology { + if getDistroFunc() != distro.Synology { return false } return distro.DSMVersion() >= 7 @@ -438,7 +440,7 @@ func (r *linuxRouter) Set(cfg *Config) error { // Issue 11405: enable IP forwarding on gokrazy. advertisingRoutes := len(cfg.SubnetRoutes) > 0 - if distro.Get() == distro.Gokrazy && advertisingRoutes { + if getDistroFunc() == distro.Gokrazy && advertisingRoutes { r.enableIPForwarding() } @@ -1181,7 +1183,9 @@ var ( tailscaleRouteTable = newRouteTable("tailscale", 52) ) -// ipRules are the policy routing rules that Tailscale uses. +// baseIPRules are the policy routing rules that Tailscale uses, when not +// running on a UDM-Pro. +// // The priority is the value represented here added to r.ipPolicyPrefBase, // which is usually 5200. // @@ -1196,7 +1200,7 @@ var ( // and 'ip rule' implementations (including busybox), don't support // checking for the lack of a fwmark, only the presence. The technique // below works even on very old kernels. -var ipRules = []netlink.Rule{ +var baseIPRules = []netlink.Rule{ // Packets from us, tagged with our fwmark, first try the kernel's // main routing table. { @@ -1232,6 +1236,34 @@ var ipRules = []netlink.Rule{ // usual rules (pref 32766 and 32767, ie. main and default). } +// udmProIPRules are the policy routing rules that Tailscale uses, when running +// on a UDM-Pro. +// +// The priority is the value represented here added to +// r.ipPolicyPrefBase, which is usually 5200. +// +// This represents an experiment that will be used to gather more information. +// If this goes well, Tailscale may opt to use this for all of Linux. +var udmProIPRules = []netlink.Rule{ + // non-fwmark packets fall through to the usual rules (pref 32766 and 32767, + // ie. main and default). + { + Priority: 70, + Invert: true, + Mark: linuxfw.TailscaleBypassMarkNum, + Table: tailscaleRouteTable.Num, + }, +} + +// ipRules returns the appropriate list of ip rules to be used by Tailscale. See +// comments on baseIPRules and udmProIPRules for more details. +func ipRules() []netlink.Rule { + if getDistroFunc() == distro.UDMPro { + return udmProIPRules + } + return baseIPRules +} + // justAddIPRules adds policy routing rule without deleting any first. func (r *linuxRouter) justAddIPRules() error { if !r.ipRuleAvailable { @@ -1243,7 +1275,7 @@ func (r *linuxRouter) justAddIPRules() error { var errAcc error for _, family := range r.addrFamilies() { - for _, ru := range ipRules { + for _, ru := range ipRules() { // Note: r is a value type here; safe to mutate it. ru.Family = family.netlinkInt() if ru.Mark != 0 { @@ -1272,7 +1304,7 @@ func (r *linuxRouter) addIPRulesWithIPCommand() error { rg := newRunGroup(nil, r.cmd) for _, family := range r.addrFamilies() { - for _, rule := range ipRules { + for _, rule := range ipRules() { args := []string{ "ip", family.dashArg(), "rule", "add", @@ -1320,7 +1352,7 @@ func (r *linuxRouter) delIPRules() error { } var errAcc error for _, family := range r.addrFamilies() { - for _, ru := range ipRules { + for _, ru := range ipRules() { // Note: r is a value type here; safe to mutate it. // When deleting rules, we want to be a bit specific (mention which // table we were routing to) but not *too* specific (fwmarks, etc). @@ -1363,7 +1395,7 @@ func (r *linuxRouter) delIPRulesWithIPCommand() error { // That leaves us some flexibility to change these values in later // versions without having ongoing hacks for every possible // combination. - for _, rule := range ipRules { + for _, rule := range ipRules() { args := []string{ "ip", family.dashArg(), "rule", "del", @@ -1500,7 +1532,7 @@ func normalizeCIDR(cidr netip.Prefix) string { // platformCanNetfilter reports whether the current distro/environment supports // running iptables/nftables commands. func platformCanNetfilter() bool { - switch distro.Get() { + switch getDistroFunc() { case distro.Synology: // Synology doesn't support iptables or nftables. Attempting to run it // just blocks for a long time while it logs about failures. @@ -1526,7 +1558,7 @@ func cleanUp(logf logger.Logf, interfaceName string) { // of the config file being present as well as a policy rule with a specific // priority (2000 + 1 - first interface mwan3 manages) and non-zero mark. func checkOpenWRTUsingMWAN3() (bool, error) { - if distro.Get() != distro.OpenWrt { + if getDistroFunc() != distro.OpenWrt { return false, nil } diff --git a/wgengine/router/router_linux_test.go b/wgengine/router/router_linux_test.go index dce69550d909a..7718f17c41706 100644 --- a/wgengine/router/router_linux_test.go +++ b/wgengine/router/router_linux_test.go @@ -28,6 +28,7 @@ import ( "tailscale.com/tstest" "tailscale.com/types/logger" "tailscale.com/util/linuxfw" + "tailscale.com/version/distro" ) func TestRouterStates(t *testing.T) { @@ -1231,3 +1232,24 @@ func adjustFwmask(t *testing.T, s string) string { return fwmaskAdjustRe.ReplaceAllString(s, "$1") } + +func TestIPRulesForUDMPro(t *testing.T) { + // Override the global getDistroFunc + getDistroFunc = func() distro.Distro { + return distro.UDMPro + } + defer func() { getDistroFunc = distro.Get }() // Restore original after the test + + expected := udmProIPRules + actual := ipRules() + + if len(expected) != len(actual) { + t.Fatalf("Expected %d rules, got %d", len(expected), len(actual)) + } + + for i, rule := range expected { + if rule != actual[i] { + t.Errorf("Rule mismatch at index %d: expected %+v, got %+v", i, rule, actual[i]) + } + } +} From c4f9f955ab281db35178a81b40c32c2828044f8f Mon Sep 17 00:00:00 2001 From: Erisa A Date: Mon, 23 Dec 2024 12:53:54 +0000 Subject: [PATCH 043/223] scripts/installer.sh: add support for PikaOS (#14461) Fixes #14460 Signed-off-by: Erisa A --- scripts/installer.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/installer.sh b/scripts/installer.sh index d2971978eebe7..6e530fefea61d 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -165,6 +165,19 @@ main() { VERSION="bullseye" fi ;; + pika) + PACKAGETYPE="apt" + # All versions of PikaOS are new enough to prefer keyring + APT_KEY_TYPE="keyring" + # Older versions of PikaOS are based on Ubuntu rather than Debian + if [ "$VERSION_ID" -lt 4 ]; then + OS="ubuntu" + VERSION="$UBUNTU_CODENAME" + else + OS="debian" + VERSION="$DEBIAN_CODENAME" + fi + ;; centos) OS="$ID" VERSION="$VERSION_ID" From 4267d0fc5b3ee065132711ab705f2fae76256906 Mon Sep 17 00:00:00 2001 From: Erisa A Date: Mon, 23 Dec 2024 14:48:35 +0000 Subject: [PATCH 044/223] .github: update matrix of installer.sh tests (#14462) Remove EOL Ubuntu versions. Add new Ubuntu LTS. Update Alpine to test latest version. Also, make the test run when its workflow is updated and installer.sh isn't. Updates #cleanup Signed-off-by: Erisa A --- .github/workflows/installer.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 48b29c6ec02cd..1c39e4d743a0d 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -6,11 +6,13 @@ on: - "main" paths: - scripts/installer.sh + - .github/workflows/installer.yml pull_request: branches: - "*" paths: - scripts/installer.sh + - .github/workflows/installer.yml jobs: test: @@ -29,10 +31,9 @@ jobs: - "debian:stable-slim" - "debian:testing-slim" - "debian:sid-slim" - - "ubuntu:18.04" - "ubuntu:20.04" - "ubuntu:22.04" - - "ubuntu:23.04" + - "ubuntu:24.04" - "elementary/docker:stable" - "elementary/docker:unstable" - "parrotsec/core:lts-amd64" @@ -48,7 +49,7 @@ jobs: - "opensuse/leap:latest" - "opensuse/tumbleweed:latest" - "archlinux:latest" - - "alpine:3.14" + - "alpine:3.21" - "alpine:latest" - "alpine:edge" deps: @@ -58,10 +59,6 @@ jobs: # Check a few images with wget rather than curl. - { image: "debian:oldstable-slim", deps: "wget" } - { image: "debian:sid-slim", deps: "wget" } - - { image: "ubuntu:23.04", deps: "wget" } - # Ubuntu 16.04 also needs apt-transport-https installed. - - { image: "ubuntu:16.04", deps: "curl apt-transport-https" } - - { image: "ubuntu:16.04", deps: "wget apt-transport-https" } runs-on: ubuntu-latest container: image: ${{ matrix.image }} From 9e2819b5d4c00e3b10802b8197b112dcb02b327a Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 22 Dec 2024 20:38:20 -0800 Subject: [PATCH 045/223] util/stringsx: add package for extra string functions, like CompareFold Noted as useful during review of #14448. Updates #14457 Change-Id: I0f16f08d5b05a8e9044b19ef6c02d3dab497f131 Signed-off-by: Brad Fitzpatrick --- util/stringsx/stringsx.go | 52 +++++++++++++++++++++++ util/stringsx/stringsx_test.go | 78 ++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 util/stringsx/stringsx.go create mode 100644 util/stringsx/stringsx_test.go diff --git a/util/stringsx/stringsx.go b/util/stringsx/stringsx.go new file mode 100644 index 0000000000000..6c7a8d20d4221 --- /dev/null +++ b/util/stringsx/stringsx.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package stringsx provides additional string manipulation functions +// that aren't in the standard library's strings package or go4.org/mem. +package stringsx + +import ( + "unicode" + "unicode/utf8" +) + +// CompareFold returns -1, 0, or 1 depending on whether a < b, a == b, or a > b, +// like cmp.Compare, but case insensitively. +func CompareFold(a, b string) int { + // Track our position in both strings + ia, ib := 0, 0 + for ia < len(a) && ib < len(b) { + ra, wa := nextRuneLower(a[ia:]) + rb, wb := nextRuneLower(b[ib:]) + if ra < rb { + return -1 + } + if ra > rb { + return 1 + } + ia += wa + ib += wb + if wa == 0 || wb == 0 { + break + } + } + + // If we've reached here, one or both strings are exhausted + // The shorter string is "less than" if they match up to this point + switch { + case ia == len(a) && ib == len(b): + return 0 + case ia == len(a): + return -1 + default: + return 1 + } +} + +// nextRuneLower returns the next rune in the string, lowercased, along with its +// original (consumed) width in bytes. If the string is empty, it returns +// (utf8.RuneError, 0) +func nextRuneLower(s string) (r rune, width int) { + r, width = utf8.DecodeRuneInString(s) + return unicode.ToLower(r), width +} diff --git a/util/stringsx/stringsx_test.go b/util/stringsx/stringsx_test.go new file mode 100644 index 0000000000000..8575c0b278fca --- /dev/null +++ b/util/stringsx/stringsx_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package stringsx + +import ( + "cmp" + "strings" + "testing" +) + +func TestCompareFold(t *testing.T) { + tests := []struct { + a, b string + }{ + // Basic ASCII cases + {"", ""}, + {"a", "a"}, + {"a", "A"}, + {"A", "a"}, + {"a", "b"}, + {"b", "a"}, + {"abc", "ABC"}, + {"ABC", "abc"}, + {"abc", "abd"}, + {"abd", "abc"}, + + // Length differences + {"abc", "ab"}, + {"ab", "abc"}, + + // Unicode cases + {"世į•Œ", "世į•Œ"}, + {"Hello世į•Œ", "hello世į•Œ"}, + {"世į•ŒHello", "世į•Œhello"}, + {"世į•Œ", "世į•Œx"}, + {"世į•Œx", "世į•Œ"}, + + // Special case folding examples + {"ß", "ss"}, // German sharp s + {"īŦ", "fi"}, // fi ligature + {"ÎŖ", "Īƒ"}, // Greek sigma + {"Ä°", "i\u0307"}, // Turkish dotted I + + // Mixed cases + {"HelloWorld", "helloworld"}, + {"HELLOWORLD", "helloworld"}, + {"helloworld", "HELLOWORLD"}, + {"HelloWorld", "helloworld"}, + {"helloworld", "HelloWorld"}, + + // Edge cases + {" ", " "}, + {"1", "1"}, + {"123", "123"}, + {"!@#", "!@#"}, + } + + wants := []int{} + for _, tt := range tests { + got := CompareFold(tt.a, tt.b) + want := cmp.Compare(strings.ToLower(tt.a), strings.ToLower(tt.b)) + if got != want { + t.Errorf("CompareFold(%q, %q) = %v, want %v", tt.a, tt.b, got, want) + } + wants = append(wants, want) + } + + if n := testing.AllocsPerRun(1000, func() { + for i, tt := range tests { + if CompareFold(tt.a, tt.b) != wants[i] { + panic("unexpected") + } + } + }); n > 0 { + t.Errorf("allocs = %v; want 0", int(n)) + } +} From 76ca1adc64ed97699968e09e0c3cfcfe80a66b9c Mon Sep 17 00:00:00 2001 From: Erisa A Date: Mon, 23 Dec 2024 16:47:55 +0000 Subject: [PATCH 046/223] scripts/installer.sh: accept different capitalisation of deepin (#14463) Newer Deepin Linux versions use `deepin` as their ID, older ones used `Deepin`. Fixes #13570 Signed-off-by: Erisa A --- scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/installer.sh b/scripts/installer.sh index 6e530fefea61d..c42ff03ea179c 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -154,7 +154,7 @@ main() { APT_KEY_TYPE="keyring" fi ;; - Deepin) # https://github.com/tailscale/tailscale/issues/7862 + Deepin|deepin) # https://github.com/tailscale/tailscale/issues/7862 OS="debian" PACKAGETYPE="apt" if [ "$VERSION_ID" -lt 20 ]; then From 3837b6cebc913a201e29b9e49abf3ea0e86cb0c0 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Sat, 21 Dec 2024 15:58:26 -0800 Subject: [PATCH 047/223] cmd/systray: rebuild menu on pref change, assorted other fixes - rebuild menu when prefs change outside of systray, such as setting an exit node - refactor onClick handler code - compare lowercase country name, the same as macOS and Windows (now sorts Ukraine before USA) - fix "connected / disconnected" menu items on stopped status - prevent nil pointer on "This Device" menu item Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 91 ++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 26316feebb6a4..8a4ee08fd64d3 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -7,7 +7,6 @@ package main import ( - "cmp" "context" "errors" "fmt" @@ -30,12 +29,15 @@ import ( "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" + "tailscale.com/util/stringsx" ) var ( localClient tailscale.LocalClient chState chan ipn.State // tailscale state changes + chRebuild chan struct{} // triggers a menu rebuild + appIcon *os.File // newMenuDelay is the amount of time to sleep after creating a new menu, @@ -111,6 +113,7 @@ func onReady() { io.Copy(appIcon, connected.renderWithBorder(3)) chState = make(chan ipn.State, 1) + chRebuild = make(chan struct{}, 1) menu := new(Menu) menu.rebuild(fetchState(ctx)) @@ -146,6 +149,10 @@ func fetchState(ctx context.Context) state { // You cannot iterate over the items in a menu, nor can you remove some items like separators. // So for now we rebuild the whole thing, and can optimize this later if needed. func (menu *Menu) rebuild(state state) { + if state.status == nil { + return + } + menu.mu.Lock() defer menu.mu.Unlock() @@ -181,25 +188,20 @@ func (menu *Menu) rebuild(state state) { item = accounts.AddSubMenuItem(title, "") } setRemoteIcon(item, profile.UserProfile.ProfilePicURL) - go func(profile ipn.LoginProfile) { - for { - select { - case <-ctx.Done(): - return - case <-item.ClickedCh: - select { - case <-ctx.Done(): - return - case menu.accountsCh <- profile.ID: - } - } + onClick(ctx, item, func(ctx context.Context) { + select { + case <-ctx.Done(): + case menu.accountsCh <- profile.ID: } - }(profile) + }) } - if state.status != nil && state.status.Self != nil { + if state.status != nil && state.status.Self != nil && len(state.status.Self.TailscaleIPs) > 0 { title := fmt.Sprintf("This Device: %s (%s)", state.status.Self.HostName, state.status.Self.TailscaleIPs[0]) menu.self = systray.AddMenuItem(title, "") + } else { + menu.self = systray.AddMenuItem("This Device: not connected", "") + menu.self.Disable() } systray.AddSeparator() @@ -266,6 +268,8 @@ func (menu *Menu) eventLoop(ctx context.Context) { select { case <-ctx.Done(): return + case <-chRebuild: + menu.rebuild(fetchState(ctx)) case state := <-chState: switch state { case ipn.Running: @@ -277,10 +281,11 @@ func (menu *Menu) eventLoop(ctx context.Context) { menu.disconnect.Show() menu.disconnect.Enable() case ipn.NoState, ipn.Stopped: + setAppIcon(disconnected) + menu.rebuild(fetchState(ctx)) menu.connect.SetTitle("Connect") menu.connect.Enable() menu.disconnect.Hide() - setAppIcon(disconnected) case ipn.Starting: setAppIcon(loading) } @@ -337,7 +342,6 @@ func (menu *Menu) eventLoop(ctx context.Context) { log.Printf("failed setting exit node: %v", err) } } - menu.rebuild(fetchState(ctx)) case <-menu.quit.ClickedCh: systray.Quit() @@ -345,6 +349,20 @@ func (menu *Menu) eventLoop(ctx context.Context) { } } +// onClick registers a click handler for a menu item. +func onClick(ctx context.Context, item *systray.MenuItem, fn func(ctx context.Context)) { + go func() { + for { + select { + case <-ctx.Done(): + return + case <-item.ClickedCh: + fn(ctx) + } + } + }() +} + // watchIPNBus subscribes to the tailscale event bus and sends state updates to chState. // This method does not return. func watchIPNBus(ctx context.Context) { @@ -383,6 +401,9 @@ func watchIPNBusInner(ctx context.Context) error { chState <- *n.State log.Printf("new state: %v", n.State) } + if n.Prefs != nil { + chRebuild <- struct{}{} + } } } } @@ -425,25 +446,17 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { time.Sleep(newMenuDelay) // register a click handler for a menu item to set nodeID as the exit node. - onClick := func(item *systray.MenuItem, nodeID tailcfg.StableNodeID) { - go func() { - for { - select { - case <-ctx.Done(): - return - case <-item.ClickedCh: - select { - case <-ctx.Done(): - return - case menu.exitNodeCh <- nodeID: - } - } + setExitNodeOnClick := func(item *systray.MenuItem, nodeID tailcfg.StableNodeID) { + onClick(ctx, item, func(ctx context.Context) { + select { + case <-ctx.Done(): + case menu.exitNodeCh <- nodeID: } - }() + }) } noExitNodeMenu := menu.exitNodes.AddSubMenuItemCheckbox("None", "", status.ExitNodeStatus == nil) - onClick(noExitNodeMenu, "") + setExitNodeOnClick(noExitNodeMenu, "") // Show recommended exit node if available. if status.Self.CapMap.Contains(tailcfg.NodeAttrSuggestExitNodeUI) { @@ -458,7 +471,7 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { } menu.exitNodes.AddSeparator() rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false) - onClick(rm, sugg.ID) + setExitNodeOnClick(rm, sugg.ID) if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID { rm.Check() } @@ -490,7 +503,7 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID { sm.Check() } - onClick(sm, ps.ID) + setExitNodeOnClick(sm, ps.ID) } } @@ -510,7 +523,7 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { // single-city country, no submenu if len(country.cities) == 1 || hideMullvadCities { - onClick(countryMenu, country.best.ID) + setExitNodeOnClick(countryMenu, country.best.ID) if status.ExitNodeStatus != nil { for _, city := range country.cities { for _, ps := range city.peers { @@ -527,12 +540,12 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { // multi-city country, build submenu with "best available" option and cities. time.Sleep(newMenuDelay) bm := countryMenu.AddSubMenuItemCheckbox("Best Available", "", false) - onClick(bm, country.best.ID) + setExitNodeOnClick(bm, country.best.ID) countryMenu.AddSeparator() for _, city := range country.sortedCities() { cityMenu := countryMenu.AddSubMenuItemCheckbox(city.name, "", false) - onClick(cityMenu, city.best.ID) + setExitNodeOnClick(cityMenu, city.best.ID) if status.ExitNodeStatus != nil { for _, ps := range city.peers { if status.ExitNodeStatus.ID == ps.ID { @@ -558,7 +571,7 @@ type mullvadPeers struct { func (mp mullvadPeers) sortedCountries() []*mvCountry { countries := slices.Collect(maps.Values(mp.countries)) slices.SortFunc(countries, func(a, b *mvCountry) int { - return cmp.Compare(a.name, b.name) + return stringsx.CompareFold(a.name, b.name) }) return countries } @@ -574,7 +587,7 @@ type mvCountry struct { func (mc *mvCountry) sortedCities() []*mvCity { cities := slices.Collect(maps.Values(mc.cities)) slices.SortFunc(cities, func(a, b *mvCity) int { - return cmp.Compare(a.name, b.name) + return stringsx.CompareFold(a.name, b.name) }) return cities } From 72b278937bfe429f4e1427a40d85eace4973feea Mon Sep 17 00:00:00 2001 From: Erisa A Date: Mon, 23 Dec 2024 17:53:06 +0000 Subject: [PATCH 048/223] scripts/installer.sh: allow CachyOS for Arch packages (#14464) Fixes #13955 Signed-off-by: Erisa A --- scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/installer.sh b/scripts/installer.sh index c42ff03ea179c..8d1fc02128200 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -237,7 +237,7 @@ main() { VERSION="leap/15.4" PACKAGETYPE="zypper" ;; - arch|archarm|endeavouros|blendos|garuda|archcraft) + arch|archarm|endeavouros|blendos|garuda|archcraft|cachyos) OS="arch" VERSION="" # rolling release PACKAGETYPE="pacman" From 68b12a74ed2379f85fa30ffe10b974cf6062c8fc Mon Sep 17 00:00:00 2001 From: James Tucker Date: Thu, 19 Dec 2024 16:32:40 -0800 Subject: [PATCH 049/223] metrics,syncs: add ShardedInt support to metrics.LabelMap metrics.LabelMap grows slightly more heavy, needing a lock to ensure proper ordering for newly initialized ShardedInt values. An Add method enables callers to use .Add for both expvar.Int and syncs.ShardedInt values, but retains the original behavior of defaulting to initializing expvar.Int values. Updates tailscale/corp#25450 Co-Authored-By: Andrew Dunham Signed-off-by: James Tucker --- cmd/stund/depaware.txt | 2 ++ metrics/metrics.go | 35 +++++++++++++++++++++++++++++++++++ metrics/metrics_test.go | 9 +++++++++ 3 files changed, 46 insertions(+) diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index 34a71c43e0010..9599f6a01b1e4 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -55,6 +55,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar tailscale.com/net/stun from tailscale.com/net/stunserver tailscale.com/net/stunserver from tailscale.com/cmd/stund tailscale.com/net/tsaddr from tailscale.com/tsweb + tailscale.com/syncs from tailscale.com/metrics tailscale.com/tailcfg from tailscale.com/version tailscale.com/tsweb from tailscale.com/cmd/stund tailscale.com/tsweb/promvarz from tailscale.com/tsweb @@ -74,6 +75,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar L đŸ’Ŗ tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/tailcfg tailscale.com/util/lineiter from tailscale.com/version/distro + tailscale.com/util/mak from tailscale.com/syncs tailscale.com/util/nocasemaps from tailscale.com/types/ipproto tailscale.com/util/rands from tailscale.com/tsweb tailscale.com/util/slicesx from tailscale.com/tailcfg diff --git a/metrics/metrics.go b/metrics/metrics.go index a07ddccae5107..d1b1c06c9dc2c 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,6 +11,9 @@ import ( "io" "slices" "strings" + "sync" + + "tailscale.com/syncs" ) // Set is a string-to-Var map variable that satisfies the expvar.Var @@ -37,6 +40,8 @@ type Set struct { type LabelMap struct { Label string expvar.Map + // shardedIntMu orders the initialization of new shardedint keys + shardedIntMu sync.Mutex } // SetInt64 sets the *Int value stored under the given map key. @@ -44,6 +49,19 @@ func (m *LabelMap) SetInt64(key string, v int64) { m.Get(key).Set(v) } +// Add adds delta to the any int-like value stored under the given map key. +func (m *LabelMap) Add(key string, delta int64) { + type intAdder interface { + Add(delta int64) + } + o := m.Map.Get(key) + if o == nil { + m.Map.Add(key, delta) + return + } + o.(intAdder).Add(delta) +} + // Get returns a direct pointer to the expvar.Int for key, creating it // if necessary. func (m *LabelMap) Get(key string) *expvar.Int { @@ -51,6 +69,23 @@ func (m *LabelMap) Get(key string) *expvar.Int { return m.Map.Get(key).(*expvar.Int) } +// GetShardedInt returns a direct pointer to the syncs.ShardedInt for key, +// creating it if necessary. +func (m *LabelMap) GetShardedInt(key string) *syncs.ShardedInt { + i := m.Map.Get(key) + if i == nil { + m.shardedIntMu.Lock() + defer m.shardedIntMu.Unlock() + i = m.Map.Get(key) + if i != nil { + return i.(*syncs.ShardedInt) + } + i = syncs.NewShardedInt() + m.Set(key, i) + } + return i.(*syncs.ShardedInt) +} + // GetIncrFunc returns a function that increments the expvar.Int named by key. // // Most callers should not need this; it exists to satisfy an diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go index 45bf39e56efd2..a808d5a73eb3e 100644 --- a/metrics/metrics_test.go +++ b/metrics/metrics_test.go @@ -21,6 +21,15 @@ func TestLabelMap(t *testing.T) { if g, w := m.Get("bar").Value(), int64(2); g != w { t.Errorf("bar = %v; want %v", g, w) } + m.GetShardedInt("sharded").Add(5) + if g, w := m.GetShardedInt("sharded").Value(), int64(5); g != w { + t.Errorf("sharded = %v; want %v", g, w) + } + m.Add("sharded", 1) + if g, w := m.GetShardedInt("sharded").Value(), int64(6); g != w { + t.Errorf("sharded = %v; want %v", g, w) + } + m.Add("neverbefore", 1) } func TestCurrentFileDescriptors(t *testing.T) { From 2bdbe5b2ab7825666378a275b4c4429c0c39f8b6 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Mon, 23 Dec 2024 13:35:27 -0800 Subject: [PATCH 050/223] cmd/systray: add icons for exit node online and offline restructure tsLogo to allow setting a mask to be used when drawing the logo dots, as well as add an overlay icon, such as the arrow when connected to an exit node. The icon is still renders as white on black, but this change also prepare for doing a black on white version, as well a fully transparent icon. I don't know if we can consistently determine which to use, so this just keeps the single icon for now. Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/logo.go | 177 +++++++++++++++++++++++++++++++++----------- 1 file changed, 133 insertions(+), 44 deletions(-) diff --git a/cmd/systray/logo.go b/cmd/systray/logo.go index ef8caca66d89e..13fd4c5646665 100644 --- a/cmd/systray/logo.go +++ b/cmd/systray/logo.go @@ -8,6 +8,7 @@ package main import ( "bytes" "context" + "image" "image/color" "image/png" "sync" @@ -17,113 +18,190 @@ import ( "github.com/fogleman/gg" ) -// tsLogo represents the state of the 3x3 dot grid in the Tailscale logo. -// A 0 represents a gray dot, any other value is a white dot. -type tsLogo [9]byte +// tsLogo represents the Tailscale logo displayed as the systray icon. +type tsLogo struct { + // dots represents the state of the 3x3 dot grid in the logo. + // A 0 represents a gray dot, any other value is a white dot. + dots [9]byte + + // dotMask returns an image mask to be used when rendering the logo dots. + dotMask func(dc *gg.Context, borderUnits int, radius int) *image.Alpha + + // overlay is called after the dots are rendered to draw an additional overlay. + overlay func(dc *gg.Context, borderUnits int, radius int) +} var ( // disconnected is all gray dots - disconnected = tsLogo{ + disconnected = tsLogo{dots: [9]byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, - } + }} // connected is the normal Tailscale logo - connected = tsLogo{ + connected = tsLogo{dots: [9]byte{ 0, 0, 0, 1, 1, 1, 0, 1, 0, - } + }} // loading is a special tsLogo value that is not meant to be rendered directly, // but indicates that the loading animation should be shown. - loading = tsLogo{'l', 'o', 'a', 'd', 'i', 'n', 'g'} + loading = tsLogo{dots: [9]byte{'l', 'o', 'a', 'd', 'i', 'n', 'g'}} // loadingIcons are shown in sequence as an animated loading icon. loadingLogos = []tsLogo{ - { + {dots: [9]byte{ 0, 1, 1, 1, 0, 1, 0, 0, 1, - }, - { + }}, + {dots: [9]byte{ 0, 1, 1, 0, 0, 1, 0, 1, 0, - }, - { + }}, + {dots: [9]byte{ 0, 1, 1, 0, 0, 0, 0, 0, 1, - }, - { + }}, + {dots: [9]byte{ 0, 0, 1, 0, 1, 0, 0, 0, 0, - }, - { + }}, + {dots: [9]byte{ 0, 1, 0, 0, 0, 0, 0, 0, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 0, 0, 1, 0, 0, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 1, 0, 0, 0, 0, 0, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 0, 0, 0, 1, 0, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 0, 0, 0, 1, 1, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 1, 0, 0, 1, 1, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 1, 1, 0, 0, 1, 0, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 1, 1, 0, 0, 1, 1, - }, - { + }}, + {dots: [9]byte{ 0, 0, 0, 1, 1, 1, 0, 0, 1, - }, - { + }}, + {dots: [9]byte{ 0, 1, 0, 0, 1, 1, 1, 0, 1, + }}, + } + + // exitNodeOnline is the Tailscale logo with an additional arrow overlay in the corner. + exitNodeOnline = tsLogo{ + dots: [9]byte{ + 0, 0, 0, + 1, 1, 1, + 0, 1, 0, + }, + dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha { + bu, r := float64(borderUnits), float64(radius) + + x1 := r * (bu + 3.5) + y := r * (bu + 7) + x2 := x1 + (r * 5) + + mc := gg.NewContext(dc.Width(), dc.Height()) + mc.DrawLine(x1, y, x2, y) + mc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) + mc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) + mc.SetLineWidth(r * 3) + mc.Stroke() + return mc.AsMask() + }, + overlay: func(dc *gg.Context, borderUnits int, radius int) { + bu, r := float64(borderUnits), float64(radius) + + x1 := r * (bu + 3.5) + y := r * (bu + 7) + x2 := x1 + (r * 5) + + dc.DrawLine(x1, y, x2, y) + dc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) + dc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) + dc.SetColor(fg) + dc.SetLineWidth(r) + dc.Stroke() + }, + } + + // exitNodeOffline is the Tailscale logo with a red "x" in the corner. + exitNodeOffline = tsLogo{ + dots: [9]byte{ + 0, 0, 0, + 1, 1, 1, + 0, 1, 0, + }, + dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha { + bu, r := float64(borderUnits), float64(radius) + x := r * (bu + 3) + + mc := gg.NewContext(dc.Width(), dc.Height()) + mc.DrawRectangle(x, x, r*6, r*6) + mc.Fill() + return mc.AsMask() + }, + overlay: func(dc *gg.Context, borderUnits int, radius int) { + bu, r := float64(borderUnits), float64(radius) + + x1 := r * (bu + 4) + x2 := x1 + (r * 3.5) + dc.DrawLine(x1, x1, x2, x2) + dc.DrawLine(x1, x2, x2, x1) + dc.SetColor(red) + dc.SetLineWidth(r) + dc.Stroke() }, } ) var ( - black = color.NRGBA{0, 0, 0, 255} - white = color.NRGBA{255, 255, 255, 255} - gray = color.NRGBA{255, 255, 255, 102} + bg = color.NRGBA{0, 0, 0, 255} + fg = color.NRGBA{255, 255, 255, 255} + gray = color.NRGBA{255, 255, 255, 102} + red = color.NRGBA{229, 111, 74, 255} ) // render returns a PNG image of the logo. @@ -140,15 +218,21 @@ func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer { dc := gg.NewContext(dim, dim) dc.DrawRectangle(0, 0, float64(dim), float64(dim)) - dc.SetColor(black) + dc.SetColor(bg) dc.Fill() + if logo.dotMask != nil { + mask := logo.dotMask(dc, borderUnits, radius) + dc.SetMask(mask) + dc.InvertMask() + } + for y := 0; y < 3; y++ { for x := 0; x < 3; x++ { px := (borderUnits + 1 + 3*x) * radius py := (borderUnits + 1 + 3*y) * radius - col := white - if logo[y*3+x] == 0 { + col := fg + if logo.dots[y*3+x] == 0 { col = gray } dc.DrawCircle(float64(px), float64(py), radius) @@ -157,6 +241,11 @@ func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer { } } + if logo.overlay != nil { + dc.ResetClip() + logo.overlay(dc, borderUnits, radius) + } + b := bytes.NewBuffer(nil) png.Encode(b, dc.Image()) return b @@ -164,7 +253,7 @@ func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer { // setAppIcon renders logo and sets it as the systray icon. func setAppIcon(icon tsLogo) { - if icon == loading { + if icon.dots == loading.dots { startLoadingAnimation() } else { stopLoadingAnimation() From 86f273d930df52440641ef2397f0f7ebca648d7c Mon Sep 17 00:00:00 2001 From: Will Norris Date: Mon, 23 Dec 2024 13:38:09 -0800 Subject: [PATCH 051/223] cmd/systray: set app icon and title consistently Refactor code to set app icon and title as part of rebuild, rather than separately in eventLoop. This fixes several cases where they weren't getting updated properly. This change also makes use of the new exit node icons. Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/logo.go | 20 ++++++++----- cmd/systray/systray.go | 67 ++++++++++++++++++++++++------------------ 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/cmd/systray/logo.go b/cmd/systray/logo.go index 13fd4c5646665..de60bcdbd2d58 100644 --- a/cmd/systray/logo.go +++ b/cmd/systray/logo.go @@ -136,6 +136,7 @@ var ( 1, 1, 1, 0, 1, 0, }, + // draw an arrow mask in the bottom right corner with a reasonably thick line width. dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha { bu, r := float64(borderUnits), float64(radius) @@ -144,13 +145,14 @@ var ( x2 := x1 + (r * 5) mc := gg.NewContext(dc.Width(), dc.Height()) - mc.DrawLine(x1, y, x2, y) - mc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) - mc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) + mc.DrawLine(x1, y, x2, y) // arrow center line + mc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip + mc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip mc.SetLineWidth(r * 3) mc.Stroke() return mc.AsMask() }, + // draw an arrow in the bottom right corner over the masked area. overlay: func(dc *gg.Context, borderUnits int, radius int) { bu, r := float64(borderUnits), float64(radius) @@ -158,9 +160,9 @@ var ( y := r * (bu + 7) x2 := x1 + (r * 5) - dc.DrawLine(x1, y, x2, y) - dc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) - dc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) + dc.DrawLine(x1, y, x2, y) // arrow center line + dc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip + dc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip dc.SetColor(fg) dc.SetLineWidth(r) dc.Stroke() @@ -174,6 +176,7 @@ var ( 1, 1, 1, 0, 1, 0, }, + // Draw a square that hides the four dots in the bottom right corner, dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha { bu, r := float64(borderUnits), float64(radius) x := r * (bu + 3) @@ -183,13 +186,14 @@ var ( mc.Fill() return mc.AsMask() }, + // draw a red "x" over the bottom right corner. overlay: func(dc *gg.Context, borderUnits int, radius int) { bu, r := float64(borderUnits), float64(radius) x1 := r * (bu + 4) x2 := x1 + (r * 3.5) - dc.DrawLine(x1, x1, x2, x2) - dc.DrawLine(x1, x2, x2, x1) + dc.DrawLine(x1, x1, x2, x2) // top-left to bottom-right stroke + dc.DrawLine(x1, x2, x2, x1) // bottom-left to top-right stroke dc.SetColor(red) dc.SetLineWidth(r) dc.Stroke() diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 8a4ee08fd64d3..0102b28a6e81d 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -34,11 +34,8 @@ import ( var ( localClient tailscale.LocalClient - chState chan ipn.State // tailscale state changes - - chRebuild chan struct{} // triggers a menu rebuild - - appIcon *os.File + rebuildCh chan struct{} // triggers a menu rebuild + appIcon *os.File // newMenuDelay is the amount of time to sleep after creating a new menu, // but before adding items to it. This works around a bug in some dbus implementations. @@ -112,8 +109,7 @@ func onReady() { appIcon, _ = os.CreateTemp("", "tailscale-systray.png") io.Copy(appIcon, connected.renderWithBorder(3)) - chState = make(chan ipn.State, 1) - chRebuild = make(chan struct{}, 1) + rebuildCh = make(chan struct{}, 1) menu := new(Menu) menu.rebuild(fetchState(ctx)) @@ -170,6 +166,34 @@ func (menu *Menu) rebuild(state state) { menu.disconnect.Hide() systray.AddSeparator() + // Set systray menu icon and title. + // Also adjust connect/disconnect menu items if needed. + switch menu.status.BackendState { + case ipn.Running.String(): + if state.status.ExitNodeStatus != nil && !state.status.ExitNodeStatus.ID.IsZero() { + if state.status.ExitNodeStatus.Online { + systray.SetTitle("Using exit node") + setAppIcon(exitNodeOnline) + } else { + systray.SetTitle("Exit node offline") + setAppIcon(exitNodeOffline) + } + } else { + systray.SetTitle(fmt.Sprintf("Connected to %s", state.status.CurrentTailnet.Name)) + setAppIcon(connected) + } + menu.connect.SetTitle("Connected") + menu.connect.Disable() + menu.disconnect.Show() + menu.disconnect.Enable() + case ipn.Starting.String(): + systray.SetTitle("Connecting") + setAppIcon(loading) + default: + systray.SetTitle("Disconnected") + setAppIcon(disconnected) + } + account := "Account" if pt := profileTitle(state.curProfile); pt != "" { account = pt @@ -268,27 +292,8 @@ func (menu *Menu) eventLoop(ctx context.Context) { select { case <-ctx.Done(): return - case <-chRebuild: + case <-rebuildCh: menu.rebuild(fetchState(ctx)) - case state := <-chState: - switch state { - case ipn.Running: - setAppIcon(loading) - menu.rebuild(fetchState(ctx)) - setAppIcon(connected) - menu.connect.SetTitle("Connected") - menu.connect.Disable() - menu.disconnect.Show() - menu.disconnect.Enable() - case ipn.NoState, ipn.Stopped: - setAppIcon(disconnected) - menu.rebuild(fetchState(ctx)) - menu.connect.SetTitle("Connect") - menu.connect.Enable() - menu.disconnect.Hide() - case ipn.Starting: - setAppIcon(loading) - } case <-menu.connect.ClickedCh: _, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{ Prefs: ipn.Prefs{ @@ -397,12 +402,16 @@ func watchIPNBusInner(ctx context.Context) error { if err != nil { return fmt.Errorf("ipnbus error: %w", err) } + var rebuild bool if n.State != nil { - chState <- *n.State log.Printf("new state: %v", n.State) + rebuild = true } if n.Prefs != nil { - chRebuild <- struct{}{} + rebuild = true + } + if rebuild { + rebuildCh <- struct{}{} } } } From 5a4148e7e81287f4914fe01cd5b270c342d29d2f Mon Sep 17 00:00:00 2001 From: Will Norris Date: Wed, 25 Dec 2024 17:30:59 -0800 Subject: [PATCH 052/223] cmd/systray: update state management and initialization Move a number of global state vars into the Menu struct, keeping things better encapsulated. The systray package still relies on its own global state, so only a single Menu instance can run at a time. Move a lot of the initialization logic out of onReady, in particular fetching the latest tailscale state. Instead, populate the state before calling systray.Run, which fixes a timing issue in GNOME (#14477). This change also creates a separate bgContext for actions not tied menu item clicks. Because we have to rebuild the entire menu regularly, we cancel that context as needed, which can cancel subsequent updateState calls. Also exit cleanly on SIGINT and SIGTERM. Updates #1708 Fixes #14477 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 220 ++++++++++++++++++++++++----------------- 1 file changed, 128 insertions(+), 92 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 0102b28a6e81d..5f498f35fdae1 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -15,10 +15,12 @@ import ( "maps" "net/http" "os" + "os/signal" "runtime" "slices" "strings" "sync" + "syscall" "time" "fyne.io/systray" @@ -33,10 +35,6 @@ import ( ) var ( - localClient tailscale.LocalClient - rebuildCh chan struct{} // triggers a menu rebuild - appIcon *os.File - // newMenuDelay is the amount of time to sleep after creating a new menu, // but before adding items to it. This works around a bug in some dbus implementations. newMenuDelay time.Duration @@ -47,26 +45,68 @@ var ( ) func main() { - systray.Run(onReady, onExit) + menu := new(Menu) + menu.updateState() + + // exit cleanly on SIGINT and SIGTERM + go func() { + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) + select { + case <-interrupt: + menu.onExit() + case <-menu.bgCtx.Done(): + } + }() + + systray.Run(menu.onReady, menu.onExit) } // Menu represents the systray menu, its items, and the current Tailscale state. type Menu struct { - mu sync.Mutex // protects the entire Menu - status *ipnstate.Status + mu sync.Mutex // protects the entire Menu + lc tailscale.LocalClient + status *ipnstate.Status + curProfile ipn.LoginProfile + allProfiles []ipn.LoginProfile + + bgCtx context.Context // ctx for background tasks not involving menu item clicks + bgCancel context.CancelFunc + + // Top-level menu items connect *systray.MenuItem disconnect *systray.MenuItem + self *systray.MenuItem + exitNodes *systray.MenuItem + more *systray.MenuItem + quit *systray.MenuItem - self *systray.MenuItem - more *systray.MenuItem - exitNodes *systray.MenuItem - quit *systray.MenuItem - + rebuildCh chan struct{} // triggers a menu rebuild accountsCh chan ipn.ProfileID exitNodeCh chan tailcfg.StableNodeID // ID of selected exit node - eventCancel func() // cancel eventLoop + eventCancel context.CancelFunc // cancel eventLoop + + notificationIcon *os.File // icon used for desktop notifications +} + +func (menu *Menu) init() { + if menu.bgCtx != nil { + // already initialized + return + } + + menu.rebuildCh = make(chan struct{}, 1) + menu.accountsCh = make(chan ipn.ProfileID) + menu.exitNodeCh = make(chan tailcfg.StableNodeID) + + // dbus wants a file path for notification icons, so copy to a temp file. + menu.notificationIcon, _ = os.CreateTemp("", "tailscale-systray.png") + io.Copy(menu.notificationIcon, connected.renderWithBorder(3)) + + menu.bgCtx, menu.bgCancel = context.WithCancel(context.Background()) + go menu.watchIPNBus() } func init() { @@ -99,44 +139,28 @@ func init() { } } -func onReady() { +// onReady is called by the systray package when the menu is ready to be built. +func (menu *Menu) onReady() { log.Printf("starting") - ctx := context.Background() - setAppIcon(disconnected) - - // dbus wants a file path for notification icons, so copy to a temp file. - appIcon, _ = os.CreateTemp("", "tailscale-systray.png") - io.Copy(appIcon, connected.renderWithBorder(3)) - - rebuildCh = make(chan struct{}, 1) - - menu := new(Menu) - menu.rebuild(fetchState(ctx)) - - go watchIPNBus(ctx) + menu.rebuild() } -type state struct { - status *ipnstate.Status - curProfile ipn.LoginProfile - allProfiles []ipn.LoginProfile -} +// updateState updates the Menu state from the Tailscale local client. +func (menu *Menu) updateState() { + menu.mu.Lock() + defer menu.mu.Unlock() + menu.init() -func fetchState(ctx context.Context) state { - status, err := localClient.Status(ctx) + var err error + menu.status, err = menu.lc.Status(menu.bgCtx) if err != nil { log.Print(err) } - curProfile, allProfiles, err := localClient.ProfileStatus(ctx) + menu.curProfile, menu.allProfiles, err = menu.lc.ProfileStatus(menu.bgCtx) if err != nil { log.Print(err) } - return state{ - status: status, - curProfile: curProfile, - allProfiles: allProfiles, - } } // rebuild the systray menu based on the current Tailscale state. @@ -144,13 +168,10 @@ func fetchState(ctx context.Context) state { // We currently rebuild the entire menu because it is not easy to update the existing menu. // You cannot iterate over the items in a menu, nor can you remove some items like separators. // So for now we rebuild the whole thing, and can optimize this later if needed. -func (menu *Menu) rebuild(state state) { - if state.status == nil { - return - } - +func (menu *Menu) rebuild() { menu.mu.Lock() defer menu.mu.Unlock() + menu.init() if menu.eventCancel != nil { menu.eventCancel() @@ -158,7 +179,6 @@ func (menu *Menu) rebuild(state state) { ctx := context.Background() ctx, menu.eventCancel = context.WithCancel(ctx) - menu.status = state.status systray.ResetMenu() menu.connect = systray.AddMenuItem("Connect", "") @@ -166,12 +186,19 @@ func (menu *Menu) rebuild(state state) { menu.disconnect.Hide() systray.AddSeparator() + // delay to prevent race setting icon on first start + time.Sleep(newMenuDelay) + // Set systray menu icon and title. // Also adjust connect/disconnect menu items if needed. - switch menu.status.BackendState { + var backendState string + if menu.status != nil { + backendState = menu.status.BackendState + } + switch backendState { case ipn.Running.String(): - if state.status.ExitNodeStatus != nil && !state.status.ExitNodeStatus.ID.IsZero() { - if state.status.ExitNodeStatus.Online { + if menu.status.ExitNodeStatus != nil && !menu.status.ExitNodeStatus.ID.IsZero() { + if menu.status.ExitNodeStatus.Online { systray.SetTitle("Using exit node") setAppIcon(exitNodeOnline) } else { @@ -179,7 +206,7 @@ func (menu *Menu) rebuild(state state) { setAppIcon(exitNodeOffline) } } else { - systray.SetTitle(fmt.Sprintf("Connected to %s", state.status.CurrentTailnet.Name)) + systray.SetTitle(fmt.Sprintf("Connected to %s", menu.status.CurrentTailnet.Name)) setAppIcon(connected) } menu.connect.SetTitle("Connected") @@ -195,18 +222,16 @@ func (menu *Menu) rebuild(state state) { } account := "Account" - if pt := profileTitle(state.curProfile); pt != "" { + if pt := profileTitle(menu.curProfile); pt != "" { account = pt } accounts := systray.AddMenuItem(account, "") - setRemoteIcon(accounts, state.curProfile.UserProfile.ProfilePicURL) + setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL) time.Sleep(newMenuDelay) - // Aggregate all clicks into a shared channel. - menu.accountsCh = make(chan ipn.ProfileID) - for _, profile := range state.allProfiles { + for _, profile := range menu.allProfiles { title := profileTitle(profile) var item *systray.MenuItem - if profile.ID == state.curProfile.ID { + if profile.ID == menu.curProfile.ID { item = accounts.AddSubMenuItemCheckbox(title, "", true) } else { item = accounts.AddSubMenuItem(title, "") @@ -220,8 +245,8 @@ func (menu *Menu) rebuild(state state) { }) } - if state.status != nil && state.status.Self != nil && len(state.status.Self.TailscaleIPs) > 0 { - title := fmt.Sprintf("This Device: %s (%s)", state.status.Self.HostName, state.status.Self.TailscaleIPs[0]) + if menu.status != nil && menu.status.Self != nil && len(menu.status.Self.TailscaleIPs) > 0 { + title := fmt.Sprintf("This Device: %s (%s)", menu.status.Self.HostName, menu.status.Self.TailscaleIPs[0]) menu.self = systray.AddMenuItem(title, "") } else { menu.self = systray.AddMenuItem("This Device: not connected", "") @@ -229,11 +254,14 @@ func (menu *Menu) rebuild(state state) { } systray.AddSeparator() - menu.exitNodeCh = make(chan tailcfg.StableNodeID) menu.rebuildExitNodeMenu(ctx) - menu.more = systray.AddMenuItem("More settings", "") - menu.more.Enable() + if menu.status != nil { + menu.more = systray.AddMenuItem("More settings", "") + onClick(ctx, menu.more, func(_ context.Context) { + webbrowser.Open("http://100.100.100.100/") + }) + } menu.quit = systray.AddMenuItem("Quit", "Quit the app") menu.quit.Enable() @@ -292,48 +320,44 @@ func (menu *Menu) eventLoop(ctx context.Context) { select { case <-ctx.Done(): return - case <-rebuildCh: - menu.rebuild(fetchState(ctx)) + case <-menu.rebuildCh: + menu.updateState() + menu.rebuild() case <-menu.connect.ClickedCh: - _, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{ + _, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ Prefs: ipn.Prefs{ WantRunning: true, }, WantRunningSet: true, }) if err != nil { - log.Print(err) - continue + log.Printf("error connecting: %v", err) } case <-menu.disconnect.ClickedCh: - _, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{ + _, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ Prefs: ipn.Prefs{ WantRunning: false, }, WantRunningSet: true, }) if err != nil { - log.Printf("disconnecting: %v", err) - continue + log.Printf("error disconnecting: %v", err) } case <-menu.self.ClickedCh: - copyTailscaleIP(menu.status.Self) - - case <-menu.more.ClickedCh: - webbrowser.Open("http://100.100.100.100/") + menu.copyTailscaleIP(menu.status.Self) case id := <-menu.accountsCh: - if err := localClient.SwitchProfile(ctx, id); err != nil { - log.Printf("failed switching to profile ID %v: %v", id, err) + if err := menu.lc.SwitchProfile(ctx, id); err != nil { + log.Printf("error switching to profile ID %v: %v", id, err) } case exitNode := <-menu.exitNodeCh: if exitNode.IsZero() { log.Print("disable exit node") - if err := localClient.SetUseExitNode(ctx, false); err != nil { - log.Printf("failed disabling exit node: %v", err) + if err := menu.lc.SetUseExitNode(ctx, false); err != nil { + log.Printf("error disabling exit node: %v", err) } } else { log.Printf("enable exit node: %v", exitNode) @@ -343,8 +367,8 @@ func (menu *Menu) eventLoop(ctx context.Context) { }, ExitNodeIDSet: true, } - if _, err := localClient.EditPrefs(ctx, mp); err != nil { - log.Printf("failed setting exit node: %v", err) + if _, err := menu.lc.EditPrefs(ctx, mp); err != nil { + log.Printf("error setting exit node: %v", err) } } @@ -370,9 +394,9 @@ func onClick(ctx context.Context, item *systray.MenuItem, fn func(ctx context.Co // watchIPNBus subscribes to the tailscale event bus and sends state updates to chState. // This method does not return. -func watchIPNBus(ctx context.Context) { +func (menu *Menu) watchIPNBus() { for { - if err := watchIPNBusInner(ctx); err != nil { + if err := menu.watchIPNBusInner(); err != nil { log.Println(err) if errors.Is(err, context.Canceled) { // If the context got canceled, we will never be able to @@ -387,15 +411,15 @@ func watchIPNBus(ctx context.Context) { } } -func watchIPNBusInner(ctx context.Context) error { - watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys) +func (menu *Menu) watchIPNBusInner() error { + watcher, err := menu.lc.WatchIPNBus(menu.bgCtx, ipn.NotifyNoPrivateKeys) if err != nil { return fmt.Errorf("watching ipn bus: %w", err) } defer watcher.Close() for { select { - case <-ctx.Done(): + case <-menu.bgCtx.Done(): return nil default: n, err := watcher.Next() @@ -411,7 +435,7 @@ func watchIPNBusInner(ctx context.Context) error { rebuild = true } if rebuild { - rebuildCh <- struct{}{} + menu.rebuildCh <- struct{}{} } } } @@ -419,7 +443,7 @@ func watchIPNBusInner(ctx context.Context) error { // copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard // and sends a notification with the copied value. -func copyTailscaleIP(device *ipnstate.PeerStatus) { +func (menu *Menu) copyTailscaleIP(device *ipnstate.PeerStatus) { if device == nil || len(device.TailscaleIPs) == 0 { return } @@ -430,11 +454,11 @@ func copyTailscaleIP(device *ipnstate.PeerStatus) { log.Printf("clipboard error: %v", err) } - sendNotification(fmt.Sprintf("Copied Address for %v", name), ip) + menu.sendNotification(fmt.Sprintf("Copied Address for %v", name), ip) } // sendNotification sends a desktop notification with the given title and content. -func sendNotification(title, content string) { +func (menu *Menu) sendNotification(title, content string) { conn, err := dbus.SessionBus() if err != nil { log.Printf("dbus: %v", err) @@ -443,13 +467,17 @@ func sendNotification(title, content string) { timeout := 3 * time.Second obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications") call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0), - appIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds())) + menu.notificationIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds())) if call.Err != nil { log.Printf("dbus: %v", call.Err) } } func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { + if menu.status == nil { + return + } + status := menu.status menu.exitNodes = systray.AddMenuItem("Exit Nodes", "") time.Sleep(newMenuDelay) @@ -469,7 +497,7 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { // Show recommended exit node if available. if status.Self.CapMap.Contains(tailcfg.NodeAttrSuggestExitNodeUI) { - sugg, err := localClient.SuggestExitNode(ctx) + sugg, err := menu.lc.SuggestExitNode(ctx) if err == nil { title := "Recommended: " if loc := sugg.Location; loc.Valid() && loc.Country() != "" { @@ -659,7 +687,15 @@ func newMullvadPeers(status *ipnstate.Status) mullvadPeers { return mullvadPeers{countries} } -func onExit() { +// onExit is called by the systray package when the menu is exiting. +func (menu *Menu) onExit() { log.Printf("exiting") - os.Remove(appIcon.Name()) + if menu.bgCancel != nil { + menu.bgCancel() + } + if menu.eventCancel != nil { + menu.eventCancel() + } + + os.Remove(menu.notificationIcon.Name()) } From c43c5ca003d64a2250aafc4530cfb074be43d535 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Fri, 27 Dec 2024 12:34:16 -0800 Subject: [PATCH 053/223] cmd/systray: properly set tooltip on different platforms On Linux, systray.SetTitle actually seems to set the tooltip on all desktops I've tested on. But on macOS, it actually does set a title that is always displayed in the systray area next to the icon. This change should properly set the tooltip across platforms. Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 5f498f35fdae1..0d6f8791689d5 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -199,14 +199,14 @@ func (menu *Menu) rebuild() { case ipn.Running.String(): if menu.status.ExitNodeStatus != nil && !menu.status.ExitNodeStatus.ID.IsZero() { if menu.status.ExitNodeStatus.Online { - systray.SetTitle("Using exit node") + setTooltip("Using exit node") setAppIcon(exitNodeOnline) } else { - systray.SetTitle("Exit node offline") + setTooltip("Exit node offline") setAppIcon(exitNodeOffline) } } else { - systray.SetTitle(fmt.Sprintf("Connected to %s", menu.status.CurrentTailnet.Name)) + setTooltip(fmt.Sprintf("Connected to %s", menu.status.CurrentTailnet.Name)) setAppIcon(connected) } menu.connect.SetTitle("Connected") @@ -214,10 +214,10 @@ func (menu *Menu) rebuild() { menu.disconnect.Show() menu.disconnect.Enable() case ipn.Starting.String(): - systray.SetTitle("Connecting") + setTooltip("Connecting") setAppIcon(loading) default: - systray.SetTitle("Disconnected") + setTooltip("Disconnected") setAppIcon(disconnected) } @@ -312,6 +312,16 @@ func setRemoteIcon(menu *systray.MenuItem, urlStr string) { } } +// setTooltip sets the tooltip text for the systray icon. +func setTooltip(text string) { + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + systray.SetTooltip(text) + } else { + // on Linux, SetTitle actually sets the tooltip + systray.SetTitle(text) + } +} + // eventLoop is the main event loop for handling click events on menu items // and responding to Tailscale state changes. // This method does not return until ctx.Done is closed. From 30d3e7b2429bc6e2226c365dd4e92f211e09147f Mon Sep 17 00:00:00 2001 From: Erisa A Date: Mon, 30 Dec 2024 17:22:48 +0000 Subject: [PATCH 054/223] scripts/install.sh: add special case for Parrot Security (#14487) Their `os-release` doesn't follow convention. Fixes #10778 Signed-off-by: Erisa A --- scripts/installer.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/installer.sh b/scripts/installer.sh index 8d1fc02128200..bdd425539e6ea 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -68,6 +68,14 @@ main() { if [ -z "${VERSION_ID:-}" ]; then # rolling release. If you haven't kept current, that's on you. APT_KEY_TYPE="keyring" + # Parrot Security is a special case that uses ID=debian + elif [ "$NAME" = "Parrot Security" ]; then + # All versions new enough to have this behaviour prefer keyring + # and their VERSION_ID is not consistent with Debian. + APT_KEY_TYPE="keyring" + # They don't specify the Debian version they're based off in os-release + # but Parrot 6 is based on Debian 12 Bookworm. + VERSION=bookworm elif [ "$VERSION_ID" -lt 11 ]; then APT_KEY_TYPE="legacy" else From ff095606ccff083160eb01a8a4cc062cacfe1a33 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 19 Nov 2024 20:17:58 -0800 Subject: [PATCH 055/223] all: add means to set device posture attributes from node Updates tailscale/corp#24690 Updates #4077 Change-Id: I05fe799beb1d2a71d1ec3ae08744cc68bcadae2a Signed-off-by: Brad Fitzpatrick --- control/controlclient/direct.go | 50 +++++++++++++++++++++++++++++++++ control/controlclient/noise.go | 7 +++-- ipn/ipnlocal/local.go | 14 +++++++++ ipn/localapi/localapi.go | 28 ++++++++++++++++++ tailcfg/tailcfg.go | 28 ++++++++++++++++++ 5 files changed, 125 insertions(+), 2 deletions(-) diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index 9cbd0e14ead52..dd361c4a20857 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -1643,6 +1643,56 @@ func (c *Direct) ReportHealthChange(w *health.Warnable, us *health.UnhealthyStat res.Body.Close() } +// SetDeviceAttrs does a synchronous call to the control plane to update +// the node's attributes. +// +// See docs on [tailcfg.SetDeviceAttributesRequest] for background. +func (c *Auto) SetDeviceAttrs(ctx context.Context, attrs tailcfg.AttrUpdate) error { + return c.direct.SetDeviceAttrs(ctx, attrs) +} + +// SetDeviceAttrs does a synchronous call to the control plane to update +// the node's attributes. +// +// See docs on [tailcfg.SetDeviceAttributesRequest] for background. +func (c *Direct) SetDeviceAttrs(ctx context.Context, attrs tailcfg.AttrUpdate) error { + nc, err := c.getNoiseClient() + if err != nil { + return err + } + nodeKey, ok := c.GetPersist().PublicNodeKeyOK() + if !ok { + return errors.New("no node key") + } + if c.panicOnUse { + panic("tainted client") + } + req := &tailcfg.SetDeviceAttributesRequest{ + NodeKey: nodeKey, + Version: tailcfg.CurrentCapabilityVersion, + Update: attrs, + } + + // TODO(bradfitz): unify the callers using doWithBody vs those using + // DoNoiseRequest. There seems to be a ~50/50 split and they're very close, + // but doWithBody sets the load balancing header and auto-JSON-encodes the + // body, but DoNoiseRequest is exported. Clean it up so they're consistent + // one way or another. + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + res, err := nc.doWithBody(ctx, "PATCH", "/machine/set-device-attr", nodeKey, req) + if err != nil { + return err + } + defer res.Body.Close() + all, _ := io.ReadAll(res.Body) + if res.StatusCode != 200 { + return fmt.Errorf("HTTP error from control plane: %v: %s", res.Status, all) + } + return nil +} + func addLBHeader(req *http.Request, nodeKey key.NodePublic) { if !nodeKey.IsZero() { req.Header.Add(tailcfg.LBHeader, nodeKey.String()) diff --git a/control/controlclient/noise.go b/control/controlclient/noise.go index 2e7c70fd1b162..db77014a6c138 100644 --- a/control/controlclient/noise.go +++ b/control/controlclient/noise.go @@ -380,17 +380,20 @@ func (nc *NoiseClient) dial(ctx context.Context) (*noiseconn.Conn, error) { // post does a POST to the control server at the given path, JSON-encoding body. // The provided nodeKey is an optional load balancing hint. func (nc *NoiseClient) post(ctx context.Context, path string, nodeKey key.NodePublic, body any) (*http.Response, error) { + return nc.doWithBody(ctx, "POST", path, nodeKey, body) +} + +func (nc *NoiseClient) doWithBody(ctx context.Context, method, path string, nodeKey key.NodePublic, body any) (*http.Response, error) { jbody, err := json.Marshal(body) if err != nil { return nil, err } - req, err := http.NewRequestWithContext(ctx, "POST", "https://"+nc.host+path, bytes.NewReader(jbody)) + req, err := http.NewRequestWithContext(ctx, method, "https://"+nc.host+path, bytes.NewReader(jbody)) if err != nil { return nil, err } addLBHeader(req, nodeKey) req.Header.Set("Content-Type", "application/json") - conn, err := nc.getConn(ctx) if err != nil { return nil, err diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index f456d49844f1e..d6daf35352d31 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -6408,6 +6408,20 @@ func (b *LocalBackend) SetExpirySooner(ctx context.Context, expiry time.Time) er return cc.SetExpirySooner(ctx, expiry) } +// SetDeviceAttrs does a synchronous call to the control plane to update +// the node's attributes. +// +// See docs on [tailcfg.SetDeviceAttributesRequest] for background. +func (b *LocalBackend) SetDeviceAttrs(ctx context.Context, attrs tailcfg.AttrUpdate) error { + b.mu.Lock() + cc := b.ccAuto + b.mu.Unlock() + if cc == nil { + return errors.New("not running") + } + return cc.SetDeviceAttrs(ctx, attrs) +} + // exitNodeCanProxyDNS reports the DoH base URL ("http://foo/dns-query") without query parameters // to exitNodeID's DoH service, if available. // diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index c14a4bdf285df..831f6a9b6b888 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -83,6 +83,7 @@ var handler = map[string]localAPIHandler{ // The other /localapi/v0/NAME handlers are exact matches and contain only NAME // without a trailing slash: + "alpha-set-device-attrs": (*Handler).serveSetDeviceAttrs, // see tailscale/corp#24690 "bugreport": (*Handler).serveBugReport, "check-ip-forwarding": (*Handler).serveCheckIPForwarding, "check-prefs": (*Handler).serveCheckPrefs, @@ -446,6 +447,33 @@ func (h *Handler) serveWhoIs(w http.ResponseWriter, r *http.Request) { h.serveWhoIsWithBackend(w, r, h.b) } +// serveSetDeviceAttrs is (as of 2024-12-30) an experimental LocalAPI handler to +// set device attributes via the control plane. +// +// See tailscale/corp#24690. +func (h *Handler) serveSetDeviceAttrs(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + if !h.PermitWrite { + http.Error(w, "set-device-attrs access denied", http.StatusForbidden) + return + } + if r.Method != "PATCH" { + http.Error(w, "only PATCH allowed", http.StatusMethodNotAllowed) + return + } + var req map[string]any + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := h.b.SetDeviceAttrs(ctx, req); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, "{}\n") +} + // localBackendWhoIsMethods is the subset of ipn.LocalBackend as needed // by the localapi WhoIs method. type localBackendWhoIsMethods interface { diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index ad07cff288a13..4c9cd59d9cbca 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2455,6 +2455,34 @@ type HealthChangeRequest struct { NodeKey key.NodePublic } +// SetDeviceAttributesRequest is a request to update the +// current node's device posture attributes. +// +// As of 2024-12-30, this is an experimental dev feature +// for internal testing. See tailscale/corp#24690. +type SetDeviceAttributesRequest struct { + // Version is the current binary's [CurrentCapabilityVersion]. + Version CapabilityVersion + + // NodeKey identifies the node to modify. It should be the currently active + // node and is an error if not. + NodeKey key.NodePublic + + // Update is a map of device posture attributes to update. + // Attributes not in the map are left unchanged. + Update AttrUpdate +} + +// AttrUpdate is a map of attributes to update. +// Attributes not in the map are left unchanged. +// The value can be a string, float64, bool, or nil to delete. +// +// See https://tailscale.com/s/api-device-posture-attrs. +// +// TODO(bradfitz): add struct type for specifying optional associated data +// for each attribute value, like an expiry time? +type AttrUpdate map[string]any + // SSHPolicy is the policy for how to handle incoming SSH connections // over Tailscale. type SSHPolicy struct { From 03b9361f479a86b39b272c6a898b84983948aec8 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 3 Jan 2025 08:35:25 -0800 Subject: [PATCH 056/223] ipn: update reference to Notify's Swift definition Updates #cleanup Signed-off-by: Brad Fitzpatrick --- ipn/backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipn/backend.go b/ipn/backend.go index 91a35df0d0da0..ef0700a70069e 100644 --- a/ipn/backend.go +++ b/ipn/backend.go @@ -147,7 +147,7 @@ type Notify struct { // any changes to the user in the UI. Health *health.State `json:",omitempty"` - // type is mirrored in xcode/Shared/IPN.swift + // type is mirrored in xcode/IPN/Core/LocalAPI/Model/LocalAPIModel.swift } func (n Notify) String() string { From e3bcb2ec83a45405f26391b8b10e47a66284d100 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 3 Jan 2025 08:49:29 -0800 Subject: [PATCH 057/223] ipn/ipnlocal: use context.CancelFunc type for doc clarity Using context.CancelFunc as the type (instead of func()) answers questions like whether it's okay to call it multiple times, whether it blocks, etc. And that's the type it actually is in this case. Updates #cleanup Signed-off-by: Brad Fitzpatrick --- ipn/ipnlocal/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index d6daf35352d31..9e8886404f0da 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -163,7 +163,7 @@ type watchSession struct { ch chan *ipn.Notify owner ipnauth.Actor // or nil sessionID string - cancel func() // call to signal that the session must be terminated + cancel context.CancelFunc // to shut down the session } // LocalBackend is the glue between the major pieces of the Tailscale From 17b881538ad4ded64b655c2b901a70628b00e921 Mon Sep 17 00:00:00 2001 From: Jason Barnett Date: Sun, 22 Dec 2024 13:18:40 -0700 Subject: [PATCH 058/223] wgengine/router: refactor udm-pro into broader ubnt support Fixes #14453 Signed-off-by: Jason Barnett --- version/distro/distro.go | 53 ++++------------------------ wgengine/router/router_linux.go | 14 ++++---- wgengine/router/router_linux_test.go | 6 ++-- 3 files changed, 17 insertions(+), 56 deletions(-) diff --git a/version/distro/distro.go b/version/distro/distro.go index 8128ce395f555..f7997e1d9f81b 100644 --- a/version/distro/distro.go +++ b/version/distro/distro.go @@ -9,7 +9,6 @@ import ( "os" "runtime" "strconv" - "strings" "tailscale.com/types/lazy" "tailscale.com/util/lineiter" @@ -31,7 +30,7 @@ const ( WDMyCloud = Distro("wdmycloud") Unraid = Distro("unraid") Alpine = Distro("alpine") - UDMPro = Distro("udmpro") + UBNT = Distro("ubnt") // Ubiquiti Networks ) var distro lazy.SyncValue[Distro] @@ -77,9 +76,12 @@ func linuxDistro() Distro { case have("/usr/local/bin/freenas-debug"): // TrueNAS Scale runs on debian return TrueNAS - case isUDMPro(): - // UDM-Pro runs on debian - return UDMPro + case have("/usr/bin/ubnt-device-info"): + // UBNT runs on Debian-based systems. This MUST be checked before Debian. + // + // Currently supported product families: + // - UDM (UniFi Dream Machine, UDM-Pro) + return UBNT case have("/etc/debian_version"): return Debian case have("/etc/arch-release"): @@ -152,44 +154,3 @@ func DSMVersion() int { return 0 }) } - -// isUDMPro checks a couple of files known to exist on a UDM-Pro and returns -// true if the expected content exists in the files. -func isUDMPro() bool { - // This is a performance guardrail against trying to load both - // /etc/board.info and /sys/firmware/devicetree/base/soc/board-cfg/id when - // not running on Debian so we don't make unnecessary calls in situations - // where we definitely are NOT on a UDM Pro. In other words, the have() call - // is much cheaper than the two os.ReadFile() in fileContainsAnyString(). - // That said, on Debian systems we will still be making the two - // os.ReadFile() in fileContainsAnyString(). - if !have("/etc/debian_version") { - return false - } - if exists, err := fileContainsAnyString("/etc/board.info", "UDMPRO", "Dream Machine PRO"); err == nil && exists { - return true - } - if exists, err := fileContainsAnyString("/sys/firmware/devicetree/base/soc/board-cfg/id", "udm pro"); err == nil && exists { - return true - } - return false -} - -// fileContainsAnyString is used to determine if one or more of the provided -// strings exists in a file. This is not efficient for larger files. If you want -// to use this function to parse large files, please refactor to use -// `io.LimitedReader`. -func fileContainsAnyString(filePath string, searchStrings ...string) (bool, error) { - data, err := os.ReadFile(filePath) - if err != nil { - return false, err - } - - content := string(data) - for _, searchString := range searchStrings { - if strings.Contains(content, searchString) { - return true, nil - } - } - return false, nil -} diff --git a/wgengine/router/router_linux.go b/wgengine/router/router_linux.go index e154a30fad91a..80191b24894ae 100644 --- a/wgengine/router/router_linux.go +++ b/wgengine/router/router_linux.go @@ -1184,7 +1184,7 @@ var ( ) // baseIPRules are the policy routing rules that Tailscale uses, when not -// running on a UDM-Pro. +// running on a UBNT device. // // The priority is the value represented here added to r.ipPolicyPrefBase, // which is usually 5200. @@ -1236,15 +1236,15 @@ var baseIPRules = []netlink.Rule{ // usual rules (pref 32766 and 32767, ie. main and default). } -// udmProIPRules are the policy routing rules that Tailscale uses, when running -// on a UDM-Pro. +// ubntIPRules are the policy routing rules that Tailscale uses, when running +// on a UBNT device. // // The priority is the value represented here added to // r.ipPolicyPrefBase, which is usually 5200. // // This represents an experiment that will be used to gather more information. // If this goes well, Tailscale may opt to use this for all of Linux. -var udmProIPRules = []netlink.Rule{ +var ubntIPRules = []netlink.Rule{ // non-fwmark packets fall through to the usual rules (pref 32766 and 32767, // ie. main and default). { @@ -1256,10 +1256,10 @@ var udmProIPRules = []netlink.Rule{ } // ipRules returns the appropriate list of ip rules to be used by Tailscale. See -// comments on baseIPRules and udmProIPRules for more details. +// comments on baseIPRules and ubntIPRules for more details. func ipRules() []netlink.Rule { - if getDistroFunc() == distro.UDMPro { - return udmProIPRules + if getDistroFunc() == distro.UBNT { + return ubntIPRules } return baseIPRules } diff --git a/wgengine/router/router_linux_test.go b/wgengine/router/router_linux_test.go index 7718f17c41706..9a159aea8b4e4 100644 --- a/wgengine/router/router_linux_test.go +++ b/wgengine/router/router_linux_test.go @@ -1233,14 +1233,14 @@ func adjustFwmask(t *testing.T, s string) string { return fwmaskAdjustRe.ReplaceAllString(s, "$1") } -func TestIPRulesForUDMPro(t *testing.T) { +func TestIPRulesForUBNT(t *testing.T) { // Override the global getDistroFunc getDistroFunc = func() distro.Distro { - return distro.UDMPro + return distro.UBNT } defer func() { getDistroFunc = distro.Get }() // Restore original after the test - expected := udmProIPRules + expected := ubntIPRules actual := ipRules() if len(expected) != len(actual) { From 1e2e319e7d261a8592744689c362fae16a3969d0 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 3 Jan 2025 10:41:02 -0800 Subject: [PATCH 059/223] util/slicesx: add MapKeys and MapValues from golang.org/x/exp/maps Importing the ~deprecated golang.org/x/exp/maps as "xmaps" to not shadow the std "maps" was getting ugly. And using slices.Collect on an iterator is verbose & allocates more. So copy (x)maps.Keys+Values into our slicesx package instead. Updates #cleanup Updates #12912 Updates #14514 (pulled out of that change) Change-Id: I5e68d12729934de93cf4a9cd87c367645f86123a Signed-off-by: Brad Fitzpatrick --- appc/appconnector.go | 7 +++-- appc/appconnector_test.go | 4 +-- cmd/systray/systray.go | 6 ++--- cmd/tailscale/cli/exitnode.go | 4 +-- cmd/tailscale/cli/serve_legacy.go | 6 ++--- cmd/tailscale/cli/serve_v2.go | 7 ++--- cmd/tailscale/depaware.txt | 2 +- cmd/tailscaled/depaware.txt | 2 +- cmd/testwrapper/testwrapper.go | 4 +-- ipn/ipnlocal/local.go | 10 +++---- net/dns/manager.go | 4 +-- util/lru/lru_test.go | 6 ++--- util/slicesx/slicesx.go | 40 ++++++++++++++++++++++++++++ util/syspolicy/source/test_store.go | 3 ++- wgengine/filter/filter_test.go | 4 +-- wgengine/magicsock/endpoint.go | 4 +-- wgengine/magicsock/magicsock_test.go | 4 +-- 17 files changed, 76 insertions(+), 41 deletions(-) diff --git a/appc/appconnector.go b/appc/appconnector.go index 671ced9534406..063381cd7e9a0 100644 --- a/appc/appconnector.go +++ b/appc/appconnector.go @@ -18,7 +18,6 @@ import ( "sync" "time" - xmaps "golang.org/x/exp/maps" "golang.org/x/net/dns/dnsmessage" "tailscale.com/types/logger" "tailscale.com/types/views" @@ -291,11 +290,11 @@ func (e *AppConnector) updateDomains(domains []string) { } } if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil { - e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err) + e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err) } } - e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards) + e.logf("handling domains: %v and wildcards: %v", slicesx.MapKeys(e.domains), e.wildcards) } // updateRoutes merges the supplied routes into the currently configured routes. The routes supplied @@ -354,7 +353,7 @@ func (e *AppConnector) Domains() views.Slice[string] { e.mu.Lock() defer e.mu.Unlock() - return views.SliceOf(xmaps.Keys(e.domains)) + return views.SliceOf(slicesx.MapKeys(e.domains)) } // DomainRoutes returns a map of domains to resolved IP diff --git a/appc/appconnector_test.go b/appc/appconnector_test.go index 7dba8cebd9e8c..36ec7a119dfbd 100644 --- a/appc/appconnector_test.go +++ b/appc/appconnector_test.go @@ -11,13 +11,13 @@ import ( "testing" "time" - xmaps "golang.org/x/exp/maps" "golang.org/x/net/dns/dnsmessage" "tailscale.com/appc/appctest" "tailscale.com/tstest" "tailscale.com/util/clientmetric" "tailscale.com/util/mak" "tailscale.com/util/must" + "tailscale.com/util/slicesx" ) func fakeStoreRoutes(*RouteInfo) error { return nil } @@ -50,7 +50,7 @@ func TestUpdateDomains(t *testing.T) { // domains are explicitly downcased on set. a.UpdateDomains([]string{"UP.EXAMPLE.COM"}) a.Wait(ctx) - if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) { + if got, want := slicesx.MapKeys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) { t.Errorf("got %v; want %v", got, want) } } diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 0d6f8791689d5..7da83a7ea0f07 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -12,7 +12,6 @@ import ( "fmt" "io" "log" - "maps" "net/http" "os" "os/signal" @@ -31,6 +30,7 @@ import ( "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" + "tailscale.com/util/slicesx" "tailscale.com/util/stringsx" ) @@ -616,7 +616,7 @@ type mullvadPeers struct { // sortedCountries returns countries containing mullvad nodes, sorted by name. func (mp mullvadPeers) sortedCountries() []*mvCountry { - countries := slices.Collect(maps.Values(mp.countries)) + countries := slicesx.MapValues(mp.countries) slices.SortFunc(countries, func(a, b *mvCountry) int { return stringsx.CompareFold(a.name, b.name) }) @@ -632,7 +632,7 @@ type mvCountry struct { // sortedCities returns cities containing mullvad nodes, sorted by name. func (mc *mvCountry) sortedCities() []*mvCity { - cities := slices.Collect(maps.Values(mc.cities)) + cities := slicesx.MapValues(mc.cities) slices.SortFunc(cities, func(a, b *mvCity) int { return stringsx.CompareFold(a.name, b.name) }) diff --git a/cmd/tailscale/cli/exitnode.go b/cmd/tailscale/cli/exitnode.go index 6b9247a7bc303..941c6be8d1add 100644 --- a/cmd/tailscale/cli/exitnode.go +++ b/cmd/tailscale/cli/exitnode.go @@ -15,10 +15,10 @@ import ( "github.com/kballard/go-shellquote" "github.com/peterbourgon/ff/v3/ffcli" - xmaps "golang.org/x/exp/maps" "tailscale.com/envknob" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" + "tailscale.com/util/slicesx" ) func exitNodeCmd() *ffcli.Command { @@ -255,7 +255,7 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) } filteredExitNodes := filteredExitNodes{ - Countries: xmaps.Values(countries), + Countries: slicesx.MapValues(countries), } for _, country := range filteredExitNodes.Countries { diff --git a/cmd/tailscale/cli/serve_legacy.go b/cmd/tailscale/cli/serve_legacy.go index 443a404abcbf7..5f55b1da68197 100644 --- a/cmd/tailscale/cli/serve_legacy.go +++ b/cmd/tailscale/cli/serve_legacy.go @@ -27,6 +27,7 @@ import ( "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" + "tailscale.com/util/slicesx" "tailscale.com/version" ) @@ -707,10 +708,7 @@ func (e *serveEnv) printWebStatusTree(sc *ipn.ServeConfig, hp ipn.HostPort) erro return "", "" } - var mounts []string - for k := range sc.Web[hp].Handlers { - mounts = append(mounts, k) - } + mounts := slicesx.MapKeys(sc.Web[hp].Handlers) sort.Slice(mounts, func(i, j int) bool { return len(mounts[i]) < len(mounts[j]) }) diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 009a61198dad8..3e173ce28d8c1 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -28,6 +28,7 @@ import ( "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" "tailscale.com/util/mak" + "tailscale.com/util/slicesx" "tailscale.com/version" ) @@ -439,11 +440,7 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN } if sc.Web[hp] != nil { - var mounts []string - - for k := range sc.Web[hp].Handlers { - mounts = append(mounts, k) - } + mounts := slicesx.MapKeys(sc.Web[hp].Handlers) sort.Slice(mounts, func(i, j int) bool { return len(mounts[i]) < len(mounts[j]) }) diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index a8496c411b99a..ff2de13c0378f 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -202,7 +202,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/sha3 from crypto/internal/mlkem768+ W golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ - golang.org/x/exp/maps from tailscale.com/cmd/tailscale/cli+ + golang.org/x/exp/maps from tailscale.com/util/syspolicy/internal/metrics+ golang.org/x/net/bpf from github.com/mdlayher/netlink+ golang.org/x/net/dns/dnsmessage from net+ golang.org/x/net/http/httpguts from net/http+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 264f8296f1e92..749c3f3100b7d 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -449,7 +449,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/crypto/sha3 from crypto/internal/mlkem768+ LD golang.org/x/crypto/ssh from github.com/pkg/sftp+ golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ - golang.org/x/exp/maps from tailscale.com/appc+ + golang.org/x/exp/maps from tailscale.com/ipn/store/mem+ golang.org/x/net/bpf from github.com/mdlayher/genetlink+ golang.org/x/net/dns/dnsmessage from net+ golang.org/x/net/http/httpguts from golang.org/x/net/http2+ diff --git a/cmd/testwrapper/testwrapper.go b/cmd/testwrapper/testwrapper.go index f6ff8f00a93ab..91aea904e62d0 100644 --- a/cmd/testwrapper/testwrapper.go +++ b/cmd/testwrapper/testwrapper.go @@ -29,8 +29,8 @@ import ( "github.com/dave/courtney/tester" "github.com/dave/patsy" "github.com/dave/patsy/vos" - xmaps "golang.org/x/exp/maps" "tailscale.com/cmd/testwrapper/flakytest" + "tailscale.com/util/slicesx" ) const ( @@ -350,7 +350,7 @@ func main() { if len(toRetry) == 0 { continue } - pkgs := xmaps.Keys(toRetry) + pkgs := slicesx.MapKeys(toRetry) sort.Strings(pkgs) nextRun := &nextRun{ attempt: thisRun.attempt + 1, diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 9e8886404f0da..c4f68e9299895 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -38,7 +38,6 @@ import ( "go4.org/mem" "go4.org/netipx" - xmaps "golang.org/x/exp/maps" "golang.org/x/net/dns/dnsmessage" "gvisor.dev/gvisor/pkg/tcpip" "tailscale.com/appc" @@ -104,6 +103,7 @@ import ( "tailscale.com/util/osuser" "tailscale.com/util/rands" "tailscale.com/util/set" + "tailscale.com/util/slicesx" "tailscale.com/util/syspolicy" "tailscale.com/util/syspolicy/rsop" "tailscale.com/util/systemd" @@ -2022,7 +2022,7 @@ func (b *LocalBackend) DisablePortMapperForTest() { func (b *LocalBackend) PeersForTest() []tailcfg.NodeView { b.mu.Lock() defer b.mu.Unlock() - ret := xmaps.Values(b.peers) + ret := slicesx.MapValues(b.peers) slices.SortFunc(ret, func(a, b tailcfg.NodeView) int { return cmp.Compare(a.ID(), b.ID()) }) @@ -7375,9 +7375,9 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug // First, try to select an exit node that has the closest DERP home, based on lastReport's DERP latency. // If there are no latency values, it returns an arbitrary region if len(candidatesByRegion) > 0 { - minRegion := minLatencyDERPRegion(xmaps.Keys(candidatesByRegion), report) + minRegion := minLatencyDERPRegion(slicesx.MapKeys(candidatesByRegion), report) if minRegion == 0 { - minRegion = selectRegion(views.SliceOf(xmaps.Keys(candidatesByRegion))) + minRegion = selectRegion(views.SliceOf(slicesx.MapKeys(candidatesByRegion))) } regionCandidates, ok := candidatesByRegion[minRegion] if !ok { @@ -7636,5 +7636,5 @@ func vipServicesFromPrefs(prefs ipn.PrefsView) []*tailcfg.VIPService { services[s].Active = true } - return slices.Collect(maps.Values(services)) + return slicesx.MapValues(services) } diff --git a/net/dns/manager.go b/net/dns/manager.go index 13cb2d84e1930..5ac2f69fc1a04 100644 --- a/net/dns/manager.go +++ b/net/dns/manager.go @@ -19,7 +19,6 @@ import ( "sync/atomic" "time" - xmaps "golang.org/x/exp/maps" "tailscale.com/control/controlknobs" "tailscale.com/health" "tailscale.com/net/dns/resolver" @@ -31,6 +30,7 @@ import ( "tailscale.com/types/logger" "tailscale.com/util/clientmetric" "tailscale.com/util/dnsname" + "tailscale.com/util/slicesx" ) var ( @@ -204,7 +204,7 @@ func compileHostEntries(cfg Config) (hosts []*HostEntry) { if len(hostsMap) == 0 { return nil } - hosts = xmaps.Values(hostsMap) + hosts = slicesx.MapValues(hostsMap) slices.SortFunc(hosts, func(a, b *HostEntry) int { if len(a.Hosts) == 0 && len(b.Hosts) == 0 { return 0 diff --git a/util/lru/lru_test.go b/util/lru/lru_test.go index fb538efbe7957..5500e5e0f309f 100644 --- a/util/lru/lru_test.go +++ b/util/lru/lru_test.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" - xmaps "golang.org/x/exp/maps" + "tailscale.com/util/slicesx" ) func TestLRU(t *testing.T) { @@ -75,7 +75,7 @@ func TestStressEvictions(t *testing.T) { for len(vm) < numKeys { vm[rand.Uint64()] = true } - vals := xmaps.Keys(vm) + vals := slicesx.MapKeys(vm) c := Cache[uint64, bool]{ MaxEntries: cacheSize, @@ -106,7 +106,7 @@ func TestStressBatchedEvictions(t *testing.T) { for len(vm) < numKeys { vm[rand.Uint64()] = true } - vals := xmaps.Keys(vm) + vals := slicesx.MapKeys(vm) c := Cache[uint64, bool]{} diff --git a/util/slicesx/slicesx.go b/util/slicesx/slicesx.go index e0b820eb71e91..1a7e18d914114 100644 --- a/util/slicesx/slicesx.go +++ b/util/slicesx/slicesx.go @@ -148,3 +148,43 @@ func FirstEqual[T comparable](s []T, v T) bool { func LastEqual[T comparable](s []T, v T) bool { return len(s) > 0 && s[len(s)-1] == v } + +// MapKeys returns the values of the map m. +// +// The keys will be in an indeterminate order. +// +// It's equivalent to golang.org/x/exp/maps.Keys, which +// unfortunately has the package name "maps", shadowing +// the std "maps" package. This version exists for clarity +// when reading call sites. +// +// As opposed to slices.Collect(maps.Keys(m)), this allocates +// the returned slice once to exactly the right size, rather than +// appending larger backing arrays as it goes. +func MapKeys[M ~map[K]V, K comparable, V any](m M) []K { + r := make([]K, 0, len(m)) + for k := range m { + r = append(r, k) + } + return r +} + +// MapValues returns the values of the map m. +// +// The values will be in an indeterminate order. +// +// It's equivalent to golang.org/x/exp/maps.Values, which +// unfortunately has the package name "maps", shadowing +// the std "maps" package. This version exists for clarity +// when reading call sites. +// +// As opposed to slices.Collect(maps.Values(m)), this allocates +// the returned slice once to exactly the right size, rather than +// appending larger backing arrays as it goes. +func MapValues[M ~map[K]V, K comparable, V any](m M) []V { + r := make([]V, 0, len(m)) + for _, v := range m { + r = append(r, v) + } + return r +} diff --git a/util/syspolicy/source/test_store.go b/util/syspolicy/source/test_store.go index 1f19bbb4386b9..e6c09d6b0430d 100644 --- a/util/syspolicy/source/test_store.go +++ b/util/syspolicy/source/test_store.go @@ -11,6 +11,7 @@ import ( xmaps "golang.org/x/exp/maps" "tailscale.com/util/mak" "tailscale.com/util/set" + "tailscale.com/util/slicesx" "tailscale.com/util/syspolicy/internal" "tailscale.com/util/syspolicy/setting" ) @@ -418,7 +419,7 @@ func (s *TestStore) NotifyPolicyChanged() { s.mu.RUnlock() return } - cbs := xmaps.Values(s.cbs) + cbs := slicesx.MapValues(s.cbs) s.mu.RUnlock() var wg sync.WaitGroup diff --git a/wgengine/filter/filter_test.go b/wgengine/filter/filter_test.go index f2796d71f6da7..7ffdd5c7b328f 100644 --- a/wgengine/filter/filter_test.go +++ b/wgengine/filter/filter_test.go @@ -18,7 +18,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "go4.org/netipx" - xmaps "golang.org/x/exp/maps" "tailscale.com/net/flowtrack" "tailscale.com/net/ipset" "tailscale.com/net/packet" @@ -30,6 +29,7 @@ import ( "tailscale.com/types/logger" "tailscale.com/types/views" "tailscale.com/util/must" + "tailscale.com/util/slicesx" "tailscale.com/wgengine/filter/filtertype" ) @@ -997,7 +997,7 @@ func TestPeerCaps(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := xmaps.Keys(filt.CapsWithValues(netip.MustParseAddr(tt.src), netip.MustParseAddr(tt.dst))) + got := slicesx.MapKeys(filt.CapsWithValues(netip.MustParseAddr(tt.src), netip.MustParseAddr(tt.dst))) slices.Sort(got) slices.Sort(tt.want) if !slices.Equal(got, tt.want) { diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index bbba3181ce453..df4299b72a4cb 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -21,7 +21,6 @@ import ( "sync/atomic" "time" - xmaps "golang.org/x/exp/maps" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" "tailscale.com/disco" @@ -34,6 +33,7 @@ import ( "tailscale.com/types/logger" "tailscale.com/util/mak" "tailscale.com/util/ringbuffer" + "tailscale.com/util/slicesx" ) var mtuProbePingSizesV4 []int @@ -587,7 +587,7 @@ func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr netip.Add needPing := len(de.endpointState) > 1 && now.Sub(oldestPing) > wireguardPingInterval if !udpAddr.IsValid() { - candidates := xmaps.Keys(de.endpointState) + candidates := slicesx.MapKeys(de.endpointState) // Randomly select an address to use until we retrieve latency information // and give it a short trustBestAddrUntil time so we avoid flapping between diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 1b3f8ec73c16e..8166004517441 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -33,7 +33,6 @@ import ( "github.com/tailscale/wireguard-go/device" "github.com/tailscale/wireguard-go/tun/tuntest" "go4.org/mem" - xmaps "golang.org/x/exp/maps" "golang.org/x/net/icmp" "golang.org/x/net/ipv4" "tailscale.com/cmd/testwrapper/flakytest" @@ -66,6 +65,7 @@ import ( "tailscale.com/util/must" "tailscale.com/util/racebuild" "tailscale.com/util/set" + "tailscale.com/util/slicesx" "tailscale.com/util/usermetric" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/wgcfg" @@ -1133,7 +1133,7 @@ func testTwoDevicePing(t *testing.T, d *devices) { } } t.Helper() - t.Errorf("missing any connection to %s from %s", wantConns, xmaps.Keys(stats)) + t.Errorf("missing any connection to %s from %s", wantConns, slicesx.MapKeys(stats)) } addrPort := netip.MustParseAddrPort From 402fc9d65f72136bb3e70b1a7e3d0f20443524ad Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 3 Jan 2025 10:10:16 -0800 Subject: [PATCH 060/223] control/controlclient: remove optimization that was more convoluted than useful While working on #13390, I ran across this non-idiomatic pointer-to-view and parallel-sorted-map accounting code that was all just to avoid a sort later. But the sort later when building a new netmap.NetworkMap is already a drop in the bucket of CPU compared to how much work & allocs mapSession.netmap and LocalBackend's spamming of the full netmap (potentially tens of thousands of peers, MBs of JSON) out to IPNBus clients for any tiny little change (node changing online status, etc). Removing the parallel sorted slice let everything be simpler to reason about, so this does that. The sort might take a bit more CPU time now in theory, but in practice for any netmap size for which it'd matter, the quadratic netmap IPN bus spam (which we need to fix soon) will overshadow that little sort. Updates #13390 Updates #1909 Change-Id: I3092d7c67dc10b2a0f141496fe0e7e98ccc07712 Signed-off-by: Brad Fitzpatrick --- control/controlclient/map.go | 68 +++++++++++-------------------- control/controlclient/map_test.go | 13 +++--- 2 files changed, 30 insertions(+), 51 deletions(-) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 7879122229e37..b20a8e17058f3 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -14,7 +14,6 @@ import ( "runtime" "runtime/debug" "slices" - "sort" "strconv" "sync" "time" @@ -31,6 +30,7 @@ import ( "tailscale.com/util/clientmetric" "tailscale.com/util/mak" "tailscale.com/util/set" + "tailscale.com/util/slicesx" "tailscale.com/wgengine/filter" ) @@ -75,8 +75,7 @@ type mapSession struct { lastPrintMap time.Time lastNode tailcfg.NodeView lastCapSet set.Set[tailcfg.NodeCapability] - peers map[tailcfg.NodeID]*tailcfg.NodeView // pointer to view (oddly). same pointers as sortedPeers. - sortedPeers []*tailcfg.NodeView // same pointers as peers, but sorted by Node.ID + peers map[tailcfg.NodeID]tailcfg.NodeView lastDNSConfig *tailcfg.DNSConfig lastDERPMap *tailcfg.DERPMap lastUserProfile map[tailcfg.UserID]tailcfg.UserProfile @@ -366,16 +365,11 @@ var ( patchifiedPeerEqual = clientmetric.NewCounter("controlclient_patchified_peer_equal") ) -// updatePeersStateFromResponseres updates ms.peers and ms.sortedPeers from res. It takes ownership of res. +// updatePeersStateFromResponseres updates ms.peers from resp. +// It takes ownership of resp. func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (stats updateStats) { - defer func() { - if stats.removed > 0 || stats.added > 0 { - ms.rebuildSorted() - } - }() - if ms.peers == nil { - ms.peers = make(map[tailcfg.NodeID]*tailcfg.NodeView) + ms.peers = make(map[tailcfg.NodeID]tailcfg.NodeView) } if len(resp.Peers) > 0 { @@ -384,12 +378,12 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s keep := make(map[tailcfg.NodeID]bool, len(resp.Peers)) for _, n := range resp.Peers { keep[n.ID] = true - if vp, ok := ms.peers[n.ID]; ok { + lenBefore := len(ms.peers) + ms.peers[n.ID] = n.View() + if len(ms.peers) == lenBefore { stats.changed++ - *vp = n.View() } else { stats.added++ - ms.peers[n.ID] = ptr.To(n.View()) } } for id := range ms.peers { @@ -410,12 +404,12 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s } for _, n := range resp.PeersChanged { - if vp, ok := ms.peers[n.ID]; ok { + lenBefore := len(ms.peers) + ms.peers[n.ID] = n.View() + if len(ms.peers) == lenBefore { stats.changed++ - *vp = n.View() } else { stats.added++ - ms.peers[n.ID] = ptr.To(n.View()) } } @@ -427,7 +421,7 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s } else { mut.LastSeen = nil } - *vp = mut.View() + ms.peers[nodeID] = mut.View() stats.changed++ } } @@ -436,7 +430,7 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s if vp, ok := ms.peers[nodeID]; ok { mut := vp.AsStruct() mut.Online = ptr.To(online) - *vp = mut.View() + ms.peers[nodeID] = mut.View() stats.changed++ } } @@ -488,31 +482,12 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s mut.CapMap = v patchCapMap.Add(1) } - *vp = mut.View() + ms.peers[pc.NodeID] = mut.View() } return } -// rebuildSorted rebuilds ms.sortedPeers from ms.peers. It should be called -// after any additions or removals from peers. -func (ms *mapSession) rebuildSorted() { - if ms.sortedPeers == nil { - ms.sortedPeers = make([]*tailcfg.NodeView, 0, len(ms.peers)) - } else { - if len(ms.sortedPeers) > len(ms.peers) { - clear(ms.sortedPeers[len(ms.peers):]) - } - ms.sortedPeers = ms.sortedPeers[:0] - } - for _, p := range ms.peers { - ms.sortedPeers = append(ms.sortedPeers, p) - } - sort.Slice(ms.sortedPeers, func(i, j int) bool { - return ms.sortedPeers[i].ID() < ms.sortedPeers[j].ID() - }) -} - func (ms *mapSession) addUserProfile(nm *netmap.NetworkMap, userID tailcfg.UserID) { if userID == 0 { return @@ -576,7 +551,7 @@ func (ms *mapSession) patchifyPeer(n *tailcfg.Node) (_ *tailcfg.PeerChange, ok b if !ok { return nil, false } - return peerChangeDiff(*was, n) + return peerChangeDiff(was, n) } // peerChangeDiff returns the difference from 'was' to 'n', if possible. @@ -778,14 +753,19 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang return ret, true } +func (ms *mapSession) sortedPeers() []tailcfg.NodeView { + ret := slicesx.MapValues(ms.peers) + slices.SortFunc(ret, func(a, b tailcfg.NodeView) int { + return cmp.Compare(a.ID(), b.ID()) + }) + return ret +} + // netmap returns a fully populated NetworkMap from the last state seen from // a call to updateStateFromResponse, filling in omitted // information from prior MapResponse values. func (ms *mapSession) netmap() *netmap.NetworkMap { - peerViews := make([]tailcfg.NodeView, len(ms.sortedPeers)) - for i, vp := range ms.sortedPeers { - peerViews[i] = *vp - } + peerViews := ms.sortedPeers() nm := &netmap.NetworkMap{ NodeKey: ms.publicNodeKey, diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index 897036a942f49..ad8f7dd6e288b 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -340,19 +340,18 @@ func TestUpdatePeersStateFromResponse(t *testing.T) { } ms := newTestMapSession(t, nil) for _, n := range tt.prev { - mak.Set(&ms.peers, n.ID, ptr.To(n.View())) + mak.Set(&ms.peers, n.ID, n.View()) } - ms.rebuildSorted() gotStats := ms.updatePeersStateFromResponse(tt.mapRes) - - got := make([]*tailcfg.Node, len(ms.sortedPeers)) - for i, vp := range ms.sortedPeers { - got[i] = vp.AsStruct() - } if gotStats != tt.wantStats { t.Errorf("got stats = %+v; want %+v", gotStats, tt.wantStats) } + + var got []*tailcfg.Node + for _, vp := range ms.sortedPeers() { + got = append(got, vp.AsStruct()) + } if !reflect.DeepEqual(got, tt.want) { t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(got), formatNodes(tt.want)) } From ad8d8e37dee33ba99f1e8bdc31faa7567ca179a9 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Fri, 3 Jan 2025 16:01:20 -0800 Subject: [PATCH 061/223] go.mod: update github.com/go-json-experiment/json (#14522) Updates tailscale/corp#11038 Signed-off-by: Joe Tsai --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e57573f188933..3c389b4de66a0 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/frankban/quicktest v1.14.6 github.com/fxamacker/cbor/v2 v2.6.0 github.com/gaissmai/bart v0.11.1 - github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 + github.com/go-json-experiment/json v0.0.0-20250103232110-6a9a0fde9288 github.com/go-logr/zapr v1.3.0 github.com/go-ole/go-ole v1.3.0 github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 diff --git a/go.sum b/go.sum index 1cbb440fa7d58..2ae4ce09df117 100644 --- a/go.sum +++ b/go.sum @@ -350,8 +350,8 @@ github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lK github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 h1:ymLjT4f35nQbASLnvxEde4XOBL+Sn7rFuV+FOJqkljg= -github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA= +github.com/go-json-experiment/json v0.0.0-20250103232110-6a9a0fde9288 h1:KbX3Z3CgiYlbaavUq3Cj9/MjpO+88S7/AGXzynVDv84= +github.com/go-json-experiment/json v0.0.0-20250103232110-6a9a0fde9288/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= From 47bd0723a08cfeb5ac04aaa23f74dd032909abbd Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 4 Jan 2025 10:14:23 -0800 Subject: [PATCH 062/223] all: use iterators in more places instead of Range funcs And misc cleanup along the way. Updates #12912 Change-Id: I0cab148b49efc668c6f5cdf09c740b84a713e388 Signed-off-by: Brad Fitzpatrick --- control/controlclient/map.go | 20 +++++++++++--------- envknob/logknob/logknob.go | 6 ++---- envknob/logknob/logknob_test.go | 7 ++----- ipn/ipnlocal/local.go | 21 +++++++++------------ ipn/ipnlocal/serve.go | 5 ++--- tailcfg/tailcfg.go | 4 ++-- types/netmap/netmap.go | 20 +++++--------------- 7 files changed, 33 insertions(+), 50 deletions(-) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index b20a8e17058f3..97d49f90d4ad3 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -663,21 +663,23 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang } case "CapMap": if len(n.CapMap) != was.CapMap().Len() { + // If they have different lengths, they're different. if n.CapMap == nil { pc().CapMap = make(tailcfg.NodeCapMap) } else { pc().CapMap = maps.Clone(n.CapMap) } - break - } - was.CapMap().Range(func(k tailcfg.NodeCapability, v views.Slice[tailcfg.RawMessage]) bool { - nv, ok := n.CapMap[k] - if !ok || !views.SliceEqual(v, views.SliceOf(nv)) { - pc().CapMap = maps.Clone(n.CapMap) - return false + } else { + // If they have the same length, check that all their keys + // have the same values. + for k, v := range was.CapMap().All() { + nv, ok := n.CapMap[k] + if !ok || !views.SliceEqual(v, views.SliceOf(nv)) { + pc().CapMap = maps.Clone(n.CapMap) + break + } } - return true - }) + } case "Tags": if !views.SliceEqual(was.Tags(), views.SliceOf(n.Tags)) { return nil, false diff --git a/envknob/logknob/logknob.go b/envknob/logknob/logknob.go index 350384b8626e3..93302d0d2bd5c 100644 --- a/envknob/logknob/logknob.go +++ b/envknob/logknob/logknob.go @@ -11,7 +11,6 @@ import ( "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/types/logger" - "tailscale.com/types/views" ) // TODO(andrew-d): should we have a package-global registry of logknobs? It @@ -59,7 +58,7 @@ func (lk *LogKnob) Set(v bool) { // about; we use this rather than a concrete type to avoid a circular // dependency. type NetMap interface { - SelfCapabilities() views.Slice[tailcfg.NodeCapability] + HasSelfCapability(tailcfg.NodeCapability) bool } // UpdateFromNetMap will enable logging if the SelfNode in the provided NetMap @@ -68,8 +67,7 @@ func (lk *LogKnob) UpdateFromNetMap(nm NetMap) { if lk.capName == "" { return } - - lk.cap.Store(views.SliceContains(nm.SelfCapabilities(), lk.capName)) + lk.cap.Store(nm.HasSelfCapability(lk.capName)) } // Do will call log with the provided format and arguments if any of the diff --git a/envknob/logknob/logknob_test.go b/envknob/logknob/logknob_test.go index b2a376a25b371..aa4fb44214e12 100644 --- a/envknob/logknob/logknob_test.go +++ b/envknob/logknob/logknob_test.go @@ -11,6 +11,7 @@ import ( "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/types/netmap" + "tailscale.com/util/set" ) var testKnob = NewLogKnob( @@ -63,11 +64,7 @@ func TestLogKnob(t *testing.T) { } testKnob.UpdateFromNetMap(&netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - Capabilities: []tailcfg.NodeCapability{ - "https://tailscale.com/cap/testing", - }, - }).View(), + AllCaps: set.Of(tailcfg.NodeCapability("https://tailscale.com/cap/testing")), }) if !testKnob.shouldLog() { t.Errorf("expected shouldLog()=true") diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index c4f68e9299895..bf88221ab3478 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1126,11 +1126,10 @@ func (b *LocalBackend) UpdateStatus(sb *ipnstate.StatusBuilder) { ss.Capabilities = make([]tailcfg.NodeCapability, 1, cm.Len()+1) ss.Capabilities[0] = "HTTPS://TAILSCALE.COM/s/DEPRECATED-NODE-CAPS#see-https://github.com/tailscale/tailscale/issues/11508" ss.CapMap = make(tailcfg.NodeCapMap, sn.CapMap().Len()) - cm.Range(func(k tailcfg.NodeCapability, v views.Slice[tailcfg.RawMessage]) bool { + for k, v := range cm.All() { ss.CapMap[k] = v.AsSlice() ss.Capabilities = append(ss.Capabilities, k) - return true - }) + } slices.Sort(ss.Capabilities[1:]) } } @@ -1192,10 +1191,9 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) { } if cm := p.CapMap(); cm.Len() > 0 { ps.CapMap = make(tailcfg.NodeCapMap, cm.Len()) - cm.Range(func(k tailcfg.NodeCapability, v views.Slice[tailcfg.RawMessage]) bool { + for k, v := range cm.All() { ps.CapMap[k] = v.AsSlice() - return true - }) + } } peerStatusFromNode(ps, p) @@ -5918,15 +5916,15 @@ func (b *LocalBackend) setServeProxyHandlersLocked() { } var backends map[string]bool b.serveConfig.RangeOverWebs(func(_ ipn.HostPort, conf ipn.WebServerConfigView) (cont bool) { - conf.Handlers().Range(func(_ string, h ipn.HTTPHandlerView) (cont bool) { + for _, h := range conf.Handlers().All() { backend := h.Proxy() if backend == "" { // Only create proxy handlers for servers with a proxy backend. - return true + continue } mak.Set(&backends, backend, true) if _, ok := b.serveProxyHandlers.Load(backend); ok { - return true + continue } b.logf("serve: creating a new proxy handler for %s", backend) @@ -5935,11 +5933,10 @@ func (b *LocalBackend) setServeProxyHandlersLocked() { // The backend endpoint (h.Proxy) should have been validated by expandProxyTarget // in the CLI, so just log the error here. b.logf("[unexpected] could not create proxy for %v: %s", backend, err) - return true + continue } b.serveProxyHandlers.Store(backend, p) - return true - }) + } return true }) diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index 61bed05527167..c144fa5299a8f 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -326,7 +326,7 @@ func (b *LocalBackend) setServeConfigLocked(config *ipn.ServeConfig, etag string if b.serveConfig.Valid() { has = b.serveConfig.Foreground().Contains } - prevConfig.Foreground().Range(func(k string, v ipn.ServeConfigView) (cont bool) { + for k := range prevConfig.Foreground().All() { if !has(k) { for _, sess := range b.notifyWatchers { if sess.sessionID == k { @@ -334,8 +334,7 @@ func (b *LocalBackend) setServeConfigLocked(config *ipn.ServeConfig, etag string } } } - return true - }) + } } return nil diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 4c9cd59d9cbca..f762d992d30c3 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -1457,7 +1457,7 @@ const ( // NodeCapMap is a map of capabilities to their optional values. It is valid for // a capability to have no values (nil slice); such capabilities can be tested -// for by using the Contains method. +// for by using the [NodeCapMap.Contains] method. // // See [NodeCapability] for more information on keys. type NodeCapMap map[NodeCapability][]RawMessage @@ -1873,7 +1873,7 @@ type MapResponse struct { // PeersChangedPatch, if non-nil, means that node(s) have changed. // This is a lighter version of the older PeersChanged support that - // only supports certain types of updates + // only supports certain types of updates. // // These are applied after Peers* above, but in practice the // control server should only send these on their own, without diff --git a/types/netmap/netmap.go b/types/netmap/netmap.go index 94e872a5593ea..b1ac612de9669 100644 --- a/types/netmap/netmap.go +++ b/types/netmap/netmap.go @@ -197,21 +197,11 @@ func (nm *NetworkMap) DomainName() string { return nm.Domain } -// SelfCapabilities returns SelfNode.Capabilities if nm and nm.SelfNode are -// non-nil. This is a method so we can use it in envknob/logknob without a -// circular dependency. -func (nm *NetworkMap) SelfCapabilities() views.Slice[tailcfg.NodeCapability] { - var zero views.Slice[tailcfg.NodeCapability] - if nm == nil || !nm.SelfNode.Valid() { - return zero - } - out := nm.SelfNode.Capabilities().AsSlice() - nm.SelfNode.CapMap().Range(func(k tailcfg.NodeCapability, _ views.Slice[tailcfg.RawMessage]) (cont bool) { - out = append(out, k) - return true - }) - - return views.SliceOf(out) +// HasSelfCapability reports whether nm.SelfNode contains capability c. +// +// It exists to satisify an unused (as of 2025-01-04) interface in the logknob package. +func (nm *NetworkMap) HasSelfCapability(c tailcfg.NodeCapability) bool { + return nm.AllCaps.Contains(c) } func (nm *NetworkMap) String() string { From 4b56bf9039eb6bce4da75aa3154a65506e38661f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 4 Jan 2025 11:50:48 -0800 Subject: [PATCH 063/223] types/views: remove various Map Range funcs; use iterators everywhere The remaining range funcs in the tree are RangeOverTCPs and RangeOverWebs in ServeConfig; those will be cleaned up separately. Updates #12912 Change-Id: Ieeae4864ab088877263c36b805f77aa8e6be938d Signed-off-by: Brad Fitzpatrick --- ipn/serve.go | 101 +++++++++++++++++++------------------------ types/views/views.go | 30 ------------- 2 files changed, 45 insertions(+), 86 deletions(-) diff --git a/ipn/serve.go b/ipn/serve.go index 49e0d9fa3d67a..32e74e6881aaa 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -568,54 +568,46 @@ func ExpandProxyTargetValue(target string, supportedSchemes []string, defaultSch // If the returned bool from the given f is false, then this function stops // iterating immediately and does not check other foreground configs. func (v ServeConfigView) RangeOverTCPs(f func(port uint16, _ TCPPortHandlerView) bool) { - parentCont := true - v.TCP().Range(func(k uint16, v TCPPortHandlerView) (cont bool) { - parentCont = f(k, v) - return parentCont - }) - v.Foreground().Range(func(k string, v ServeConfigView) (cont bool) { - if !parentCont { - return false + for k, v := range v.TCP().All() { + if !f(k, v) { + return } - v.TCP().Range(func(k uint16, v TCPPortHandlerView) (cont bool) { - parentCont = f(k, v) - return parentCont - }) - return parentCont - }) + } + for _, conf := range v.Foreground().All() { + for k, v := range conf.TCP().All() { + if !f(k, v) { + return + } + } + } } // RangeOverWebs ranges over both background and foreground Webs. // If the returned bool from the given f is false, then this function stops // iterating immediately and does not check other foreground configs. -func (v ServeConfigView) RangeOverWebs(f func(_ HostPort, conf WebServerConfigView) bool) { - parentCont := true - v.Web().Range(func(k HostPort, v WebServerConfigView) (cont bool) { - parentCont = f(k, v) - return parentCont - }) - v.Foreground().Range(func(k string, v ServeConfigView) (cont bool) { - if !parentCont { - return false +func (v ServeConfigView) RangeOverWebs(f func(HostPort, WebServerConfigView) bool) { + for k, v := range v.Web().All() { + if !f(k, v) { + return + } + } + for _, conf := range v.Foreground().All() { + for k, v := range conf.Web().All() { + if !f(k, v) { + return + } } - v.Web().Range(func(k HostPort, v WebServerConfigView) (cont bool) { - parentCont = f(k, v) - return parentCont - }) - return parentCont - }) + } } // FindTCP returns the first TCP that matches with the given port. It // prefers a foreground match first followed by a background search if none // existed. func (v ServeConfigView) FindTCP(port uint16) (res TCPPortHandlerView, ok bool) { - v.Foreground().Range(func(_ string, v ServeConfigView) (cont bool) { - res, ok = v.TCP().GetOk(port) - return !ok - }) - if ok { - return res, ok + for _, conf := range v.Foreground().All() { + if res, ok := conf.TCP().GetOk(port); ok { + return res, ok + } } return v.TCP().GetOk(port) } @@ -624,12 +616,10 @@ func (v ServeConfigView) FindTCP(port uint16) (res TCPPortHandlerView, ok bool) // prefers a foreground match first followed by a background search if none // existed. func (v ServeConfigView) FindWeb(hp HostPort) (res WebServerConfigView, ok bool) { - v.Foreground().Range(func(_ string, v ServeConfigView) (cont bool) { - res, ok = v.Web().GetOk(hp) - return !ok - }) - if ok { - return res, ok + for _, conf := range v.Foreground().All() { + if res, ok := conf.Web().GetOk(hp); ok { + return res, ok + } } return v.Web().GetOk(hp) } @@ -637,14 +627,15 @@ func (v ServeConfigView) FindWeb(hp HostPort) (res WebServerConfigView, ok bool) // HasAllowFunnel returns whether this config has at least one AllowFunnel // set in the background or foreground configs. func (v ServeConfigView) HasAllowFunnel() bool { - return v.AllowFunnel().Len() > 0 || func() bool { - var exists bool - v.Foreground().Range(func(k string, v ServeConfigView) (cont bool) { - exists = v.AllowFunnel().Len() > 0 - return !exists - }) - return exists - }() + if v.AllowFunnel().Len() > 0 { + return true + } + for _, conf := range v.Foreground().All() { + if conf.AllowFunnel().Len() > 0 { + return true + } + } + return false } // FindFunnel reports whether target exists in either the background AllowFunnel @@ -653,12 +644,10 @@ func (v ServeConfigView) HasFunnelForTarget(target HostPort) bool { if v.AllowFunnel().Get(target) { return true } - var exists bool - v.Foreground().Range(func(_ string, v ServeConfigView) (cont bool) { - if exists = v.AllowFunnel().Get(target); exists { - return false + for _, conf := range v.Foreground().All() { + if conf.AllowFunnel().Get(target) { + return true } - return true - }) - return exists + } + return false } diff --git a/types/views/views.go b/types/views/views.go index 19aa69d4a8edb..eae8c0b169404 100644 --- a/types/views/views.go +++ b/types/views/views.go @@ -415,16 +415,6 @@ func (m *MapSlice[K, V]) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &m.Đļ) } -// Range calls f for every k,v pair in the underlying map. -// It stops iteration immediately if f returns false. -func (m MapSlice[K, V]) Range(f MapRangeFn[K, Slice[V]]) { - for k, v := range m.Đļ { - if !f(k, SliceOf(v)) { - return - } - } -} - // AsMap returns a shallow-clone of the underlying map. // // If V is a pointer type, it is the caller's responsibility to make sure the @@ -527,16 +517,6 @@ func (m Map[K, V]) AsMap() map[K]V { // Implementations should return false to stop range. type MapRangeFn[K comparable, V any] func(k K, v V) (cont bool) -// Range calls f for every k,v pair in the underlying map. -// It stops iteration immediately if f returns false. -func (m Map[K, V]) Range(f MapRangeFn[K, V]) { - for k, v := range m.Đļ { - if !f(k, v) { - return - } - } -} - // All returns an iterator iterating over the keys // and values of m. func (m Map[K, V]) All() iter.Seq2[K, V] { @@ -600,16 +580,6 @@ func (m MapFn[K, T, V]) GetOk(k K) (V, bool) { return m.wrapv(v), ok } -// Range calls f for every k,v pair in the underlying map. -// It stops iteration immediately if f returns false. -func (m MapFn[K, T, V]) Range(f MapRangeFn[K, V]) { - for k, v := range m.Đļ { - if !f(k, m.wrapv(v)) { - return - } - } -} - // All returns an iterator iterating over the keys and value views of m. func (m MapFn[K, T, V]) All() iter.Seq2[K, V] { return func(yield func(K, V) bool) { From 2b8f02b407934602e120f3e3b096bbb6d32e61ad Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 4 Jan 2025 13:46:09 -0800 Subject: [PATCH 064/223] ipn: convert ServeConfig Range methods to iterators These were the last two Range funcs in this repo. Updates #12912 Change-Id: I6ba0a911933cb5fc4e43697a9aac58a8035f9622 Signed-off-by: Brad Fitzpatrick --- ipn/ipnlocal/local.go | 10 ++++----- ipn/serve.go | 52 +++++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index bf88221ab3478..fc7b997bc94a0 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -5882,12 +5882,11 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn. b.reloadServeConfigLocked(prefs) if b.serveConfig.Valid() { servePorts := make([]uint16, 0, 3) - b.serveConfig.RangeOverTCPs(func(port uint16, _ ipn.TCPPortHandlerView) bool { + for port := range b.serveConfig.TCPs() { if port > 0 { servePorts = append(servePorts, uint16(port)) } - return true - }) + } handlePorts = append(handlePorts, servePorts...) b.setServeProxyHandlersLocked() @@ -5915,7 +5914,7 @@ func (b *LocalBackend) setServeProxyHandlersLocked() { return } var backends map[string]bool - b.serveConfig.RangeOverWebs(func(_ ipn.HostPort, conf ipn.WebServerConfigView) (cont bool) { + for _, conf := range b.serveConfig.Webs() { for _, h := range conf.Handlers().All() { backend := h.Proxy() if backend == "" { @@ -5937,8 +5936,7 @@ func (b *LocalBackend) setServeProxyHandlersLocked() { } b.serveProxyHandlers.Store(backend, p) } - return true - }) + } // Clean up handlers for proxy backends that are no longer present // in configuration. diff --git a/ipn/serve.go b/ipn/serve.go index 32e74e6881aaa..e82279db8c604 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -6,6 +6,7 @@ package ipn import ( "errors" "fmt" + "iter" "net" "net/netip" "net/url" @@ -564,39 +565,42 @@ func ExpandProxyTargetValue(target string, supportedSchemes []string, defaultSch return u.String(), nil } -// RangeOverTCPs ranges over both background and foreground TCPs. -// If the returned bool from the given f is false, then this function stops -// iterating immediately and does not check other foreground configs. -func (v ServeConfigView) RangeOverTCPs(f func(port uint16, _ TCPPortHandlerView) bool) { - for k, v := range v.TCP().All() { - if !f(k, v) { - return - } - } - for _, conf := range v.Foreground().All() { - for k, v := range conf.TCP().All() { - if !f(k, v) { +// TCPs returns an iterator over both background and foreground TCP +// listeners. +// +// The key is the port number. +func (v ServeConfigView) TCPs() iter.Seq2[uint16, TCPPortHandlerView] { + return func(yield func(uint16, TCPPortHandlerView) bool) { + for k, v := range v.TCP().All() { + if !yield(k, v) { return } } + for _, conf := range v.Foreground().All() { + for k, v := range conf.TCP().All() { + if !yield(k, v) { + return + } + } + } } } -// RangeOverWebs ranges over both background and foreground Webs. -// If the returned bool from the given f is false, then this function stops -// iterating immediately and does not check other foreground configs. -func (v ServeConfigView) RangeOverWebs(f func(HostPort, WebServerConfigView) bool) { - for k, v := range v.Web().All() { - if !f(k, v) { - return - } - } - for _, conf := range v.Foreground().All() { - for k, v := range conf.Web().All() { - if !f(k, v) { +// Webs returns an iterator over both background and foreground Web configurations. +func (v ServeConfigView) Webs() iter.Seq2[HostPort, WebServerConfigView] { + return func(yield func(HostPort, WebServerConfigView) bool) { + for k, v := range v.Web().All() { + if !yield(k, v) { return } } + for _, conf := range v.Foreground().All() { + for k, v := range conf.Web().All() { + if !yield(k, v) { + return + } + } + } } } From 60930d19c06de9469f4b65f4fd79eacdba3e3ee1 Mon Sep 17 00:00:00 2001 From: Marc Paquette Date: Sat, 28 Dec 2024 19:46:23 -0500 Subject: [PATCH 065/223] Update README to reference correct Commit Style URL Change-Id: I2981c685a8905ad58536a8d9b01511d04c3017d1 Signed-off-by: Marc Paquette --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4627d9780f0b5..a20132a6a3838 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin) `Signed-off-by` lines in commits. See `git log` for our commit message style. It's basically the same as -[Go's style](https://github.com/golang/go/wiki/CommitMessage). +[Go's style](https://go.dev/wiki/CommitMessage). ## About Us From 36ea792f06f7871ebd9f0f092e9950835b280f7a Mon Sep 17 00:00:00 2001 From: Marc Paquette Date: Sat, 28 Dec 2024 01:29:34 -0500 Subject: [PATCH 066/223] Fix various linting, vet & static check issues Fixes #14492 ----- Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. Change-Id: I6dc1068d34bbfa7477e7b7a56a4325b3868c92e1 Signed-off-by: Marc Paquette --- client/tailscale/localclient.go | 2 +- version-embed.go | 1 + wgengine/filter/filter_test.go | 8 ++++---- wgengine/netstack/gro/gro.go | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/client/tailscale/localclient.go b/client/tailscale/localclient.go index 34c094a63fbf9..4e452f894d8e8 100644 --- a/client/tailscale/localclient.go +++ b/client/tailscale/localclient.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build go1.19 +//go:build go1.22 package tailscale diff --git a/version-embed.go b/version-embed.go index 2d517339d571c..17bf578dd33f1 100644 --- a/version-embed.go +++ b/version-embed.go @@ -26,6 +26,7 @@ var AlpineDockerTag string //go:embed go.toolchain.rev var GoToolchainRev string +//lint:ignore U1000 used by tests + assert_ts_toolchain_match.go w/ right build tags func tailscaleToolchainRev() (gitHash string, ok bool) { bi, ok := debug.ReadBuildInfo() if !ok { diff --git a/wgengine/filter/filter_test.go b/wgengine/filter/filter_test.go index 7ffdd5c7b328f..e7f71e6a45310 100644 --- a/wgengine/filter/filter_test.go +++ b/wgengine/filter/filter_test.go @@ -768,7 +768,7 @@ func ports(s string) PortRange { if err != nil { panic(fmt.Sprintf("invalid NetPortRange %q", s)) } - return PortRange{uint16(first), uint16(last)} + return PortRange{First: uint16(first), Last: uint16(last)} } func netports(netPorts ...string) (ret []NetPortRange) { @@ -814,11 +814,11 @@ func TestMatchesFromFilterRules(t *testing.T) { Dsts: []NetPortRange{ { Net: netip.MustParsePrefix("0.0.0.0/0"), - Ports: PortRange{22, 22}, + Ports: PortRange{First: 22, Last: 22}, }, { Net: netip.MustParsePrefix("::0/0"), - Ports: PortRange{22, 22}, + Ports: PortRange{First: 22, Last: 22}, }, }, Srcs: []netip.Prefix{ @@ -848,7 +848,7 @@ func TestMatchesFromFilterRules(t *testing.T) { Dsts: []NetPortRange{ { Net: netip.MustParsePrefix("1.2.0.0/16"), - Ports: PortRange{22, 22}, + Ports: PortRange{First: 22, Last: 22}, }, }, Srcs: []netip.Prefix{ diff --git a/wgengine/netstack/gro/gro.go b/wgengine/netstack/gro/gro.go index b268534eb46c8..654d170566f0d 100644 --- a/wgengine/netstack/gro/gro.go +++ b/wgengine/netstack/gro/gro.go @@ -6,6 +6,7 @@ package gro import ( "bytes" + "github.com/tailscale/wireguard-go/tun" "gvisor.dev/gvisor/pkg/buffer" "gvisor.dev/gvisor/pkg/tcpip" From 2fb361a3cf1146112b80b56ebcaf33be9474c21b Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 4 Jan 2025 15:33:29 -0800 Subject: [PATCH 067/223] ipn: declare NotifyWatchOpt consts without using iota Updates #cleanup Updates #1909 (noticed while working on that) Change-Id: I505001e5294287ad2a937b4db61d9e67de70fa14 Signed-off-by: Brad Fitzpatrick --- ipn/backend.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ipn/backend.go b/ipn/backend.go index ef0700a70069e..3e956f4734f5f 100644 --- a/ipn/backend.go +++ b/ipn/backend.go @@ -58,23 +58,29 @@ type EngineStatus struct { // to subscribe to. type NotifyWatchOpt uint64 +// NotifyWatchOpt values. +// +// These aren't declared using Go's iota because they're not purely internal to +// the process and iota should not be used for values that are serialized to +// disk or network. In this case, these values come over the network via the +// LocalAPI, a mostly stable API. const ( // NotifyWatchEngineUpdates, if set, causes Engine updates to be sent to the // client either regularly or when they change, without having to ask for // each one via Engine.RequestStatus. - NotifyWatchEngineUpdates NotifyWatchOpt = 1 << iota + NotifyWatchEngineUpdates NotifyWatchOpt = 1 << 0 - NotifyInitialState // if set, the first Notify message (sent immediately) will contain the current State + BrowseToURL + SessionID - NotifyInitialPrefs // if set, the first Notify message (sent immediately) will contain the current Prefs - NotifyInitialNetMap // if set, the first Notify message (sent immediately) will contain the current NetMap + NotifyInitialState NotifyWatchOpt = 1 << 1 // if set, the first Notify message (sent immediately) will contain the current State + BrowseToURL + SessionID + NotifyInitialPrefs NotifyWatchOpt = 1 << 2 // if set, the first Notify message (sent immediately) will contain the current Prefs + NotifyInitialNetMap NotifyWatchOpt = 1 << 3 // if set, the first Notify message (sent immediately) will contain the current NetMap - NotifyNoPrivateKeys // if set, private keys that would normally be sent in updates are zeroed out - NotifyInitialDriveShares // if set, the first Notify message (sent immediately) will contain the current Taildrive Shares - NotifyInitialOutgoingFiles // if set, the first Notify message (sent immediately) will contain the current Taildrop OutgoingFiles + NotifyNoPrivateKeys NotifyWatchOpt = 1 << 4 // if set, private keys that would normally be sent in updates are zeroed out + NotifyInitialDriveShares NotifyWatchOpt = 1 << 5 // if set, the first Notify message (sent immediately) will contain the current Taildrive Shares + NotifyInitialOutgoingFiles NotifyWatchOpt = 1 << 6 // if set, the first Notify message (sent immediately) will contain the current Taildrop OutgoingFiles - NotifyInitialHealthState // if set, the first Notify message (sent immediately) will contain the current health.State of the client + NotifyInitialHealthState NotifyWatchOpt = 1 << 7 // if set, the first Notify message (sent immediately) will contain the current health.State of the client - NotifyRateLimit // if set, rate limit spammy netmap updates to every few seconds + NotifyRateLimit NotifyWatchOpt = 1 << 8 // if set, rate limit spammy netmap updates to every few seconds ) // Notify is a communication from a backend (e.g. tailscaled) to a frontend From f13b2bce93b9bd5631df7a86e322c4c7ef1299de Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 6 Jan 2025 08:27:40 -0800 Subject: [PATCH 068/223] tailcfg: flesh out docs Updates #cleanup Updates #14542 Change-Id: I41f7ce69d43032e0ba3c866d9c89d2a7eccbf090 Signed-off-by: Brad Fitzpatrick --- tailcfg/tailcfg.go | 71 +++++++++++++++++++++++++++++++++------- tailcfg/tailcfg_clone.go | 4 +-- tailcfg/tailcfg_view.go | 8 ++--- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index f762d992d30c3..fb643a6df7037 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -155,35 +155,70 @@ type CapabilityVersion int // - 110: 2024-12-12: removed never-before-used Tailscale SSH public key support (#14373) const CurrentCapabilityVersion CapabilityVersion = 110 -type StableID string - +// ID is an integer ID for a user, node, or login allocated by the +// control plane. +// +// To be nice, control plane servers should not use int64s that are too large to +// fit in a JavaScript number (see JavaScript's Number.MAX_SAFE_INTEGER). +// The Tailscale-hosted control plane stopped allocating large integers in +// March 2023 but nodes prior to that may have IDs larger than +// MAX_SAFE_INTEGER (2^53 – 1). +// +// IDs must not be zero or negative. type ID int64 +// UserID is an [ID] for a [User]. type UserID ID func (u UserID) IsZero() bool { return u == 0 } +// LoginID is an [ID] for a [Login]. +// +// It is not used in the Tailscale client, but is used in the control plane. type LoginID ID func (u LoginID) IsZero() bool { return u == 0 } +// NodeID is a unique integer ID for a node. +// +// It's global within a control plane URL ("tailscale up --login-server") and is +// (as of 2025-01-06) never re-used even after a node is deleted. +// +// To be nice, control plane servers should not use int64s that are too large to +// fit in a JavaScript number (see JavaScript's Number.MAX_SAFE_INTEGER). +// The Tailscale-hosted control plane stopped allocating large integers in +// March 2023 but nodes prior to that may have node IDs larger than +// MAX_SAFE_INTEGER (2^53 – 1). +// +// NodeIDs are not stable across control plane URLs. For more stable URLs, +// see [StableNodeID]. type NodeID ID func (u NodeID) IsZero() bool { return u == 0 } -type StableNodeID StableID +// StableNodeID is a string form of [NodeID]. +// +// Different control plane servers should ideally have different StableNodeID +// suffixes for different sites or regions. +// +// Being a string, it's safer to use in JavaScript without worrying about the +// size of the integer, as documented on [NodeID]. +// +// But in general, Tailscale APIs can accept either a [NodeID] integer or a +// [StableNodeID] string when referring to a node. +type StableNodeID string func (u StableNodeID) IsZero() bool { return u == "" } -// User is an IPN user. +// User is a Tailscale user. // // A user can have multiple logins associated with it (e.g. gmail and github oauth). // (Note: none of our UIs support this yet.) @@ -196,23 +231,29 @@ func (u StableNodeID) IsZero() bool { // have a general gmail address login associated with the user. type User struct { ID UserID - LoginName string `json:"-"` // not stored, filled from Login // TODO REMOVE DisplayName string // if non-empty overrides Login field ProfilePicURL string // if non-empty overrides Login field - Logins []LoginID Created time.Time + + // Old, unused fields... + // TODO(bradfitz): remove, once verifying old clients don't need them. + + LoginName string `json:"-"` // not stored, filled from Login // TODO REMOVE + Logins []LoginID } +// Login is a user from a specific identity provider, not associated with any +// particular tailnet. type Login struct { _ structs.Incomparable - ID LoginID - Provider string - LoginName string - DisplayName string - ProfilePicURL string + ID LoginID // unused in the Tailscale client + Provider string // "google", "github", "okta_foo", etc. + LoginName string // an email address or "email-ish" string (like alice@github) + DisplayName string // from the IdP + ProfilePicURL string // from the IdP } -// A UserProfile is display-friendly data for a user. +// A UserProfile is display-friendly data for a [User]. // It includes the LoginName for display purposes but *not* the Provider. // It also includes derived data from one of the user's logins. type UserProfile struct { @@ -283,6 +324,7 @@ func MarshalCapJSON[T any](capRule T) (RawMessage, error) { return RawMessage(string(bs)), nil } +// Node is a Tailscale device in a tailnet. type Node struct { ID NodeID StableID StableNodeID @@ -563,6 +605,11 @@ func (n *Node) InitDisplayNames(networkMagicDNSSuffix string) { n.ComputedNameWithHost = nameWithHost } +// MachineStatus is the state of a [Node]'s approval into a tailnet. +// +// A "node" and a "machine" are often 1:1, but technically a Tailscale +// daemon has one machine key and can have multiple nodes (e.g. different +// users on Windows) for that one machine key. type MachineStatus int const ( diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index bf9bac2980df9..78da0aea6dad6 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -33,11 +33,11 @@ func (src *User) Clone() *User { // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _UserCloneNeedsRegeneration = User(struct { ID UserID - LoginName string DisplayName string ProfilePicURL string - Logins []LoginID Created time.Time + LoginName string + Logins []LoginID }{}) // Clone makes a deep copy of Node. diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index 6c21e5f450340..1c5fda627ce31 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -67,20 +67,20 @@ func (v *UserView) UnmarshalJSON(b []byte) error { } func (v UserView) ID() UserID { return v.Đļ.ID } -func (v UserView) LoginName() string { return v.Đļ.LoginName } func (v UserView) DisplayName() string { return v.Đļ.DisplayName } func (v UserView) ProfilePicURL() string { return v.Đļ.ProfilePicURL } -func (v UserView) Logins() views.Slice[LoginID] { return views.SliceOf(v.Đļ.Logins) } func (v UserView) Created() time.Time { return v.Đļ.Created } +func (v UserView) LoginName() string { return v.Đļ.LoginName } +func (v UserView) Logins() views.Slice[LoginID] { return views.SliceOf(v.Đļ.Logins) } // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _UserViewNeedsRegeneration = User(struct { ID UserID - LoginName string DisplayName string ProfilePicURL string - Logins []LoginID Created time.Time + LoginName string + Logins []LoginID }{}) // View returns a readonly view of Node. From 5da772c6700c81431924d52c493590aecf9c4163 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 6 Jan 2025 10:11:15 -0800 Subject: [PATCH 069/223] cmd/tailscale/cli: fix TestUpdatePrefs on macOS It was failing about an unaccepted risk ("mac-app-connector") because it was checking runtime.GOOS ("darwin") instead of the test's env.goos string value ("linux", which doesn't have the warning). Fixes #14544 Change-Id: I470d86a6ad4bb18e1dd99d334538e56556147835 Signed-off-by: Brad Fitzpatrick --- cmd/tailscale/cli/up.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index e86687527e5e0..b907257cf5a3e 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -379,7 +379,7 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus return false, nil, err } - if runtime.GOOS == "darwin" && env.upArgs.advertiseConnector { + if env.goos == "darwin" && env.upArgs.advertiseConnector { if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, env.upArgs.acceptedRisks); err != nil { return false, nil, err } From b90707665ee7626ee9834ecb0cbd4960a9a52754 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 6 Jan 2025 09:54:11 -0800 Subject: [PATCH 070/223] tailcfg: remove unused User fields Fixes #14542 Change-Id: Ifeb0f90c570c1b555af761161f79df75f18ae3f9 Signed-off-by: Brad Fitzpatrick --- tailcfg/tailcfg.go | 6 ---- tailcfg/tailcfg_clone.go | 4 --- tailcfg/tailcfg_test.go | 1 - tailcfg/tailcfg_view.go | 14 ++++------ tstest/integration/testcontrol/testcontrol.go | 28 ++++++++++--------- 5 files changed, 20 insertions(+), 33 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index fb643a6df7037..1ede0bd9b08a5 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -234,12 +234,6 @@ type User struct { DisplayName string // if non-empty overrides Login field ProfilePicURL string // if non-empty overrides Login field Created time.Time - - // Old, unused fields... - // TODO(bradfitz): remove, once verifying old clients don't need them. - - LoginName string `json:"-"` // not stored, filled from Login // TODO REMOVE - Logins []LoginID } // Login is a user from a specific identity provider, not associated with any diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index 78da0aea6dad6..d282719b7d182 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -26,7 +26,6 @@ func (src *User) Clone() *User { } dst := new(User) *dst = *src - dst.Logins = append(src.Logins[:0:0], src.Logins...) return dst } @@ -36,8 +35,6 @@ var _UserCloneNeedsRegeneration = User(struct { DisplayName string ProfilePicURL string Created time.Time - LoginName string - Logins []LoginID }{}) // Clone makes a deep copy of Node. @@ -302,7 +299,6 @@ func (src *RegisterResponse) Clone() *RegisterResponse { } dst := new(RegisterResponse) *dst = *src - dst.User = *src.User.Clone() dst.NodeKeySignature = append(src.NodeKeySignature[:0:0], src.NodeKeySignature...) return dst } diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index 9f8c418a1ccf9..b9a204eadf17a 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -666,7 +666,6 @@ func TestCloneUser(t *testing.T) { u *User }{ {"nil_logins", &User{}}, - {"zero_logins", &User{Logins: make([]LoginID, 0)}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index 1c5fda627ce31..774a18258ce6c 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -66,12 +66,10 @@ func (v *UserView) UnmarshalJSON(b []byte) error { return nil } -func (v UserView) ID() UserID { return v.Đļ.ID } -func (v UserView) DisplayName() string { return v.Đļ.DisplayName } -func (v UserView) ProfilePicURL() string { return v.Đļ.ProfilePicURL } -func (v UserView) Created() time.Time { return v.Đļ.Created } -func (v UserView) LoginName() string { return v.Đļ.LoginName } -func (v UserView) Logins() views.Slice[LoginID] { return views.SliceOf(v.Đļ.Logins) } +func (v UserView) ID() UserID { return v.Đļ.ID } +func (v UserView) DisplayName() string { return v.Đļ.DisplayName } +func (v UserView) ProfilePicURL() string { return v.Đļ.ProfilePicURL } +func (v UserView) Created() time.Time { return v.Đļ.Created } // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _UserViewNeedsRegeneration = User(struct { @@ -79,8 +77,6 @@ var _UserViewNeedsRegeneration = User(struct { DisplayName string ProfilePicURL string Created time.Time - LoginName string - Logins []LoginID }{}) // View returns a readonly view of Node. @@ -637,7 +633,7 @@ func (v *RegisterResponseView) UnmarshalJSON(b []byte) error { return nil } -func (v RegisterResponseView) User() UserView { return v.Đļ.User.View() } +func (v RegisterResponseView) User() User { return v.Đļ.User } func (v RegisterResponseView) Login() Login { return v.Đļ.Login } func (v RegisterResponseView) NodeKeyExpired() bool { return v.Đļ.NodeKeyExpired } func (v RegisterResponseView) MachineAuthorized() bool { return v.Đļ.MachineAuthorized } diff --git a/tstest/integration/testcontrol/testcontrol.go b/tstest/integration/testcontrol/testcontrol.go index a6b2e1828b8fe..92f74e24440be 100644 --- a/tstest/integration/testcontrol/testcontrol.go +++ b/tstest/integration/testcontrol/testcontrol.go @@ -6,6 +6,7 @@ package testcontrol import ( "bytes" + "cmp" "context" "encoding/binary" "encoding/json" @@ -476,13 +477,22 @@ func (s *Server) AddFakeNode() { // TODO: send updates to other (non-fake?) nodes } -func (s *Server) AllUsers() (users []*tailcfg.User) { +func (s *Server) allUserProfiles() (res []tailcfg.UserProfile) { s.mu.Lock() defer s.mu.Unlock() - for _, u := range s.users { - users = append(users, u.Clone()) + for k, u := range s.users { + up := tailcfg.UserProfile{ + ID: u.ID, + DisplayName: u.DisplayName, + } + if login, ok := s.logins[k]; ok { + up.LoginName = login.LoginName + up.ProfilePicURL = cmp.Or(up.ProfilePicURL, login.ProfilePicURL) + up.DisplayName = cmp.Or(up.DisplayName, login.DisplayName) + } + res = append(res, up) } - return users + return res } func (s *Server) AllNodes() (nodes []*tailcfg.Node) { @@ -523,9 +533,7 @@ func (s *Server) getUser(nodeKey key.NodePublic) (*tailcfg.User, *tailcfg.Login) } user := &tailcfg.User{ ID: id, - LoginName: loginName, DisplayName: displayName, - Logins: []tailcfg.LoginID{login.ID}, } s.users[nodeKey] = user s.logins[nodeKey] = login @@ -1001,13 +1009,7 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, sort.Slice(res.Peers, func(i, j int) bool { return res.Peers[i].ID < res.Peers[j].ID }) - for _, u := range s.AllUsers() { - res.UserProfiles = append(res.UserProfiles, tailcfg.UserProfile{ - ID: u.ID, - LoginName: u.LoginName, - DisplayName: u.DisplayName, - }) - } + res.UserProfiles = s.allUserProfiles() v4Prefix := netip.PrefixFrom(netaddr.IPv4(100, 64, uint8(tailcfg.NodeID(user.ID)>>8), uint8(tailcfg.NodeID(user.ID))), 32) v6Prefix := netip.PrefixFrom(tsaddr.Tailscale4To6(v4Prefix.Addr()), 128) From 07aae18bca5a26e459b3d3f906f4c099c3f75df0 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 3 Jan 2025 14:24:32 -0800 Subject: [PATCH 071/223] ipn/ipnlocal, util/goroutines: track goroutines for tests, shutdown Updates #14520 Updates #14517 (in that I pulled this out of there) Change-Id: Ibc28162816e083fcadf550586c06805c76e378fc Signed-off-by: Brad Fitzpatrick --- ipn/ipnlocal/local.go | 64 ++++++++++++++++++++++++++------- util/goroutines/goroutines.go | 2 +- util/goroutines/tracker.go | 66 +++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 14 deletions(-) create mode 100644 util/goroutines/tracker.go diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index fc7b997bc94a0..0a11263092ead 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -96,6 +96,7 @@ import ( "tailscale.com/types/views" "tailscale.com/util/deephash" "tailscale.com/util/dnsname" + "tailscale.com/util/goroutines" "tailscale.com/util/httpm" "tailscale.com/util/mak" "tailscale.com/util/multierr" @@ -178,7 +179,7 @@ type watchSession struct { // state machine generates events back out to zero or more components. type LocalBackend struct { // Elements that are thread-safe or constant after construction. - ctx context.Context // canceled by Close + ctx context.Context // canceled by [LocalBackend.Shutdown] ctxCancel context.CancelFunc // cancels ctx logf logger.Logf // general logging keyLogf logger.Logf // for printing list of peers on change @@ -231,6 +232,10 @@ type LocalBackend struct { shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool] numClientStatusCalls atomic.Uint32 + // goTracker accounts for all goroutines started by LocalBacked, primarily + // for testing and graceful shutdown purposes. + goTracker goroutines.Tracker + // The mutex protects the following elements. mu sync.Mutex conf *conffile.Config // latest parsed config, or nil if not in declarative mode @@ -866,7 +871,7 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { // TODO(raggi,tailscale/corp#22574): authReconfig should be refactored such that we can call the // necessary operations here and avoid the need for asynchronous behavior that is racy and hard // to test here, and do less extra work in these conditions. - go b.authReconfig() + b.goTracker.Go(b.authReconfig) } } @@ -879,7 +884,7 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { want := b.netMap.GetAddresses().Len() if len(b.peerAPIListeners) < want { b.logf("linkChange: peerAPIListeners too low; trying again") - go b.initPeerAPIListener() + b.goTracker.Go(b.initPeerAPIListener) } } } @@ -1004,6 +1009,33 @@ func (b *LocalBackend) Shutdown() { b.ctxCancel() b.e.Close() <-b.e.Done() + b.awaitNoGoroutinesInTest() +} + +func (b *LocalBackend) awaitNoGoroutinesInTest() { + if !testenv.InTest() { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + + ch := make(chan bool, 1) + defer b.goTracker.AddDoneCallback(func() { ch <- true })() + + for { + n := b.goTracker.RunningGoroutines() + if n == 0 { + return + } + select { + case <-ctx.Done(): + // TODO(bradfitz): pass down some TB-like failer interface from + // tests, without depending on testing from here? + // But this is fine in tests too: + panic(fmt.Sprintf("timeout waiting for %d goroutines to stop", n)) + case <-ch: + } + } } func stripKeysFromPrefs(p ipn.PrefsView) ipn.PrefsView { @@ -2152,7 +2184,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error { if b.portpoll != nil { b.portpollOnce.Do(func() { - go b.readPoller() + b.goTracker.Go(b.readPoller) }) } @@ -2366,7 +2398,7 @@ func (b *LocalBackend) updateFilterLocked(netMap *netmap.NetworkMap, prefs ipn.P b.e.SetJailedFilter(filter.NewShieldsUpFilter(localNets, logNets, oldJailedFilter, b.logf)) if b.sshServer != nil { - go b.sshServer.OnPolicyChange() + b.goTracker.Go(b.sshServer.OnPolicyChange) } } @@ -2843,7 +2875,7 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A // request every 2 seconds. // TODO(bradfitz): plumb this further and only send a Notify on change. if mask&ipn.NotifyWatchEngineUpdates != 0 { - go b.pollRequestEngineStatus(ctx) + b.goTracker.Go(func() { b.pollRequestEngineStatus(ctx) }) } // TODO(marwan-at-work): streaming background logs? @@ -3850,7 +3882,7 @@ func (b *LocalBackend) editPrefsLockedOnEntry(mp *ipn.MaskedPrefs, unlock unlock if mp.EggSet { mp.EggSet = false b.egg = true - go b.doSetHostinfoFilterServices() + b.goTracker.Go(b.doSetHostinfoFilterServices) } p0 := b.pm.CurrentPrefs() p1 := b.pm.CurrentPrefs().AsStruct() @@ -3943,7 +3975,7 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce) if oldp.ShouldSSHBeRunning() && !newp.ShouldSSHBeRunning() { if b.sshServer != nil { - go b.sshServer.Shutdown() + b.goTracker.Go(b.sshServer.Shutdown) b.sshServer = nil } } @@ -4285,8 +4317,14 @@ func (b *LocalBackend) authReconfig() { dcfg := dnsConfigForNetmap(nm, b.peers, prefs, b.keyExpired, b.logf, version.OS()) // If the current node is an app connector, ensure the app connector machine is started b.reconfigAppConnectorLocked(nm, prefs) + closing := b.shutdownCalled b.mu.Unlock() + if closing { + b.logf("[v1] authReconfig: skipping because in shutdown") + return + } + if blocked { b.logf("[v1] authReconfig: blocked, skipping.") return @@ -4751,7 +4789,7 @@ func (b *LocalBackend) initPeerAPIListener() { b.peerAPIListeners = append(b.peerAPIListeners, pln) } - go b.doSetHostinfoFilterServices() + b.goTracker.Go(b.doSetHostinfoFilterServices) } // magicDNSRootDomains returns the subset of nm.DNS.Domains that are the search domains for MagicDNS. @@ -5020,7 +5058,7 @@ func (b *LocalBackend) enterStateLockedOnEntry(newState ipn.State, unlock unlock // can be shut down if we transition away from Running. if b.captiveCancel == nil { b.captiveCtx, b.captiveCancel = context.WithCancel(b.ctx) - go b.checkCaptivePortalLoop(b.captiveCtx) + b.goTracker.Go(func() { b.checkCaptivePortalLoop(b.captiveCtx) }) } } else if oldState == ipn.Running { // Transitioning away from running. @@ -5272,7 +5310,7 @@ func (b *LocalBackend) requestEngineStatusAndWait() { b.statusLock.Lock() defer b.statusLock.Unlock() - go b.e.RequestStatus() + b.goTracker.Go(b.e.RequestStatus) b.logf("requestEngineStatusAndWait: waiting...") b.statusChanged.Wait() // temporarily releases lock while waiting b.logf("requestEngineStatusAndWait: got status update.") @@ -5383,7 +5421,7 @@ func (b *LocalBackend) setWebClientAtomicBoolLocked(nm *netmap.NetworkMap) { shouldRun := !nm.HasCap(tailcfg.NodeAttrDisableWebClient) wasRunning := b.webClientAtomicBool.Swap(shouldRun) if wasRunning && !shouldRun { - go b.webClientShutdown() // stop web client + b.goTracker.Go(b.webClientShutdown) // stop web client } } @@ -5900,7 +5938,7 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn. if wire := b.wantIngressLocked(); b.hostinfo != nil && b.hostinfo.WireIngress != wire { b.logf("Hostinfo.WireIngress changed to %v", wire) b.hostinfo.WireIngress = wire - go b.doSetHostinfoFilterServices() + b.goTracker.Go(b.doSetHostinfoFilterServices) } b.setTCPPortsIntercepted(handlePorts) diff --git a/util/goroutines/goroutines.go b/util/goroutines/goroutines.go index 9758b07586613..d40cbecb10876 100644 --- a/util/goroutines/goroutines.go +++ b/util/goroutines/goroutines.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -// The goroutines package contains utilities for getting active goroutines. +// The goroutines package contains utilities for tracking and getting active goroutines. package goroutines import ( diff --git a/util/goroutines/tracker.go b/util/goroutines/tracker.go new file mode 100644 index 0000000000000..044843d33d155 --- /dev/null +++ b/util/goroutines/tracker.go @@ -0,0 +1,66 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package goroutines + +import ( + "sync" + "sync/atomic" + + "tailscale.com/util/set" +) + +// Tracker tracks a set of goroutines. +type Tracker struct { + started atomic.Int64 // counter + running atomic.Int64 // gauge + + mu sync.Mutex + onDone set.HandleSet[func()] +} + +func (t *Tracker) Go(f func()) { + t.started.Add(1) + t.running.Add(1) + go t.goAndDecr(f) +} + +func (t *Tracker) goAndDecr(f func()) { + defer t.decr() + f() +} + +func (t *Tracker) decr() { + t.running.Add(-1) + + t.mu.Lock() + defer t.mu.Unlock() + for _, f := range t.onDone { + go f() + } +} + +// AddDoneCallback adds a callback to be called in a new goroutine +// whenever a goroutine managed by t (excluding ones from this method) +// finishes. It returns a function to remove the callback. +func (t *Tracker) AddDoneCallback(f func()) (remove func()) { + t.mu.Lock() + defer t.mu.Unlock() + if t.onDone == nil { + t.onDone = set.HandleSet[func()]{} + } + h := t.onDone.Add(f) + return func() { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.onDone, h) + } +} + +func (t *Tracker) RunningGoroutines() int64 { + return t.running.Load() +} + +func (t *Tracker) StartedGoroutines() int64 { + return t.started.Load() +} From 041622c92f124491c7d9ece71efa310adb0f238c Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 3 Jan 2025 14:30:02 -0800 Subject: [PATCH 072/223] ipn/ipnlocal: move where auto exit node selection happens In the process, because I needed it for testing, make all LocalBackend-managed goroutines be accounted for. And then in tests, verify they're no longer running during LocalBackend.Shutdown. Updates tailscale/corp#19681 Change-Id: Iad873d4df7d30103a4a7863dfacf9e078c77e6a3 Signed-off-by: Brad Fitzpatrick --- ipn/ipnlocal/local.go | 48 ++++++++++++++++++-------- ipn/ipnlocal/local_test.go | 69 +++++++++++++++++++++++++++----------- 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 0a11263092ead..4c58ae8ecfb19 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -367,7 +367,7 @@ type LocalBackend struct { allowedSuggestedExitNodes set.Set[tailcfg.StableNodeID] // refreshAutoExitNode indicates if the exit node should be recomputed when the next netcheck report is available. - refreshAutoExitNode bool + refreshAutoExitNode bool // guarded by mu // captiveCtx and captiveCancel are used to control captive portal // detection. They are protected by 'mu' and can be changed during the @@ -1812,8 +1812,9 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo b.send(*notify) } }() - unlock := b.lockAndGetUnlock() - defer unlock() + b.mu.Lock() + defer b.mu.Unlock() + if !b.updateNetmapDeltaLocked(muts) { return false } @@ -1821,14 +1822,8 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo if b.netMap != nil && mutationsAreWorthyOfTellingIPNBus(muts) { nm := ptr.To(*b.netMap) // shallow clone nm.Peers = make([]tailcfg.NodeView, 0, len(b.peers)) - shouldAutoExitNode := shouldAutoExitNode() for _, p := range b.peers { nm.Peers = append(nm.Peers, p) - // If the auto exit node currently set goes offline, find another auto exit node. - if shouldAutoExitNode && b.pm.prefs.ExitNodeID() == p.StableID() && p.Online() != nil && !*p.Online() { - b.setAutoExitNodeIDLockedOnEntry(unlock) - return false - } } slices.SortFunc(nm.Peers, func(a, b tailcfg.NodeView) int { return cmp.Compare(a.ID(), b.ID()) @@ -1859,6 +1854,20 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool { return false } +// pickNewAutoExitNode picks a new automatic exit node if needed. +func (b *LocalBackend) pickNewAutoExitNode() { + unlock := b.lockAndGetUnlock() + defer unlock() + + newPrefs := b.setAutoExitNodeIDLockedOnEntry(unlock) + if !newPrefs.Valid() { + // Unchanged. + return + } + + b.send(ipn.Notify{Prefs: &newPrefs}) +} + func (b *LocalBackend) updateNetmapDeltaLocked(muts []netmap.NodeMutation) (handled bool) { if b.netMap == nil || len(b.peers) == 0 { return false @@ -1881,6 +1890,12 @@ func (b *LocalBackend) updateNetmapDeltaLocked(muts []netmap.NodeMutation) (hand mak.Set(&mutableNodes, nv.ID(), n) } m.Apply(n) + + // If our exit node went offline, we need to schedule picking + // a new one. + if mo, ok := m.(netmap.NodeMutationOnline); ok && !mo.Online && n.StableID == b.pm.prefs.ExitNodeID() && shouldAutoExitNode() { + b.goTracker.Go(b.pickNewAutoExitNode) + } } for nid, n := range mutableNodes { b.peers[nid] = n.View() @@ -5542,29 +5557,34 @@ func (b *LocalBackend) setNetInfo(ni *tailcfg.NetInfo) { } } -func (b *LocalBackend) setAutoExitNodeIDLockedOnEntry(unlock unlockOnce) { +func (b *LocalBackend) setAutoExitNodeIDLockedOnEntry(unlock unlockOnce) (newPrefs ipn.PrefsView) { + var zero ipn.PrefsView defer unlock() prefs := b.pm.CurrentPrefs() if !prefs.Valid() { b.logf("[unexpected]: received tailnet exit node ID pref change callback but current prefs are nil") - return + return zero } prefsClone := prefs.AsStruct() newSuggestion, err := b.suggestExitNodeLocked(nil) if err != nil { b.logf("setAutoExitNodeID: %v", err) - return + return zero + } + if prefsClone.ExitNodeID == newSuggestion.ID { + return zero } prefsClone.ExitNodeID = newSuggestion.ID - _, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ + newPrefs, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ Prefs: *prefsClone, ExitNodeIDSet: true, }, unlock) if err != nil { b.logf("setAutoExitNodeID: failed to apply exit node ID preference: %v", err) - return + return zero } + return newPrefs } // setNetMapLocked updates the LocalBackend state to reflect the newly diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index b1be86392185d..15766741b754e 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1867,16 +1867,16 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { PreferredDERP: 2, } tests := []struct { - name string - lastSuggestedExitNode tailcfg.StableNodeID - netmap *netmap.NetworkMap - muts []*tailcfg.PeerChange - exitNodeIDWant tailcfg.StableNodeID - updateNetmapDeltaResponse bool - report *netcheck.Report + name string + lastSuggestedExitNode tailcfg.StableNodeID + netmap *netmap.NetworkMap + muts []*tailcfg.PeerChange + exitNodeIDWant tailcfg.StableNodeID + report *netcheck.Report }{ { - name: "selected auto exit node goes offline", + // selected auto exit node goes offline + name: "exit-node-goes-offline", lastSuggestedExitNode: peer1.StableID(), netmap: &netmap.NetworkMap{ Peers: []tailcfg.NodeView{ @@ -1895,12 +1895,12 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { Online: ptr.To(true), }, }, - exitNodeIDWant: peer2.StableID(), - updateNetmapDeltaResponse: false, - report: report, + exitNodeIDWant: peer2.StableID(), + report: report, }, { - name: "other exit node goes offline doesn't change selected auto exit node that's still online", + // other exit node goes offline doesn't change selected auto exit node that's still online + name: "other-node-goes-offline", lastSuggestedExitNode: peer2.StableID(), netmap: &netmap.NetworkMap{ Peers: []tailcfg.NodeView{ @@ -1919,9 +1919,8 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { Online: ptr.To(true), }, }, - exitNodeIDWant: peer2.StableID(), - updateNetmapDeltaResponse: true, - report: report, + exitNodeIDWant: peer2.StableID(), + report: report, }, } @@ -1939,6 +1938,20 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { b.lastSuggestedExitNode = tt.lastSuggestedExitNode b.sys.MagicSock.Get().SetLastNetcheckReportForTest(b.ctx, tt.report) b.SetPrefsForTest(b.pm.CurrentPrefs().AsStruct()) + + allDone := make(chan bool, 1) + defer b.goTracker.AddDoneCallback(func() { + b.mu.Lock() + defer b.mu.Unlock() + if b.goTracker.RunningGoroutines() > 0 { + return + } + select { + case allDone <- true: + default: + } + })() + someTime := time.Unix(123, 0) muts, ok := netmap.MutationsFromMapResponse(&tailcfg.MapResponse{ PeersChangedPatch: tt.muts, @@ -1946,16 +1959,34 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { if !ok { t.Fatal("netmap.MutationsFromMapResponse failed") } + if b.pm.prefs.ExitNodeID() != tt.lastSuggestedExitNode { t.Fatalf("did not set exit node ID to last suggested exit node despite auto policy") } + was := b.goTracker.StartedGoroutines() got := b.UpdateNetmapDelta(muts) - if got != tt.updateNetmapDeltaResponse { - t.Fatalf("got %v expected %v from UpdateNetmapDelta", got, tt.updateNetmapDeltaResponse) + if !got { + t.Error("got false from UpdateNetmapDelta") + } + startedGoroutine := b.goTracker.StartedGoroutines() != was + + wantChange := tt.exitNodeIDWant != tt.lastSuggestedExitNode + if startedGoroutine != wantChange { + t.Errorf("got startedGoroutine %v, want %v", startedGoroutine, wantChange) } - if b.pm.prefs.ExitNodeID() != tt.exitNodeIDWant { - t.Fatalf("did not get expected exit node id after UpdateNetmapDelta") + if startedGoroutine { + select { + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for goroutine to finish") + case <-allDone: + } + } + b.mu.Lock() + gotExitNode := b.pm.prefs.ExitNodeID() + b.mu.Unlock() + if gotExitNode != tt.exitNodeIDWant { + t.Fatalf("exit node ID after UpdateNetmapDelta = %v; want %v", gotExitNode, tt.exitNodeIDWant) } }) } From 82e99fcf84e72cd49cea72adb5b20d7888bd6f6c Mon Sep 17 00:00:00 2001 From: Will Norris Date: Mon, 6 Jan 2025 16:02:53 -0800 Subject: [PATCH 073/223] client/systray: move cmd/systray to client/systray Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- {cmd => client}/systray/logo.go | 2 +- {cmd => client}/systray/systray.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename {cmd => client}/systray/logo.go (99%) rename {cmd => client}/systray/systray.go (99%) diff --git a/cmd/systray/logo.go b/client/systray/logo.go similarity index 99% rename from cmd/systray/logo.go rename to client/systray/logo.go index de60bcdbd2d58..857a8a9372158 100644 --- a/cmd/systray/logo.go +++ b/client/systray/logo.go @@ -3,7 +3,7 @@ //go:build cgo || !darwin -package main +package systray import ( "bytes" diff --git a/cmd/systray/systray.go b/client/systray/systray.go similarity index 99% rename from cmd/systray/systray.go rename to client/systray/systray.go index 7da83a7ea0f07..782fc54202408 100644 --- a/cmd/systray/systray.go +++ b/client/systray/systray.go @@ -3,8 +3,8 @@ //go:build cgo || !darwin -// The systray command is a minimal Tailscale systray application for Linux. -package main +// Package systray provides a minimal Tailscale systray application. +package systray import ( "context" @@ -44,8 +44,8 @@ var ( hideMullvadCities bool ) -func main() { - menu := new(Menu) +// Run starts the systray menu and blocks until the menu exits. +func (menu *Menu) Run() { menu.updateState() // exit cleanly on SIGINT and SIGTERM From b36984cb16ba297293a1b542e52f9bd23cb31042 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Mon, 6 Jan 2025 16:05:14 -0800 Subject: [PATCH 074/223] cmd/systray: add cmd/systray back as a small client/systray wrapper Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- cmd/systray/systray.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 cmd/systray/systray.go diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go new file mode 100644 index 0000000000000..0185a1bc2dc5e --- /dev/null +++ b/cmd/systray/systray.go @@ -0,0 +1,15 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build cgo || !darwin + +// systray is a minimal Tailscale systray application. +package main + +import ( + "tailscale.com/client/systray" +) + +func main() { + new(systray.Menu).Run() +} From cc4aa435eff873510f7e86e55ead12d7bd6d2346 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 7 Jan 2025 06:43:50 -0800 Subject: [PATCH 075/223] go.mod: bump github.com/tailscale/peercred for Solaris This pulls in Solaris/Illumos-specific: https://github.com/tailscale/peercred/pull/10 https://go-review.googlesource.com/c/sys/+/639755 Updates tailscale/peercred#10 (from @nshalman) Change-Id: I8211035fdcf84417009da352927149d68905c0f1 Signed-off-by: Brad Fitzpatrick --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 3c389b4de66a0..e2164768411fb 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 - github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4 + github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3 @@ -101,7 +101,7 @@ require ( golang.org/x/net v0.32.0 golang.org/x/oauth2 v0.16.0 golang.org/x/sync v0.10.0 - golang.org/x/sys v0.28.0 + golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab golang.org/x/term v0.27.0 golang.org/x/time v0.5.0 golang.org/x/tools v0.23.0 diff --git a/go.sum b/go.sum index 2ae4ce09df117..353d8d5b8de94 100644 --- a/go.sum +++ b/go.sum @@ -935,8 +935,8 @@ github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10 h1:ZB47BgnHcEHQJOD github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10/go.mod h1:iDx/0Rr9VV/KanSUDpJ6I/ROf0sQ7OqljXc/esl0UIA= github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 h1:uFsXVBE9Qr4ZoF094vE6iYTLDl0qCiKzYXlL6UeWObU= github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0= -github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4 h1:Gz0rz40FvFVLTBk/K8UNAenb36EbDSnh+q7Z9ldcC8w= -github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4/go.mod h1:phI29ccmHQBc+wvroosENp1IF9195449VDnFDhJ4rJU= +github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc h1:24heQPtnFR+yfntqhI3oAu9i27nEojcQ4NuBQOo5ZFA= +github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc/go.mod h1:f93CXfllFsO9ZQVq+Zocb1Gp4G5Fz0b0rXHLOzt/Djc= github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:tdUdyPqJ0C97SJfjB9tW6EylTtreyee9C44de+UBG0g= github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= @@ -1239,8 +1239,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab h1:BMkEEWYOjkvOX7+YKOGbp6jCyQ5pR2j0Ah47p1Vdsx4= +golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 6e45a8304eadf1b6aef9898a7484c846ff65e01d Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 6 Jan 2025 15:39:41 -0800 Subject: [PATCH 076/223] cmd/derper: improve logging on derp mesh connect Include the mesh log prefix in all mesh connection setup. Updates tailscale/corp#25653 Signed-off-by: James Tucker --- cmd/derper/mesh.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/derper/mesh.go b/cmd/derper/mesh.go index ee1807f001202..c4218dd9401f5 100644 --- a/cmd/derper/mesh.go +++ b/cmd/derper/mesh.go @@ -47,6 +47,7 @@ func startMeshWithHost(s *derp.Server, host string) error { c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) { host, port, err := net.SplitHostPort(addr) if err != nil { + logf("failed to split %q: %v", addr, err) return nil, err } var d net.Dialer @@ -55,15 +56,18 @@ func startMeshWithHost(s *derp.Server, host string) error { subCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() vpcHost := base + "-vpc.tailscale.com" - ips, _ := r.LookupIP(subCtx, "ip", vpcHost) + ips, err := r.LookupIP(subCtx, "ip", vpcHost) + if err != nil { + logf("failed to resolve %v: %v", vpcHost, err) + } if len(ips) > 0 { vpcAddr := net.JoinHostPort(ips[0].String(), port) c, err := d.DialContext(subCtx, network, vpcAddr) if err == nil { - log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base) + logf("connected to %v (%v) instead of %v", vpcHost, ips[0], base) return c, nil } - log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err) + logf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err) } } return d.DialContext(ctx, network, addr) From f4f57b815bf9804badf449e91a42ff80a08ea59d Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 6 Jan 2025 12:32:13 -0800 Subject: [PATCH 077/223] wgengine/magicsock: rebind on EPIPE/ECONNRESET Observed in the wild some macOS machines gain broken sockets coming out of sleep (we observe "time jumped", followed by EPIPE on sendto). The cause of this in the platform is unclear, but the fix is clear: always rebind if the socket is broken. This can also be created artificially on Linux via `ss -K`, and other conditions or software on a system could also lead to the same outcomes. Updates tailscale/corp#25648 Signed-off-by: James Tucker --- wgengine/magicsock/magicsock.go | 29 -------------- wgengine/magicsock/magicsock_notplan9.go | 49 ++++++++++++++++++++++++ wgengine/magicsock/magicsock_plan9.go | 12 ++++++ 3 files changed, 61 insertions(+), 29 deletions(-) create mode 100644 wgengine/magicsock/magicsock_notplan9.go create mode 100644 wgengine/magicsock/magicsock_plan9.go diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index bff905caa5ae4..188933c0e3e0c 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -21,7 +21,6 @@ import ( "strings" "sync" "sync/atomic" - "syscall" "time" "github.com/tailscale/wireguard-go/conn" @@ -1290,34 +1289,6 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool) (sent bool, e return } -// maybeRebindOnError performs a rebind and restun if the error is defined and -// any conditionals are met. -func (c *Conn) maybeRebindOnError(os string, err error) bool { - switch { - case errors.Is(err, syscall.EPERM): - why := "operation-not-permitted-rebind" - switch os { - // We currently will only rebind and restun on a syscall.EPERM if it is experienced - // on a client running darwin. - // TODO(charlotte, raggi): expand os options if required. - case "darwin": - // TODO(charlotte): implement a backoff, so we don't end up in a rebind loop for persistent - // EPERMs. - if c.lastEPERMRebind.Load().Before(time.Now().Add(-5 * time.Second)) { - c.logf("magicsock: performing %q", why) - c.lastEPERMRebind.Store(time.Now()) - c.Rebind() - go c.ReSTUN(why) - return true - } - default: - c.logf("magicsock: not performing %q", why) - return false - } - } - return false -} - // sendUDPNetcheck sends b via UDP to addr. It is used exclusively by netcheck. // It returns the number of bytes sent along with any error encountered. It // returns errors.ErrUnsupported if the client is explicitly configured to only diff --git a/wgengine/magicsock/magicsock_notplan9.go b/wgengine/magicsock/magicsock_notplan9.go new file mode 100644 index 0000000000000..44f08cb1ccccb --- /dev/null +++ b/wgengine/magicsock/magicsock_notplan9.go @@ -0,0 +1,49 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package magicsock + +import ( + "errors" + "syscall" + "time" +) + +// maybeRebindOnError performs a rebind and restun if the error is defined and +// any conditionals are met. +func (c *Conn) maybeRebindOnError(os string, err error) bool { + switch { + case errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ENOTCONN): + // EPIPE/ENOTCONN are common errors when a send fails due to a closed + // socket. There is some platform and version inconsistency in which + // error is returned, but the meaning is the same. + why := "broken-pipe-rebind" + c.logf("magicsock: performing %q", why) + c.Rebind() + go c.ReSTUN(why) + return true + case errors.Is(err, syscall.EPERM): + why := "operation-not-permitted-rebind" + switch os { + // We currently will only rebind and restun on a syscall.EPERM if it is experienced + // on a client running darwin. + // TODO(charlotte, raggi): expand os options if required. + case "darwin": + // TODO(charlotte): implement a backoff, so we don't end up in a rebind loop for persistent + // EPERMs. + if c.lastEPERMRebind.Load().Before(time.Now().Add(-5 * time.Second)) { + c.logf("magicsock: performing %q", why) + c.lastEPERMRebind.Store(time.Now()) + c.Rebind() + go c.ReSTUN(why) + return true + } + default: + c.logf("magicsock: not performing %q", why) + return false + } + } + return false +} diff --git a/wgengine/magicsock/magicsock_plan9.go b/wgengine/magicsock/magicsock_plan9.go new file mode 100644 index 0000000000000..23f710430cb58 --- /dev/null +++ b/wgengine/magicsock/magicsock_plan9.go @@ -0,0 +1,12 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build plan9 + +package magicsock + +// maybeRebindOnError performs a rebind and restun if the error is defined and +// any conditionals are met. +func (c *Conn) maybeRebindOnError(os string, err error) bool { + return false +} From 6db220b47834d8c45fd2b9ab34eff860aa5a6d72 Mon Sep 17 00:00:00 2001 From: Andrea Gottardo Date: Tue, 7 Jan 2025 10:24:32 -0800 Subject: [PATCH 078/223] controlclient: do not set HTTPS port for any private coordination server IP (#14564) Fixes tailscale/tailscale#14563 When creating a NoiseClient, ensure that if any private IP address is provided, with both an `http` scheme and an explicit port number, we do not ever attempt to use HTTPS. We were only handling the case of `127.0.0.1` and `localhost`, but `192.168.x.y` is a private IP as well. This uses the `netip` package to check and adds some logging in case we ever need to troubleshoot this. Signed-off-by: Andrea Gottardo --- control/controlclient/noise.go | 28 +++++-- control/controlclient/noise_test.go | 118 ++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 6 deletions(-) diff --git a/control/controlclient/noise.go b/control/controlclient/noise.go index db77014a6c138..4bd8cfc25ee96 100644 --- a/control/controlclient/noise.go +++ b/control/controlclient/noise.go @@ -11,6 +11,7 @@ import ( "errors" "math" "net/http" + "net/netip" "net/url" "sync" "time" @@ -111,24 +112,39 @@ type NoiseOpts struct { // netMon may be nil, if non-nil it's used to do faster interface lookups. // dialPlan may be nil func NewNoiseClient(opts NoiseOpts) (*NoiseClient, error) { + logf := opts.Logf u, err := url.Parse(opts.ServerURL) if err != nil { return nil, err } + + if u.Scheme != "http" && u.Scheme != "https" { + return nil, errors.New("invalid ServerURL scheme, must be http or https") + } + var httpPort string var httpsPort string + addr, _ := netip.ParseAddr(u.Hostname()) + isPrivateHost := addr.IsPrivate() || addr.IsLoopback() || u.Hostname() == "localhost" if port := u.Port(); port != "" { - // If there is an explicit port specified, trust the scheme and hope for the best - if u.Scheme == "http" { + // If there is an explicit port specified, entirely rely on the scheme, + // unless it's http with a private host in which case we never try using HTTPS. + if u.Scheme == "https" { + httpPort = "" + httpsPort = port + } else if u.Scheme == "http" { httpPort = port httpsPort = "443" - if u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost" { + if isPrivateHost { + logf("setting empty HTTPS port with http scheme and private host %s", u.Hostname()) httpsPort = "" } - } else { - httpPort = "80" - httpsPort = port } + } else if u.Scheme == "http" && isPrivateHost { + // Whenever the scheme is http and the hostname is an IP address, do not set the HTTPS port, + // as there cannot be a TLS certificate issued for an IP, unless it's a public IP. + httpPort = "80" + httpsPort = "" } else { // Otherwise, use the standard ports httpPort = "80" diff --git a/control/controlclient/noise_test.go b/control/controlclient/noise_test.go index 69a3a6a36551d..dadf237df94ad 100644 --- a/control/controlclient/noise_test.go +++ b/control/controlclient/noise_test.go @@ -54,6 +54,123 @@ func TestNoiseClientHTTP2Upgrade_earlyPayload(t *testing.T) { }.run(t) } +func makeClientWithURL(t *testing.T, url string) *NoiseClient { + nc, err := NewNoiseClient(NoiseOpts{ + Logf: t.Logf, + ServerURL: url, + }) + if err != nil { + t.Fatal(err) + } + return nc +} + +func TestNoiseClientPortsAreSet(t *testing.T) { + tests := []struct { + name string + url string + wantHTTPS string + wantHTTP string + }{ + { + name: "https-url", + url: "https://example.com", + wantHTTPS: "443", + wantHTTP: "80", + }, + { + name: "http-url", + url: "http://example.com", + wantHTTPS: "443", // TODO(bradfitz): questionable; change? + wantHTTP: "80", + }, + { + name: "https-url-custom-port", + url: "https://example.com:123", + wantHTTPS: "123", + wantHTTP: "", + }, + { + name: "http-url-custom-port", + url: "http://example.com:123", + wantHTTPS: "443", // TODO(bradfitz): questionable; change? + wantHTTP: "123", + }, + { + name: "http-loopback-no-port", + url: "http://127.0.0.1", + wantHTTPS: "", + wantHTTP: "80", + }, + { + name: "http-loopback-custom-port", + url: "http://127.0.0.1:8080", + wantHTTPS: "", + wantHTTP: "8080", + }, + { + name: "http-localhost-no-port", + url: "http://localhost", + wantHTTPS: "", + wantHTTP: "80", + }, + { + name: "http-localhost-custom-port", + url: "http://localhost:8080", + wantHTTPS: "", + wantHTTP: "8080", + }, + { + name: "http-private-ip-no-port", + url: "http://192.168.2.3", + wantHTTPS: "", + wantHTTP: "80", + }, + { + name: "http-private-ip-custom-port", + url: "http://192.168.2.3:8080", + wantHTTPS: "", + wantHTTP: "8080", + }, + { + name: "http-public-ip", + url: "http://1.2.3.4", + wantHTTPS: "443", // TODO(bradfitz): questionable; change? + wantHTTP: "80", + }, + { + name: "http-public-ip-custom-port", + url: "http://1.2.3.4:8080", + wantHTTPS: "443", // TODO(bradfitz): questionable; change? + wantHTTP: "8080", + }, + { + name: "https-public-ip", + url: "https://1.2.3.4", + wantHTTPS: "443", + wantHTTP: "80", + }, + { + name: "https-public-ip-custom-port", + url: "https://1.2.3.4:8080", + wantHTTPS: "8080", + wantHTTP: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nc := makeClientWithURL(t, tt.url) + if nc.httpsPort != tt.wantHTTPS { + t.Errorf("nc.httpsPort = %q; want %q", nc.httpsPort, tt.wantHTTPS) + } + if nc.httpPort != tt.wantHTTP { + t.Errorf("nc.httpPort = %q; want %q", nc.httpPort, tt.wantHTTP) + } + }) + } +} + func (tt noiseClientTest) run(t *testing.T) { serverPrivate := key.NewMachine() clientPrivate := key.NewMachine() @@ -81,6 +198,7 @@ func (tt noiseClientTest) run(t *testing.T) { ServerPubKey: serverPrivate.Public(), ServerURL: hs.URL, Dialer: dialer, + Logf: t.Logf, }) if err != nil { t.Fatal(err) From 2c07f5dfcd3bffd32aa70a08a9d85a90add474f0 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 6 Jan 2025 13:10:56 -0800 Subject: [PATCH 079/223] wgengine/magicsock: refactor maybeRebindOnError Remove the platform specificity, it is unnecessary complexity. Deduplicate repeated code as a result of reduced complexity. Split out error identification code. Update call-sites and tests. Updates #14551 Updates tailscale/corp#25648 Signed-off-by: James Tucker --- wgengine/magicsock/magicsock.go | 27 ++++++-- wgengine/magicsock/magicsock_notplan9.go | 48 +++++--------- wgengine/magicsock/magicsock_plan9.go | 8 +-- wgengine/magicsock/magicsock_test.go | 81 ++++++++++++++++-------- 4 files changed, 97 insertions(+), 67 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 188933c0e3e0c..d3075f55d63e3 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -364,9 +364,9 @@ type Conn struct { // wireguard state by its public key. If nil, it's not used. getPeerByKey func(key.NodePublic) (_ wgint.Peer, ok bool) - // lastEPERMRebind tracks the last time a rebind was performed - // after experiencing a syscall.EPERM. - lastEPERMRebind syncs.AtomicValue[time.Time] + // lastErrRebind tracks the last time a rebind was performed after + // experiencing a write error, and is used to throttle the rate of rebinds. + lastErrRebind syncs.AtomicValue[time.Time] // staticEndpoints are user set endpoints that this node should // advertise amongst its wireguard endpoints. It is user's @@ -1258,7 +1258,7 @@ func (c *Conn) sendUDPBatch(addr netip.AddrPort, buffs [][]byte) (sent bool, err c.logf("magicsock: %s", errGSO.Error()) err = errGSO.RetryErr } else { - _ = c.maybeRebindOnError(runtime.GOOS, err) + c.maybeRebindOnError(err) } } return err == nil, err @@ -1273,7 +1273,7 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool) (sent bool, e sent, err = c.sendUDPStd(ipp, b) if err != nil { metricSendUDPError.Add(1) - _ = c.maybeRebindOnError(runtime.GOOS, err) + c.maybeRebindOnError(err) } else { if sent && !isDisco { switch { @@ -1289,6 +1289,23 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool) (sent bool, e return } +// maybeRebindOnError performs a rebind and restun if the error is one that is +// known to be healed by a rebind, and the rebind is not throttled. +func (c *Conn) maybeRebindOnError(err error) { + ok, reason := shouldRebind(err) + if !ok { + return + } + + if c.lastErrRebind.Load().Before(time.Now().Add(-5 * time.Second)) { + c.logf("magicsock: performing rebind due to %q", reason) + c.Rebind() + go c.ReSTUN(reason) + } else { + c.logf("magicsock: not performing %q rebind due to throttle", reason) + } +} + // sendUDPNetcheck sends b via UDP to addr. It is used exclusively by netcheck. // It returns the number of bytes sent along with any error encountered. It // returns errors.ErrUnsupported if the client is explicitly configured to only diff --git a/wgengine/magicsock/magicsock_notplan9.go b/wgengine/magicsock/magicsock_notplan9.go index 44f08cb1ccccb..86d099ee7f48c 100644 --- a/wgengine/magicsock/magicsock_notplan9.go +++ b/wgengine/magicsock/magicsock_notplan9.go @@ -8,42 +8,24 @@ package magicsock import ( "errors" "syscall" - "time" ) -// maybeRebindOnError performs a rebind and restun if the error is defined and -// any conditionals are met. -func (c *Conn) maybeRebindOnError(os string, err error) bool { +// shouldRebind returns if the error is one that is known to be healed by a +// rebind, and if so also returns a resason string for the rebind. +func shouldRebind(err error) (ok bool, reason string) { switch { - case errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ENOTCONN): - // EPIPE/ENOTCONN are common errors when a send fails due to a closed - // socket. There is some platform and version inconsistency in which - // error is returned, but the meaning is the same. - why := "broken-pipe-rebind" - c.logf("magicsock: performing %q", why) - c.Rebind() - go c.ReSTUN(why) - return true + // EPIPE/ENOTCONN are common errors when a send fails due to a closed + // socket. There is some platform and version inconsistency in which + // error is returned, but the meaning is the same. + case errors.Is(err, syscall.EPIPE), errors.Is(err, syscall.ENOTCONN): + return true, "broken-pipe" + + // EPERM is typically caused by EDR software, and has been observed to be + // transient, it seems that some versions of some EDR lose track of sockets + // at times, and return EPERM, but reconnects will establish appropriate + // rights associated with a new socket. case errors.Is(err, syscall.EPERM): - why := "operation-not-permitted-rebind" - switch os { - // We currently will only rebind and restun on a syscall.EPERM if it is experienced - // on a client running darwin. - // TODO(charlotte, raggi): expand os options if required. - case "darwin": - // TODO(charlotte): implement a backoff, so we don't end up in a rebind loop for persistent - // EPERMs. - if c.lastEPERMRebind.Load().Before(time.Now().Add(-5 * time.Second)) { - c.logf("magicsock: performing %q", why) - c.lastEPERMRebind.Store(time.Now()) - c.Rebind() - go c.ReSTUN(why) - return true - } - default: - c.logf("magicsock: not performing %q", why) - return false - } + return true, "operation-not-permitted" } - return false + return false, "" } diff --git a/wgengine/magicsock/magicsock_plan9.go b/wgengine/magicsock/magicsock_plan9.go index 23f710430cb58..65714c3e13c33 100644 --- a/wgengine/magicsock/magicsock_plan9.go +++ b/wgengine/magicsock/magicsock_plan9.go @@ -5,8 +5,8 @@ package magicsock -// maybeRebindOnError performs a rebind and restun if the error is defined and -// any conditionals are met. -func (c *Conn) maybeRebindOnError(os string, err error) bool { - return false +// shouldRebind returns if the error is one that is known to be healed by a +// rebind, and if so also returns a resason string for the rebind. +func shouldRebind(err error) (ok bool, reason string) { + return false, "" } diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 8166004517441..d4c9f0cbb9a4d 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3050,37 +3050,68 @@ func TestMaybeSetNearestDERP(t *testing.T) { } } +func TestShouldRebind(t *testing.T) { + tests := []struct { + err error + ok bool + reason string + }{ + {nil, false, ""}, + {io.EOF, false, ""}, + {io.ErrUnexpectedEOF, false, ""}, + {io.ErrShortBuffer, false, ""}, + {&net.OpError{Err: syscall.EPERM}, true, "operation-not-permitted"}, + {&net.OpError{Err: syscall.EPIPE}, true, "broken-pipe"}, + {&net.OpError{Err: syscall.ENOTCONN}, true, "broken-pipe"}, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%s-%v", tt.err, tt.ok), func(t *testing.T) { + if got, reason := shouldRebind(tt.err); got != tt.ok || reason != tt.reason { + t.Errorf("errShouldRebind(%v) = %v, %q; want %v, %q", tt.err, got, reason, tt.ok, tt.reason) + } + }) + } +} + func TestMaybeRebindOnError(t *testing.T) { tstest.PanicOnLog() tstest.ResourceCheck(t) - err := fmt.Errorf("outer err: %w", syscall.EPERM) - - t.Run("darwin-rebind", func(t *testing.T) { - conn := newTestConn(t) - defer conn.Close() - rebound := conn.maybeRebindOnError("darwin", err) - if !rebound { - t.Errorf("darwin should rebind on syscall.EPERM") - } - }) - - t.Run("linux-not-rebind", func(t *testing.T) { - conn := newTestConn(t) - defer conn.Close() - rebound := conn.maybeRebindOnError("linux", err) - if rebound { - t.Errorf("linux should not rebind on syscall.EPERM") - } - }) + var rebindErrs []error + if runtime.GOOS != "plan9" { + rebindErrs = append(rebindErrs, + &net.OpError{Err: syscall.EPERM}, + &net.OpError{Err: syscall.EPIPE}, + &net.OpError{Err: syscall.ENOTCONN}, + ) + } + + for _, rebindErr := range rebindErrs { + t.Run(fmt.Sprintf("rebind-%s", rebindErr), func(t *testing.T) { + conn := newTestConn(t) + defer conn.Close() + + before := metricRebindCalls.Value() + conn.maybeRebindOnError(rebindErr) + after := metricRebindCalls.Value() + if before+1 != after { + t.Errorf("should rebind on %#v", rebindErr) + } + }) + } t.Run("no-frequent-rebind", func(t *testing.T) { - conn := newTestConn(t) - defer conn.Close() - conn.lastEPERMRebind.Store(time.Now().Add(-1 * time.Second)) - rebound := conn.maybeRebindOnError("darwin", err) - if rebound { - t.Errorf("darwin should not rebind on syscall.EPERM within 5 seconds of last") + if runtime.GOOS != "plan9" { + err := fmt.Errorf("outer err: %w", syscall.EPERM) + conn := newTestConn(t) + defer conn.Close() + conn.lastErrRebind.Store(time.Now().Add(-1 * time.Second)) + before := metricRebindCalls.Value() + conn.maybeRebindOnError(err) + after := metricRebindCalls.Value() + if before != after { + t.Errorf("should not rebind within 5 seconds of last") + } } }) } From 220dc56f01fc2e6bc9974ad3e3949c15e1d406e1 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 7 Jan 2025 11:18:05 -0800 Subject: [PATCH 080/223] go.mod: bump tailscale/wireguard-go for Solaris/Illumos Updates #14565 Change-Id: Ifb88ab2ee1997c00c3d4316be04f6f4cc71b2cd3 Signed-off-by: Brad Fitzpatrick --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e2164768411fb..650ec4557c2cf 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 - github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3 + github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e github.com/tc-hib/winres v0.2.1 github.com/tcnksm/go-httpstat v0.2.0 diff --git a/go.sum b/go.sum index 353d8d5b8de94..ae6a09262454a 100644 --- a/go.sum +++ b/go.sum @@ -941,8 +941,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= -github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3 h1:dmoPb3dG27tZgMtrvqfD/LW4w7gA6BSWl8prCPNmkCQ= -github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19 h1:BcEJP2ewTIK2ZCsqgl6YGpuO6+oKqqag5HHb7ehljKw= +github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= From de9d4b2f886b6bf5cf0fe9be6c17d080267acef1 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Thu, 5 Dec 2024 14:02:30 -0800 Subject: [PATCH 081/223] net/netmon: remove extra panic guard around ParseRIB This was an extra defense added for #14201 that is no longer required. Fixes #14201 Signed-off-by: James Tucker --- net/netmon/netmon_darwin.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/net/netmon/netmon_darwin.go b/net/netmon/netmon_darwin.go index e89e2d04794e5..cc630112523fa 100644 --- a/net/netmon/netmon_darwin.go +++ b/net/netmon/netmon_darwin.go @@ -56,18 +56,7 @@ func (m *darwinRouteMon) Receive() (message, error) { if err != nil { return nil, err } - msgs, err := func() (msgs []route.Message, err error) { - defer func() { - // TODO(raggi,#14201): remove once we've got a fix from - // golang/go#70528. - msg := recover() - if msg != nil { - msgs = nil - err = fmt.Errorf("panic in route.ParseRIB: %s", msg) - } - }() - return route.ParseRIB(route.RIBTypeRoute, m.buf[:n]) - }() + msgs, err := route.ParseRIB(route.RIBTypeRoute, m.buf[:n]) if err != nil { if debugRouteMessages { m.logf("read %d bytes (% 02x), failed to parse RIB: %v", n, m.buf[:n], err) From 60daa2adb8eb2d496f3dd037d87f34060db3a072 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Mon, 6 Jan 2025 16:34:27 -0800 Subject: [PATCH 082/223] all: fix golangci-lint errors These erroneously blocked a recent PR, which I fixed by simply re-running CI. But we might as well fix them anyway. These are mostly `printf` to `print` and a couple of `!=` to `!Equal()` Updates #cleanup Signed-off-by: Will Norris --- cmd/addlicense/main.go | 4 ++-- cmd/k8s-operator/proxy.go | 2 +- cmd/tailscale/cli/risks.go | 2 +- cmd/tsconnect/tsconnect.go | 4 ++-- net/tshttpproxy/tshttpproxy_synology.go | 2 +- net/tshttpproxy/tshttpproxy_synology_test.go | 2 +- ssh/tailssh/incubator.go | 8 ++++---- tstest/integration/testcontrol/testcontrol.go | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/addlicense/main.go b/cmd/addlicense/main.go index a8fd9dd4ab96a..1cd1b0f19354a 100644 --- a/cmd/addlicense/main.go +++ b/cmd/addlicense/main.go @@ -18,12 +18,12 @@ var ( ) func usage() { - fmt.Fprintf(os.Stderr, ` + fmt.Fprint(os.Stderr, ` usage: addlicense -file FILE `[1:]) flag.PrintDefaults() - fmt.Fprintf(os.Stderr, ` + fmt.Fprint(os.Stderr, ` addlicense adds a Tailscale license to the beginning of file. It is intended for use with 'go generate', so it also runs a subcommand, diff --git a/cmd/k8s-operator/proxy.go b/cmd/k8s-operator/proxy.go index 672f07b1f1608..4509c0dd83fc1 100644 --- a/cmd/k8s-operator/proxy.go +++ b/cmd/k8s-operator/proxy.go @@ -311,7 +311,7 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { // Now add the impersonation headers that we want. if err := addImpersonationHeaders(r, h.log); err != nil { - log.Printf("failed to add impersonation headers: " + err.Error()) + log.Print("failed to add impersonation headers: ", err.Error()) } } diff --git a/cmd/tailscale/cli/risks.go b/cmd/tailscale/cli/risks.go index acb50e723c585..c36ffafaeb11a 100644 --- a/cmd/tailscale/cli/risks.go +++ b/cmd/tailscale/cli/risks.go @@ -77,7 +77,7 @@ func presentRiskToUser(riskType, riskMessage, acceptedRisks string) error { for left := riskAbortTimeSeconds; left > 0; left-- { msg := fmt.Sprintf("\rContinuing in %d seconds...", left) msgLen = len(msg) - printf(msg) + printf("%s", msg) select { case <-interrupt: printf("\r%s\r", strings.Repeat("x", msgLen+1)) diff --git a/cmd/tsconnect/tsconnect.go b/cmd/tsconnect/tsconnect.go index 4c8a0a52ece34..ef55593b49268 100644 --- a/cmd/tsconnect/tsconnect.go +++ b/cmd/tsconnect/tsconnect.go @@ -53,12 +53,12 @@ func main() { } func usage() { - fmt.Fprintf(os.Stderr, ` + fmt.Fprint(os.Stderr, ` usage: tsconnect {dev|build|serve} `[1:]) flag.PrintDefaults() - fmt.Fprintf(os.Stderr, ` + fmt.Fprint(os.Stderr, ` tsconnect implements development/build/serving workflows for Tailscale Connect. It can be invoked with one of three subcommands: diff --git a/net/tshttpproxy/tshttpproxy_synology.go b/net/tshttpproxy/tshttpproxy_synology.go index 2e50d26d3a655..e28844f7dbf67 100644 --- a/net/tshttpproxy/tshttpproxy_synology.go +++ b/net/tshttpproxy/tshttpproxy_synology.go @@ -47,7 +47,7 @@ func synologyProxyFromConfigCached(req *http.Request) (*url.URL, error) { var err error modtime := mtime(synologyProxyConfigPath) - if modtime != cache.updated { + if !modtime.Equal(cache.updated) { cache.httpProxy, cache.httpsProxy, err = synologyProxiesFromConfig() cache.updated = modtime } diff --git a/net/tshttpproxy/tshttpproxy_synology_test.go b/net/tshttpproxy/tshttpproxy_synology_test.go index 3061740f3beff..b6e8b948c3ae9 100644 --- a/net/tshttpproxy/tshttpproxy_synology_test.go +++ b/net/tshttpproxy/tshttpproxy_synology_test.go @@ -41,7 +41,7 @@ func TestSynologyProxyFromConfigCached(t *testing.T) { t.Fatalf("got %s, %v; want nil, nil", val, err) } - if got, want := cache.updated, time.Unix(0, 0); got != want { + if got, want := cache.updated.UTC(), time.Unix(0, 0).UTC(); !got.Equal(want) { t.Fatalf("got %s, want %s", got, want) } if cache.httpProxy != nil { diff --git a/ssh/tailssh/incubator.go b/ssh/tailssh/incubator.go index 3ff676d519898..986b60bd385d6 100644 --- a/ssh/tailssh/incubator.go +++ b/ssh/tailssh/incubator.go @@ -1014,10 +1014,10 @@ func (ss *sshSession) startWithStdPipes() (err error) { func envForUser(u *userMeta) []string { return []string{ - fmt.Sprintf("SHELL=" + u.LoginShell()), - fmt.Sprintf("USER=" + u.Username), - fmt.Sprintf("HOME=" + u.HomeDir), - fmt.Sprintf("PATH=" + defaultPathForUser(&u.User)), + fmt.Sprintf("SHELL=%s", u.LoginShell()), + fmt.Sprintf("USER=%s", u.Username), + fmt.Sprintf("HOME=%s", u.HomeDir), + fmt.Sprintf("PATH=%s", defaultPathForUser(&u.User)), } } diff --git a/tstest/integration/testcontrol/testcontrol.go b/tstest/integration/testcontrol/testcontrol.go index 92f74e24440be..386359f19fa11 100644 --- a/tstest/integration/testcontrol/testcontrol.go +++ b/tstest/integration/testcontrol/testcontrol.go @@ -955,7 +955,7 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, if dns != nil && s.MagicDNSDomain != "" { dns = dns.Clone() dns.CertDomains = []string{ - fmt.Sprintf(node.Hostinfo.Hostname() + "." + s.MagicDNSDomain), + node.Hostinfo.Hostname() + "." + s.MagicDNSDomain, } } From 009da8a364ae2b1c807fe195b4ff55a8219890aa Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Mon, 6 Jan 2025 11:27:11 -0500 Subject: [PATCH 083/223] ipn/ipnlocal: connect serve config to c2n endpoint This commit updates the VIPService c2n endpoint on client to response with actual VIPService configuration stored in the serve config. Fixes tailscale/corp#24510 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- ipn/ipnlocal/local.go | 45 +++++++++------- ipn/ipnlocal/local_test.go | 103 ++++++++++++++++++++++++++++++++----- ipn/serve.go | 40 ++++++++++++++ 3 files changed, 154 insertions(+), 34 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 4c58ae8ecfb19..0af40cfc7782f 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -11,6 +11,7 @@ import ( "context" "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -5017,13 +5018,7 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip } hi.SSH_HostKeys = sshHostKeys - services := vipServicesFromPrefs(prefs) - if len(services) > 0 { - buf, _ := json.Marshal(services) - hi.ServicesHash = fmt.Sprintf("%02x", sha256.Sum256(buf)) - } else { - hi.ServicesHash = "" - } + hi.ServicesHash = b.vipServiceHashLocked(prefs) // The Hostinfo.WantIngress field tells control whether this node wants to // be wired up for ingress connections. If harmless if it's accidentally @@ -7659,28 +7654,38 @@ func maybeUsernameOf(actor ipnauth.Actor) string { func (b *LocalBackend) VIPServices() []*tailcfg.VIPService { b.mu.Lock() defer b.mu.Unlock() - return vipServicesFromPrefs(b.pm.CurrentPrefs()) + return b.vipServicesFromPrefsLocked(b.pm.CurrentPrefs()) +} + +func (b *LocalBackend) vipServiceHashLocked(prefs ipn.PrefsView) string { + services := b.vipServicesFromPrefsLocked(prefs) + if len(services) == 0 { + return "" + } + buf, err := json.Marshal(services) + if err != nil { + b.logf("vipServiceHashLocked: %v", err) + return "" + } + hash := sha256.Sum256(buf) + return hex.EncodeToString(hash[:]) } -func vipServicesFromPrefs(prefs ipn.PrefsView) []*tailcfg.VIPService { +func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcfg.VIPService { // keyed by service name var services map[string]*tailcfg.VIPService - - // TODO(naman): this envknob will be replaced with service-specific port - // information once we start storing that. - var allPortsServices []string - if env := envknob.String("TS_DEBUG_ALLPORTS_SERVICES"); env != "" { - allPortsServices = strings.Split(env, ",") + if !b.serveConfig.Valid() { + return nil } - for _, s := range allPortsServices { - mak.Set(&services, s, &tailcfg.VIPService{ - Name: s, - Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}}, + for svc, config := range b.serveConfig.Services().All() { + mak.Set(&services, svc, &tailcfg.VIPService{ + Name: svc, + Ports: config.ServicePortRange(), }) } - for _, s := range prefs.AdvertiseServices().AsSlice() { + for _, s := range prefs.AdvertiseServices().All() { if services == nil || services[s] == nil { mak.Set(&services, s, &tailcfg.VIPService{ Name: s, diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 15766741b754e..f3ee24a6bd8aa 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -30,7 +30,6 @@ import ( "tailscale.com/control/controlclient" "tailscale.com/drive" "tailscale.com/drive/driveimpl" - "tailscale.com/envknob" "tailscale.com/health" "tailscale.com/hostinfo" "tailscale.com/ipn" @@ -4509,15 +4508,15 @@ func TestConfigFileReload(t *testing.T) { func TestGetVIPServices(t *testing.T) { tests := []struct { - name string - advertised []string - mapped []string - want []*tailcfg.VIPService + name string + advertised []string + serveConfig *ipn.ServeConfig + want []*tailcfg.VIPService }{ { "advertised-only", []string{"svc:abc", "svc:def"}, - []string{}, + &ipn.ServeConfig{}, []*tailcfg.VIPService{ { Name: "svc:abc", @@ -4530,9 +4529,13 @@ func TestGetVIPServices(t *testing.T) { }, }, { - "mapped-only", + "served-only", []string{}, - []string{"svc:abc"}, + &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:abc": {Tun: true}, + }, + }, []*tailcfg.VIPService{ { Name: "svc:abc", @@ -4541,9 +4544,13 @@ func TestGetVIPServices(t *testing.T) { }, }, { - "mapped-and-advertised", - []string{"svc:abc"}, + "served-and-advertised", []string{"svc:abc"}, + &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:abc": {Tun: true}, + }, + }, []*tailcfg.VIPService{ { Name: "svc:abc", @@ -4553,9 +4560,13 @@ func TestGetVIPServices(t *testing.T) { }, }, { - "mapped-and-advertised-separately", + "served-and-advertised-different-service", []string{"svc:def"}, - []string{"svc:abc"}, + &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:abc": {Tun: true}, + }, + }, []*tailcfg.VIPService{ { Name: "svc:abc", @@ -4567,14 +4578,78 @@ func TestGetVIPServices(t *testing.T) { }, }, }, + { + "served-with-port-ranges-one-range-single", + []string{}, + &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTPS: true}, + }}, + }, + }, + []*tailcfg.VIPService{ + { + Name: "svc:abc", + Ports: []tailcfg.ProtoPortRange{{Proto: 6, Ports: tailcfg.PortRange{First: 80, Last: 80}}}, + }, + }, + }, + { + "served-with-port-ranges-one-range-multiple", + []string{}, + &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTPS: true}, + 81: {HTTPS: true}, + 82: {HTTPS: true}, + }}, + }, + }, + []*tailcfg.VIPService{ + { + Name: "svc:abc", + Ports: []tailcfg.ProtoPortRange{{Proto: 6, Ports: tailcfg.PortRange{First: 80, Last: 82}}}, + }, + }, + }, + { + "served-with-port-ranges-multiple-ranges", + []string{}, + &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTPS: true}, + 81: {HTTPS: true}, + 82: {HTTPS: true}, + 1212: {HTTPS: true}, + 1213: {HTTPS: true}, + 1214: {HTTPS: true}, + }}, + }, + }, + []*tailcfg.VIPService{ + { + Name: "svc:abc", + Ports: []tailcfg.ProtoPortRange{ + {Proto: 6, Ports: tailcfg.PortRange{First: 80, Last: 82}}, + {Proto: 6, Ports: tailcfg.PortRange{First: 1212, Last: 1214}}, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - envknob.Setenv("TS_DEBUG_ALLPORTS_SERVICES", strings.Join(tt.mapped, ",")) + lb := newLocalBackendWithTestControl(t, false, func(tb testing.TB, opts controlclient.Options) controlclient.Client { + return newClient(tb, opts) + }) + lb.serveConfig = tt.serveConfig.View() prefs := &ipn.Prefs{ AdvertiseServices: tt.advertised, } - got := vipServicesFromPrefs(prefs.View()) + got := lb.vipServicesFromPrefsLocked(prefs.View()) slices.SortFunc(got, func(a, b *tailcfg.VIPService) int { return strings.Compare(a.Name, b.Name) }) diff --git a/ipn/serve.go b/ipn/serve.go index e82279db8c604..b7effa874c136 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -16,7 +16,9 @@ import ( "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" + "tailscale.com/types/ipproto" "tailscale.com/util/mak" + "tailscale.com/util/set" ) // ServeConfigKey returns a StateKey that stores the @@ -655,3 +657,41 @@ func (v ServeConfigView) HasFunnelForTarget(target HostPort) bool { } return false } + +// ServicePortRange returns the list of tailcfg.ProtoPortRange that represents +// the proto/ports pairs that are being served by the service. +// +// Right now Tun mode is the only thing supports UDP, otherwise serve only supports TCP. +func (v ServiceConfigView) ServicePortRange() []tailcfg.ProtoPortRange { + if v.Tun() { + // If the service is in Tun mode, means service accept TCP/UDP on all ports. + return []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}} + } + tcp := int(ipproto.TCP) + + // Deduplicate the ports. + servePorts := make(set.Set[uint16]) + for port := range v.TCP().All() { + if port > 0 { + servePorts.Add(uint16(port)) + } + } + dedupedServePorts := servePorts.Slice() + slices.Sort(dedupedServePorts) + + var ranges []tailcfg.ProtoPortRange + for _, p := range dedupedServePorts { + if n := len(ranges); n > 0 && p == ranges[n-1].Ports.Last+1 { + ranges[n-1].Ports.Last = p + continue + } + ranges = append(ranges, tailcfg.ProtoPortRange{ + Proto: tcp, + Ports: tailcfg.PortRange{ + First: p, + Last: p, + }, + }) + } + return ranges +} From 8d4ca13cf8093fdcff06140feedb0e32d42cbc91 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Wed, 8 Jan 2025 13:43:17 +0000 Subject: [PATCH 084/223] cmd/k8s-operator,k8s-operator: support ingress ProxyGroup type (#14548) Currently this does not yet do anything apart from creating the ProxyGroup resources like StatefulSet. Updates tailscale/corp#24795 Signed-off-by: Irbe Krumina --- .../crds/tailscale.com_proxygroups.yaml | 13 +- .../deploy/manifests/operator.yaml | 13 +- cmd/k8s-operator/egress-services.go | 14 +- cmd/k8s-operator/operator.go | 2 +- cmd/k8s-operator/proxygroup.go | 42 +++++- cmd/k8s-operator/proxygroup_specs.go | 15 +- cmd/k8s-operator/proxygroup_test.go | 138 +++++++++++++++++- k8s-operator/api.md | 6 +- .../apis/v1alpha1/types_proxygroup.go | 11 +- 9 files changed, 222 insertions(+), 32 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 5e6b537853794..d6a4fe7415dd1 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -20,6 +20,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: ProxyGroup type. + jsonPath: .spec.type + name: Type + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -84,6 +88,7 @@ spec: Defaults to 2. type: integer format: int32 + minimum: 0 tags: description: |- Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s]. @@ -97,10 +102,16 @@ spec: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: - description: Type of the ProxyGroup proxies. Currently the only supported type is egress. + description: |- + Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type is immutable once a ProxyGroup is created. type: string enum: - egress + - ingress + x-kubernetes-validations: + - rule: self == oldSelf + message: ProxyGroup type is immutable status: description: |- ProxyGroupStatus describes the status of the ProxyGroup resources. This is diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index dd34c2a1e5a5c..2f5100ab688eb 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2721,6 +2721,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: ProxyGroup type. + jsonPath: .spec.type + name: Type + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -2778,6 +2782,7 @@ spec: Replicas specifies how many replicas to create the StatefulSet with. Defaults to 2. format: int32 + minimum: 0 type: integer tags: description: |- @@ -2792,10 +2797,16 @@ spec: type: string type: array type: - description: Type of the ProxyGroup proxies. Currently the only supported type is egress. + description: |- + Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type is immutable once a ProxyGroup is created. enum: - egress + - ingress type: string + x-kubernetes-validations: + - message: ProxyGroup type is immutable + rule: self == oldSelf required: - type type: object diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 7544376fb2e65..55003ee91de1b 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -495,13 +495,6 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, err } - if !tsoperator.ProxyGroupIsReady(pg) { - l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) - tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) - return false, nil - } - if violations := validateEgressService(svc, pg); len(violations) > 0 { msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", ")) esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) @@ -510,6 +503,13 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } + if !tsoperator.ProxyGroupIsReady(pg) { + l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) + return false, nil + } + l.Debugf("egress service is valid") tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l) return true, nil diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index ebb2c4578ab93..b2483908229d1 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -499,7 +499,7 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not create Recorder reconciler: %v", err) } - // Recorder reconciler. + // ProxyGroup reconciler. ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{}) proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 60f470fc28bb5..194474fb23b9c 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -51,7 +51,10 @@ const ( optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) -var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) +var ( + gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) + gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) +) // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition. type ProxyGroupReconciler struct { @@ -68,8 +71,9 @@ type ProxyGroupReconciler struct { tsFirewallMode string defaultProxyClass string - mu sync.Mutex // protects following - proxyGroups set.Slice[types.UID] // for proxygroups gauge + mu sync.Mutex // protects following + egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge + ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge } func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger { @@ -203,8 +207,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error { logger := r.logger(pg.Name) r.mu.Lock() - r.proxyGroups.Add(pg.UID) - gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len())) + r.ensureAddedToGaugeForProxyGroup(pg) r.mu.Unlock() cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass) @@ -358,8 +361,7 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy logger.Infof("cleaned up ProxyGroup resources") r.mu.Lock() - r.proxyGroups.Remove(pg.UID) - gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len())) + r.ensureRemovedFromGaugeForProxyGroup(pg) r.mu.Unlock() return true, nil } @@ -469,6 +471,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p return configSHA256Sum, nil } +// ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup +// is created. r.mu must be held. +func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGroup) { + switch pg.Spec.Type { + case tsapi.ProxyGroupTypeEgress: + r.egressProxyGroups.Add(pg.UID) + case tsapi.ProxyGroupTypeIngress: + r.ingressProxyGroups.Add(pg.UID) + } + gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) + gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) +} + +// ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the +// ProxyGroup is deleted. r.mu must be held. +func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.ProxyGroup) { + switch pg.Spec.Type { + case tsapi.ProxyGroupTypeEgress: + r.egressProxyGroups.Remove(pg.UID) + case tsapi.ProxyGroupTypeIngress: + r.ingressProxyGroups.Remove(pg.UID) + } + gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) + gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) +} + func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ Version: "alpha0", diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index b47cb39b1e9c6..d602be8147ed6 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -138,10 +138,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)", }, - { - Name: "TS_INTERNAL_APP", - Value: kubetypes.AppProxyGroupEgress, - }, } if tsFirewallMode != "" { @@ -155,9 +151,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa envs = append(envs, corev1.EnvVar{ Name: "TS_EGRESS_SERVICES_CONFIG_PATH", Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices), + }, + corev1.EnvVar{ + Name: "TS_INTERNAL_APP", + Value: kubetypes.AppProxyGroupEgress, + }, + ) + } else { + envs = append(envs, corev1.EnvVar{ + Name: "TS_INTERNAL_APP", + Value: kubetypes.AppProxyGroupIngress, }) } - return append(c.Env, envs...) }() diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 9c4df9e4f9302..bc0dccdff00d2 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -25,6 +25,8 @@ import ( "tailscale.com/client/tailscale" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/egressservices" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/types/ptr" ) @@ -53,6 +55,9 @@ func TestProxyGroup(t *testing.T) { Name: "test", Finalizers: []string{"tailscale.com/finalizer"}, }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + }, } fc := fake.NewClientBuilder(). @@ -112,8 +117,8 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg, nil) expectProxyGroupResources(t, fc, pg, true, initialCfgHash) - if expected := 1; reconciler.proxyGroups.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len()) + if expected := 1; reconciler.egressProxyGroups.Len() != expected { + t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } expectProxyGroupResources(t, fc, pg, true, initialCfgHash) keyReq := tailscale.KeyCapabilities{ @@ -227,8 +232,8 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name) - if expected := 0; reconciler.proxyGroups.Len() != expected { - t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len()) + if expected := 0; reconciler.egressProxyGroups.Len() != expected { + t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } // 2 nodes should get deleted as part of the scale down, and then finally // the first node gets deleted with the ProxyGroup cleanup. @@ -241,6 +246,131 @@ func TestProxyGroup(t *testing.T) { }) } +func TestProxyGroupTypes(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + Build() + + zl, _ := zap.NewDevelopment() + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + proxyImage: testProxyImage, + Client: fc, + l: zl.Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + } + + t.Run("egress_type", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-egress", + UID: "test-egress-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + Replicas: ptr.To[int32](0), + }, + } + if err := fc.Create(context.Background(), pg); err != nil { + t.Fatal(err) + } + + expectReconciled(t, reconciler, "", pg.Name) + verifyProxyGroupCounts(t, reconciler, 0, 1) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress) + verifyEnvVar(t, sts, "TS_EGRESS_SERVICES_CONFIG_PATH", fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices)) + + // Verify that egress configuration has been set up. + cm := &corev1.ConfigMap{} + cmName := fmt.Sprintf("%s-egress-config", pg.Name) + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil { + t.Fatalf("failed to get ConfigMap: %v", err) + } + + expectedVolumes := []corev1.Volume{ + { + Name: cmName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmName, + }, + }, + }, + }, + } + + expectedVolumeMounts := []corev1.VolumeMount{ + { + Name: cmName, + MountPath: "/etc/proxies", + ReadOnly: true, + }, + } + + if diff := cmp.Diff(expectedVolumes, sts.Spec.Template.Spec.Volumes); diff != "" { + t.Errorf("unexpected volumes (-want +got):\n%s", diff) + } + + if diff := cmp.Diff(expectedVolumeMounts, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" { + t.Errorf("unexpected volume mounts (-want +got):\n%s", diff) + } + }) + + t.Run("ingress_type", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + UID: "test-ingress-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + } + if err := fc.Create(context.Background(), pg); err != nil { + t.Fatal(err) + } + + expectReconciled(t, reconciler, "", pg.Name) + verifyProxyGroupCounts(t, reconciler, 1, 1) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) + }) +} + +func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) { + t.Helper() + if r.ingressProxyGroups.Len() != wantIngress { + t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len()) + } + if r.egressProxyGroups.Len() != wantEgress { + t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len()) + } +} + +func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) { + t.Helper() + for _, env := range sts.Spec.Template.Spec.Containers[0].Env { + if env.Name == name { + if env.Value != expectedValue { + t.Errorf("expected %s=%s, got %s", name, expectedValue, env.Value) + } + return + } + } + t.Errorf("%s environment variable not found", name) +} + func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) { t.Helper() diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 327f95ea9eb82..f526069893b43 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -568,9 +568,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. | | Enum: [egress]
Type: string
| +| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress]
Type: string
| | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
If you specify custom tags here, make sure you also make the operator
an owner of these tags.
See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
Tags cannot be changed once a ProxyGroup device has been created.
Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
Type: string
| -| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2. | | | +| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2. | | Minimum: 0
| | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
by the ProxyGroup. Each device will have the integer number from its
StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains
configuration options that should be applied to the resources created
for this ProxyGroup. If unset, and there is no default ProxyClass
configured, the operator will create resources with the default
configuration. | | | @@ -599,7 +599,7 @@ _Underlying type:_ _string_ _Validation:_ -- Enum: [egress] +- Enum: [egress ingress] - Type: string _Appears in:_ diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index e7397f33ec5ba..f95fc58d0f35c 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -13,6 +13,7 @@ import ( // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster,shortName=pg // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyGroupReady")].reason`,description="Status of the deployed ProxyGroup resources." +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=`.spec.type`,description="ProxyGroup type." // ProxyGroup defines a set of Tailscale devices that will act as proxies. // Currently only egress ProxyGroups are supported. @@ -47,7 +48,9 @@ type ProxyGroupList struct { } type ProxyGroupSpec struct { - // Type of the ProxyGroup proxies. Currently the only supported type is egress. + // Type of the ProxyGroup proxies. Supported types are egress and ingress. + // Type is immutable once a ProxyGroup is created. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable" Type ProxyGroupType `json:"type"` // Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s]. @@ -62,6 +65,7 @@ type ProxyGroupSpec struct { // Replicas specifies how many replicas to create the StatefulSet with. // Defaults to 2. // +optional + // +kubebuilder:validation:Minimum=0 Replicas *int32 `json:"replicas,omitempty"` // HostnamePrefix is the hostname prefix to use for tailnet devices created @@ -109,11 +113,12 @@ type TailnetDevice struct { } // +kubebuilder:validation:Type=string -// +kubebuilder:validation:Enum=egress +// +kubebuilder:validation:Enum=egress;ingress type ProxyGroupType string const ( - ProxyGroupTypeEgress ProxyGroupType = "egress" + ProxyGroupTypeEgress ProxyGroupType = "egress" + ProxyGroupTypeIngress ProxyGroupType = "ingress" ) // +kubebuilder:validation:Type=string From c81a95dd53d905b95c67568dd520db16ee1bdb8a Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Wed, 8 Jan 2025 11:44:10 -0600 Subject: [PATCH 085/223] prober: clone histogram buckets before handing to Prometheus for derp_qd_probe_delays_seconds Updates tailscale/corp#25697 Signed-off-by: Percy Wegmann --- prober/derp.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prober/derp.go b/prober/derp.go index 5adc0c0b408d0..3cd6394ad8e18 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -15,6 +15,7 @@ import ( "fmt" "io" "log" + "maps" "net" "net/http" "net/netip" @@ -350,7 +351,7 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa qdh.mx.Lock() result := []prometheus.Metric{ prometheus.MustNewConstMetric(prometheus.NewDesc("derp_qd_probe_dropped_packets", "Total packets dropped", nil, l), prometheus.CounterValue, float64(packetsDropped.Value())), - prometheus.MustNewConstHistogram(prometheus.NewDesc("derp_qd_probe_delays_seconds", "Distribution of queuing delays", nil, l), qdh.count, qdh.sum, qdh.bucketedCounts), + prometheus.MustNewConstHistogram(prometheus.NewDesc("derp_qd_probe_delays_seconds", "Distribution of queuing delays", nil, l), qdh.count, qdh.sum, maps.Clone(qdh.bucketedCounts)), } qdh.mx.Unlock() return result From 8d6b9964831185fe07be5d420fa67b2b197421e6 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 8 Jan 2025 10:47:33 -0800 Subject: [PATCH 086/223] ipn/ipnlocal: add client metric gauge for number of IPNBus connections Updates #1708 Change-Id: Ic7e28d692b4c48e78c842c26234b861fe42a916e Signed-off-by: Brad Fitzpatrick --- ipn/ipnlocal/local.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 0af40cfc7782f..8d2652e0abe92 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -95,6 +95,7 @@ import ( "tailscale.com/types/preftype" "tailscale.com/types/ptr" "tailscale.com/types/views" + "tailscale.com/util/clientmetric" "tailscale.com/util/deephash" "tailscale.com/util/dnsname" "tailscale.com/util/goroutines" @@ -2863,6 +2864,9 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A mak.Set(&b.notifyWatchers, sessionID, session) b.mu.Unlock() + metricCurrentWatchIPNBus.Add(1) + defer metricCurrentWatchIPNBus.Add(-1) + defer func() { b.mu.Lock() delete(b.notifyWatchers, sessionID) @@ -7696,3 +7700,7 @@ func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcf return slicesx.MapValues(services) } + +var ( + metricCurrentWatchIPNBus = clientmetric.NewGauge("localbackend_current_watch_ipn_bus") +) From 1d4fd2fb34d0a731308f276924992e6b879c0fd7 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 8 Jan 2025 11:06:36 -0800 Subject: [PATCH 087/223] hostinfo: improve accuracy of Linux desktop detection heuristic DBus doesn't imply desktop. Updates #1708 Change-Id: Id43205aafb293533119256adf372a7d762aa7aca Signed-off-by: Brad Fitzpatrick --- hostinfo/hostinfo.go | 1 - 1 file changed, 1 deletion(-) diff --git a/hostinfo/hostinfo.go b/hostinfo/hostinfo.go index 3d4216922a12b..89968e1e6db87 100644 --- a/hostinfo/hostinfo.go +++ b/hostinfo/hostinfo.go @@ -233,7 +233,6 @@ func desktop() (ret opt.Bool) { seenDesktop := false for lr := range lineiter.File("/proc/net/unix") { line, _ := lr.Value() - seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S(" @/tmp/dbus-")) seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S(".X11-unix")) seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S("/wayland-1")) } From 9f17260e216905cce04d0e938f6767a391317275 Mon Sep 17 00:00:00 2001 From: Andrew Dunham Date: Wed, 8 Jan 2025 13:21:54 -0500 Subject: [PATCH 088/223] types/views: add MapViewsEqual and MapViewsEqualFunc Extracted from some code written in the other repo. Updates tailscale/corp#25479 Signed-off-by: Andrew Dunham Change-Id: I92c97a63a8f35cace6e89a730938ea587dcefd9b --- types/views/views.go | 41 +++++++++++++++++++ types/views/views_test.go | 85 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/types/views/views.go b/types/views/views.go index eae8c0b169404..4addc64487f76 100644 --- a/types/views/views.go +++ b/types/views/views.go @@ -513,6 +513,47 @@ func (m Map[K, V]) AsMap() map[K]V { return maps.Clone(m.Đļ) } +// NOTE: the type constraints for MapViewsEqual and MapViewsEqualFunc are based +// on those for maps.Equal and maps.EqualFunc. + +// MapViewsEqual returns whether the two given [Map]s are equal. Both K and V +// must be comparable; if V is non-comparable, use [MapViewsEqualFunc] instead. +func MapViewsEqual[K, V comparable](a, b Map[K, V]) bool { + if a.Len() != b.Len() || a.IsNil() != b.IsNil() { + return false + } + if a.IsNil() { + return true // both nil; can exit early + } + + for k, v := range a.All() { + bv, ok := b.GetOk(k) + if !ok || v != bv { + return false + } + } + return true +} + +// MapViewsEqualFunc returns whether the two given [Map]s are equal, using the +// given function to compare two values. +func MapViewsEqualFunc[K comparable, V1, V2 any](a Map[K, V1], b Map[K, V2], eq func(V1, V2) bool) bool { + if a.Len() != b.Len() || a.IsNil() != b.IsNil() { + return false + } + if a.IsNil() { + return true // both nil; can exit early + } + + for k, v := range a.All() { + bv, ok := b.GetOk(k) + if !ok || !eq(v, bv) { + return false + } + } + return true +} + // MapRangeFn is the func called from a Map.Range call. // Implementations should return false to stop range. type MapRangeFn[K comparable, V any] func(k K, v V) (cont bool) diff --git a/types/views/views_test.go b/types/views/views_test.go index 8a1ff3fddfc9e..51b086a4e05d3 100644 --- a/types/views/views_test.go +++ b/types/views/views_test.go @@ -15,6 +15,7 @@ import ( "unsafe" qt "github.com/frankban/quicktest" + "tailscale.com/types/structs" ) type viewStruct struct { @@ -501,3 +502,87 @@ func TestMapFnIter(t *testing.T) { t.Errorf("got %q; want %q", got, want) } } + +func TestMapViewsEqual(t *testing.T) { + testCases := []struct { + name string + a, b map[string]string + want bool + }{ + { + name: "both_nil", + a: nil, + b: nil, + want: true, + }, + { + name: "both_empty", + a: map[string]string{}, + b: map[string]string{}, + want: true, + }, + { + name: "one_nil", + a: nil, + b: map[string]string{"a": "1"}, + want: false, + }, + { + name: "different_length", + a: map[string]string{"a": "1"}, + b: map[string]string{"a": "1", "b": "2"}, + want: false, + }, + { + name: "different_values", + a: map[string]string{"a": "1"}, + b: map[string]string{"a": "2"}, + want: false, + }, + { + name: "different_keys", + a: map[string]string{"a": "1"}, + b: map[string]string{"b": "1"}, + want: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := MapViewsEqual(MapOf(tc.a), MapOf(tc.b)) + if got != tc.want { + t.Errorf("MapViewsEqual: got=%v, want %v", got, tc.want) + } + + got = MapViewsEqualFunc(MapOf(tc.a), MapOf(tc.b), func(a, b string) bool { + return a == b + }) + if got != tc.want { + t.Errorf("MapViewsEqualFunc: got=%v, want %v", got, tc.want) + } + }) + } +} + +func TestMapViewsEqualFunc(t *testing.T) { + // Test that we can compare maps with two different non-comparable + // values using a custom comparison function. + type customStruct1 struct { + _ structs.Incomparable + Field1 string + } + type customStruct2 struct { + _ structs.Incomparable + Field2 string + } + + a := map[string]customStruct1{"a": {Field1: "1"}} + b := map[string]customStruct2{"a": {Field2: "1"}} + + got := MapViewsEqualFunc(MapOf(a), MapOf(b), func(a customStruct1, b customStruct2) bool { + return a.Field1 == b.Field2 + }) + if !got { + t.Errorf("MapViewsEqualFunc: got=%v, want true", got) + } +} From fa52035574fd6e9d6896b39f683ed67946a73dbd Mon Sep 17 00:00:00 2001 From: Will Norris Date: Wed, 8 Jan 2025 11:08:53 -0800 Subject: [PATCH 089/223] client/systray: record that systray is running Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- client/systray/systray.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/systray/systray.go b/client/systray/systray.go index 782fc54202408..de2a37d8d1e6e 100644 --- a/client/systray/systray.go +++ b/client/systray/systray.go @@ -58,6 +58,7 @@ func (menu *Menu) Run() { case <-menu.bgCtx.Done(): } }() + go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1) systray.Run(menu.onReady, menu.onExit) } From 0b4ba4074f584af28fd34945a4bf62d559b2c64d Mon Sep 17 00:00:00 2001 From: Mario Minardi Date: Wed, 8 Jan 2025 13:20:31 -0700 Subject: [PATCH 090/223] client/web: properly show "Log In" for web client on fresh install (#14569) Change the type of the `IPv4` and `IPv6` members in the `nodeData` struct to be `netip.Addr` instead of `string`. We were previously calling `String()` on this struct, which returns "invalid IP" when the `netip.Addr` is its zero value, and passing this value into the aforementioned attributes. This caused rendering issues on the frontend as we were assuming that the value for `IPv4` and `IPv6` would be falsy in this case. The zero value for a `netip.Addr` marshalls to an empty string instead which is the behaviour we want downstream. Updates https://github.com/tailscale/tailscale/issues/14568 Signed-off-by: Mario Minardi --- client/web/web.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/web/web.go b/client/web/web.go index 56c5c92e808bb..1e338b735bc0e 100644 --- a/client/web/web.go +++ b/client/web/web.go @@ -804,8 +804,8 @@ type nodeData struct { DeviceName string TailnetName string // TLS cert name DomainName string - IPv4 string - IPv6 string + IPv4 netip.Addr + IPv6 netip.Addr OS string IPNVersion string @@ -864,10 +864,14 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) { return } filterRules, _ := s.lc.DebugPacketFilterRules(r.Context()) + ipv4, ipv6 := s.selfNodeAddresses(r, st) + data := &nodeData{ ID: st.Self.ID, Status: st.BackendState, DeviceName: strings.Split(st.Self.DNSName, ".")[0], + IPv4: ipv4, + IPv6: ipv6, OS: st.Self.OS, IPNVersion: strings.Split(st.Version, "-")[0], Profile: st.User[st.Self.UserID], @@ -887,10 +891,6 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) { ACLAllowsAnyIncomingTraffic: s.aclsAllowAccess(filterRules), } - ipv4, ipv6 := s.selfNodeAddresses(r, st) - data.IPv4 = ipv4.String() - data.IPv6 = ipv6.String() - if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn && data.URLPrefix == "" { // X-Ingress-Path is the path prefix in use for Home Assistant // https://developers.home-assistant.io/docs/add-ons/presentation#ingress From d8579a48b9ca1cf4636d646baf7ec51c945b9e70 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Wed, 8 Jan 2025 12:44:49 -0800 Subject: [PATCH 091/223] go.mod: bump go-git to v5.13.1 (#14584) govulncheck flagged a couple fresh vulns in that package: * https://pkg.go.dev/vuln/GO-2025-3367 * https://pkg.go.dev/vuln/GO-2025-3368 I don't believe these affect us, as we only do any git stuff from release tooling which is all internal and with hardcoded repo URLs. Updates #cleanup Signed-off-by: Andrew Lytvynov --- go.mod | 20 ++++++++++---------- go.sum | 59 ++++++++++++++++++++++++++-------------------------------- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 650ec4557c2cf..62a431d189205 100644 --- a/go.mod +++ b/go.mod @@ -95,10 +95,10 @@ require ( go.uber.org/zap v1.27.0 go4.org/mem v0.0.0-20220726221520-4f986261bf13 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba - golang.org/x/crypto v0.30.0 - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a + golang.org/x/crypto v0.31.0 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/mod v0.19.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/oauth2 v0.16.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab @@ -135,7 +135,7 @@ require ( github.com/catenacyber/perfsprint v0.7.1 // indirect github.com/ccojocar/zxcvbn-go v1.0.2 // indirect github.com/ckaznocha/intrange v0.1.0 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/cyphar/filepath-securejoin v0.3.6 // indirect github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect github.com/dave/brenda v1.1.0 // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -183,7 +183,7 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect - github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/ProtonMail/go-crypto v1.1.3 // indirect github.com/alexkohler/prealloc v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/ashanbrown/forbidigo v1.6.0 // indirect @@ -236,8 +236,8 @@ require ( github.com/fzipp/gocyclo v0.6.0 // indirect github.com/go-critic/go-critic v0.11.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.5.0 // indirect - github.com/go-git/go-git/v5 v5.11.0 // indirect + github.com/go-git/go-billy/v5 v5.6.1 // indirect + github.com/go-git/go-git/v5 v5.13.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect @@ -343,13 +343,13 @@ require ( github.com/sashamelentyev/interfacebloat v1.1.0 // indirect github.com/sashamelentyev/usestdlibvars v1.25.0 // indirect github.com/securego/gosec/v2 v2.19.0 // indirect - github.com/sergi/go-diff v1.3.1 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sivchari/containedctx v1.0.3 // indirect github.com/sivchari/tenv v1.7.1 // indirect - github.com/skeema/knownhosts v1.2.1 // indirect + github.com/skeema/knownhosts v1.3.0 // indirect github.com/sonatard/noctx v0.0.2 // indirect github.com/sourcegraph/go-diff v0.7.0 // indirect github.com/spf13/afero v1.11.0 // indirect @@ -361,7 +361,7 @@ require ( github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/subosito/gotenv v1.4.2 // indirect github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 diff --git a/go.sum b/go.sum index ae6a09262454a..efb20e63a82f2 100644 --- a/go.sum +++ b/go.sum @@ -83,8 +83,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA= github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ= -github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= -github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk= +github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= github.com/ProtonMail/gopenpgp/v2 v2.7.1 h1:Awsg7MPc2gD3I7IFac2qE3Gdls0lZW8SzrFZ3k1oz0s= @@ -200,7 +200,6 @@ github.com/butuzov/ireturn v0.3.0 h1:hTjMqWw3y5JC3kpnC5vXmFJAWI/m31jaCYQqzkS6PL0 github.com/butuzov/ireturn v0.3.0/go.mod h1:A09nIiwiqzN/IoVo9ogpa0Hzi9fex1kd9PSD6edP5ZA= github.com/butuzov/mirror v1.1.0 h1:ZqX54gBVMXu78QLoiqdwpl2mgmoOJTk7s4p4o+0avZI= github.com/butuzov/mirror v1.1.0/go.mod h1:8Q0BdQU6rC6WILDiBM60DBfvV78OLJmMmixe7GF45AE= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/caarlos0/go-rpmutils v0.2.1-0.20211112020245-2cd62ff89b11 h1:IRrDwVlWQr6kS1U8/EtyA1+EHcc4yl8pndcqXWrEamg= github.com/caarlos0/go-rpmutils v0.2.1-0.20211112020245-2cd62ff89b11/go.mod h1:je2KZ+LxaCNvCoKg32jtOIULcFogJKcL1ZWUaIBjKj0= github.com/caarlos0/testfs v0.4.4 h1:3PHvzHi5Lt+g332CiShwS8ogTgS3HjrmzZxCm6JCDr8= @@ -231,7 +230,6 @@ github.com/cilium/ebpf v0.15.0/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P github.com/ckaznocha/intrange v0.1.0 h1:ZiGBhvrdsKpoEfzh9CjBfDSZof6QB0ORY5tXasUtiew= github.com/ckaznocha/intrange v0.1.0/go.mod h1:Vwa9Ekex2BrEQMg6zlrWwbs/FtYw7eS5838Q7UjK7TQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -251,8 +249,8 @@ github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0= github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM= +github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/daixiang0/gci v0.12.3 h1:yOZI7VAxAGPQmkb1eqt5g/11SUlwoat1fSblGLmdiQc= github.com/daixiang0/gci v0.12.3/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 h1:YI1gOOdmMk3xodBao7fehcvoZsEeOyy/cfhlpCSPgM4= @@ -293,8 +291,8 @@ github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI= github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40= github.com/elastic/crd-ref-docs v0.0.12 h1:F3seyncbzUz3rT3d+caeYWhumb5ojYQ6Bl0Z+zOp16M= github.com/elastic/crd-ref-docs v0.0.12/go.mod h1:X83mMBdJt05heJUYiS3T0yJ/JkCuliuhSUNav5Gjo/U= -github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= -github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy v1.2.3 h1:xwIyKHbaP5yfT6O9KIeYJR5549MXRQkoQMRXGztz8YQ= +github.com/elazarl/goproxy v1.2.3/go.mod h1:YfEbZtqP4AetfO6d40vWchF3znWX7C7Vd6ZMfdL8z64= github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU= github.com/emicklei/go-restful/v3 v3.11.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -335,18 +333,18 @@ github.com/ghostiam/protogetter v0.3.5 h1:+f7UiF8XNd4w3a//4DnusQ2SZjPkUjxkMEfjbx github.com/ghostiam/protogetter v0.3.5/go.mod h1:7lpeDnEJ1ZjL/YtyoN99ljO4z0pd3H0d18/t2dPBxHw= github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I= github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-critic/go-critic v0.11.2 h1:81xH/2muBphEgPtcwH1p6QD+KzXl2tMSi3hXjBSxDnM= github.com/go-critic/go-critic v0.11.2/go.mod h1:OePaicfjsf+KPy33yq4gzv6CO7TEQ9Rom6ns1KsJnl8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= -github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-billy/v5 v5.6.1 h1:u+dcrgaguSSkbjzHwelEjc0Yj300NUevrrPphk/SoRA= +github.com/go-git/go-billy/v5 v5.6.1/go.mod h1:0AsLr1z2+Uksi4NlElmMblP5rPcDZNRCD8ujZCRR2BE= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= -github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= +github.com/go-git/go-git/v5 v5.13.1 h1:DAQ9APonnlvSWpvolXWIuV6Q6zXy2wHbN4cVlNR5Q+M= +github.com/go-git/go-git/v5 v5.13.1/go.mod h1:qryJB4cSBoq3FRoBRf5A77joojuBcmPJ0qu3XXXVixc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -745,8 +743,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -846,8 +844,8 @@ github.com/sashamelentyev/usestdlibvars v1.25.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7 github.com/securego/gosec/v2 v2.19.0 h1:gl5xMkOI0/E6Hxx0XCY2XujA3V7SNSefA8sC+3f1gnk= github.com/securego/gosec/v2 v2.19.0/go.mod h1:hOkDcHz9J/XIgIlPDXalxjeVYsHxoWUc5zJSHxcB8YM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -865,8 +863,8 @@ github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+W github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= github.com/sivchari/tenv v1.7.1 h1:PSpuD4bu6fSmtWMxSGWcvqUUgIn7k3yOJhOIzVWn8Ak= github.com/sivchari/tenv v1.7.1/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg= -github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= -github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= +github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= @@ -909,8 +907,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/studio-b12/gowebdav v0.9.0 h1:1j1sc9gQnNxbXXM4M/CebPOX4aXYtr7MojAVcN4dHjU= github.com/studio-b12/gowebdav v0.9.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= @@ -1060,10 +1059,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1074,8 +1071,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8= @@ -1152,9 +1149,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1234,7 +1230,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1247,7 +1242,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1261,7 +1255,6 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 68997e0dfac4b78ac2ebaa9ea9f0f075c250aae7 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Thu, 9 Jan 2025 07:15:19 +0000 Subject: [PATCH 092/223] cmd/k8s-operator,k8s-operator: allow users to set custom labels for the optional ServiceMonitor (#14475) * cmd/k8s-operator,k8s-operator: allow users to set custom labels for the optional ServiceMonitor Updates tailscale/tailscale#14381 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/connector_test.go | 2 +- .../crds/tailscale.com_proxyclasses.yaml | 14 +++ .../deploy/manifests/operator.yaml | 14 +++ cmd/k8s-operator/ingress_test.go | 115 ++++++++++-------- cmd/k8s-operator/metrics_resources.go | 37 ++++-- cmd/k8s-operator/operator_test.go | 103 +++++++++++++++- cmd/k8s-operator/proxyclass.go | 9 +- cmd/k8s-operator/proxyclass_test.go | 23 +++- cmd/k8s-operator/proxygroup_test.go | 1 + cmd/k8s-operator/sts.go | 4 +- cmd/k8s-operator/sts_test.go | 47 ++++--- cmd/k8s-operator/testutils_test.go | 21 ++-- k8s-operator/api.md | 36 +++++- .../apis/v1alpha1/types_proxyclass.go | 30 ++++- .../apis/v1alpha1/zz_generated.deepcopy.go | 34 +++++- 15 files changed, 389 insertions(+), 101 deletions(-) diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 7cdd83115e877..242f1f99fdd90 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -203,7 +203,7 @@ func TestConnectorWithProxyClass(t *testing.T) { pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ - Labels: map[string]string{"foo": "bar"}, + Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, } diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 9b45deedb62b7..2e53d5ee801fe 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -99,6 +99,16 @@ spec: enable: description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled. type: boolean + labels: + description: |- + Labels to add to the ServiceMonitor. + Labels must be valid Kubernetes labels. + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ x-kubernetes-validations: - rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable && !self.enable)' message: ServiceMonitor can only be enabled if metrics are enabled @@ -133,6 +143,8 @@ spec: type: object additionalProperties: type: string + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ pod: description: Configuration for the proxy Pod. type: object @@ -1062,6 +1074,8 @@ spec: type: object additionalProperties: type: string + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ nodeName: description: |- Proxy Pod's node name. diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 2f5100ab688eb..0026ffef57c26 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -563,6 +563,16 @@ spec: enable: description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled. type: boolean + labels: + additionalProperties: + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ + type: string + description: |- + Labels to add to the ServiceMonitor. + Labels must be valid Kubernetes labels. + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object required: - enable type: object @@ -592,6 +602,8 @@ spec: type: object labels: additionalProperties: + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ type: string description: |- Labels that will be added to the StatefulSet created for the proxy. @@ -1522,6 +1534,8 @@ spec: type: array labels: additionalProperties: + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ type: string description: |- Labels that will be added to the proxy Pod. diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index c4332908a08f9..955258cc3b1ed 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -295,7 +295,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ - Labels: map[string]string{"foo": "bar"}, + Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, } @@ -424,12 +424,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { func TestTailscaleIngressWithServiceMonitor(t *testing.T) { pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1}, - Spec: tsapi.ProxyClassSpec{ - Metrics: &tsapi.Metrics{ - Enable: true, - ServiceMonitor: &tsapi.ServiceMonitor{Enable: true}, - }, - }, + Spec: tsapi.ProxyClassSpec{}, Status: tsapi.ProxyClassStatus{ Conditions: []metav1.Condition{{ Status: metav1.ConditionTrue, @@ -437,32 +432,6 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { ObservedGeneration: 1, }}}, } - crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} - tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}} - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). - WithObjects(pc, tsIngressClass). - WithStatusSubresource(pc). - Build() - ft := &fakeTSClient{} - fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } - ingR := &IngressReconciler{ - Client: fc, - ssr: &tailscaleSTSReconciler{ - Client: fc, - tsClient: ft, - tsnetServer: fakeTsnetServer, - defaultTags: []string{"tag:k8s"}, - operatorNamespace: "operator-ns", - proxyImage: "tailscale/tailscale", - }, - logger: zl.Sugar(), - } - // 1. Enable metrics- expect metrics Service to be created ing := &networkingv1.Ingress{ TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -491,8 +460,7 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { }, }, } - mustCreate(t, fc, ing) - mustCreate(t, fc, &corev1.Service{ + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", @@ -504,11 +472,38 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { Name: "http"}, }, }, - }) - + } + crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} + tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}} + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc, tsIngressClass, ing, svc). + WithStatusSubresource(pc). + Build() + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + ingR := &IngressReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } expectReconciled(t, ingR, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + serveConfig := &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, + } opts := configOpts{ stsName: shortName, secretName: fullName, @@ -517,27 +512,51 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { parentType: "ingress", hostname: "default-test", app: kubetypes.AppIngressResource, - enableMetrics: true, namespaced: true, proxyType: proxyTypeIngressResource, + serveConfig: serveConfig, + resourceVersion: "1", } - serveConfig := &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, - Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, - } - opts.serveConfig = serveConfig - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil) + // 1. Enable metrics- expect metrics Service to be created + mustUpdate(t, fc, "", "metrics", func(proxyClass *tsapi.ProxyClass) { + proxyClass.Spec.Metrics = &tsapi.Metrics{Enable: true} + }) + opts.enableMetrics = true + + expectReconciled(t, ingR, "default", "test") + expectEqual(t, fc, expectedMetricsService(opts), nil) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) + // 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) { - pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true} + pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true, Labels: tsapi.Labels{"foo": "bar"}} }) expectReconciled(t, ingR, "default", "test") + expectEqual(t, fc, expectedMetricsService(opts), nil) + // 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created mustCreate(t, fc, crd) expectReconciled(t, ingR, "default", "test") + opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"} + expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) + + // 4. Update ServiceMonitor CRD and reconcile- ServiceMonitor should get updated + mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) { + proxyClass.Spec.Metrics.ServiceMonitor.Labels = nil + }) + expectReconciled(t, ingR, "default", "test") + opts.serviceMonitorLabels = nil + opts.resourceVersion = "2" + expectEqual(t, fc, expectedMetricsService(opts), nil) expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) + + // 5. Disable metrics - metrics resources should get deleted. + mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) { + proxyClass.Spec.Metrics = nil + }) + expectReconciled(t, ingR, "default", "test") + expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName)) + // ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here. } diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go index 4881436e8e184..8516cf8be07fa 100644 --- a/cmd/k8s-operator/metrics_resources.go +++ b/cmd/k8s-operator/metrics_resources.go @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "reflect" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -115,15 +116,15 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o return maybeCleanupServiceMonitor(ctx, cl, opts.proxyStsName, opts.tsNamespace) } - logger.Info("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name) - svcMonitor, err := newServiceMonitor(metricsSvc) + logger.Infof("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name) + svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor) if err != nil { return fmt.Errorf("error creating ServiceMonitor: %w", err) } - // We don't use createOrUpdate here because that does not work with unstructured types. We also do not update - // the ServiceMonitor because it is not expected that any of its fields would change. Currently this is good - // enough, but in future we might want to add logic to create-or-update unstructured types. - err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), svcMonitor.DeepCopy()) + + // We don't use createOrUpdate here because that does not work with unstructured types. + existing := svcMonitor.DeepCopy() + err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), existing) if apierrors.IsNotFound(err) { if err := cl.Create(ctx, svcMonitor); err != nil { return fmt.Errorf("error creating ServiceMonitor: %w", err) @@ -133,6 +134,13 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o if err != nil { return fmt.Errorf("error getting ServiceMonitor: %w", err) } + // Currently, we only update labels on the ServiceMonitor as those are the only values that can change. + if !reflect.DeepEqual(existing.GetLabels(), svcMonitor.GetLabels()) { + existing.SetLabels(svcMonitor.GetLabels()) + if err := cl.Update(ctx, existing); err != nil { + return fmt.Errorf("error updating ServiceMonitor: %w", err) + } + } return nil } @@ -165,9 +173,13 @@ func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName, // newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that // proxy that can be applied to the kube API server. // The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema. -func newServiceMonitor(metricsSvc *corev1.Service) (*unstructured.Unstructured, error) { +func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) (*unstructured.Unstructured, error) { sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace) sm.ObjectMeta.Labels = metricsSvc.Labels + if spec != nil && len(spec.Labels) > 0 { + sm.ObjectMeta.Labels = mergeMapKeys(sm.ObjectMeta.Labels, spec.Labels.Parse()) + } + sm.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(metricsSvc, corev1.SchemeGroupVersion.WithKind("Service"))} sm.Spec = ServiceMonitorSpec{ Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels}, @@ -270,3 +282,14 @@ type metricsOpts struct { func isNamespacedProxyType(typ string) bool { return typ == proxyTypeIngressResource || typ == proxyTypeIngressService } + +func mergeMapKeys(a, b map[string]string) map[string]string { + m := make(map[string]string, len(a)+len(b)) + for key, val := range b { + m[key] = val + } + for key, val := range a { + m[key] = val + } + return m +} diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index e46cdd7fe6e45..d53269f05a77a 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -16,6 +16,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" @@ -1129,7 +1130,7 @@ func TestProxyClassForService(t *testing.T) { AcceptRoutes: true, }, StatefulSet: &tsapi.StatefulSet{ - Labels: map[string]string{"foo": "bar"}, + Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, } @@ -1766,6 +1767,106 @@ func Test_externalNameService(t *testing.T) { expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) } +func Test_metricsResourceCreation(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1}, + Spec: tsapi.ProxyClassSpec{}, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Status: metav1.ConditionTrue, + Type: string(tsapi.ProxyClassReady), + ObservedGeneration: 1, + }}}, + } + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + UID: types.UID("1234-UID"), + Labels: map[string]string{LabelProxyClass: "metrics"}, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: ptr.To("tailscale"), + }, + } + crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc, svc). + WithStatusSubresource(pc). + Build() + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + clock := tstest.NewClock(tstest.ClockOpts{}) + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + operatorNamespace: "operator-ns", + }, + logger: zl.Sugar(), + clock: clock, + } + expectReconciled(t, sr, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") + opts := configOpts{ + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "svc", + tailscaleNamespace: "operator-ns", + hostname: "default-test", + namespaced: true, + proxyType: proxyTypeIngressService, + app: kubetypes.AppIngressProxy, + resourceVersion: "1", + } + + // 1. Enable metrics- expect metrics Service to be created + mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) { + pc.Spec = tsapi.ProxyClassSpec{Metrics: &tsapi.Metrics{Enable: true}} + }) + expectReconciled(t, sr, "default", "test") + opts.enableMetrics = true + expectEqual(t, fc, expectedMetricsService(opts), nil) + + // 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster + mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) { + pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true} + }) + expectReconciled(t, sr, "default", "test") + + // 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created + mustCreate(t, fc, crd) + expectReconciled(t, sr, "default", "test") + expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) + + // 4. A change to ServiceMonitor config gets reflected in the ServiceMonitor resource + mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) { + pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar"} + }) + expectReconciled(t, sr, "default", "test") + opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"} + opts.resourceVersion = "2" + expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) + + // 5. Disable metrics- expect metrics Service to be deleted + mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) { + pc.Spec.Metrics = nil + }) + expectReconciled(t, sr, "default", "test") + expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(opts.stsName)) + // ServiceMonitor gets garbage collected when Service gets deleted (it has OwnerReference of the Service + // object). We cannot test this using the fake client. +} + func toFQDN(t *testing.T, s string) dnsname.FQDN { t.Helper() fqdn, err := dnsname.ToFQDN(s) diff --git a/cmd/k8s-operator/proxyclass.go b/cmd/k8s-operator/proxyclass.go index b781af05adaaa..5ec9897d0a8b7 100644 --- a/cmd/k8s-operator/proxyclass.go +++ b/cmd/k8s-operator/proxyclass.go @@ -115,7 +115,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) { if sts := pc.Spec.StatefulSet; sts != nil { if len(sts.Labels) > 0 { - if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil { + if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil { violations = append(violations, errs...) } } @@ -126,7 +126,7 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl } if pod := sts.Pod; pod != nil { if len(pod.Labels) > 0 { - if errs := metavalidation.ValidateLabels(pod.Labels, field.NewPath(".spec.statefulSet.pod.labels")); errs != nil { + if errs := metavalidation.ValidateLabels(pod.Labels.Parse(), field.NewPath(".spec.statefulSet.pod.labels")); errs != nil { violations = append(violations, errs...) } } @@ -178,6 +178,11 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl violations = append(violations, field.TypeInvalid(field.NewPath("spec", "metrics", "serviceMonitor"), "enable", msg)) } } + if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && len(pc.Spec.Metrics.ServiceMonitor.Labels) > 0 { + if errs := metavalidation.ValidateLabels(pc.Spec.Metrics.ServiceMonitor.Labels.Parse(), field.NewPath(".spec.metrics.serviceMonitor.labels")); errs != nil { + violations = append(violations, errs...) + } + } // We do not validate embedded fields (security context, resource // requirements etc) as we inherit upstream validation for those fields. // Invalid values would get rejected by upstream validations at apply diff --git a/cmd/k8s-operator/proxyclass_test.go b/cmd/k8s-operator/proxyclass_test.go index e6e16e9f9d59f..78828107a09e0 100644 --- a/cmd/k8s-operator/proxyclass_test.go +++ b/cmd/k8s-operator/proxyclass_test.go @@ -36,10 +36,10 @@ func TestProxyClass(t *testing.T) { }, Spec: tsapi.ProxyClassSpec{ StatefulSet: &tsapi.StatefulSet{ - Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"}, + Labels: tsapi.Labels{"foo": "bar", "xyz1234": "abc567"}, Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"}, Pod: &tsapi.Pod{ - Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"}, + Labels: tsapi.Labels{"foo": "bar", "xyz1234": "abc567"}, Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"}, TailscaleContainer: &tsapi.Container{ Env: []tsapi.Env{{Name: "FOO", Value: "BAR"}}, @@ -155,6 +155,25 @@ func TestProxyClass(t *testing.T) { expectReconciled(t, pcr, "", "test") tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar()) expectEqual(t, fc, pc, nil) + + // 7. A ProxyClass with invalid ServiceMonitor labels gets its status updated to Invalid with an error message. + pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar!"} + mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) { + proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels + }) + expectReconciled(t, pcr, "", "test") + msg = `ProxyClass is not valid: .spec.metrics.serviceMonitor.labels: Invalid value: "bar!": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')` + tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar()) + expectEqual(t, fc, pc, nil) + + // 8. A ProxyClass with valid ServiceMonitor labels gets its status updated to Valid. + pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar", "xyz1234": "abc567", "empty": "", "onechar": "a"} + mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) { + proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels + }) + expectReconciled(t, pcr, "", "test") + tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar()) + expectEqual(t, fc, pc, nil) } func TestValidateProxyClass(t *testing.T) { diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index bc0dccdff00d2..6464a0b2daaa9 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -88,6 +88,7 @@ func TestProxyGroup(t *testing.T) { stsName: pg.Name, parentType: "proxygroup", tailscaleNamespace: "tailscale", + resourceVersion: "1", } t.Run("proxyclass_not_ready", func(t *testing.T) { diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index ff7c074a8b425..b861bdffff90d 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -761,7 +761,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, } // Update StatefulSet metadata. - if wantsSSLabels := pc.Spec.StatefulSet.Labels; len(wantsSSLabels) > 0 { + if wantsSSLabels := pc.Spec.StatefulSet.Labels.Parse(); len(wantsSSLabels) > 0 { ss.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Labels, wantsSSLabels, tailscaleManagedLabels) } if wantsSSAnnots := pc.Spec.StatefulSet.Annotations; len(wantsSSAnnots) > 0 { @@ -773,7 +773,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, return ss } wantsPod := pc.Spec.StatefulSet.Pod - if wantsPodLabels := wantsPod.Labels; len(wantsPodLabels) > 0 { + if wantsPodLabels := wantsPod.Labels.Parse(); len(wantsPodLabels) > 0 { ss.Spec.Template.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, tailscaleManagedLabels) } if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 { diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 05aafaee6a5d4..3d0cecc043b9f 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -61,10 +61,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { proxyClassAllOpts := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ StatefulSet: &tsapi.StatefulSet{ - Labels: map[string]string{"foo": "bar"}, + Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"foo.io/bar": "foo"}, Pod: &tsapi.Pod{ - Labels: map[string]string{"bar": "foo"}, + Labels: tsapi.Labels{"bar": "foo"}, Annotations: map[string]string{"bar.io/foo": "foo"}, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: ptr.To(int64(0)), @@ -116,10 +116,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { proxyClassJustLabels := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ StatefulSet: &tsapi.StatefulSet{ - Labels: map[string]string{"foo": "bar"}, + Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"foo.io/bar": "foo"}, Pod: &tsapi.Pod{ - Labels: map[string]string{"bar": "foo"}, + Labels: tsapi.Labels{"bar": "foo"}, Annotations: map[string]string{"bar.io/foo": "foo"}, }, }, @@ -146,7 +146,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { }, } } - var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil { t.Fatalf("unmarshaling userspace proxy template: %v", err) @@ -176,9 +175,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { // 1. Test that a ProxyClass with all fields set gets correctly applied // to a Statefulset built from non-userspace proxy template. wantSS := nonUserspaceProxySS.DeepCopy() - wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels) - wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations) - wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels + updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse()) + updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse() wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets @@ -207,9 +206,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { // StatefulSet and Pod set gets correctly applied to a Statefulset built // from non-userspace proxy template. wantSS = nonUserspaceProxySS.DeepCopy() - wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels) - wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) - wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels + updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse()) + updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse() wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { @@ -219,9 +218,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { // 3. Test that a ProxyClass with all fields set gets correctly applied // to a Statefulset built from a userspace proxy template. wantSS = userspaceProxySS.DeepCopy() - wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels) - wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations) - wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels + updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse()) + updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse() wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets @@ -243,9 +242,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { // 4. Test that a ProxyClass with custom labels and annotations gets correctly applied // to a Statefulset built from a userspace proxy template. wantSS = userspaceProxySS.DeepCopy() - wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels) - wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) - wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels + updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse()) + updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse() wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { @@ -294,13 +293,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { } } -func mergeMapKeys(a, b map[string]string) map[string]string { - for key, val := range b { - a[key] = val - } - return a -} - func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { tests := []struct { name string @@ -392,3 +384,10 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { }) } } + +// updateMap updates map a with the values from map b. +func updateMap(a, b map[string]string) { + for key, val := range b { + a[key] = val + } +} diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index f6ae29b62fefc..d43e75b1e3cbc 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -61,7 +61,10 @@ type configOpts struct { app string shouldRemoveAuthKey bool secretExtraData map[string][]byte - enableMetrics bool + resourceVersion string + + enableMetrics bool + serviceMonitorLabels tsapi.Labels } func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { @@ -431,14 +434,17 @@ func metricsLabels(opts configOpts) map[string]string { func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstructured { t.Helper() - labels := metricsLabels(opts) + smLabels := metricsLabels(opts) + if len(opts.serviceMonitorLabels) != 0 { + smLabels = mergeMapKeys(smLabels, opts.serviceMonitorLabels.Parse()) + } name := metricsResourceName(opts.stsName) sm := &ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: opts.tailscaleNamespace, - Labels: labels, - ResourceVersion: "1", + Labels: smLabels, + ResourceVersion: opts.resourceVersion, OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: ptr.To(true), Controller: ptr.To(true)}}, }, TypeMeta: metav1.TypeMeta{ @@ -446,7 +452,7 @@ func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstruc APIVersion: "monitoring.coreos.com/v1", }, Spec: ServiceMonitorSpec{ - Selector: metav1.LabelSelector{MatchLabels: labels}, + Selector: metav1.LabelSelector{MatchLabels: metricsLabels(opts)}, Endpoints: []ServiceMonitorEndpoint{{ Port: "metrics", }}, @@ -653,10 +659,11 @@ func expectEqualUnstructured(t *testing.T, client client.Client, want *unstructu func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string) { t.Helper() obj := O(new(T)) - if err := client.Get(context.Background(), types.NamespacedName{ + err := client.Get(context.Background(), types.NamespacedName{ Name: name, Namespace: ns, - }, obj); !apierrors.IsNotFound(err) { + }, obj) + if !apierrors.IsNotFound(err) { t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name) } } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index f526069893b43..fae25b1f61708 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -313,6 +313,37 @@ _Appears in:_ +#### LabelValue + +_Underlying type:_ _string_ + + + +_Validation:_ +- MaxLength: 63 +- Pattern: `^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$` +- Type: string + +_Appears in:_ +- [Labels](#labels) + + + +#### Labels + +_Underlying type:_ _[map[string]LabelValue](#map[string]labelvalue)_ + + + + + +_Appears in:_ +- [Pod](#pod) +- [ServiceMonitor](#servicemonitor) +- [StatefulSet](#statefulset) + + + #### Metrics @@ -407,7 +438,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `labels` _object (keys:string, values:string)_ | Labels that will be added to the proxy Pod.
Any labels specified here will be merged with the default labels
applied to the Pod by the Tailscale Kubernetes operator.
Label keys and values must be valid Kubernetes label keys and values.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | | +| `labels` _[Labels](#labels)_ | Labels that will be added to the proxy Pod.
Any labels specified here will be merged with the default labels
applied to the Pod by the Tailscale Kubernetes operator.
Label keys and values must be valid Kubernetes label keys and values.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | | | `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the proxy Pod.
Any annotations specified here will be merged with the default
annotations applied to the Pod by the Tailscale Kubernetes operator.
Annotations must be valid Kubernetes annotations.
https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | | | `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#affinity-v1-core)_ | Proxy Pod's affinity rules.
By default, the Tailscale Kubernetes operator does not apply any affinity rules.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity | | | | `tailscaleContainer` _[Container](#container)_ | Configuration for the proxy container running tailscale. | | | @@ -864,6 +895,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `enable` _boolean_ | If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled. | | | +| `labels` _[Labels](#labels)_ | Labels to add to the ServiceMonitor.
Labels must be valid Kubernetes labels.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | | #### StatefulSet @@ -879,7 +911,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `labels` _object (keys:string, values:string)_ | Labels that will be added to the StatefulSet created for the proxy.
Any labels specified here will be merged with the default labels
applied to the StatefulSet by the Tailscale Kubernetes operator as
well as any other labels that might have been applied by other
actors.
Label keys and values must be valid Kubernetes label keys and values.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | | +| `labels` _[Labels](#labels)_ | Labels that will be added to the StatefulSet created for the proxy.
Any labels specified here will be merged with the default labels
applied to the StatefulSet by the Tailscale Kubernetes operator as
well as any other labels that might have been applied by other
actors.
Label keys and values must be valid Kubernetes label keys and values.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | | | `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the StatefulSet created for the proxy.
Any Annotations specified here will be merged with the default annotations
applied to the StatefulSet by the Tailscale Kubernetes operator as
well as any other annotations that might have been applied by other
actors.
Annotations must be valid Kubernetes annotations.
https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | | | `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. | | | diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index ef9a071d02bbe..549234fef7b78 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -87,7 +87,7 @@ type StatefulSet struct { // Label keys and values must be valid Kubernetes label keys and values. // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set // +optional - Labels map[string]string `json:"labels,omitempty"` + Labels Labels `json:"labels,omitempty"` // Annotations that will be added to the StatefulSet created for the proxy. // Any Annotations specified here will be merged with the default annotations // applied to the StatefulSet by the Tailscale Kubernetes operator as @@ -109,7 +109,7 @@ type Pod struct { // Label keys and values must be valid Kubernetes label keys and values. // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set // +optional - Labels map[string]string `json:"labels,omitempty"` + Labels Labels `json:"labels,omitempty"` // Annotations that will be added to the proxy Pod. // Any annotations specified here will be merged with the default // annotations applied to the Pod by the Tailscale Kubernetes operator. @@ -188,8 +188,34 @@ type Metrics struct { type ServiceMonitor struct { // If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled. Enable bool `json:"enable"` + // Labels to add to the ServiceMonitor. + // Labels must be valid Kubernetes labels. + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + // +optional + Labels Labels `json:"labels"` +} + +type Labels map[string]LabelValue + +func (l Labels) Parse() map[string]string { + if l == nil { + return nil + } + m := make(map[string]string, len(l)) + for k, v := range l { + m[k] = string(v) + } + return m } +// We do not validate the values of the label keys here - it is done by the ProxyClass +// reconciler because the validation rules are too complex for a CRD validation markers regex. + +// +kubebuilder:validation:Type=string +// +kubebuilder:validation:Pattern=`^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$` +// +kubebuilder:validation:MaxLength=63 +type LabelValue string + type Container struct { // List of environment variables to set in the container. // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 29c71cb90f309..5e7e7455cdba6 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -316,13 +316,34 @@ func (in *Env) DeepCopy() *Env { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Labels) DeepCopyInto(out *Labels) { + { + in := &in + *out = make(Labels, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Labels. +func (in Labels) DeepCopy() Labels { + if in == nil { + return nil + } + out := new(Labels) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Metrics) DeepCopyInto(out *Metrics) { *out = *in if in.ServiceMonitor != nil { in, out := &in.ServiceMonitor, &out.ServiceMonitor *out = new(ServiceMonitor) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -391,7 +412,7 @@ func (in *Pod) DeepCopyInto(out *Pod) { *out = *in if in.Labels != nil { in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) + *out = make(Labels, len(*in)) for key, val := range *in { (*out)[key] = val } @@ -999,6 +1020,13 @@ func (in *S3Secret) DeepCopy() *S3Secret { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceMonitor) DeepCopyInto(out *ServiceMonitor) { *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(Labels, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceMonitor. @@ -1016,7 +1044,7 @@ func (in *StatefulSet) DeepCopyInto(out *StatefulSet) { *out = *in if in.Labels != nil { in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) + *out = make(Labels, len(*in)) for key, val := range *in { (*out)[key] = val } From a51672cafd8b6c4e87915a55bda1491eb7cbee84 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Wed, 8 Jan 2025 10:36:35 -0600 Subject: [PATCH 093/223] prober: record total bytes transferred in DERP bandwidth probes This will enable Prometheus queries to look at the bandwidth over time windows, for example 'increase(derp_bw_bytes_total)[1h] / increase(derp_bw_transfer_time_seconds_total)[1h]'. Updates tailscale/corp#25503 Signed-off-by: Percy Wegmann --- prober/derp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/prober/derp.go b/prober/derp.go index 3cd6394ad8e18..6bad3584579a8 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -318,6 +318,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { return []prometheus.Metric{ prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_probe_size_bytes", "Payload size of the bandwidth prober", nil, l), prometheus.GaugeValue, float64(size)), prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_transfer_time_seconds_total", "Time it took to transfer data", nil, l), prometheus.CounterValue, transferTimeSeconds.Value()), + prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_bytes_total", "Amount of data transferred", nil, l), prometheus.CounterValue, float64(size)), } }, } From 7fa07f34169b11a02e44165fd875e48b2e2e211d Mon Sep 17 00:00:00 2001 From: Andrew Dunham Date: Thu, 9 Jan 2025 16:03:52 -0500 Subject: [PATCH 094/223] types/views: add SliceEqualAnyOrderFunc Extracted from some code written in the other repo. Updates tailscale/corp#25479 Signed-off-by: Andrew Dunham Change-Id: I6df062fdffa1705524caa44ac3b6f2788cf64595 --- types/views/views.go | 35 +++++++++++++++++++++++++++++++++++ types/views/views_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/types/views/views.go b/types/views/views.go index 4addc64487f76..0f53313c76fe1 100644 --- a/types/views/views.go +++ b/types/views/views.go @@ -360,6 +360,41 @@ func SliceEqualAnyOrder[T comparable](a, b Slice[T]) bool { return true } +// SliceEqualAnyOrderFunc reports whether a and b contain the same elements, +// regardless of order. The underlying slices for a and b can be nil. +// +// The provided function should return a comparable value for each element. +func SliceEqualAnyOrderFunc[T any, V comparable](a, b Slice[T], cmp func(T) V) bool { + if a.Len() != b.Len() { + return false + } + + var diffStart int // beginning index where a and b differ + for n := a.Len(); diffStart < n; diffStart++ { + av := cmp(a.At(diffStart)) + bv := cmp(b.At(diffStart)) + if av != bv { + break + } + } + if diffStart == a.Len() { + return true + } + + // count the occurrences of remaining values and compare + valueCount := make(map[V]int) + for i, n := diffStart, a.Len(); i < n; i++ { + valueCount[cmp(a.At(i))]++ + valueCount[cmp(b.At(i))]-- + } + for _, count := range valueCount { + if count != 0 { + return false + } + } + return true +} + // MapSlice is a view over a map whose values are slices. type MapSlice[K comparable, V any] struct { // Đļ is the underlying mutable value, named with a hard-to-type diff --git a/types/views/views_test.go b/types/views/views_test.go index 51b086a4e05d3..f290670fb12fc 100644 --- a/types/views/views_test.go +++ b/types/views/views_test.go @@ -153,6 +153,43 @@ func TestViewUtils(t *testing.T) { qt.Equals, true) } +func TestSliceEqualAnyOrderFunc(t *testing.T) { + type nc struct { + _ structs.Incomparable + v string + } + + // ncFrom returns a Slice[nc] from a slice of []string + ncFrom := func(s ...string) Slice[nc] { + var out []nc + for _, v := range s { + out = append(out, nc{v: v}) + } + return SliceOf(out) + } + + // cmp returns a comparable value for a nc + cmp := func(a nc) string { return a.v } + + v := ncFrom("foo", "bar") + c := qt.New(t) + + // Simple case of slice equal to itself. + c.Check(SliceEqualAnyOrderFunc(v, v, cmp), qt.Equals, true) + + // Different order. + c.Check(SliceEqualAnyOrderFunc(v, ncFrom("bar", "foo"), cmp), qt.Equals, true) + + // Different values, same length + c.Check(SliceEqualAnyOrderFunc(v, ncFrom("foo", "baz"), cmp), qt.Equals, false) + + // Different values, different length + c.Check(SliceEqualAnyOrderFunc(v, ncFrom("foo"), cmp), qt.Equals, false) + + // Nothing shared + c.Check(SliceEqualAnyOrderFunc(v, ncFrom("baz", "qux"), cmp), qt.Equals, false) +} + func TestSliceEqual(t *testing.T) { a := SliceOf([]string{"foo", "bar"}) b := SliceOf([]string{"foo", "bar"}) From 6ddeae755695de850c56c788cba10bd4736934bb Mon Sep 17 00:00:00 2001 From: Andrew Dunham Date: Thu, 9 Jan 2025 16:55:07 -0500 Subject: [PATCH 095/223] types/views: optimize SliceEqualAnyOrderFunc for small slices If the total number of differences is less than a small amount, just do the dumb quadratic thing and compare every single object instead of allocating a map. Updates tailscale/corp#25479 Signed-off-by: Andrew Dunham Change-Id: I8931b4355a2da4ec0f19739927311cf88711a840 --- types/views/views.go | 23 +++++++++++++++++++++++ types/views/views_test.go | 9 +++++++++ 2 files changed, 32 insertions(+) diff --git a/types/views/views.go b/types/views/views.go index 0f53313c76fe1..40d8811f542ee 100644 --- a/types/views/views.go +++ b/types/views/views.go @@ -381,6 +381,29 @@ func SliceEqualAnyOrderFunc[T any, V comparable](a, b Slice[T], cmp func(T) V) b return true } + // For a small number of items, avoid the allocation of a map and just + // do the quadratic thing. We can also only check the items between + // diffStart and the end. + nRemain := a.Len() - diffStart + if nRemain <= 5 { + maxLen := a.Len() // same as b.Len() + for i := diffStart; i < maxLen; i++ { + av := cmp(a.At(i)) + found := false + for j := diffStart; j < maxLen; j++ { + bv := cmp(b.At(j)) + if av == bv { + found = true + break + } + } + if !found { + return false + } + } + return true + } + // count the occurrences of remaining values and compare valueCount := make(map[V]int) for i, n := diffStart, a.Len(); i < n; i++ { diff --git a/types/views/views_test.go b/types/views/views_test.go index f290670fb12fc..70e021aa4f8bf 100644 --- a/types/views/views_test.go +++ b/types/views/views_test.go @@ -188,6 +188,15 @@ func TestSliceEqualAnyOrderFunc(t *testing.T) { // Nothing shared c.Check(SliceEqualAnyOrderFunc(v, ncFrom("baz", "qux"), cmp), qt.Equals, false) + + // Long slice that matches + longSlice := ncFrom("a", "b", "c", "d", "e", "f", "g", "h", "i", "j") + longSame := ncFrom("b", "a", "c", "d", "e", "f", "g", "h", "i", "j") // first 2 elems swapped + c.Check(SliceEqualAnyOrderFunc(longSlice, longSame, cmp), qt.Equals, true) + + // Long difference; past the quadratic limit + longDiff := ncFrom("b", "a", "c", "d", "e", "f", "g", "h", "i", "k") // differs at end + c.Check(SliceEqualAnyOrderFunc(longSlice, longDiff, cmp), qt.Equals, false) } func TestSliceEqual(t *testing.T) { From 9373a1b9026cf3419cbab202bde9d93e44e82091 Mon Sep 17 00:00:00 2001 From: Nahum Shalman Date: Sun, 23 Apr 2023 15:57:35 +0000 Subject: [PATCH 096/223] all: illumos/solaris userspace only support Updates #14565 Change-Id: I743148144938794db0a224873ce76c10dbe6fa5f Signed-off-by: Nahum Shalman --- .github/workflows/test.yml | 6 ++++++ cmd/tailscaled/tailscaled.go | 4 ++-- ipn/ipnlocal/local.go | 2 +- ipn/ipnserver/actor.go | 2 +- ipn/ipnstate/ipnstate.go | 2 ++ ipn/localapi/localapi.go | 4 ++-- ipn/localapi/localapi_test.go | 2 +- net/dns/manager_default.go | 2 +- net/dns/manager_solaris.go | 14 ++++++++++++++ net/dns/resolver/tsdns.go | 2 +- net/netutil/ip_forward.go | 26 ++++++++++++++++++++++++++ net/tstun/tstun_stub.go | 2 +- net/tstun/tun.go | 2 +- paths/paths_unix.go | 2 +- 14 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 net/dns/manager_solaris.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4dccd103d237..d4c73ab7c7724 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -313,6 +313,12 @@ jobs: # AIX - goos: aix goarch: ppc64 + # Solaris + - goos: solaris + goarch: amd64 + # illumos + - goos: illumos + goarch: amd64 runs-on: ubuntu-22.04 steps: diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 7a5ee03983f44..9dd00ddd95624 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -81,7 +81,7 @@ func defaultTunName() string { // "utun" is recognized by wireguard-go/tun/tun_darwin.go // as a magic value that uses/creates any free number. return "utun" - case "plan9", "aix": + case "plan9", "aix", "solaris", "illumos": return "userspace-networking" case "linux": switch distro.Get() { @@ -665,7 +665,7 @@ func handleSubnetsInNetstack() bool { return true } switch runtime.GOOS { - case "windows", "darwin", "freebsd", "openbsd": + case "windows", "darwin", "freebsd", "openbsd", "solaris", "illumos": // Enable on Windows and tailscaled-on-macOS (this doesn't // affect the GUI clients), and on FreeBSD. return true diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 8d2652e0abe92..ad3bbaef32d24 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4176,7 +4176,7 @@ func (b *LocalBackend) peerAPIServicesLocked() (ret []tailcfg.Service) { }) } switch runtime.GOOS { - case "linux", "freebsd", "openbsd", "illumos", "darwin", "windows", "android", "ios": + case "linux", "freebsd", "openbsd", "illumos", "solaris", "darwin", "windows", "android", "ios": // These are the platforms currently supported by // net/dns/resolver/tsdns.go:Resolver.HandleExitNodeDNSQuery. ret = append(ret, tailcfg.Service{ diff --git a/ipn/ipnserver/actor.go b/ipn/ipnserver/actor.go index 63d4b183ca11d..0e716009cc976 100644 --- a/ipn/ipnserver/actor.go +++ b/ipn/ipnserver/actor.go @@ -96,7 +96,7 @@ func (a *actor) Username() (string, error) { } defer tok.Close() return tok.Username() - case "darwin", "linux": + case "darwin", "linux", "illumos", "solaris": uid, ok := a.ci.Creds().UserID() if !ok { return "", errors.New("missing user ID") diff --git a/ipn/ipnstate/ipnstate.go b/ipn/ipnstate/ipnstate.go index 9f8bd34f61033..37ab47714ef64 100644 --- a/ipn/ipnstate/ipnstate.go +++ b/ipn/ipnstate/ipnstate.go @@ -650,6 +650,8 @@ func osEmoji(os string) string { return "🐡" case "illumos": return "☀ī¸" + case "solaris": + return "🌤ī¸" } return "đŸ‘Ŋ" } diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 831f6a9b6b888..157f72a65be03 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -1097,7 +1097,7 @@ func (h *Handler) serveServeConfig(w http.ResponseWriter, r *http.Request) { func authorizeServeConfigForGOOSAndUserContext(goos string, configIn *ipn.ServeConfig, h *Handler) error { switch goos { - case "windows", "linux", "darwin": + case "windows", "linux", "darwin", "illumos", "solaris": default: return nil } @@ -1117,7 +1117,7 @@ func authorizeServeConfigForGOOSAndUserContext(goos string, configIn *ipn.ServeC switch goos { case "windows": return errors.New("must be a Windows local admin to serve a path") - case "linux", "darwin": + case "linux", "darwin", "illumos", "solaris": return errors.New("must be root, or be an operator and able to run 'sudo tailscale' to serve a path") default: // We filter goos at the start of the func, this default case diff --git a/ipn/localapi/localapi_test.go b/ipn/localapi/localapi_test.go index 145910830e80f..b7f0c416c8cf0 100644 --- a/ipn/localapi/localapi_test.go +++ b/ipn/localapi/localapi_test.go @@ -237,7 +237,7 @@ func TestShouldDenyServeConfigForGOOSAndUserContext(t *testing.T) { } for _, tt := range tests { - for _, goos := range []string{"linux", "windows", "darwin"} { + for _, goos := range []string{"linux", "windows", "darwin", "illumos", "solaris"} { t.Run(goos+"-"+tt.name, func(t *testing.T) { err := authorizeServeConfigForGOOSAndUserContext(goos, tt.configIn, tt.h) gotErr := err != nil diff --git a/net/dns/manager_default.go b/net/dns/manager_default.go index 11dea5ca888b1..99ff017da84e2 100644 --- a/net/dns/manager_default.go +++ b/net/dns/manager_default.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build !linux && !freebsd && !openbsd && !windows && !darwin +//go:build !linux && !freebsd && !openbsd && !windows && !darwin && !illumos && !solaris package dns diff --git a/net/dns/manager_solaris.go b/net/dns/manager_solaris.go new file mode 100644 index 0000000000000..1f48efb9e61a1 --- /dev/null +++ b/net/dns/manager_solaris.go @@ -0,0 +1,14 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package dns + +import ( + "tailscale.com/control/controlknobs" + "tailscale.com/health" + "tailscale.com/types/logger" +) + +func NewOSConfigurator(logf logger.Logf, health *health.Tracker, _ *controlknobs.Knobs, iface string) (OSConfigurator, error) { + return newDirectManager(logf, health), nil +} diff --git a/net/dns/resolver/tsdns.go b/net/dns/resolver/tsdns.go index 43ba0acf194f2..107740b136d54 100644 --- a/net/dns/resolver/tsdns.go +++ b/net/dns/resolver/tsdns.go @@ -384,7 +384,7 @@ func (r *Resolver) HandlePeerDNSQuery(ctx context.Context, q []byte, from netip. // but for now that's probably good enough. Later we'll // want to blend in everything from scutil --dns. fallthrough - case "linux", "freebsd", "openbsd", "illumos", "ios": + case "linux", "freebsd", "openbsd", "illumos", "solaris", "ios": nameserver, err := stubResolverForOS() if err != nil { r.logf("stubResolverForOS: %v", err) diff --git a/net/netutil/ip_forward.go b/net/netutil/ip_forward.go index 48cee68eaff88..c64a9e4269ae0 100644 --- a/net/netutil/ip_forward.go +++ b/net/netutil/ip_forward.go @@ -63,6 +63,11 @@ func CheckIPForwarding(routes []netip.Prefix, state *netmon.State) (warn, err er switch runtime.GOOS { case "dragonfly", "freebsd", "netbsd", "openbsd": return fmt.Errorf("Subnet routing and exit nodes only work with additional manual configuration on %v, and is not currently officially supported.", runtime.GOOS), nil + case "illumos", "solaris": + _, err := ipForwardingEnabledSunOS(ipv4, "") + if err != nil { + return nil, fmt.Errorf("Couldn't check system's IP forwarding configuration, subnet routing/exit nodes may not work: %w%s", err, "") + } } return nil, nil } @@ -325,3 +330,24 @@ func reversePathFilterValueLinux(iface string) (int, error) { } return v, nil } + +func ipForwardingEnabledSunOS(p protocol, iface string) (bool, error) { + var proto string + if p == ipv4 { + proto = "ipv4" + } else if p == ipv6 { + proto = "ipv6" + } else { + return false, fmt.Errorf("unknown protocol") + } + + ipadmCmd := "\"ipadm show-prop " + proto + " -p forwarding -o CURRENT -c\"" + bs, err := exec.Command("ipadm", "show-prop", proto, "-p", "forwarding", "-o", "CURRENT", "-c").Output() + if err != nil { + return false, fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", ipadmCmd, err) + } + if string(bs) != "on\n" { + return false, fmt.Errorf("IP forwarding is set to off. Subnet routes won't work. Try 'routeadm -u -e %s-forwarding'", proto) + } + return true, nil +} diff --git a/net/tstun/tstun_stub.go b/net/tstun/tstun_stub.go index 7a4f71a099fd5..3119d647cc55a 100644 --- a/net/tstun/tstun_stub.go +++ b/net/tstun/tstun_stub.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build plan9 || aix +//go:build plan9 || aix || solaris || illumos package tstun diff --git a/net/tstun/tun.go b/net/tstun/tun.go index 9f5d42ecc3269..56c66c83a27e5 100644 --- a/net/tstun/tun.go +++ b/net/tstun/tun.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build !wasm && !plan9 && !tamago && !aix +//go:build !wasm && !plan9 && !tamago && !aix && !solaris && !illumos // Package tun creates a tuntap device, working around OS-specific // quirks if necessary. diff --git a/paths/paths_unix.go b/paths/paths_unix.go index 6a2b28733a93b..50a8b7ca502f7 100644 --- a/paths/paths_unix.go +++ b/paths/paths_unix.go @@ -22,7 +22,7 @@ func init() { func statePath() string { switch runtime.GOOS { - case "linux": + case "linux", "illumos", "solaris": return "/var/lib/tailscale/tailscaled.state" case "freebsd", "openbsd": return "/var/db/tailscale/tailscaled.state" From fc8b6d9c6a6c510227b21abf00bdfd24e53ab176 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 10 Jan 2025 06:33:58 +0000 Subject: [PATCH 097/223] ipn/conf.go: add VIPServices to tailscaled configfile (#14345) Updates tailscale/corp#24795 Signed-off-by: Irbe Krumina --- ipn/conf.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ipn/conf.go b/ipn/conf.go index 1b2831b03b6c6..addeea79e4dc2 100644 --- a/ipn/conf.go +++ b/ipn/conf.go @@ -32,6 +32,8 @@ type ConfigVAlpha struct { AdvertiseRoutes []netip.Prefix `json:",omitempty"` DisableSNAT opt.Bool `json:",omitempty"` + AdvertiseServices []string `json:",omitempty"` + AppConnector *AppConnectorPrefs `json:",omitempty"` // advertise app connector; defaults to false (if nil or explicitly set to false) NetfilterMode *string `json:",omitempty"` // "on", "off", "nodivert" @@ -143,5 +145,9 @@ func (c *ConfigVAlpha) ToPrefs() (MaskedPrefs, error) { mp.AppConnector = *c.AppConnector mp.AppConnectorSet = true } + if c.AdvertiseServices != nil { + mp.AdvertiseServices = c.AdvertiseServices + mp.AdvertiseServicesSet = true + } return mp, nil } From 48a95c422ae5eb304a68ef95f4b62e14870f641a Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 10 Jan 2025 07:29:11 +0000 Subject: [PATCH 098/223] cmd/containerboot,cmd/k8s-operator: reload tailscaled config (#14342) cmd/{k8s-operator,containerboot}: reload tailscaled configfile when its contents have changed Instead of restarting the Kubernetes Operator proxies each time tailscaled config has changed, this dynamically reloads the configfile using the new reload endpoint. Older annotation based mechanism will be supported till 1.84 to ensure that proxy versions prior to 1.80 keep working with operator 1.80 and newer. Updates tailscale/tailscale#13032 Updates tailscale/corp#24795 Signed-off-by: Irbe Krumina --- cmd/containerboot/main.go | 8 +++ cmd/containerboot/tailscaled.go | 70 ++++++++++++++++++++++++++ cmd/k8s-operator/operator_test.go | 3 +- cmd/k8s-operator/proxygroup.go | 73 +++++++++++++++++++++++++--- cmd/k8s-operator/proxygroup_specs.go | 5 +- cmd/k8s-operator/proxygroup_test.go | 10 ++-- cmd/k8s-operator/sts.go | 62 ++++++++++++++++------- cmd/k8s-operator/testutils_test.go | 15 +++--- 8 files changed, 207 insertions(+), 39 deletions(-) diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 7411ea9496cfd..895be108b0090 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -359,6 +359,12 @@ authLoop: log.Fatalf("rewatching tailscaled for updates after auth: %v", err) } + // If tailscaled config was read from a mounted file, watch the file for updates and reload. + cfgWatchErrChan := make(chan error) + if cfg.TailscaledConfigFilePath != "" { + go watchTailscaledConfigChanges(ctx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan) + } + var ( startupTasksDone = false currentIPs deephash.Sum // tailscale IPs assigned to device @@ -452,6 +458,8 @@ runLoop: break runLoop case err := <-errChan: log.Fatalf("failed to read from tailscaled: %v", err) + case err := <-cfgWatchErrChan: + log.Fatalf("failed to watch tailscaled config: %v", err) case n := <-notifyChan: if n.State != nil && *n.State != ipn.Running { // Something's gone wrong and we've left the authenticated state. diff --git a/cmd/containerboot/tailscaled.go b/cmd/containerboot/tailscaled.go index d8da49b033d06..fc209247723df 100644 --- a/cmd/containerboot/tailscaled.go +++ b/cmd/containerboot/tailscaled.go @@ -13,10 +13,13 @@ import ( "log" "os" "os/exec" + "path/filepath" + "reflect" "strings" "syscall" "time" + "github.com/fsnotify/fsnotify" "tailscale.com/client/tailscale" ) @@ -166,3 +169,70 @@ func tailscaleSet(ctx context.Context, cfg *settings) error { } return nil } + +func watchTailscaledConfigChanges(ctx context.Context, path string, lc *tailscale.LocalClient, errCh chan<- error) { + var ( + tickChan <-chan time.Time + tailscaledCfgDir = filepath.Dir(path) + prevTailscaledCfg []byte + ) + w, err := fsnotify.NewWatcher() + if err != nil { + log.Printf("tailscaled config watch: failed to create fsnotify watcher, timer-only mode: %v", err) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + tickChan = ticker.C + } else { + defer w.Close() + if err := w.Add(tailscaledCfgDir); err != nil { + errCh <- fmt.Errorf("failed to add fsnotify watch: %w", err) + return + } + } + b, err := os.ReadFile(path) + if err != nil { + errCh <- fmt.Errorf("error reading configfile: %w", err) + return + } + prevTailscaledCfg = b + // kubelet mounts Secrets to Pods using a series of symlinks, one of + // which is /..data that Kubernetes recommends consumers to + // use if they need to monitor changes + // https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61 + const kubeletMountedCfg = "..data" + toWatch := filepath.Join(tailscaledCfgDir, kubeletMountedCfg) + for { + select { + case <-ctx.Done(): + return + case err := <-w.Errors: + errCh <- fmt.Errorf("watcher error: %w", err) + return + case <-tickChan: + case event := <-w.Events: + if event.Name != toWatch { + continue + } + } + b, err := os.ReadFile(path) + if err != nil { + errCh <- fmt.Errorf("error reading configfile: %w", err) + return + } + // For some proxy types the mounted volume also contains tailscaled state and other files. We + // don't want to reload config unnecessarily on unrelated changes to these files. + if reflect.DeepEqual(b, prevTailscaledCfg) { + continue + } + prevTailscaledCfg = b + log.Printf("tailscaled config watch: ensuring that config is up to date") + ok, err := lc.ReloadConfig(ctx) + if err != nil { + errCh <- fmt.Errorf("error reloading tailscaled config: %w", err) + return + } + if ok { + log.Printf("tailscaled config watch: config was reloaded") + } + } +} diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index d53269f05a77a..1998fe3bcc36d 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -1379,6 +1379,7 @@ func TestTailscaledConfigfileHash(t *testing.T) { }, }) + expectReconciled(t, sr, "default", "test") expectReconciled(t, sr, "default", "test") fullName, shortName := findGenName(t, fc, "default", "test", "svc") @@ -1389,7 +1390,7 @@ func TestTailscaledConfigfileHash(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "acf3467364b0a3ba9b8ee0dd772cb7c2f0bf585e288fa99b7fe4566009ed6041", + confFileHash: "848bff4b5ba83ac999e6984c8464e597156daba961ae045e7dbaef606d54ab5e", app: kubetypes.AppIngressProxy, } expectEqual(t, fc, expectedSTS(t, fc, o), nil) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 194474fb23b9c..a4befa039a820 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -261,17 +261,44 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return fmt.Errorf("error provisioning ConfigMap: %w", err) } } - ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash) + ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode) if err != nil { return fmt.Errorf("error generating StatefulSet spec: %w", err) } ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { + capver, err := r.capVerForPG(ctx, pg, logger) + if err != nil { + return fmt.Errorf("error getting device info: %w", err) + } + + updateSS := func(s *appsv1.StatefulSet) { + + // This is a temporary workaround to ensure that egress ProxyGroup proxies with capver older than 110 + // are restarted when tailscaled configfile contents have changed. + // This workaround ensures that: + // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. + // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. + // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid + // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a + // restarting Pod and the hash is re-added again. + // Note that this workaround is only applied to egress ProxyGroups, because ingress ProxyGroup was added after capver 110. + // Note also that the hash annotation is only set on updates, not creation, because if the StatefulSet is + // being created, there is no need for a restart. + // TODO(irbekrm): remove this in 1.84. + hash := cfgHash + if capver >= 110 { + hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] + } + s.Spec = ss.Spec + if hash != "" && pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) + } + s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences - s.Spec = ss.Spec - }); err != nil { + } + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil { return fmt.Errorf("error provisioning StatefulSet: %w", err) } mo := &metricsOpts{ @@ -564,12 +591,19 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr continue } - metadata = append(metadata, nodeMetadata{ + nm := nodeMetadata{ ordinal: ordinal, stateSecret: &secret, tsID: id, dnsName: dnsName, - }) + } + pod := &corev1.Pod{} + if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } else if err == nil { + nm.podUID = string(pod.UID) + } + metadata = append(metadata, nm) } return metadata, nil @@ -601,6 +635,29 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.Prox type nodeMetadata struct { ordinal int stateSecret *corev1.Secret - tsID tailcfg.StableNodeID - dnsName string + // podUID is the UID of the current Pod or empty if the Pod does not exist. + podUID string + tsID tailcfg.StableNodeID + dnsName string +} + +// capVerForPG returns best effort capability version for the given ProxyGroup. It attempts to find it by looking at the +// Secret + Pod for the replica with ordinal 0. Returns -1 if it is not possible to determine the capability version +// (i.e there is no Pod yet). +func (r *ProxyGroupReconciler) capVerForPG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (tailcfg.CapabilityVersion, error) { + metas, err := r.getNodeMetadata(ctx, pg) + if err != nil { + return -1, fmt.Errorf("error getting node metadata: %w", err) + } + if len(metas) == 0 { + return -1, nil + } + dev, err := deviceInfo(metas[0].stateSecret, metas[0].podUID, logger) + if err != nil { + return -1, fmt.Errorf("error getting device info: %w", err) + } + if dev == nil { + return -1, nil + } + return dev.capver, nil } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index d602be8147ed6..dc58b9f0e6ff0 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -21,7 +21,7 @@ import ( // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) (*appsv1.StatefulSet, error) { +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) @@ -53,9 +53,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa Namespace: namespace, Labels: pgLabels(pg.Name, nil), DeletionGracePeriodSeconds: ptr.To[int64](10), - Annotations: map[string]string{ - podAnnotationLastSetConfigFileHash: cfgHash, - }, } tmpl.Spec.ServiceAccountName = pg.Name tmpl.Spec.InitContainers[0].Image = image diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 6464a0b2daaa9..96ffefbed1d7d 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -29,6 +29,7 @@ import ( "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/types/ptr" + "tailscale.com/util/mak" ) const testProxyImage = "tailscale/tailscale:test" @@ -117,11 +118,11 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg, nil) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash) + expectProxyGroupResources(t, fc, pg, true, "") if expected := 1; reconciler.egressProxyGroups.Len() != expected { t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } - expectProxyGroupResources(t, fc, pg, true, initialCfgHash) + expectProxyGroupResources(t, fc, pg, true, "") keyReq := tailscale.KeyCapabilities{ Devices: tailscale.KeyDeviceCapabilities{ Create: tailscale.KeyDeviceCreateCapabilities{ @@ -378,11 +379,14 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", cfgHash) + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto") if err != nil { t.Fatal(err) } statefulSet.Annotations = defaultProxyClassAnnotations + if cfgHash != "" { + mak.Set(&statefulSet.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, cfgHash) + } if shouldExist { expectEqual(t, fc, role, nil) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index b861bdffff90d..c2b9250589936 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -437,10 +437,10 @@ func sanitizeConfigBytes(c ipn.ConfigVAlpha) string { return string(sanitizedBytes) } -// DeviceInfo returns the device ID, hostname and IPs for the Tailscale device -// that acts as an operator proxy. It retrieves info from a Kubernetes Secret -// labeled with the provided labels. -// Either of device ID, hostname and IPs can be empty string if not found in the Secret. +// DeviceInfo returns the device ID, hostname, IPs and capver for the Tailscale device that acts as an operator proxy. +// It retrieves info from a Kubernetes Secret labeled with the provided labels. Capver is cross-validated against the +// Pod to ensure that it is the currently running Pod that set the capver. If the Pod or the Secret does not exist, the +// returned capver is -1. Either of device ID, hostname and IPs can be empty string if not found in the Secret. func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) (dev *device, err error) { sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels) if err != nil { @@ -449,12 +449,14 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map if sec == nil { return dev, nil } + podUID := "" pod := new(corev1.Pod) if err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod); err != nil && !apierrors.IsNotFound(err) { - return dev, nil + return dev, err + } else if err == nil { + podUID = string(pod.ObjectMeta.UID) } - - return deviceInfo(sec, pod, logger) + return deviceInfo(sec, podUID, logger) } // device contains tailscale state of a proxy device as gathered from its tailscale state Secret. @@ -465,9 +467,10 @@ type device struct { // ingressDNSName is the L7 Ingress DNS name. In practice this will be the same value as hostname, but only set // when the device has been configured to serve traffic on it via 'tailscale serve'. ingressDNSName string + capver tailcfg.CapabilityVersion } -func deviceInfo(sec *corev1.Secret, pod *corev1.Pod, log *zap.SugaredLogger) (dev *device, err error) { +func deviceInfo(sec *corev1.Secret, podUID string, log *zap.SugaredLogger) (dev *device, err error) { id := tailcfg.StableNodeID(sec.Data[kubetypes.KeyDeviceID]) if id == "" { return dev, nil @@ -484,10 +487,12 @@ func deviceInfo(sec *corev1.Secret, pod *corev1.Pod, log *zap.SugaredLogger) (de // operator to clean up such devices. return dev, nil } + dev.ingressDNSName = dev.hostname + pcv := proxyCapVer(sec, podUID, log) + dev.capver = pcv // TODO(irbekrm): we fall back to using the hostname field to determine Ingress's hostname to ensure backwards // compatibility. In 1.82 we can remove this fallback mechanism. - dev.ingressDNSName = dev.hostname - if proxyCapVer(sec, pod, log) >= 109 { + if pcv >= 109 { dev.ingressDNSName = strings.TrimSuffix(string(sec.Data[kubetypes.KeyHTTPSEndpoint]), ".") if strings.EqualFold(dev.ingressDNSName, kubetypes.ValueNoHTTPS) { dev.ingressDNSName = "" @@ -584,8 +589,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: "true", }) } - // Configure containeboot to run tailscaled with a configfile read from the state Secret. - mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash) configVolume := corev1.Volume{ Name: "tailscaledconfig", @@ -655,6 +658,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }, }) } + + dev, err := a.DeviceInfo(ctx, sts.ChildResourceLabels, logger) + if err != nil { + return nil, fmt.Errorf("failed to get device info: %w", err) + } + app, err := appInfoForProxy(sts) if err != nil { // No need to error out if now or in future we end up in a @@ -673,7 +682,25 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { + // This is a temporary workaround to ensure that proxies with capver older than 110 + // are restarted when tailscaled configfile contents have changed. + // This workaround ensures that: + // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. + // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. + // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid + // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a + // restarting Pod and the hash is re-added again. + // Note that the hash annotation is only set on updates not creation, because if the StatefulSet is + // being created, there is no need for a restart. + // TODO(irbekrm): remove this in 1.84. + hash := tsConfigHash + if dev != nil && dev.capver >= 110 { + hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] + } s.Spec = ss.Spec + if hash != "" { + mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) + } s.ObjectMeta.Labels = ss.Labels s.ObjectMeta.Annotations = ss.Annotations } @@ -1112,10 +1139,11 @@ func isValidFirewallMode(m string) bool { return m == "auto" || m == "nftables" || m == "iptables" } -// proxyCapVer accepts a proxy state Secret and a proxy Pod returns the capability version of a proxy Pod. -// This is best effort - if the capability version can not (currently) be determined, it returns -1. -func proxyCapVer(sec *corev1.Secret, pod *corev1.Pod, log *zap.SugaredLogger) tailcfg.CapabilityVersion { - if sec == nil || pod == nil { +// proxyCapVer accepts a proxy state Secret and UID of the current proxy Pod returns the capability version of the +// tailscale running in that Pod. This is best effort - if the capability version can not (currently) be determined, it +// returns -1. +func proxyCapVer(sec *corev1.Secret, podUID string, log *zap.SugaredLogger) tailcfg.CapabilityVersion { + if sec == nil || podUID == "" { return tailcfg.CapabilityVersion(-1) } if len(sec.Data[kubetypes.KeyCapVer]) == 0 || len(sec.Data[kubetypes.KeyPodUID]) == 0 { @@ -1126,7 +1154,7 @@ func proxyCapVer(sec *corev1.Secret, pod *corev1.Pod, log *zap.SugaredLogger) ta log.Infof("[unexpected]: unexpected capability version in proxy's state Secret, expected an integer, got %q", string(sec.Data[kubetypes.KeyCapVer])) return tailcfg.CapabilityVersion(-1) } - if !strings.EqualFold(string(pod.ObjectMeta.UID), string(sec.Data[kubetypes.KeyPodUID])) { + if !strings.EqualFold(podUID, string(sec.Data[kubetypes.KeyPodUID])) { return tailcfg.CapabilityVersion(-1) } return tailcfg.CapabilityVersion(capVer) diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index d43e75b1e3cbc..277bd16dfbc47 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -95,7 +95,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef Value: "true", }) } - annots := make(map[string]string) + var annots map[string]string var volumes []corev1.Volume volumes = []corev1.Volume{ { @@ -113,7 +113,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef MountPath: "/etc/tsconfig", }} if opts.confFileHash != "" { - annots["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash + mak.Set(&annots, "tailscale.com/operator-last-set-config-file-hash", opts.confFileHash) } if opts.firewallMode != "" { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ @@ -122,13 +122,13 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef }) } if opts.tailnetTargetIP != "" { - annots["tailscale.com/operator-last-set-ts-tailnet-target-ip"] = opts.tailnetTargetIP + mak.Set(&annots, "tailscale.com/operator-last-set-ts-tailnet-target-ip", opts.tailnetTargetIP) tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_TAILNET_TARGET_IP", Value: opts.tailnetTargetIP, }) } else if opts.tailnetTargetFQDN != "" { - annots["tailscale.com/operator-last-set-ts-tailnet-target-fqdn"] = opts.tailnetTargetFQDN + mak.Set(&annots, "tailscale.com/operator-last-set-ts-tailnet-target-fqdn", opts.tailnetTargetFQDN) tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_TAILNET_TARGET_FQDN", Value: opts.tailnetTargetFQDN, @@ -139,13 +139,13 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef Name: "TS_DEST_IP", Value: opts.clusterTargetIP, }) - annots["tailscale.com/operator-last-set-cluster-ip"] = opts.clusterTargetIP + mak.Set(&annots, "tailscale.com/operator-last-set-cluster-ip", opts.clusterTargetIP) } else if opts.clusterTargetDNS != "" { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_EXPERIMENTAL_DEST_DNS_NAME", Value: opts.clusterTargetDNS, }) - annots["tailscale.com/operator-last-set-cluster-dns-name"] = opts.clusterTargetDNS + mak.Set(&annots, "tailscale.com/operator-last-set-cluster-dns-name", opts.clusterTargetDNS) } if opts.serveConfig != nil { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ @@ -794,6 +794,9 @@ func (c *fakeTSClient) Deleted() []string { // change to the configfile contents). func removeHashAnnotation(sts *appsv1.StatefulSet) { delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash) + if len(sts.Spec.Template.Annotations) == 0 { + sts.Spec.Template.Annotations = nil + } } func removeTargetPortsFromSvc(svc *corev1.Service) { From 77017bae59c2b89d80a2428524abd53042948e9c Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 10 Jan 2025 07:31:28 +0000 Subject: [PATCH 099/223] cmd/containerboot: load containerboot serve config that does not contain HTTPS endpoint in tailnets with HTTPS disabled (#14538) cmd/containerboot: load containerboot serve config that does not contain HTTPS endpoint in tailnets with HTTPS disabled Fixes an issue where, if a tailnet has HTTPS disabled, no serve config set via TS_SERVE_CONFIG was loaded, even if it does not contain an HTTPS endpoint. Now for tailnets with HTTPS disabled serve config provided to containerboot is considered invalid (and therefore not loaded) only if there is an HTTPS endpoint defined in the config. Fixes tailscale/tailscale#14495 Signed-off-by: Irbe Krumina --- cmd/containerboot/serve.go | 34 ++-- cmd/containerboot/serve_test.go | 267 ++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+), 11 deletions(-) create mode 100644 cmd/containerboot/serve_test.go diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 14c7f00d7450f..1729e65b5594c 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -68,7 +68,6 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) { continue } - validateHTTPSServe(certDomain, sc) if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil { log.Fatalf("serve proxy: error updating serve config: %v", err) } @@ -88,27 +87,34 @@ func certDomainFromNetmap(nm *netmap.NetworkMap) string { return nm.DNS.CertDomains[0] } -func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc *tailscale.LocalClient) error { - // TODO(irbekrm): This means that serve config that does not expose HTTPS endpoint will not be set for a tailnet - // that does not have HTTPS enabled. We probably want to fix this. - if certDomain == kubetypes.ValueNoHTTPS { +// localClient is a subset of tailscale.LocalClient that can be mocked for testing. +type localClient interface { + SetServeConfig(context.Context, *ipn.ServeConfig) error +} + +func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error { + if !isValidHTTPSConfig(certDomain, sc) { return nil } log.Printf("serve proxy: applying serve config") return lc.SetServeConfig(ctx, sc) } -func validateHTTPSServe(certDomain string, sc *ipn.ServeConfig) { - if certDomain != kubetypes.ValueNoHTTPS || !hasHTTPSEndpoint(sc) { - return - } - log.Printf( - `serve proxy: this node is configured as a proxy that exposes an HTTPS endpoint to tailnet, +func isValidHTTPSConfig(certDomain string, sc *ipn.ServeConfig) bool { + if certDomain == kubetypes.ValueNoHTTPS && hasHTTPSEndpoint(sc) { + log.Printf( + `serve proxy: this node is configured as a proxy that exposes an HTTPS endpoint to tailnet, (perhaps a Kubernetes operator Ingress proxy) but it is not able to issue TLS certs, so this will likely not work. To make it work, ensure that HTTPS is enabled for your tailnet, see https://tailscale.com/kb/1153/enabling-https for more details.`) + return false + } + return true } func hasHTTPSEndpoint(cfg *ipn.ServeConfig) bool { + if cfg == nil { + return false + } for _, tcpCfg := range cfg.TCP { if tcpCfg.HTTPS { return true @@ -127,6 +133,12 @@ func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) { if err != nil { return nil, err } + // Serve config can be provided by users as well as the Kubernetes Operator (for its proxies). User-provided + // config could be empty for reasons. + if len(j) == 0 { + log.Printf("serve proxy: serve config file is empty, skipping") + return nil, nil + } j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain)) var sc ipn.ServeConfig if err := json.Unmarshal(j, &sc); err != nil { diff --git a/cmd/containerboot/serve_test.go b/cmd/containerboot/serve_test.go new file mode 100644 index 0000000000000..4563c52fcad69 --- /dev/null +++ b/cmd/containerboot/serve_test.go @@ -0,0 +1,267 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/client/tailscale" + "tailscale.com/ipn" + "tailscale.com/kube/kubetypes" +) + +func TestUpdateServeConfig(t *testing.T) { + tests := []struct { + name string + sc *ipn.ServeConfig + certDomain string + wantCall bool + }{ + { + name: "no_https_no_cert_domain", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + }, + certDomain: kubetypes.ValueNoHTTPS, // tailnet has HTTPS disabled + wantCall: true, // should set serve config as it doesn't have HTTPS endpoints + }, + { + name: "https_with_cert_domain", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://10.0.1.100:8080"}, + }, + }, + }, + }, + certDomain: "test-node.tailnet.ts.net", + wantCall: true, + }, + { + name: "https_without_cert_domain", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + }, + }, + certDomain: kubetypes.ValueNoHTTPS, + wantCall: false, // incorrect configuration- should not set serve config + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeLC := &fakeLocalClient{} + err := updateServeConfig(context.Background(), tt.sc, tt.certDomain, fakeLC) + if err != nil { + t.Errorf("updateServeConfig() error = %v", err) + } + if fakeLC.setServeCalled != tt.wantCall { + t.Errorf("SetServeConfig() called = %v, want %v", fakeLC.setServeCalled, tt.wantCall) + } + }) + } +} + +func TestReadServeConfig(t *testing.T) { + tests := []struct { + name string + gotSC string + certDomain string + wantSC *ipn.ServeConfig + wantErr bool + }{ + { + name: "empty_file", + }, + { + name: "valid_config_with_cert_domain_placeholder", + gotSC: `{ + "TCP": { + "443": { + "HTTPS": true + } + }, + "Web": { + "${TS_CERT_DOMAIN}:443": { + "Handlers": { + "/api": { + "Proxy": "https://10.2.3.4/api" + }}}}}`, + certDomain: "example.com", + wantSC: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + HTTPS: true, + }, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + ipn.HostPort("example.com:443"): { + Handlers: map[string]*ipn.HTTPHandler{ + "/api": { + Proxy: "https://10.2.3.4/api", + }, + }, + }, + }, + }, + }, + { + name: "valid_config_for_http_proxy", + gotSC: `{ + "TCP": { + "80": { + "HTTP": true + } + }}`, + wantSC: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: { + HTTP: true, + }, + }, + }, + }, + { + name: "config_without_cert_domain", + gotSC: `{ + "TCP": { + "443": { + "HTTPS": true + } + }, + "Web": { + "localhost:443": { + "Handlers": { + "/api": { + "Proxy": "https://10.2.3.4/api" + }}}}}`, + certDomain: "", + wantErr: false, + wantSC: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + HTTPS: true, + }, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + ipn.HostPort("localhost:443"): { + Handlers: map[string]*ipn.HTTPHandler{ + "/api": { + Proxy: "https://10.2.3.4/api", + }, + }, + }, + }, + }, + }, + { + name: "invalid_json", + gotSC: "invalid json", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "serve-config.json") + if err := os.WriteFile(path, []byte(tt.gotSC), 0644); err != nil { + t.Fatal(err) + } + + got, err := readServeConfig(path, tt.certDomain) + if (err != nil) != tt.wantErr { + t.Errorf("readServeConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !cmp.Equal(got, tt.wantSC) { + t.Errorf("readServeConfig() diff (-got +want):\n%s", cmp.Diff(got, tt.wantSC)) + } + }) + } +} + +type fakeLocalClient struct { + *tailscale.LocalClient + setServeCalled bool +} + +func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConfig) error { + m.setServeCalled = true + return nil +} + +func TestHasHTTPSEndpoint(t *testing.T) { + tests := []struct { + name string + cfg *ipn.ServeConfig + want bool + }{ + { + name: "nil_config", + cfg: nil, + want: false, + }, + { + name: "empty_config", + cfg: &ipn.ServeConfig{}, + want: false, + }, + { + name: "no_https_endpoints", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: { + HTTPS: false, + }, + }, + }, + want: false, + }, + { + name: "has_https_endpoint", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + HTTPS: true, + }, + }, + }, + want: true, + }, + { + name: "mixed_endpoints", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTPS: false}, + 443: {HTTPS: true}, + }, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hasHTTPSEndpoint(tt.cfg) + if got != tt.want { + t.Errorf("hasHTTPSEndpoint() = %v, want %v", got, tt.want) + } + }) + } +} From a841f9d87be9490c0c94bcfcbe62549a2d2dfefd Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 10 Jan 2025 07:59:31 -0800 Subject: [PATCH 100/223] go.mod: bump some deps Most of these are effectively no-ops, but appease security scanners. At least one (x/net for x/net/html) only affect builds from the open source repo, since we already had it updated in our "corp" repo: golang.org/x/net v0.33.1-0.20241230221519-e9d95ba163f7 ... and that's where we do the official releases from. e.g. tailscale.io % go install tailscale.com/cmd/tailscaled tailscale.io % go version -m ~/go/bin/tailscaled | grep x/net dep golang.org/x/net v0.33.1-0.20241230221519-e9d95ba163f7 h1:raAbYgZplPuXQ6s7jPklBFBmmLh6LjnFaJdp3xR2ljY= tailscale.io % cd ../tailscale.com tailscale.com % go install tailscale.com/cmd/tailscaled tailscale.com % go version -m ~/go/bin/tailscaled | grep x/net dep golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= Updates #8043 Updates #14599 Change-Id: I6e238cef62ca22444145a5313554aab8709b33c9 Signed-off-by: Brad Fitzpatrick --- go.mod | 21 ++++++++++----------- go.sum | 43 ++++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 62a431d189205..c28338caf22b9 100644 --- a/go.mod +++ b/go.mod @@ -93,18 +93,18 @@ require ( github.com/u-root/u-root v0.12.0 github.com/vishvananda/netns v0.0.4 go.uber.org/zap v1.27.0 - go4.org/mem v0.0.0-20220726221520-4f986261bf13 + go4.org/mem v0.0.0-20240501181205-ae6ca9944745 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba - golang.org/x/crypto v0.31.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/mod v0.19.0 - golang.org/x/net v0.33.0 - golang.org/x/oauth2 v0.16.0 + golang.org/x/crypto v0.32.0 + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 + golang.org/x/mod v0.22.0 + golang.org/x/net v0.34.0 + golang.org/x/oauth2 v0.25.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab - golang.org/x/term v0.27.0 - golang.org/x/time v0.5.0 - golang.org/x/tools v0.23.0 + golang.org/x/term v0.28.0 + golang.org/x/time v0.9.0 + golang.org/x/tools v0.29.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wireguard/windows v0.5.3 gopkg.in/square/go-jose.v2 v2.6.0 @@ -385,10 +385,9 @@ require ( gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect - golang.org/x/image v0.18.0 // indirect + golang.org/x/image v0.23.0 // indirect golang.org/x/text v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index efb20e63a82f2..be8f291a41f61 100644 --- a/go.sum +++ b/go.sum @@ -1045,8 +1045,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go4.org/mem v0.0.0-20220726221520-4f986261bf13 h1:CbZeCBZ0aZj8EfVgnqQcYZgf0lpZ3H9rmp5nkDTAst8= -go4.org/mem v0.0.0-20220726221520-4f986261bf13/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= +go4.org/mem v0.0.0-20240501181205-ae6ca9944745 h1:Tl++JLUCe4sxGu8cTpDzRLd3tN7US4hOxG5YpKCzkek= +go4.org/mem v0.0.0-20240501181205-ae6ca9944745/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1059,8 +1059,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1071,16 +1071,16 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8= golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68= +golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1108,8 +1108,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1149,16 +1149,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1242,8 +1242,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1251,7 +1251,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -1260,8 +1259,8 @@ golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1326,8 +1325,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1362,8 +1361,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= From cd795d8a7f47f06fe3cc1aaf67a7a6cd26509649 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Fri, 10 Jan 2025 12:23:51 -0600 Subject: [PATCH 101/223] prober: support filtering regions by region ID in addition to code Updates tailscale/corp#25758 Signed-off-by: Percy Wegmann --- cmd/derpprobe/derpprobe.go | 6 +++--- prober/derp.go | 14 +++++++------- prober/derp_test.go | 24 ++++++++++++------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 62b7d47a4e6bf..6e8c603b943d7 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -32,7 +32,7 @@ var ( bwTUNIPv4Address = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP; we will use a /30 subnet including this IP address") qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second") qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings") - regionCode = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed") + regionCodeOrID = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed") ) func main() { @@ -52,8 +52,8 @@ func main() { if *bwInterval > 0 { opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address)) } - if *regionCode != "" { - opts = append(opts, prober.WithRegion(*regionCode)) + if *regionCodeOrID != "" { + opts = append(opts, prober.WithRegionCodeOrID(*regionCodeOrID)) } dp, err := prober.DERP(p, *derpMapURL, opts...) if err != nil { diff --git a/prober/derp.go b/prober/derp.go index 6bad3584579a8..f405549ff8532 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -60,8 +60,8 @@ type derpProber struct { qdPacketsPerSecond int // in packets per second qdPacketTimeout time.Duration - // Optionally restrict probes to a single regionCode. - regionCode string + // Optionally restrict probes to a single regionCodeOrID. + regionCodeOrID string // Probe class for fetching & updating the DERP map. ProbeMap ProbeClass @@ -135,11 +135,11 @@ func WithTLSProbing(interval time.Duration) DERPOpt { } } -// WithRegion restricts probing to the specified region identified by its code -// (e.g. "lax"). This is case sensitive. -func WithRegion(regionCode string) DERPOpt { +// WithRegionCodeOrID restricts probing to the specified region identified by its code +// (e.g. "lax") or its id (e.g. "17"). This is case sensitive. +func WithRegionCodeOrID(regionCode string) DERPOpt { return func(d *derpProber) { - d.regionCode = regionCode + d.regionCodeOrID = regionCode } } @@ -598,7 +598,7 @@ func (d *derpProber) ProbeUDP(ipaddr string, port int) ProbeClass { } func (d *derpProber) skipRegion(region *tailcfg.DERPRegion) bool { - return d.regionCode != "" && region.RegionCode != d.regionCode + return d.regionCodeOrID != "" && region.RegionCode != d.regionCodeOrID && strconv.Itoa(region.RegionID) != d.regionCodeOrID } func derpProbeUDP(ctx context.Context, ipStr string, port int) error { diff --git a/prober/derp_test.go b/prober/derp_test.go index c084803e94f6a..93b8d760b3f18 100644 --- a/prober/derp_test.go +++ b/prober/derp_test.go @@ -71,17 +71,17 @@ func TestDerpProber(t *testing.T) { clk := newFakeTime() p := newForTest(clk.Now, clk.NewTicker) dp := &derpProber{ - p: p, - derpMapURL: srv.URL, - tlsInterval: time.Second, - tlsProbeFn: func(_ string) ProbeClass { return FuncProbe(func(context.Context) error { return nil }) }, - udpInterval: time.Second, - udpProbeFn: func(_ string, _ int) ProbeClass { return FuncProbe(func(context.Context) error { return nil }) }, - meshInterval: time.Second, - meshProbeFn: func(_, _ string) ProbeClass { return FuncProbe(func(context.Context) error { return nil }) }, - nodes: make(map[string]*tailcfg.DERPNode), - probes: make(map[string]*Probe), - regionCode: "zero", + p: p, + derpMapURL: srv.URL, + tlsInterval: time.Second, + tlsProbeFn: func(_ string) ProbeClass { return FuncProbe(func(context.Context) error { return nil }) }, + udpInterval: time.Second, + udpProbeFn: func(_ string, _ int) ProbeClass { return FuncProbe(func(context.Context) error { return nil }) }, + meshInterval: time.Second, + meshProbeFn: func(_, _ string) ProbeClass { return FuncProbe(func(context.Context) error { return nil }) }, + nodes: make(map[string]*tailcfg.DERPNode), + probes: make(map[string]*Probe), + regionCodeOrID: "zero", } if err := dp.probeMapFn(context.Background()); err != nil { t.Errorf("unexpected probeMapFn() error: %s", err) @@ -129,7 +129,7 @@ func TestDerpProber(t *testing.T) { } // Stop filtering regions. - dp.regionCode = "" + dp.regionCodeOrID = "" if err := dp.probeMapFn(context.Background()); err != nil { t.Errorf("unexpected probeMapFn() error: %s", err) } From 2af255790dd561ddbca8e1b8264b2b0e7f5f8976 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:45:04 -0500 Subject: [PATCH 102/223] ipn/ipnlocal: add VIPServices hash to return body of vip-services c2n endpoint This commit updates the return body of c2n endpoint /vip-services to keep hash generation logic on client side. Updates tailscale/corp#24510 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- ipn/ipnlocal/c2n.go | 6 +++++- ipn/ipnlocal/local.go | 5 ++--- tailcfg/c2ntypes.go | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ipn/ipnlocal/c2n.go b/ipn/ipnlocal/c2n.go index f3a4a3a3d2b29..04f91954ff441 100644 --- a/ipn/ipnlocal/c2n.go +++ b/ipn/ipnlocal/c2n.go @@ -274,8 +274,12 @@ func handleC2NSetNetfilterKind(b *LocalBackend, w http.ResponseWriter, r *http.R func handleC2NVIPServicesGet(b *LocalBackend, w http.ResponseWriter, r *http.Request) { b.logf("c2n: GET /vip-services received") + var res tailcfg.C2NVIPServicesResponse + res.VIPServices = b.VIPServices() + res.ServicesHash = b.vipServiceHash(res.VIPServices) - json.NewEncoder(w).Encode(b.VIPServices()) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(res) } func handleC2NUpdateGet(b *LocalBackend, w http.ResponseWriter, r *http.Request) { diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index ad3bbaef32d24..088f1ef7588e5 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -5022,7 +5022,7 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip } hi.SSH_HostKeys = sshHostKeys - hi.ServicesHash = b.vipServiceHashLocked(prefs) + hi.ServicesHash = b.vipServiceHash(b.vipServicesFromPrefsLocked(prefs)) // The Hostinfo.WantIngress field tells control whether this node wants to // be wired up for ingress connections. If harmless if it's accidentally @@ -7661,8 +7661,7 @@ func (b *LocalBackend) VIPServices() []*tailcfg.VIPService { return b.vipServicesFromPrefsLocked(b.pm.CurrentPrefs()) } -func (b *LocalBackend) vipServiceHashLocked(prefs ipn.PrefsView) string { - services := b.vipServicesFromPrefsLocked(prefs) +func (b *LocalBackend) vipServiceHash(services []*tailcfg.VIPService) string { if len(services) == 0 { return "" } diff --git a/tailcfg/c2ntypes.go b/tailcfg/c2ntypes.go index 54efb736e6db9..66f95785c4a83 100644 --- a/tailcfg/c2ntypes.go +++ b/tailcfg/c2ntypes.go @@ -102,3 +102,18 @@ type C2NTLSCertInfo struct { // TODO(bradfitz): add fields for whether an ACME fetch is currently in // process and when it started, etc. } + +// C2NVIPServicesResponse is the response (from node to control) from the +// /vip-services handler. +// +// It returns the list of VIPServices that the node is currently serving with +// their port info and whether they are active or not. It also returns a hash of +// the response to allow the control server to detect changes. +type C2NVIPServicesResponse struct { + // VIPServices is the list of VIP services that the node is currently serving. + VIPServices []*VIPService `json:",omitempty"` + + // ServicesHash is the hash of VIPServices to allow the control server to detect + // changes. This value matches what is reported in latest [Hostinfo.ServicesHash]. + ServicesHash string +} From 5fdb4f83ad23f0ee7a9dc08ecc2a0ceeabd81fc3 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Fri, 10 Jan 2025 17:21:39 -0800 Subject: [PATCH 103/223] Dockerfile: bump base alpine image (#14604) Bump the versions to pick up some CVE patches. They don't affect us, but customer scanners will complain. Updates #cleanup Signed-off-by: Andrew Lytvynov --- ALPINE.txt | 2 +- Dockerfile | 2 +- Dockerfile.base | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ALPINE.txt b/ALPINE.txt index 55b698c77f5d2..f29702326bba5 100644 --- a/ALPINE.txt +++ b/ALPINE.txt @@ -1 +1 @@ -3.18 \ No newline at end of file +3.21 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4ad3d88d9577a..7a5dbce5a9eeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,7 @@ RUN GOARCH=$TARGETARCH go install -ldflags="\ -X tailscale.com/version.gitCommitStamp=$VERSION_GIT_HASH" \ -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot -FROM alpine:3.18 +FROM alpine:3.21 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables COPY --from=build-env /go/bin/* /usr/local/bin/ diff --git a/Dockerfile.base b/Dockerfile.base index eb4f0a02a8b75..5186746a42529 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,5 +1,5 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause -FROM alpine:3.18 +FROM alpine:3.21 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables iputils From 69b90742fe852cb83e0106b8f4fe36976c3ed3c8 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 12 Jan 2025 19:14:04 -0800 Subject: [PATCH 104/223] util/uniq,types/lazy,*: delete code that's now in Go std sync.OnceValue and slices.Compact were both added in Go 1.21. cmp.Or was added in Go 1.22. Updates #8632 Updates #11058 Change-Id: I89ba4c404f40188e1f8a9566c8aaa049be377754 Signed-off-by: Brad Fitzpatrick --- cmd/k8s-operator/depaware.txt | 1 - cmd/tailscaled/depaware.txt | 1 - ipn/ipnlocal/local.go | 3 +- ipn/localapi/debugderp.go | 23 +++---- types/lazy/lazy.go | 35 ----------- types/lazy/sync_test.go | 43 -------------- util/uniq/slice.go | 62 ------------------- util/uniq/slice_test.go | 102 -------------------------------- version/print.go | 5 +- version/prop.go | 3 +- version/version.go | 3 +- wgengine/magicsock/magicsock.go | 4 +- wgengine/pendopen.go | 4 +- 13 files changed, 18 insertions(+), 271 deletions(-) delete mode 100644 util/uniq/slice.go delete mode 100644 util/uniq/slice_test.go diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 0e42fe2b60dd0..3489e5a60c3eb 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -817,7 +817,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/util/systemd from tailscale.com/control/controlclient+ tailscale.com/util/testenv from tailscale.com/control/controlclient+ tailscale.com/util/truncate from tailscale.com/logtail - tailscale.com/util/uniq from tailscale.com/ipn/ipnlocal+ tailscale.com/util/usermetric from tailscale.com/health+ tailscale.com/util/vizerror from tailscale.com/tailcfg+ đŸ’Ŗ tailscale.com/util/winutil from tailscale.com/clientupdate+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 749c3f3100b7d..4dad47421e87d 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -406,7 +406,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/systemd from tailscale.com/control/controlclient+ tailscale.com/util/testenv from tailscale.com/ipn/ipnlocal+ tailscale.com/util/truncate from tailscale.com/logtail - tailscale.com/util/uniq from tailscale.com/ipn/ipnlocal+ tailscale.com/util/usermetric from tailscale.com/health+ tailscale.com/util/vizerror from tailscale.com/tailcfg+ đŸ’Ŗ tailscale.com/util/winutil from tailscale.com/clientupdate+ diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 088f1ef7588e5..3a2a22c581825 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -111,7 +111,6 @@ import ( "tailscale.com/util/syspolicy/rsop" "tailscale.com/util/systemd" "tailscale.com/util/testenv" - "tailscale.com/util/uniq" "tailscale.com/util/usermetric" "tailscale.com/version" "tailscale.com/version/distro" @@ -3346,7 +3345,7 @@ func (b *LocalBackend) clearMachineKeyLocked() error { // incoming packet. func (b *LocalBackend) setTCPPortsIntercepted(ports []uint16) { slices.Sort(ports) - uniq.ModifySlice(&ports) + ports = slices.Compact(ports) var f func(uint16) bool switch len(ports) { case 0: diff --git a/ipn/localapi/debugderp.go b/ipn/localapi/debugderp.go index 85eb031e6fd0a..dbdf5cf794a8e 100644 --- a/ipn/localapi/debugderp.go +++ b/ipn/localapi/debugderp.go @@ -4,6 +4,7 @@ package localapi import ( + "cmp" "context" "crypto/tls" "encoding/json" @@ -81,7 +82,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { client *http.Client = http.DefaultClient ) checkConn := func(derpNode *tailcfg.DERPNode) bool { - port := firstNonzero(derpNode.DERPPort, 443) + port := cmp.Or(derpNode.DERPPort, 443) var ( hasIPv4 bool @@ -89,7 +90,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { ) // Check IPv4 first - addr := net.JoinHostPort(firstNonzero(derpNode.IPv4, derpNode.HostName), strconv.Itoa(port)) + addr := net.JoinHostPort(cmp.Or(derpNode.IPv4, derpNode.HostName), strconv.Itoa(port)) conn, err := dialer.DialContext(ctx, "tcp4", addr) if err != nil { st.Errors = append(st.Errors, fmt.Sprintf("Error connecting to node %q @ %q over IPv4: %v", derpNode.HostName, addr, err)) @@ -98,7 +99,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { // Upgrade to TLS and verify that works properly. tlsConn := tls.Client(conn, &tls.Config{ - ServerName: firstNonzero(derpNode.CertName, derpNode.HostName), + ServerName: cmp.Or(derpNode.CertName, derpNode.HostName), }) if err := tlsConn.HandshakeContext(ctx); err != nil { st.Errors = append(st.Errors, fmt.Sprintf("Error upgrading connection to node %q @ %q to TLS over IPv4: %v", derpNode.HostName, addr, err)) @@ -108,7 +109,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { } // Check IPv6 - addr = net.JoinHostPort(firstNonzero(derpNode.IPv6, derpNode.HostName), strconv.Itoa(port)) + addr = net.JoinHostPort(cmp.Or(derpNode.IPv6, derpNode.HostName), strconv.Itoa(port)) conn, err = dialer.DialContext(ctx, "tcp6", addr) if err != nil { st.Errors = append(st.Errors, fmt.Sprintf("Error connecting to node %q @ %q over IPv6: %v", derpNode.HostName, addr, err)) @@ -117,7 +118,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { // Upgrade to TLS and verify that works properly. tlsConn := tls.Client(conn, &tls.Config{ - ServerName: firstNonzero(derpNode.CertName, derpNode.HostName), + ServerName: cmp.Or(derpNode.CertName, derpNode.HostName), // TODO(andrew-d): we should print more // detailed failure information on if/why TLS // verification fails @@ -166,7 +167,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { addr = addrs[0] } - addrPort := netip.AddrPortFrom(addr, uint16(firstNonzero(derpNode.STUNPort, 3478))) + addrPort := netip.AddrPortFrom(addr, uint16(cmp.Or(derpNode.STUNPort, 3478))) txID := stun.NewTxID() req := stun.Request(txID) @@ -292,13 +293,3 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { // issued in the first place, tell them specifically that the // cert is bad not just that the connection failed. } - -func firstNonzero[T comparable](items ...T) T { - var zero T - for _, item := range items { - if item != zero { - return item - } - } - return zero -} diff --git a/types/lazy/lazy.go b/types/lazy/lazy.go index 43325512d9cb0..c29a03db4452a 100644 --- a/types/lazy/lazy.go +++ b/types/lazy/lazy.go @@ -120,41 +120,6 @@ func (z *SyncValue[T]) PeekErr() (v T, err error, ok bool) { return zero, nil, false } -// SyncFunc wraps a function to make it lazy. -// -// The returned function calls fill the first time it's called, and returns -// fill's result on every subsequent call. -// -// The returned function is safe for concurrent use. -func SyncFunc[T any](fill func() T) func() T { - var ( - once sync.Once - v T - ) - return func() T { - once.Do(func() { v = fill() }) - return v - } -} - -// SyncFuncErr wraps a function to make it lazy. -// -// The returned function calls fill the first time it's called, and returns -// fill's results on every subsequent call. -// -// The returned function is safe for concurrent use. -func SyncFuncErr[T any](fill func() (T, error)) func() (T, error) { - var ( - once sync.Once - v T - err error - ) - return func() (T, error) { - once.Do(func() { v, err = fill() }) - return v, err - } -} - // TB is a subset of testing.TB that we use to set up test helpers. // It's defined here to avoid pulling in the testing package. type TB interface { diff --git a/types/lazy/sync_test.go b/types/lazy/sync_test.go index 5578eee0cfed9..4d1278253955b 100644 --- a/types/lazy/sync_test.go +++ b/types/lazy/sync_test.go @@ -354,46 +354,3 @@ func TestSyncValueSetForTest(t *testing.T) { }) } } - -func TestSyncFunc(t *testing.T) { - f := SyncFunc(fortyTwo) - - n := int(testing.AllocsPerRun(1000, func() { - got := f() - if got != 42 { - t.Fatalf("got %v; want 42", got) - } - })) - if n != 0 { - t.Errorf("allocs = %v; want 0", n) - } -} - -func TestSyncFuncErr(t *testing.T) { - f := SyncFuncErr(func() (int, error) { - return 42, nil - }) - n := int(testing.AllocsPerRun(1000, func() { - got, err := f() - if got != 42 || err != nil { - t.Fatalf("got %v, %v; want 42, nil", got, err) - } - })) - if n != 0 { - t.Errorf("allocs = %v; want 0", n) - } - - wantErr := errors.New("test error") - f = SyncFuncErr(func() (int, error) { - return 0, wantErr - }) - n = int(testing.AllocsPerRun(1000, func() { - got, err := f() - if got != 0 || err != wantErr { - t.Fatalf("got %v, %v; want 0, %v", got, err, wantErr) - } - })) - if n != 0 { - t.Errorf("allocs = %v; want 0", n) - } -} diff --git a/util/uniq/slice.go b/util/uniq/slice.go deleted file mode 100644 index 4ab933a9d82d1..0000000000000 --- a/util/uniq/slice.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -// Package uniq provides removal of adjacent duplicate elements in slices. -// It is similar to the unix command uniq. -package uniq - -// ModifySlice removes adjacent duplicate elements from the given slice. It -// adjusts the length of the slice appropriately and zeros the tail. -// -// ModifySlice does O(len(*slice)) operations. -func ModifySlice[E comparable](slice *[]E) { - // Remove duplicates - dst := 0 - for i := 1; i < len(*slice); i++ { - if (*slice)[i] == (*slice)[dst] { - continue - } - dst++ - (*slice)[dst] = (*slice)[i] - } - - // Zero out the elements we removed at the end of the slice - end := dst + 1 - var zero E - for i := end; i < len(*slice); i++ { - (*slice)[i] = zero - } - - // Truncate the slice - if end < len(*slice) { - *slice = (*slice)[:end] - } -} - -// ModifySliceFunc is the same as ModifySlice except that it allows using a -// custom comparison function. -// -// eq should report whether the two provided elements are equal. -func ModifySliceFunc[E any](slice *[]E, eq func(i, j E) bool) { - // Remove duplicates - dst := 0 - for i := 1; i < len(*slice); i++ { - if eq((*slice)[dst], (*slice)[i]) { - continue - } - dst++ - (*slice)[dst] = (*slice)[i] - } - - // Zero out the elements we removed at the end of the slice - end := dst + 1 - var zero E - for i := end; i < len(*slice); i++ { - (*slice)[i] = zero - } - - // Truncate the slice - if end < len(*slice) { - *slice = (*slice)[:end] - } -} diff --git a/util/uniq/slice_test.go b/util/uniq/slice_test.go deleted file mode 100644 index 564fc08660332..0000000000000 --- a/util/uniq/slice_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package uniq_test - -import ( - "reflect" - "strconv" - "testing" - - "tailscale.com/util/uniq" -) - -func runTests(t *testing.T, cb func(*[]uint32)) { - tests := []struct { - // Use uint32 to be different from an int-typed slice index - in []uint32 - want []uint32 - }{ - {in: []uint32{0, 1, 2}, want: []uint32{0, 1, 2}}, - {in: []uint32{0, 1, 2, 2}, want: []uint32{0, 1, 2}}, - {in: []uint32{0, 0, 1, 2}, want: []uint32{0, 1, 2}}, - {in: []uint32{0, 1, 0, 2}, want: []uint32{0, 1, 0, 2}}, - {in: []uint32{0}, want: []uint32{0}}, - {in: []uint32{0, 0}, want: []uint32{0}}, - {in: []uint32{}, want: []uint32{}}, - } - - for _, test := range tests { - in := make([]uint32, len(test.in)) - copy(in, test.in) - cb(&test.in) - if !reflect.DeepEqual(test.in, test.want) { - t.Errorf("uniq.Slice(%v) = %v, want %v", in, test.in, test.want) - } - start := len(test.in) - test.in = test.in[:cap(test.in)] - for i := start; i < len(in); i++ { - if test.in[i] != 0 { - t.Errorf("uniq.Slice(%v): non-0 in tail of %v at index %v", in, test.in, i) - } - } - } -} - -func TestModifySlice(t *testing.T) { - runTests(t, func(slice *[]uint32) { - uniq.ModifySlice(slice) - }) -} - -func TestModifySliceFunc(t *testing.T) { - runTests(t, func(slice *[]uint32) { - uniq.ModifySliceFunc(slice, func(i, j uint32) bool { - return i == j - }) - }) -} - -func Benchmark(b *testing.B) { - benches := []struct { - name string - reset func(s []byte) - }{ - {name: "AllDups", - reset: func(s []byte) { - for i := range s { - s[i] = '*' - } - }, - }, - {name: "NoDups", - reset: func(s []byte) { - for i := range s { - s[i] = byte(i) - } - }, - }, - } - - for _, bb := range benches { - b.Run(bb.name, func(b *testing.B) { - for size := 1; size <= 4096; size *= 16 { - b.Run(strconv.Itoa(size), func(b *testing.B) { - benchmark(b, 64, bb.reset) - }) - } - }) - } -} - -func benchmark(b *testing.B, size int64, reset func(s []byte)) { - b.ReportAllocs() - b.SetBytes(size) - s := make([]byte, size) - b.ResetTimer() - for range b.N { - s = s[:size] - reset(s) - uniq.ModifySlice(&s) - } -} diff --git a/version/print.go b/version/print.go index 7d8554279f255..be90432cc85df 100644 --- a/version/print.go +++ b/version/print.go @@ -7,11 +7,10 @@ import ( "fmt" "runtime" "strings" - - "tailscale.com/types/lazy" + "sync" ) -var stringLazy = lazy.SyncFunc(func() string { +var stringLazy = sync.OnceValue(func() string { var ret strings.Builder ret.WriteString(Short()) ret.WriteByte('\n') diff --git a/version/prop.go b/version/prop.go index fee76c65fe0f2..6026d1179ce2e 100644 --- a/version/prop.go +++ b/version/prop.go @@ -9,6 +9,7 @@ import ( "runtime" "strconv" "strings" + "sync" "tailscale.com/tailcfg" "tailscale.com/types/lazy" @@ -174,7 +175,7 @@ func IsUnstableBuild() bool { }) } -var isDev = lazy.SyncFunc(func() bool { +var isDev = sync.OnceValue(func() bool { return strings.Contains(Short(), "-dev") }) diff --git a/version/version.go b/version/version.go index 5edea22ca6df0..2add25689e1dd 100644 --- a/version/version.go +++ b/version/version.go @@ -9,6 +9,7 @@ import ( "runtime/debug" "strconv" "strings" + "sync" tailscaleroot "tailscale.com" "tailscale.com/types/lazy" @@ -117,7 +118,7 @@ func (i embeddedInfo) commitAbbrev() string { return i.commit } -var getEmbeddedInfo = lazy.SyncFunc(func() embeddedInfo { +var getEmbeddedInfo = sync.OnceValue(func() embeddedInfo { bi, ok := debug.ReadBuildInfo() if !ok { return embeddedInfo{} diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index d3075f55d63e3..6a49f091ee370 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -17,6 +17,7 @@ import ( "net/netip" "reflect" "runtime" + "slices" "strconv" "strings" "sync" @@ -59,7 +60,6 @@ import ( "tailscale.com/util/ringbuffer" "tailscale.com/util/set" "tailscale.com/util/testenv" - "tailscale.com/util/uniq" "tailscale.com/util/usermetric" "tailscale.com/wgengine/capture" "tailscale.com/wgengine/wgint" @@ -2666,7 +2666,7 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur } ports = append(ports, 0) // Remove duplicates. (All duplicates are consecutive.) - uniq.ModifySlice(&ports) + ports = slices.Compact(ports) if debugBindSocket() { c.logf("magicsock: bindSocket: candidate ports: %+v", ports) diff --git a/wgengine/pendopen.go b/wgengine/pendopen.go index 7db07c685aa75..308c3ede28a2c 100644 --- a/wgengine/pendopen.go +++ b/wgengine/pendopen.go @@ -8,6 +8,7 @@ import ( "net/netip" "runtime" "strings" + "sync" "time" "github.com/gaissmai/bart" @@ -15,7 +16,6 @@ import ( "tailscale.com/net/packet" "tailscale.com/net/tstun" "tailscale.com/types/ipproto" - "tailscale.com/types/lazy" "tailscale.com/util/mak" "tailscale.com/wgengine/filter" ) @@ -91,7 +91,7 @@ func (e *userspaceEngine) trackOpenPreFilterIn(pp *packet.Parsed, t *tstun.Wrapp var ( appleIPRange = netip.MustParsePrefix("17.0.0.0/8") - canonicalIPs = lazy.SyncFunc(func() (checkIPFunc func(netip.Addr) bool) { + canonicalIPs = sync.OnceValue(func() (checkIPFunc func(netip.Addr) bool) { // https://bgp.he.net/AS41231#_prefixes t := &bart.Table[bool]{} for _, s := range strings.Fields(` From 60d19fa00d93f756fe9244a6c6b7ceb806696477 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 12 Jan 2025 21:03:43 -0800 Subject: [PATCH 105/223] all: use Go 1.21's binary.NativeEndian We still use josharian/native (hi @josharian!) via netlink, but I also sent https://github.com/mdlayher/netlink/pull/220 Updates #8632 Change-Id: I2eedcb7facb36ec894aee7f152c8a1f56d7fc8ba Signed-off-by: Brad Fitzpatrick --- go.mod | 2 +- net/dns/nm.go | 4 ++-- util/cstruct/cstruct.go | 9 ++++----- util/linuxfw/nftables.go | 6 +++--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index c28338caf22b9..f3adfd47aa426 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,6 @@ require ( github.com/inetaf/tcpproxy v0.0.0-20240214030015-3ce58045626c github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 github.com/jellydator/ttlcache/v3 v3.1.0 - github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 github.com/jsimonetti/rtnetlink v1.4.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/klauspost/compress v1.17.11 @@ -152,6 +151,7 @@ require ( github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/jjti/go-spancheck v0.5.3 // indirect + github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 // indirect github.com/karamaru-alpha/copyloopvar v1.0.8 // indirect github.com/macabu/inamedparam v0.1.3 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect diff --git a/net/dns/nm.go b/net/dns/nm.go index adb33cdb7967a..ef07a90d88470 100644 --- a/net/dns/nm.go +++ b/net/dns/nm.go @@ -7,6 +7,7 @@ package dns import ( "context" + "encoding/binary" "fmt" "net" "net/netip" @@ -14,7 +15,6 @@ import ( "time" "github.com/godbus/dbus/v5" - "github.com/josharian/native" "tailscale.com/net/tsaddr" "tailscale.com/util/dnsname" ) @@ -137,7 +137,7 @@ func (m *nmManager) trySet(ctx context.Context, config OSConfig) error { for _, ip := range config.Nameservers { b := ip.As16() if ip.Is4() { - dnsv4 = append(dnsv4, native.Endian.Uint32(b[12:])) + dnsv4 = append(dnsv4, binary.NativeEndian.Uint32(b[12:])) } else { dnsv6 = append(dnsv6, b[:]) } diff --git a/util/cstruct/cstruct.go b/util/cstruct/cstruct.go index 464dc5dc3cadf..4d1d0a98b8032 100644 --- a/util/cstruct/cstruct.go +++ b/util/cstruct/cstruct.go @@ -6,10 +6,9 @@ package cstruct import ( + "encoding/binary" "errors" "io" - - "github.com/josharian/native" ) // Size of a pointer-typed value, in bits @@ -120,7 +119,7 @@ func (d *Decoder) Uint16() uint16 { d.err = err return 0 } - return native.Endian.Uint16(d.dbuf[0:2]) + return binary.NativeEndian.Uint16(d.dbuf[0:2]) } // Uint32 returns a uint32 decoded from the buffer. @@ -133,7 +132,7 @@ func (d *Decoder) Uint32() uint32 { d.err = err return 0 } - return native.Endian.Uint32(d.dbuf[0:4]) + return binary.NativeEndian.Uint32(d.dbuf[0:4]) } // Uint64 returns a uint64 decoded from the buffer. @@ -146,7 +145,7 @@ func (d *Decoder) Uint64() uint64 { d.err = err return 0 } - return native.Endian.Uint64(d.dbuf[0:8]) + return binary.NativeEndian.Uint64(d.dbuf[0:8]) } // Uintptr returns a uintptr decoded from the buffer. diff --git a/util/linuxfw/nftables.go b/util/linuxfw/nftables.go index 056563071479f..e8b267b5e42ae 100644 --- a/util/linuxfw/nftables.go +++ b/util/linuxfw/nftables.go @@ -8,6 +8,7 @@ package linuxfw import ( "cmp" + "encoding/binary" "fmt" "sort" "strings" @@ -15,7 +16,6 @@ import ( "github.com/google/nftables" "github.com/google/nftables/expr" "github.com/google/nftables/xt" - "github.com/josharian/native" "golang.org/x/sys/unix" "tailscale.com/types/logger" ) @@ -235,8 +235,8 @@ func printMatchInfo(name string, info xt.InfoAny) string { break } - pkttype := int(native.Endian.Uint32(data[0:4])) - invert := int(native.Endian.Uint32(data[4:8])) + pkttype := int(binary.NativeEndian.Uint32(data[0:4])) + invert := int(binary.NativeEndian.Uint32(data[4:8])) var invertPrefix string if invert != 0 { invertPrefix = "!" From 377127c20ca5cb5660e7086d64bc1fcb3e9da9d8 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Mon, 13 Jan 2025 10:02:26 -0800 Subject: [PATCH 106/223] Revert "Dockerfile: bump base alpine image (#14604)" (#14620) This reverts commit 5fdb4f83ad23f0ee7a9dc08ecc2a0ceeabd81fc3. Signed-off-by: Andrew Lytvynov --- ALPINE.txt | 2 +- Dockerfile | 2 +- Dockerfile.base | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ALPINE.txt b/ALPINE.txt index f29702326bba5..55b698c77f5d2 100644 --- a/ALPINE.txt +++ b/ALPINE.txt @@ -1 +1 @@ -3.21 \ No newline at end of file +3.18 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7a5dbce5a9eeb..4ad3d88d9577a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,7 @@ RUN GOARCH=$TARGETARCH go install -ldflags="\ -X tailscale.com/version.gitCommitStamp=$VERSION_GIT_HASH" \ -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot -FROM alpine:3.21 +FROM alpine:3.18 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables COPY --from=build-env /go/bin/* /usr/local/bin/ diff --git a/Dockerfile.base b/Dockerfile.base index 5186746a42529..eb4f0a02a8b75 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,5 +1,5 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause -FROM alpine:3.21 +FROM alpine:3.18 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables iputils From 6ccde369ffa0aa381fdaf6a735f3e3bbaba179b9 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Thu, 9 Jan 2025 11:50:11 -0600 Subject: [PATCH 107/223] prober: record total bytes transferred in DERP bandwidth probes This will enable Prometheus queries to look at the bandwidth over time windows, for example 'increase(derp_bw_bytes_total)[1h] / increase(derp_bw_transfer_time_seconds_total)[1h]'. Fixes commit a51672cafd8b6c4e87915a55bda1491eb7cbee84. Updates tailscale/corp#25503 Signed-off-by: Percy Wegmann --- prober/derp.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/prober/derp.go b/prober/derp.go index f405549ff8532..870460d964a70 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -301,13 +301,14 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { derpPath = "single" } var transferTimeSeconds expvar.Float + var totalBytesTransferred expvar.Float return ProbeClass{ Probe: func(ctx context.Context) error { fromN, toN, err := d.getNodePair(from, to) if err != nil { return err } - return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, d.bwTUNIPv4Prefix) + return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix) }, Class: "derp_bw", Labels: Labels{ @@ -315,11 +316,15 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { "tcp_in_tcp": strconv.FormatBool(d.bwTUNIPv4Prefix != nil), }, Metrics: func(l prometheus.Labels) []prometheus.Metric { - return []prometheus.Metric{ + metrics := []prometheus.Metric{ prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_probe_size_bytes", "Payload size of the bandwidth prober", nil, l), prometheus.GaugeValue, float64(size)), prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_transfer_time_seconds_total", "Time it took to transfer data", nil, l), prometheus.CounterValue, transferTimeSeconds.Value()), - prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_bytes_total", "Amount of data transferred", nil, l), prometheus.CounterValue, float64(size)), } + if d.bwTUNIPv4Prefix != nil { + // For TCP-in-TCP probes, also record cumulative bytes transferred. + metrics = append(metrics, prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_bytes_total", "Amount of data transferred", nil, l), prometheus.CounterValue, totalBytesTransferred.Value())) + } + return metrics }, } } @@ -655,7 +660,7 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error { // DERP clients connected to two DERP servers.If tunIPv4Address is specified, // probes will use a TCP connection over a TUN device at this address in order // to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. -func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) { +func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper logs with every packet // sent by the bandwidth probe. fromc, err := newConn(ctx, dm, from, false) @@ -677,7 +682,7 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail } if tunIPv4Prefix != nil { - err = derpProbeBandwidthTUN(ctx, transferTimeSeconds, from, to, fromc, toc, size, tunIPv4Prefix) + err = derpProbeBandwidthTUN(ctx, transferTimeSeconds, totalBytesTransferred, from, to, fromc, toc, size, tunIPv4Prefix) } else { err = derpProbeBandwidthDirect(ctx, transferTimeSeconds, from, to, fromc, toc, size) } @@ -848,7 +853,7 @@ var derpProbeBandwidthTUNMu sync.Mutex // to another over a TUN device at an address at the start of the usable host IP // range that the given tunAddress lives in. The time taken to finish the transfer // is recorded in `transferTimeSeconds`. -func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds *expvar.Float, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, size int64, prefix *netip.Prefix) error { +func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesTransferred *expvar.Float, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, size int64, prefix *netip.Prefix) error { // Make sure all goroutines have finished. var wg sync.WaitGroup defer wg.Wait() @@ -1046,9 +1051,10 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds *expvar.Floa readFinishedC <- fmt.Errorf("unable to set read deadline: %w", err) } } - _, err = io.CopyN(io.Discard, readConn, size) - // Measure transfer time irrespective of whether it succeeded or failed. + n, err := io.CopyN(io.Discard, readConn, size) + // Measure transfer time and bytes transferred irrespective of whether it succeeded or failed. transferTimeSeconds.Add(time.Since(start).Seconds()) + totalBytesTransferred.Add(float64(n)) readFinishedC <- err }() From 64ab0ddff14cfcae55b66017ba104afd3640e422 Mon Sep 17 00:00:00 2001 From: Michael Stapelberg Date: Sun, 12 Jan 2025 09:57:38 +0100 Subject: [PATCH 108/223] cmd/tailscale/cli: only exit silently if len(args) == 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This amends commit b7e48058c8d243adf1ff687e3e92d3fb02b035ea. That commit broke all documented ways of starting Tailscale on gokrazy: https://gokrazy.org/packages/tailscale/ — both Option A (tailscale up) and Option B (tailscale up --auth-key) rely on the tailscale CLI working. I verified that the tailscale CLI just prints it help when started without arguments, i.e. it does not stay running and is not restarted. I verified that the tailscale CLI successfully exits when started with tailscale up --auth-key, regardless of whether the node has joined the tailnet yet or not. I verified that the tailscale CLI successfully waits and exits when started with tailscale up, as expected. fixes https://github.com/gokrazy/gokrazy/issues/286 Signed-off-by: Michael Stapelberg --- cmd/tailscale/cli/cli.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index 66961b2e0086d..542a2e4644c3a 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -84,9 +84,9 @@ var localClient = tailscale.LocalClient{ // Run runs the CLI. The args do not include the binary name. func Run(args []string) (err error) { - if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 { - // We're running on gokrazy and it's the first start. - // Don't run the tailscale CLI as a service; just exit. + if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 && len(args) == 0 { + // We're running on gokrazy and the user did not specify 'up'. + // Don't run the tailscale CLI and spam logs with usage; just exit. // See https://gokrazy.org/development/process-interface/ os.Exit(0) } From e4385f1c022dcce6809758c1bf0c06001a69c5dd Mon Sep 17 00:00:00 2001 From: Anton Tolchanov Date: Tue, 14 Jan 2025 12:12:05 +0000 Subject: [PATCH 109/223] cmd/tailscale/cli: add --posture-checking to tailscale up This will prevent `tailscale up` from resetting the posture checking client pref. Fixes #12154 Signed-off-by: Anton Tolchanov --- cmd/tailscale/cli/cli_test.go | 14 ++++++++++++++ cmd/tailscale/cli/up.go | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index 0444e914c7260..dccb69876d3b4 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -601,6 +601,19 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) { goos: "linux", want: "", }, + { + name: "losing_posture_checking", + flags: []string{"--accept-dns"}, + curPrefs: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + WantRunning: false, + CorpDNS: true, + PostureChecking: true, + NetfilterMode: preftype.NetfilterOn, + NoStatefulFiltering: opt.NewBool(true), + }, + want: accidentalUpPrefix + " --accept-dns --posture-checking", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1045,6 +1058,7 @@ func TestUpdatePrefs(t *testing.T) { NoSNATSet: true, NoStatefulFilteringSet: true, OperatorUserSet: true, + PostureCheckingSet: true, RouteAllSet: true, RunSSHSet: true, ShieldsUpSet: true, diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index b907257cf5a3e..4af264d73a991 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -116,6 +116,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes") upf.BoolVar(&upArgs.advertiseConnector, "advertise-connector", false, "advertise this node as an app connector") upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet") + upf.BoolVar(&upArgs.postureChecking, "posture-checking", false, hidden+"allow management plane to gather device posture information") if safesocket.GOOSUsesPeerCreds(goos) { upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo") @@ -194,6 +195,7 @@ type upArgsT struct { timeout time.Duration acceptedRisks string profileName string + postureChecking bool } func (a upArgsT) getAuthKey() (string, error) { @@ -304,6 +306,7 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo prefs.OperatorUser = upArgs.opUser prefs.ProfileName = upArgs.profileName prefs.AppConnector.Advertise = upArgs.advertiseConnector + prefs.PostureChecking = upArgs.postureChecking if goos == "linux" { prefs.NoSNAT = !upArgs.snat @@ -1053,6 +1056,8 @@ func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]any) { set(prefs.NetfilterMode.String()) case "unattended": set(prefs.ForceDaemon) + case "posture-checking": + set(prefs.PostureChecking) } }) return ret From da9965d51ca89a393bbf4c52818a579320dca93c Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 8 Jan 2025 17:21:44 -0600 Subject: [PATCH 110/223] cmd/viewer,types/views,various: avoid allocations in pointer field getters whenever possible In this PR, we add a generic views.ValuePointer type that can be used as a view for pointers to basic types and struct types that do not require deep cloning and do not have corresponding view types. Its Get/GetOk methods return stack-allocated shallow copies of the underlying value. We then update the cmd/viewer codegen to produce getters that return either concrete views when available or ValuePointer views when not, for pointer fields in generated view types. This allows us to avoid unnecessary allocations compared to returning pointers to newly allocated shallow copies. Updates #14570 Signed-off-by: Nick Khyl --- cmd/tsconnect/wasm/wasm_js.go | 2 +- cmd/viewer/tests/tests.go | 9 +++- cmd/viewer/tests/tests_clone.go | 4 ++ cmd/viewer/tests/tests_view.go | 45 +++++-------------- cmd/viewer/viewer.go | 72 ++++++++++++++++++++++++----- control/controlclient/map.go | 14 +++--- ipn/ipnlocal/drive.go | 3 +- ipn/ipnlocal/expiry_test.go | 8 ++-- ipn/ipnlocal/local.go | 39 +++++++--------- ipn/ipnlocal/peerapi.go | 8 ++-- tailcfg/tailcfg_view.go | 75 +++++++------------------------ types/prefs/prefs_view_test.go | 11 +---- types/views/views.go | 80 +++++++++++++++++++++++++++++++++ wgengine/pendopen.go | 8 ++-- wgengine/wgcfg/nmcfg/nmcfg.go | 4 +- 15 files changed, 219 insertions(+), 163 deletions(-) diff --git a/cmd/tsconnect/wasm/wasm_js.go b/cmd/tsconnect/wasm/wasm_js.go index 4ea1cd89713cd..a7e3e506bc9b8 100644 --- a/cmd/tsconnect/wasm/wasm_js.go +++ b/cmd/tsconnect/wasm/wasm_js.go @@ -282,7 +282,7 @@ func (i *jsIPN) run(jsCallbacks js.Value) { MachineKey: p.Machine().String(), NodeKey: p.Key().String(), }, - Online: p.Online(), + Online: p.Online().Clone(), TailscaleSSHEnabled: p.Hostinfo().TailscaleSSHEnabled(), } }), diff --git a/cmd/viewer/tests/tests.go b/cmd/viewer/tests/tests.go index 14a4888615bc1..ac094c53b7893 100644 --- a/cmd/viewer/tests/tests.go +++ b/cmd/viewer/tests/tests.go @@ -37,9 +37,14 @@ type Map struct { StructWithPtrKey map[StructWithPtrs]int `json:"-"` } +type StructWithNoView struct { + Value int +} + type StructWithPtrs struct { - Value *StructWithoutPtrs - Int *int + Value *StructWithoutPtrs + Int *int + NoView *StructWithNoView NoCloneValue *StructWithoutPtrs `codegen:"noclone"` } diff --git a/cmd/viewer/tests/tests_clone.go b/cmd/viewer/tests/tests_clone.go index 9131f5040c45d..106a9b6843b56 100644 --- a/cmd/viewer/tests/tests_clone.go +++ b/cmd/viewer/tests/tests_clone.go @@ -28,6 +28,9 @@ func (src *StructWithPtrs) Clone() *StructWithPtrs { if dst.Int != nil { dst.Int = ptr.To(*src.Int) } + if dst.NoView != nil { + dst.NoView = ptr.To(*src.NoView) + } return dst } @@ -35,6 +38,7 @@ func (src *StructWithPtrs) Clone() *StructWithPtrs { var _StructWithPtrsCloneNeedsRegeneration = StructWithPtrs(struct { Value *StructWithoutPtrs Int *int + NoView *StructWithNoView NoCloneValue *StructWithoutPtrs }{}) diff --git a/cmd/viewer/tests/tests_view.go b/cmd/viewer/tests/tests_view.go index 9c74c94261e08..41c1338ff4366 100644 --- a/cmd/viewer/tests/tests_view.go +++ b/cmd/viewer/tests/tests_view.go @@ -61,20 +61,11 @@ func (v *StructWithPtrsView) UnmarshalJSON(b []byte) error { return nil } -func (v StructWithPtrsView) Value() *StructWithoutPtrs { - if v.Đļ.Value == nil { - return nil - } - x := *v.Đļ.Value - return &x -} +func (v StructWithPtrsView) Value() StructWithoutPtrsView { return v.Đļ.Value.View() } +func (v StructWithPtrsView) Int() views.ValuePointer[int] { return views.ValuePointerOf(v.Đļ.Int) } -func (v StructWithPtrsView) Int() *int { - if v.Đļ.Int == nil { - return nil - } - x := *v.Đļ.Int - return &x +func (v StructWithPtrsView) NoView() views.ValuePointer[StructWithNoView] { + return views.ValuePointerOf(v.Đļ.NoView) } func (v StructWithPtrsView) NoCloneValue() *StructWithoutPtrs { return v.Đļ.NoCloneValue } @@ -85,6 +76,7 @@ func (v StructWithPtrsView) Equal(v2 StructWithPtrsView) bool { return v.Đļ.Equa var _StructWithPtrsViewNeedsRegeneration = StructWithPtrs(struct { Value *StructWithoutPtrs Int *int + NoView *StructWithNoView NoCloneValue *StructWithoutPtrs }{}) @@ -424,12 +416,8 @@ func (v *GenericIntStructView[T]) UnmarshalJSON(b []byte) error { } func (v GenericIntStructView[T]) Value() T { return v.Đļ.Value } -func (v GenericIntStructView[T]) Pointer() *T { - if v.Đļ.Pointer == nil { - return nil - } - x := *v.Đļ.Pointer - return &x +func (v GenericIntStructView[T]) Pointer() views.ValuePointer[T] { + return views.ValuePointerOf(v.Đļ.Pointer) } func (v GenericIntStructView[T]) Slice() views.Slice[T] { return views.SliceOf(v.Đļ.Slice) } @@ -500,12 +488,8 @@ func (v *GenericNoPtrsStructView[T]) UnmarshalJSON(b []byte) error { } func (v GenericNoPtrsStructView[T]) Value() T { return v.Đļ.Value } -func (v GenericNoPtrsStructView[T]) Pointer() *T { - if v.Đļ.Pointer == nil { - return nil - } - x := *v.Đļ.Pointer - return &x +func (v GenericNoPtrsStructView[T]) Pointer() views.ValuePointer[T] { + return views.ValuePointerOf(v.Đļ.Pointer) } func (v GenericNoPtrsStructView[T]) Slice() views.Slice[T] { return views.SliceOf(v.Đļ.Slice) } @@ -722,19 +706,14 @@ func (v *StructWithTypeAliasFieldsView) UnmarshalJSON(b []byte) error { return nil } -func (v StructWithTypeAliasFieldsView) WithPtr() StructWithPtrsView { return v.Đļ.WithPtr.View() } +func (v StructWithTypeAliasFieldsView) WithPtr() StructWithPtrsAliasView { return v.Đļ.WithPtr.View() } func (v StructWithTypeAliasFieldsView) WithoutPtr() StructWithoutPtrsAlias { return v.Đļ.WithoutPtr } func (v StructWithTypeAliasFieldsView) WithPtrByPtr() StructWithPtrsAliasView { return v.Đļ.WithPtrByPtr.View() } -func (v StructWithTypeAliasFieldsView) WithoutPtrByPtr() *StructWithoutPtrsAlias { - if v.Đļ.WithoutPtrByPtr == nil { - return nil - } - x := *v.Đļ.WithoutPtrByPtr - return &x +func (v StructWithTypeAliasFieldsView) WithoutPtrByPtr() StructWithoutPtrsAliasView { + return v.Đļ.WithoutPtrByPtr.View() } - func (v StructWithTypeAliasFieldsView) SliceWithPtrs() views.SliceView[*StructWithPtrsAlias, StructWithPtrsAliasView] { return views.SliceOfViews[*StructWithPtrsAlias, StructWithPtrsAliasView](v.Đļ.SliceWithPtrs) } diff --git a/cmd/viewer/viewer.go b/cmd/viewer/viewer.go index 0c5868f3a86e6..e265defe0c522 100644 --- a/cmd/viewer/viewer.go +++ b/cmd/viewer/viewer.go @@ -79,13 +79,7 @@ func (v *{{.ViewName}}{{.TypeParamNames}}) UnmarshalJSON(b []byte) error { {{end}} {{define "makeViewField"}}func (v {{.ViewName}}{{.TypeParamNames}}) {{.FieldName}}() {{.FieldViewName}} { return {{.MakeViewFnName}}(&v.Đļ.{{.FieldName}}) } {{end}} -{{define "valuePointerField"}}func (v {{.ViewName}}{{.TypeParamNames}}) {{.FieldName}}() {{.FieldType}} { - if v.Đļ.{{.FieldName}} == nil { - return nil - } - x := *v.Đļ.{{.FieldName}} - return &x -} +{{define "valuePointerField"}}func (v {{.ViewName}}{{.TypeParamNames}}) {{.FieldName}}() views.ValuePointer[{{.FieldType}}] { return views.ValuePointerOf(v.Đļ.{{.FieldName}}) } {{end}} {{define "mapField"}} @@ -126,7 +120,7 @@ func requiresCloning(t types.Type) (shallow, deep bool, base types.Type) { return p, p, t } -func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thisPkg *types.Package) { +func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, _ *types.Package) { t, ok := typ.Underlying().(*types.Struct) if !ok || codegen.IsViewType(t) { return @@ -354,10 +348,32 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi } else { writeTemplate("unsupportedField") } - } else { - args.FieldType = it.QualifiedName(ptr) - writeTemplate("valuePointerField") + continue + } + + // If a view type is already defined for the base type, use it as the field's view type. + if viewType := viewTypeForValueType(base); viewType != nil { + args.FieldType = it.QualifiedName(base) + args.FieldViewName = it.QualifiedName(viewType) + writeTemplate("viewField") + continue + } + + // Otherwise, if the unaliased base type is a named type whose view type will be generated by this viewer invocation, + // append the "View" suffix to the unaliased base type name and use it as the field's view type. + if base, ok := types.Unalias(base).(*types.Named); ok && slices.Contains(typeNames, it.QualifiedName(base)) { + baseTypeName := it.QualifiedName(base) + args.FieldType = baseTypeName + args.FieldViewName = appendNameSuffix(args.FieldType, "View") + writeTemplate("viewField") + continue } + + // Otherwise, if the base type does not require deep cloning, has no existing view type, + // and will not have a generated view type, use views.ValuePointer[T] as the field's view type. + // Its Get/GetOk methods return stack-allocated shallow copies of the field's value. + args.FieldType = it.QualifiedName(base) + writeTemplate("valuePointerField") continue case *types.Interface: // If fieldType is an interface with a "View() {ViewType}" method, it can be used to clone the field. @@ -405,6 +421,33 @@ func appendNameSuffix(name, suffix string) string { return name + suffix } +func typeNameOf(typ types.Type) (name *types.TypeName, ok bool) { + switch t := typ.(type) { + case *types.Alias: + return t.Obj(), true + case *types.Named: + return t.Obj(), true + default: + return nil, false + } +} + +func lookupViewType(typ types.Type) types.Type { + for { + if typeName, ok := typeNameOf(typ); ok && typeName.Pkg() != nil { + if viewTypeObj := typeName.Pkg().Scope().Lookup(typeName.Name() + "View"); viewTypeObj != nil { + return viewTypeObj.Type() + } + } + switch alias := typ.(type) { + case *types.Alias: + typ = alias.Rhs() + default: + return nil + } + } +} + func viewTypeForValueType(typ types.Type) types.Type { if ptr, ok := typ.(*types.Pointer); ok { return viewTypeForValueType(ptr.Elem()) @@ -417,7 +460,12 @@ func viewTypeForValueType(typ types.Type) types.Type { if !ok || sig.Results().Len() != 1 { return nil } - return sig.Results().At(0).Type() + viewType := sig.Results().At(0).Type() + // Check if the typ's package defines an alias for the view type, and use it if so. + if viewTypeAlias, ok := lookupViewType(typ).(*types.Alias); ok && types.AssignableTo(viewType, viewTypeAlias) { + viewType = viewTypeAlias + } + return viewType } func viewTypeForContainerType(typ types.Type) (*types.Named, *types.Func) { diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 97d49f90d4ad3..30c1da672b9dc 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -689,13 +689,11 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang return nil, false } case "Online": - wasOnline := was.Online() - if n.Online != nil && wasOnline != nil && *n.Online != *wasOnline { + if wasOnline, ok := was.Online().GetOk(); ok && n.Online != nil && *n.Online != wasOnline { pc().Online = ptr.To(*n.Online) } case "LastSeen": - wasSeen := was.LastSeen() - if n.LastSeen != nil && wasSeen != nil && !wasSeen.Equal(*n.LastSeen) { + if wasSeen, ok := was.LastSeen().GetOk(); ok && n.LastSeen != nil && !wasSeen.Equal(*n.LastSeen) { pc().LastSeen = ptr.To(*n.LastSeen) } case "MachineAuthorized": @@ -720,18 +718,18 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang } case "SelfNodeV4MasqAddrForThisPeer": va, vb := was.SelfNodeV4MasqAddrForThisPeer(), n.SelfNodeV4MasqAddrForThisPeer - if va == nil && vb == nil { + if !va.Valid() && vb == nil { continue } - if va == nil || vb == nil || *va != *vb { + if va, ok := va.GetOk(); !ok || vb == nil || va != *vb { return nil, false } case "SelfNodeV6MasqAddrForThisPeer": va, vb := was.SelfNodeV6MasqAddrForThisPeer(), n.SelfNodeV6MasqAddrForThisPeer - if va == nil && vb == nil { + if !va.Valid() && vb == nil { continue } - if va == nil || vb == nil || *va != *vb { + if va, ok := va.GetOk(); !ok || vb == nil || va != *vb { return nil, false } case "ExitNodeDNSResolvers": diff --git a/ipn/ipnlocal/drive.go b/ipn/ipnlocal/drive.go index fe3622ba40e3e..8ae813ff239d2 100644 --- a/ipn/ipnlocal/drive.go +++ b/ipn/ipnlocal/drive.go @@ -347,8 +347,7 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem // TODO(oxtoacart): for some reason, this correctly // catches when a node goes from offline to online, // but not the other way around... - online := peer.Online() - if online == nil || !*online { + if !peer.Online().Get() { return false } diff --git a/ipn/ipnlocal/expiry_test.go b/ipn/ipnlocal/expiry_test.go index af1aa337bbe0c..a2b10fe325b8a 100644 --- a/ipn/ipnlocal/expiry_test.go +++ b/ipn/ipnlocal/expiry_test.go @@ -283,11 +283,11 @@ func formatNodes(nodes []tailcfg.NodeView) string { } fmt.Fprintf(&sb, "(%d, %q", n.ID(), n.Name()) - if n.Online() != nil { - fmt.Fprintf(&sb, ", online=%v", *n.Online()) + if online, ok := n.Online().GetOk(); ok { + fmt.Fprintf(&sb, ", online=%v", online) } - if n.LastSeen() != nil { - fmt.Fprintf(&sb, ", lastSeen=%v", n.LastSeen().Unix()) + if lastSeen, ok := n.LastSeen().GetOk(); ok { + fmt.Fprintf(&sb, ", lastSeen=%v", lastSeen.Unix()) } if n.Key() != (key.NodePublic{}) { fmt.Fprintf(&sb, ", key=%v", n.Key().String()) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 3a2a22c581825..4ebcd5d6d0869 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1117,13 +1117,9 @@ func (b *LocalBackend) UpdateStatus(sb *ipnstate.StatusBuilder) { } if !prefs.ExitNodeID().IsZero() { if exitPeer, ok := b.netMap.PeerWithStableID(prefs.ExitNodeID()); ok { - online := false - if v := exitPeer.Online(); v != nil { - online = *v - } s.ExitNodeStatus = &ipnstate.ExitNodeStatus{ ID: prefs.ExitNodeID(), - Online: online, + Online: exitPeer.Online().Get(), TailscaleIPs: exitPeer.Addresses().AsSlice(), } } @@ -1194,10 +1190,6 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) { } exitNodeID := b.pm.CurrentPrefs().ExitNodeID() for _, p := range b.peers { - var lastSeen time.Time - if p.LastSeen() != nil { - lastSeen = *p.LastSeen() - } tailscaleIPs := make([]netip.Addr, 0, p.Addresses().Len()) for i := range p.Addresses().Len() { addr := p.Addresses().At(i) @@ -1205,7 +1197,6 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) { tailscaleIPs = append(tailscaleIPs, addr.Addr()) } } - online := p.Online() ps := &ipnstate.PeerStatus{ InNetworkMap: true, UserID: p.User(), @@ -1214,12 +1205,12 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) { HostName: p.Hostinfo().Hostname(), DNSName: p.Name(), OS: p.Hostinfo().OS(), - LastSeen: lastSeen, - Online: online != nil && *online, + LastSeen: p.LastSeen().Get(), + Online: p.Online().Get(), ShareeNode: p.Hostinfo().ShareeNode(), ExitNode: p.StableID() != "" && p.StableID() == exitNodeID, SSH_HostKeys: p.Hostinfo().SSH_HostKeys().AsSlice(), - Location: p.Hostinfo().Location(), + Location: p.Hostinfo().Location().AsStruct(), Capabilities: p.Capabilities().AsSlice(), } if cm := p.CapMap(); cm.Len() > 0 { @@ -7369,8 +7360,8 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug if len(candidates) == 1 { peer := candidates[0] if hi := peer.Hostinfo(); hi.Valid() { - if loc := hi.Location(); loc != nil { - res.Location = loc.View() + if loc := hi.Location(); loc.Valid() { + res.Location = loc } } res.ID = peer.StableID() @@ -7414,10 +7405,10 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug continue } loc := hi.Location() - if loc == nil { + if !loc.Valid() { continue } - distance := longLatDistance(preferredDERP.Latitude, preferredDERP.Longitude, loc.Latitude, loc.Longitude) + distance := longLatDistance(preferredDERP.Latitude, preferredDERP.Longitude, loc.Latitude(), loc.Longitude()) if distance < minDistance { minDistance = distance } @@ -7438,8 +7429,8 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug res.ID = chosen.StableID() res.Name = chosen.Name() if hi := chosen.Hostinfo(); hi.Valid() { - if loc := hi.Location(); loc != nil { - res.Location = loc.View() + if loc := hi.Location(); loc.Valid() { + res.Location = loc } } return res, nil @@ -7468,8 +7459,8 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug res.ID = chosen.StableID() res.Name = chosen.Name() if hi := chosen.Hostinfo(); hi.Valid() { - if loc := hi.Location(); loc != nil { - res.Location = loc.View() + if loc := hi.Location(); loc.Valid() { + res.Location = loc } } return res, nil @@ -7485,13 +7476,13 @@ func pickWeighted(candidates []tailcfg.NodeView) []tailcfg.NodeView { continue } loc := hi.Location() - if loc == nil || loc.Priority < maxWeight { + if !loc.Valid() || loc.Priority() < maxWeight { continue } - if maxWeight != loc.Priority { + if maxWeight != loc.Priority() { best = best[:0] } - maxWeight = loc.Priority + maxWeight = loc.Priority() best = append(best, c) } return best diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index aa18c35886648..7aa677640b81f 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -233,11 +233,11 @@ func (h *peerAPIHandler) logf(format string, a ...any) { // isAddressValid reports whether addr is a valid destination address for this // node originating from the peer. func (h *peerAPIHandler) isAddressValid(addr netip.Addr) bool { - if v := h.peerNode.SelfNodeV4MasqAddrForThisPeer(); v != nil { - return *v == addr + if v, ok := h.peerNode.SelfNodeV4MasqAddrForThisPeer().GetOk(); ok { + return v == addr } - if v := h.peerNode.SelfNodeV6MasqAddrForThisPeer(); v != nil { - return *v == addr + if v, ok := h.peerNode.SelfNodeV6MasqAddrForThisPeer().GetOk(); ok { + return v == addr } pfx := netip.PrefixFrom(addr, addr.BitLen()) return views.SliceContains(h.selfNode.Addresses(), pfx) diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index 774a18258ce6c..53df3dcef0d99 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -145,21 +145,11 @@ func (v NodeView) Created() time.Time { return v.Đļ.Create func (v NodeView) Cap() CapabilityVersion { return v.Đļ.Cap } func (v NodeView) Tags() views.Slice[string] { return views.SliceOf(v.Đļ.Tags) } func (v NodeView) PrimaryRoutes() views.Slice[netip.Prefix] { return views.SliceOf(v.Đļ.PrimaryRoutes) } -func (v NodeView) LastSeen() *time.Time { - if v.Đļ.LastSeen == nil { - return nil - } - x := *v.Đļ.LastSeen - return &x +func (v NodeView) LastSeen() views.ValuePointer[time.Time] { + return views.ValuePointerOf(v.Đļ.LastSeen) } -func (v NodeView) Online() *bool { - if v.Đļ.Online == nil { - return nil - } - x := *v.Đļ.Online - return &x -} +func (v NodeView) Online() views.ValuePointer[bool] { return views.ValuePointerOf(v.Đļ.Online) } func (v NodeView) MachineAuthorized() bool { return v.Đļ.MachineAuthorized } func (v NodeView) Capabilities() views.Slice[NodeCapability] { return views.SliceOf(v.Đļ.Capabilities) } @@ -172,20 +162,12 @@ func (v NodeView) ComputedName() string { return v.Đļ.ComputedName } func (v NodeView) ComputedNameWithHost() string { return v.Đļ.ComputedNameWithHost } func (v NodeView) DataPlaneAuditLogID() string { return v.Đļ.DataPlaneAuditLogID } func (v NodeView) Expired() bool { return v.Đļ.Expired } -func (v NodeView) SelfNodeV4MasqAddrForThisPeer() *netip.Addr { - if v.Đļ.SelfNodeV4MasqAddrForThisPeer == nil { - return nil - } - x := *v.Đļ.SelfNodeV4MasqAddrForThisPeer - return &x +func (v NodeView) SelfNodeV4MasqAddrForThisPeer() views.ValuePointer[netip.Addr] { + return views.ValuePointerOf(v.Đļ.SelfNodeV4MasqAddrForThisPeer) } -func (v NodeView) SelfNodeV6MasqAddrForThisPeer() *netip.Addr { - if v.Đļ.SelfNodeV6MasqAddrForThisPeer == nil { - return nil - } - x := *v.Đļ.SelfNodeV6MasqAddrForThisPeer - return &x +func (v NodeView) SelfNodeV6MasqAddrForThisPeer() views.ValuePointer[netip.Addr] { + return views.ValuePointerOf(v.Đļ.SelfNodeV6MasqAddrForThisPeer) } func (v NodeView) IsWireGuardOnly() bool { return v.Đļ.IsWireGuardOnly } @@ -315,15 +297,8 @@ func (v HostinfoView) Userspace() opt.Bool { return v.Đļ.User func (v HostinfoView) UserspaceRouter() opt.Bool { return v.Đļ.UserspaceRouter } func (v HostinfoView) AppConnector() opt.Bool { return v.Đļ.AppConnector } func (v HostinfoView) ServicesHash() string { return v.Đļ.ServicesHash } -func (v HostinfoView) Location() *Location { - if v.Đļ.Location == nil { - return nil - } - x := *v.Đļ.Location - return &x -} - -func (v HostinfoView) Equal(v2 HostinfoView) bool { return v.Đļ.Equal(v2.Đļ) } +func (v HostinfoView) Location() LocationView { return v.Đļ.Location.View() } +func (v HostinfoView) Equal(v2 HostinfoView) bool { return v.Đļ.Equal(v2.Đļ) } // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _HostinfoViewNeedsRegeneration = Hostinfo(struct { @@ -699,12 +674,8 @@ func (v *RegisterResponseAuthView) UnmarshalJSON(b []byte) error { return nil } -func (v RegisterResponseAuthView) Oauth2Token() *Oauth2Token { - if v.Đļ.Oauth2Token == nil { - return nil - } - x := *v.Đļ.Oauth2Token - return &x +func (v RegisterResponseAuthView) Oauth2Token() views.ValuePointer[Oauth2Token] { + return views.ValuePointerOf(v.Đļ.Oauth2Token) } func (v RegisterResponseAuthView) AuthKey() string { return v.Đļ.AuthKey } @@ -774,12 +745,8 @@ func (v RegisterRequestView) NodeKeySignature() views.ByteSlice[tkatype.Marshale return views.ByteSliceOf(v.Đļ.NodeKeySignature) } func (v RegisterRequestView) SignatureType() SignatureType { return v.Đļ.SignatureType } -func (v RegisterRequestView) Timestamp() *time.Time { - if v.Đļ.Timestamp == nil { - return nil - } - x := *v.Đļ.Timestamp - return &x +func (v RegisterRequestView) Timestamp() views.ValuePointer[time.Time] { + return views.ValuePointerOf(v.Đļ.Timestamp) } func (v RegisterRequestView) DeviceCert() views.ByteSlice[[]byte] { @@ -1110,12 +1077,8 @@ func (v *SSHRuleView) UnmarshalJSON(b []byte) error { return nil } -func (v SSHRuleView) RuleExpires() *time.Time { - if v.Đļ.RuleExpires == nil { - return nil - } - x := *v.Đļ.RuleExpires - return &x +func (v SSHRuleView) RuleExpires() views.ValuePointer[time.Time] { + return views.ValuePointerOf(v.Đļ.RuleExpires) } func (v SSHRuleView) Principals() views.SliceView[*SSHPrincipal, SSHPrincipalView] { @@ -1189,12 +1152,8 @@ func (v SSHActionView) HoldAndDelegate() string { return v.Đļ.Hol func (v SSHActionView) AllowLocalPortForwarding() bool { return v.Đļ.AllowLocalPortForwarding } func (v SSHActionView) AllowRemotePortForwarding() bool { return v.Đļ.AllowRemotePortForwarding } func (v SSHActionView) Recorders() views.Slice[netip.AddrPort] { return views.SliceOf(v.Đļ.Recorders) } -func (v SSHActionView) OnRecordingFailure() *SSHRecorderFailureAction { - if v.Đļ.OnRecordingFailure == nil { - return nil - } - x := *v.Đļ.OnRecordingFailure - return &x +func (v SSHActionView) OnRecordingFailure() views.ValuePointer[SSHRecorderFailureAction] { + return views.ValuePointerOf(v.Đļ.OnRecordingFailure) } // A compilation failure here means this code must be regenerated, with the command at the top of this file. diff --git a/types/prefs/prefs_view_test.go b/types/prefs/prefs_view_test.go index d76eebb43e9ef..ef9f09603ea71 100644 --- a/types/prefs/prefs_view_test.go +++ b/types/prefs/prefs_view_test.go @@ -162,15 +162,8 @@ func (v *TestBundleView) UnmarshalJSON(b []byte) error { return nil } -func (v TestBundleView) Name() string { return v.Đļ.Name } -func (v TestBundleView) Nested() *TestValueStruct { - if v.Đļ.Nested == nil { - return nil - } - x := *v.Đļ.Nested - return &x -} - +func (v TestBundleView) Name() string { return v.Đļ.Name } +func (v TestBundleView) Nested() TestValueStructView { return v.Đļ.Nested.View() } func (v TestBundleView) Equal(v2 TestBundleView) bool { return v.Đļ.Equal(v2.Đļ) } // A compilation failure here means this code must be regenerated, with the command at the top of this file. diff --git a/types/views/views.go b/types/views/views.go index 40d8811f542ee..d8acf27ce4fca 100644 --- a/types/views/views.go +++ b/types/views/views.go @@ -16,6 +16,7 @@ import ( "slices" "go4.org/mem" + "tailscale.com/types/ptr" ) func unmarshalSliceFromJSON[T any](b []byte, x *[]T) error { @@ -690,6 +691,85 @@ func (m MapFn[K, T, V]) All() iter.Seq2[K, V] { } } +// ValuePointer provides a read-only view of a pointer to a value type, +// such as a primitive type or an immutable struct. Its Value and ValueOk +// methods return a stack-allocated shallow copy of the underlying value. +// It is the caller's responsibility to ensure that T +// is free from memory aliasing/mutation concerns. +type ValuePointer[T any] struct { + // Đļ is the underlying value, named with a hard-to-type + // character that looks pointy like a pointer. + // It is named distinctively to make you think of how dangerous it is to escape + // to callers. You must not let callers be able to mutate it. + Đļ *T +} + +// Valid reports whether the underlying pointer is non-nil. +func (p ValuePointer[T]) Valid() bool { + return p.Đļ != nil +} + +// Get returns a shallow copy of the value if the underlying pointer is non-nil. +// Otherwise, it returns a zero value. +func (p ValuePointer[T]) Get() T { + v, _ := p.GetOk() + return v +} + +// GetOk returns a shallow copy of the underlying value and true if the underlying +// pointer is non-nil. Otherwise, it returns a zero value and false. +func (p ValuePointer[T]) GetOk() (value T, ok bool) { + if p.Đļ == nil { + return value, false // value holds a zero value + } + return *p.Đļ, true +} + +// GetOr returns a shallow copy of the underlying value if it is non-nil. +// Otherwise, it returns the provided default value. +func (p ValuePointer[T]) GetOr(def T) T { + if p.Đļ == nil { + return def + } + return *p.Đļ +} + +// Clone returns a shallow copy of the underlying value. +func (p ValuePointer[T]) Clone() *T { + if p.Đļ == nil { + return nil + } + return ptr.To(*p.Đļ) +} + +// String implements [fmt.Stringer]. +func (p ValuePointer[T]) String() string { + if p.Đļ == nil { + return "nil" + } + return fmt.Sprint(p.Đļ) +} + +// ValuePointerOf returns an immutable view of a pointer to an immutable value. +// It is the caller's responsibility to ensure that T +// is free from memory aliasing/mutation concerns. +func ValuePointerOf[T any](v *T) ValuePointer[T] { + return ValuePointer[T]{v} +} + +// MarshalJSON implements [json.Marshaler]. +func (p ValuePointer[T]) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Đļ) +} + +// UnmarshalJSON implements [json.Unmarshaler]. +func (p *ValuePointer[T]) UnmarshalJSON(b []byte) error { + if p.Đļ != nil { + return errors.New("already initialized") + } + return json.Unmarshal(b, &p.Đļ) +} + // ContainsPointers reports whether T contains any pointers, // either explicitly or implicitly. // It has special handling for some types that contain pointers diff --git a/wgengine/pendopen.go b/wgengine/pendopen.go index 308c3ede28a2c..f8e9198a516db 100644 --- a/wgengine/pendopen.go +++ b/wgengine/pendopen.go @@ -239,15 +239,15 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) { if n.IsWireGuardOnly() { online = "wg" } else { - if v := n.Online(); v != nil { - if *v { + if v, ok := n.Online().GetOk(); ok { + if v { online = "yes" } else { online = "no" } } - if n.LastSeen() != nil && online != "yes" { - online += fmt.Sprintf(", lastseen=%v", durFmt(*n.LastSeen())) + if lastSeen, ok := n.LastSeen().GetOk(); ok && online != "yes" { + online += fmt.Sprintf(", lastseen=%v", durFmt(lastSeen)) } } e.logf("open-conn-track: timeout opening %v to node %v; online=%v, lastRecv=%v", diff --git a/wgengine/wgcfg/nmcfg/nmcfg.go b/wgengine/wgcfg/nmcfg/nmcfg.go index e7d5edf150537..97304aa415edb 100644 --- a/wgengine/wgcfg/nmcfg/nmcfg.go +++ b/wgengine/wgcfg/nmcfg/nmcfg.go @@ -106,8 +106,8 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags, cpeer := &cfg.Peers[len(cfg.Peers)-1] didExitNodeWarn := false - cpeer.V4MasqAddr = peer.SelfNodeV4MasqAddrForThisPeer() - cpeer.V6MasqAddr = peer.SelfNodeV6MasqAddrForThisPeer() + cpeer.V4MasqAddr = peer.SelfNodeV4MasqAddrForThisPeer().Clone() + cpeer.V6MasqAddr = peer.SelfNodeV6MasqAddrForThisPeer().Clone() cpeer.IsJailed = peer.IsJailed() for _, allowedIP := range peer.AllowedIPs().All() { if allowedIP.Bits() == 0 && peer.StableID() != exitNode { From 414a01126a1c75564980d9077b1899f7e7956c9e Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 14 Jan 2025 08:02:58 -0800 Subject: [PATCH 111/223] go.mod: bump mdlayher/netlink and u-root/uio to use Go 1.21 NativeEndian This finishes the work started in #14616. Updates #8632 Change-Id: I4dc07d45b1e00c3db32217c03b21b8b1ec19e782 Signed-off-by: Brad Fitzpatrick --- cmd/derper/depaware.txt | 3 +-- cmd/k8s-operator/depaware.txt | 3 +-- cmd/tailscale/depaware.txt | 3 +-- cmd/tailscaled/depaware.txt | 3 +-- go.mod | 5 ++--- go.sum | 14 ++++---------- 6 files changed, 10 insertions(+), 21 deletions(-) diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 076074f2554a1..d4b406d9dd67e 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -28,7 +28,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ github.com/hdevalence/ed25519consensus from tailscale.com/tka - L github.com/josharian/native from github.com/mdlayher/netlink+ L đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink L đŸ’Ŗ github.com/mdlayher/netlink from github.com/google/nftables+ @@ -204,7 +203,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa golang.org/x/net/proxy from tailscale.com/net/netns D golang.org/x/net/route from net+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ - golang.org/x/sys/cpu from github.com/josharian/native+ + golang.org/x/sys/cpu from golang.org/x/crypto/argon2+ LD golang.org/x/sys/unix from github.com/google/nftables+ W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 3489e5a60c3eb..f757cda185c82 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -147,7 +147,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ L github.com/insomniacslk/dhcp/rfc1035label from github.com/insomniacslk/dhcp/dhcpv4 L github.com/jmespath/go-jmespath from github.com/aws/aws-sdk-go-v2/service/ssm github.com/josharian/intern from github.com/mailru/easyjson/jlexer - L github.com/josharian/native from github.com/mdlayher/netlink+ L đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink đŸ’Ŗ github.com/json-iterator/go from sigs.k8s.io/structured-merge-diff/v4/fieldpath+ @@ -877,7 +876,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/oauth2/clientcredentials from tailscale.com/cmd/k8s-operator golang.org/x/oauth2/internal from golang.org/x/oauth2+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ - golang.org/x/sys/cpu from github.com/josharian/native+ + golang.org/x/sys/cpu from github.com/tailscale/certstore+ LD golang.org/x/sys/unix from github.com/fsnotify/fsnotify+ W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index ff2de13c0378f..e894e06742b05 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -26,7 +26,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep github.com/gorilla/csrf from tailscale.com/client/web github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ - L github.com/josharian/native from github.com/mdlayher/netlink+ L đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink github.com/kballard/go-shellquote from tailscale.com/cmd/tailscale/cli @@ -219,7 +218,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep golang.org/x/oauth2/clientcredentials from tailscale.com/cmd/tailscale/cli golang.org/x/oauth2/internal from golang.org/x/oauth2+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ - golang.org/x/sys/cpu from github.com/josharian/native+ + golang.org/x/sys/cpu from golang.org/x/crypto/argon2+ LD golang.org/x/sys/unix from github.com/google/nftables+ W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 4dad47421e87d..19254b6164800 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -118,7 +118,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/insomniacslk/dhcp/rfc1035label from github.com/insomniacslk/dhcp/dhcpv4 github.com/jellydator/ttlcache/v3 from tailscale.com/drive/driveimpl/compositedav L github.com/jmespath/go-jmespath from github.com/aws/aws-sdk-go-v2/service/ssm - L github.com/josharian/native from github.com/mdlayher/netlink+ L đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink github.com/klauspost/compress from github.com/klauspost/compress/zstd @@ -464,7 +463,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de D golang.org/x/net/route from net+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ golang.org/x/sync/singleflight from github.com/jellydator/ttlcache/v3 - golang.org/x/sys/cpu from github.com/josharian/native+ + golang.org/x/sys/cpu from github.com/tailscale/certstore+ LD golang.org/x/sys/unix from github.com/google/nftables+ W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ diff --git a/go.mod b/go.mod index f3adfd47aa426..79374eb9c198a 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/mdlayher/genetlink v1.3.2 - github.com/mdlayher/netlink v1.7.2 + github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 github.com/mdlayher/sdnotify v1.0.0 github.com/miekg/dns v1.1.58 github.com/mitchellh/go-ps v1.0.0 @@ -151,7 +151,6 @@ require ( github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/jjti/go-spancheck v0.5.3 // indirect - github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 // indirect github.com/karamaru-alpha/copyloopvar v1.0.8 // indirect github.com/macabu/inamedparam v0.1.3 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -371,7 +370,7 @@ require ( github.com/timonwong/loggercheck v0.9.4 // indirect github.com/tomarrell/wrapcheck/v2 v2.8.3 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect - github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e // indirect + github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/ultraware/funlen v0.1.0 // indirect github.com/ultraware/whitespace v0.1.0 // indirect diff --git a/go.sum b/go.sum index be8f291a41f61..28315ad1eff46 100644 --- a/go.sum +++ b/go.sum @@ -594,9 +594,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= -github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 h1:elKwZS1OcdQ0WwEDBeqxKwb7WB62QX8bvZ/FJnVXIfk= -github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86/go.mod h1:aFAMtuldEgx/4q7iSGazk22+IcgvtiC+HIimFO9XlS8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jsimonetti/rtnetlink v1.4.0 h1:Z1BF0fRgcETPEa0Kt0MRk3yV5+kF1FWTni6KUFKrq2I= github.com/jsimonetti/rtnetlink v1.4.0/go.mod h1:5W1jDvWdnthFJ7fxYX1GMK07BUpI4oskfOqvPteYS6E= @@ -684,8 +681,8 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o= -github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= -github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o= github.com/mdlayher/sdnotify v1.0.0 h1:Ma9XeLVN/l0qpyx1tNeMSeTjCPH6NtuD6/N9XdTlQ3c= github.com/mdlayher/sdnotify v1.0.0/go.mod h1:HQUmpM4XgYkhDLtd+Uad8ZFK1T9D5+pNxnXQjCeJlGE= github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= @@ -760,7 +757,6 @@ github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeB github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= -github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= @@ -970,8 +966,8 @@ github.com/u-root/gobusybox/src v0.0.0-20231228173702-b69f654846aa h1:unMPGGK/CR github.com/u-root/gobusybox/src v0.0.0-20231228173702-b69f654846aa/go.mod h1:Zj4Tt22fJVn/nz/y6Ergm1SahR9dio1Zm/D2/S0TmXM= github.com/u-root/u-root v0.12.0 h1:K0AuBFriwr0w/PGS3HawiAw89e3+MU7ks80GpghAsNs= github.com/u-root/u-root v0.12.0/go.mod h1:FYjTOh4IkIZHhjsd17lb8nYW6udgXdJhG1c0r6u0arI= -github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e h1:BA9O3BmlTmpjbvajAwzWx4Wo2TRVdpPXZEeemGQcajw= -github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= +github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM= +github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ultraware/funlen v0.1.0 h1:BuqclbkY6pO+cvxoq7OsktIXZpgBSkYTQtmwhAK81vI= @@ -1222,7 +1218,6 @@ golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1231,7 +1226,6 @@ golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab h1:BMkEEWYOjkvOX7+YKOGbp6jCyQ5pR2j0Ah47p1Vdsx4= From cfda1ff70982f13594e1ae781e0464cb048fa931 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 14 Jan 2025 08:15:27 -0800 Subject: [PATCH 112/223] cmd/viewer,all: consistently use "read-only" instead of "readonly" Updates #cleanup Change-Id: I8e4e3497d3d0ec5b16a73aedda500fe5cfa37a67 Signed-off-by: Brad Fitzpatrick --- client/web/web.go | 14 ++-- cmd/viewer/tests/tests.go | 4 +- cmd/viewer/tests/tests_view.go | 44 +++++------ cmd/viewer/viewer.go | 6 +- drive/drive_view.go | 4 +- ipn/ipn_view.go | 24 +++--- ipn/ipnlocal/local.go | 2 +- tailcfg/tailcfg_view.go | 76 +++++++++---------- types/dnstype/dnstype_view.go | 4 +- types/persist/persist_view.go | 4 +- types/prefs/prefs.go | 4 +- .../prefs/prefs_example/prefs_example_view.go | 12 +-- types/prefs/prefs_view_test.go | 20 ++--- types/prefs/struct_map.go | 2 +- 14 files changed, 110 insertions(+), 110 deletions(-) diff --git a/client/web/web.go b/client/web/web.go index 1e338b735bc0e..4e48669230db3 100644 --- a/client/web/web.go +++ b/client/web/web.go @@ -89,8 +89,8 @@ type Server struct { type ServerMode string const ( - // LoginServerMode serves a readonly login client for logging a - // node into a tailnet, and viewing a readonly interface of the + // LoginServerMode serves a read-only login client for logging a + // node into a tailnet, and viewing a read-only interface of the // node's current Tailscale settings. // // In this mode, API calls are authenticated via platform auth. @@ -110,7 +110,7 @@ const ( // This mode restricts the app to only being assessible over Tailscale, // and API calls are authenticated via browser sessions associated with // the source's Tailscale identity. If the source browser does not have - // a valid session, a readonly version of the app is displayed. + // a valid session, a read-only version of the app is displayed. ManageServerMode ServerMode = "manage" ) @@ -695,16 +695,16 @@ func (s *Server) serveAPIAuth(w http.ResponseWriter, r *http.Request) { switch { case sErr != nil && errors.Is(sErr, errNotUsingTailscale): s.lc.IncrementCounter(r.Context(), "web_client_viewing_local", 1) - resp.Authorized = false // restricted to the readonly view + resp.Authorized = false // restricted to the read-only view case sErr != nil && errors.Is(sErr, errNotOwner): s.lc.IncrementCounter(r.Context(), "web_client_viewing_not_owner", 1) - resp.Authorized = false // restricted to the readonly view + resp.Authorized = false // restricted to the read-only view case sErr != nil && errors.Is(sErr, errTaggedLocalSource): s.lc.IncrementCounter(r.Context(), "web_client_viewing_local_tag", 1) - resp.Authorized = false // restricted to the readonly view + resp.Authorized = false // restricted to the read-only view case sErr != nil && errors.Is(sErr, errTaggedRemoteSource): s.lc.IncrementCounter(r.Context(), "web_client_viewing_remote_tag", 1) - resp.Authorized = false // restricted to the readonly view + resp.Authorized = false // restricted to the read-only view case sErr != nil && !errors.Is(sErr, errNoSession): // Any other error. http.Error(w, sErr.Error(), http.StatusInternalServerError) diff --git a/cmd/viewer/tests/tests.go b/cmd/viewer/tests/tests.go index ac094c53b7893..4020e5651978a 100644 --- a/cmd/viewer/tests/tests.go +++ b/cmd/viewer/tests/tests.go @@ -140,7 +140,7 @@ func (c *Container[T]) Clone() *Container[T] { panic(fmt.Errorf("%T contains pointers, but is not cloneable", c.Item)) } -// ContainerView is a pre-defined readonly view of a Container[T]. +// ContainerView is a pre-defined read-only view of a Container[T]. type ContainerView[T views.ViewCloner[T, V], V views.StructView[T]] struct { // Đļ is the underlying mutable value, named with a hard-to-type // character that looks pointy like a pointer. @@ -178,7 +178,7 @@ func (c *MapContainer[K, V]) Clone() *MapContainer[K, V] { return &MapContainer[K, V]{m} } -// MapContainerView is a pre-defined readonly view of a [MapContainer][K, T]. +// MapContainerView is a pre-defined read-only view of a [MapContainer][K, T]. type MapContainerView[K comparable, T views.ViewCloner[T, V], V views.StructView[T]] struct { // Đļ is the underlying mutable value, named with a hard-to-type // character that looks pointy like a pointer. diff --git a/cmd/viewer/tests/tests_view.go b/cmd/viewer/tests/tests_view.go index 41c1338ff4366..f1d8f424ff01b 100644 --- a/cmd/viewer/tests/tests_view.go +++ b/cmd/viewer/tests/tests_view.go @@ -16,7 +16,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=false -type=StructWithPtrs,StructWithoutPtrs,Map,StructWithSlices,OnlyGetClone,StructWithEmbedded,GenericIntStruct,GenericNoPtrsStruct,GenericCloneableStruct,StructWithContainers,StructWithTypeAliasFields,GenericTypeAliasStruct -// View returns a readonly view of StructWithPtrs. +// View returns a read-only view of StructWithPtrs. func (p *StructWithPtrs) View() StructWithPtrsView { return StructWithPtrsView{Đļ: p} } @@ -32,7 +32,7 @@ type StructWithPtrsView struct { Đļ *StructWithPtrs } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v StructWithPtrsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -80,7 +80,7 @@ var _StructWithPtrsViewNeedsRegeneration = StructWithPtrs(struct { NoCloneValue *StructWithoutPtrs }{}) -// View returns a readonly view of StructWithoutPtrs. +// View returns a read-only view of StructWithoutPtrs. func (p *StructWithoutPtrs) View() StructWithoutPtrsView { return StructWithoutPtrsView{Đļ: p} } @@ -96,7 +96,7 @@ type StructWithoutPtrsView struct { Đļ *StructWithoutPtrs } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v StructWithoutPtrsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -134,7 +134,7 @@ var _StructWithoutPtrsViewNeedsRegeneration = StructWithoutPtrs(struct { Pfx netip.Prefix }{}) -// View returns a readonly view of Map. +// View returns a read-only view of Map. func (p *Map) View() MapView { return MapView{Đļ: p} } @@ -150,7 +150,7 @@ type MapView struct { Đļ *Map } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v MapView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -240,7 +240,7 @@ var _MapViewNeedsRegeneration = Map(struct { StructWithPtrKey map[StructWithPtrs]int }{}) -// View returns a readonly view of StructWithSlices. +// View returns a read-only view of StructWithSlices. func (p *StructWithSlices) View() StructWithSlicesView { return StructWithSlicesView{Đļ: p} } @@ -256,7 +256,7 @@ type StructWithSlicesView struct { Đļ *StructWithSlices } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v StructWithSlicesView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -314,7 +314,7 @@ var _StructWithSlicesViewNeedsRegeneration = StructWithSlices(struct { Ints []*int }{}) -// View returns a readonly view of StructWithEmbedded. +// View returns a read-only view of StructWithEmbedded. func (p *StructWithEmbedded) View() StructWithEmbeddedView { return StructWithEmbeddedView{Đļ: p} } @@ -330,7 +330,7 @@ type StructWithEmbeddedView struct { Đļ *StructWithEmbedded } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v StructWithEmbeddedView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -370,7 +370,7 @@ var _StructWithEmbeddedViewNeedsRegeneration = StructWithEmbedded(struct { StructWithSlices }{}) -// View returns a readonly view of GenericIntStruct. +// View returns a read-only view of GenericIntStruct. func (p *GenericIntStruct[T]) View() GenericIntStructView[T] { return GenericIntStructView[T]{Đļ: p} } @@ -386,7 +386,7 @@ type GenericIntStructView[T constraints.Integer] struct { Đļ *GenericIntStruct[T] } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v GenericIntStructView[T]) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -442,7 +442,7 @@ func _GenericIntStructViewNeedsRegeneration[T constraints.Integer](GenericIntStr }{}) } -// View returns a readonly view of GenericNoPtrsStruct. +// View returns a read-only view of GenericNoPtrsStruct. func (p *GenericNoPtrsStruct[T]) View() GenericNoPtrsStructView[T] { return GenericNoPtrsStructView[T]{Đļ: p} } @@ -458,7 +458,7 @@ type GenericNoPtrsStructView[T StructWithoutPtrs | netip.Prefix | BasicType] str Đļ *GenericNoPtrsStruct[T] } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v GenericNoPtrsStructView[T]) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -514,7 +514,7 @@ func _GenericNoPtrsStructViewNeedsRegeneration[T StructWithoutPtrs | netip.Prefi }{}) } -// View returns a readonly view of GenericCloneableStruct. +// View returns a read-only view of GenericCloneableStruct. func (p *GenericCloneableStruct[T, V]) View() GenericCloneableStructView[T, V] { return GenericCloneableStructView[T, V]{Đļ: p} } @@ -530,7 +530,7 @@ type GenericCloneableStructView[T views.ViewCloner[T, V], V views.StructView[T]] Đļ *GenericCloneableStruct[T, V] } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v GenericCloneableStructView[T, V]) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -589,7 +589,7 @@ func _GenericCloneableStructViewNeedsRegeneration[T views.ViewCloner[T, V], V vi }{}) } -// View returns a readonly view of StructWithContainers. +// View returns a read-only view of StructWithContainers. func (p *StructWithContainers) View() StructWithContainersView { return StructWithContainersView{Đļ: p} } @@ -605,7 +605,7 @@ type StructWithContainersView struct { Đļ *StructWithContainers } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v StructWithContainersView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -661,7 +661,7 @@ var _StructWithContainersViewNeedsRegeneration = StructWithContainers(struct { CloneableGenericMap MapContainer[int, *GenericNoPtrsStruct[int]] }{}) -// View returns a readonly view of StructWithTypeAliasFields. +// View returns a read-only view of StructWithTypeAliasFields. func (p *StructWithTypeAliasFields) View() StructWithTypeAliasFieldsView { return StructWithTypeAliasFieldsView{Đļ: p} } @@ -677,7 +677,7 @@ type StructWithTypeAliasFieldsView struct { Đļ *StructWithTypeAliasFields } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v StructWithTypeAliasFieldsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -759,7 +759,7 @@ var _StructWithTypeAliasFieldsViewNeedsRegeneration = StructWithTypeAliasFields( MapOfSlicesWithoutPtrs map[string][]*StructWithoutPtrsAlias }{}) -// View returns a readonly view of GenericTypeAliasStruct. +// View returns a read-only view of GenericTypeAliasStruct. func (p *GenericTypeAliasStruct[T, T2, V2]) View() GenericTypeAliasStructView[T, T2, V2] { return GenericTypeAliasStructView[T, T2, V2]{Đļ: p} } @@ -775,7 +775,7 @@ type GenericTypeAliasStructView[T integer, T2 views.ViewCloner[T2, V2], V2 views Đļ *GenericTypeAliasStruct[T, T2, V2] } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v GenericTypeAliasStructView[T, T2, V2]) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/cmd/viewer/viewer.go b/cmd/viewer/viewer.go index e265defe0c522..2d30cc2eb1f2d 100644 --- a/cmd/viewer/viewer.go +++ b/cmd/viewer/viewer.go @@ -21,7 +21,7 @@ import ( ) const viewTemplateStr = `{{define "common"}} -// View returns a readonly view of {{.StructName}}. +// View returns a read-only view of {{.StructName}}. func (p *{{.StructName}}{{.TypeParamNames}}) View() {{.ViewName}}{{.TypeParamNames}} { return {{.ViewName}}{{.TypeParamNames}}{Đļ: p} } @@ -37,7 +37,7 @@ type {{.ViewName}}{{.TypeParams}} struct { Đļ *{{.StructName}}{{.TypeParamNames}} } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v {{.ViewName}}{{.TypeParamNames}}) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -143,7 +143,7 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, _ * MapValueView string MapFn string - // MakeViewFnName is the name of the function that accepts a value and returns a readonly view of it. + // MakeViewFnName is the name of the function that accepts a value and returns a read-only view of it. MakeViewFnName string }{ StructName: typ.Obj().Name(), diff --git a/drive/drive_view.go b/drive/drive_view.go index a6adfbc705378..0f6686f24da68 100644 --- a/drive/drive_view.go +++ b/drive/drive_view.go @@ -14,7 +14,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=true -type=Share -// View returns a readonly view of Share. +// View returns a read-only view of Share. func (p *Share) View() ShareView { return ShareView{Đļ: p} } @@ -30,7 +30,7 @@ type ShareView struct { Đļ *Share } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v ShareView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/ipn/ipn_view.go b/ipn/ipn_view.go index bc67531e4253d..9cd5a466a6840 100644 --- a/ipn/ipn_view.go +++ b/ipn/ipn_view.go @@ -20,7 +20,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=false -type=Prefs,ServeConfig,ServiceConfig,TCPPortHandler,HTTPHandler,WebServerConfig -// View returns a readonly view of Prefs. +// View returns a read-only view of Prefs. func (p *Prefs) View() PrefsView { return PrefsView{Đļ: p} } @@ -36,7 +36,7 @@ type PrefsView struct { Đļ *Prefs } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v PrefsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -138,7 +138,7 @@ var _PrefsViewNeedsRegeneration = Prefs(struct { Persist *persist.Persist }{}) -// View returns a readonly view of ServeConfig. +// View returns a read-only view of ServeConfig. func (p *ServeConfig) View() ServeConfigView { return ServeConfigView{Đļ: p} } @@ -154,7 +154,7 @@ type ServeConfigView struct { Đļ *ServeConfig } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v ServeConfigView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -222,7 +222,7 @@ var _ServeConfigViewNeedsRegeneration = ServeConfig(struct { ETag string }{}) -// View returns a readonly view of ServiceConfig. +// View returns a read-only view of ServiceConfig. func (p *ServiceConfig) View() ServiceConfigView { return ServiceConfigView{Đļ: p} } @@ -238,7 +238,7 @@ type ServiceConfigView struct { Đļ *ServiceConfig } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v ServiceConfigView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -287,7 +287,7 @@ var _ServiceConfigViewNeedsRegeneration = ServiceConfig(struct { Tun bool }{}) -// View returns a readonly view of TCPPortHandler. +// View returns a read-only view of TCPPortHandler. func (p *TCPPortHandler) View() TCPPortHandlerView { return TCPPortHandlerView{Đļ: p} } @@ -303,7 +303,7 @@ type TCPPortHandlerView struct { Đļ *TCPPortHandler } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v TCPPortHandlerView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -345,7 +345,7 @@ var _TCPPortHandlerViewNeedsRegeneration = TCPPortHandler(struct { TerminateTLS string }{}) -// View returns a readonly view of HTTPHandler. +// View returns a read-only view of HTTPHandler. func (p *HTTPHandler) View() HTTPHandlerView { return HTTPHandlerView{Đļ: p} } @@ -361,7 +361,7 @@ type HTTPHandlerView struct { Đļ *HTTPHandler } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v HTTPHandlerView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -401,7 +401,7 @@ var _HTTPHandlerViewNeedsRegeneration = HTTPHandler(struct { Text string }{}) -// View returns a readonly view of WebServerConfig. +// View returns a read-only view of WebServerConfig. func (p *WebServerConfig) View() WebServerConfigView { return WebServerConfigView{Đļ: p} } @@ -417,7 +417,7 @@ type WebServerConfigView struct { Đļ *WebServerConfig } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v WebServerConfigView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 4ebcd5d6d0869..d33e2c9eef039 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -3952,7 +3952,7 @@ func (b *LocalBackend) wantIngressLocked() bool { // setPrefsLockedOnEntry requires b.mu be held to call it, but it // unlocks b.mu when done. newp ownership passes to this function. -// It returns a readonly copy of the new prefs. +// It returns a read-only copy of the new prefs. func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce) ipn.PrefsView { defer unlock() diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index 53df3dcef0d99..8edd19c832092 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -21,7 +21,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=true -type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse,RegisterResponseAuth,RegisterRequest,DERPHomeParams,DERPRegion,DERPMap,DERPNode,SSHRule,SSHAction,SSHPrincipal,ControlDialPlan,Location,UserProfile -// View returns a readonly view of User. +// View returns a read-only view of User. func (p *User) View() UserView { return UserView{Đļ: p} } @@ -37,7 +37,7 @@ type UserView struct { Đļ *User } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v UserView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -79,7 +79,7 @@ var _UserViewNeedsRegeneration = User(struct { Created time.Time }{}) -// View returns a readonly view of Node. +// View returns a read-only view of Node. func (p *Node) View() NodeView { return NodeView{Đļ: p} } @@ -95,7 +95,7 @@ type NodeView struct { Đļ *Node } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v NodeView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -216,7 +216,7 @@ var _NodeViewNeedsRegeneration = Node(struct { ExitNodeDNSResolvers []*dnstype.Resolver }{}) -// View returns a readonly view of Hostinfo. +// View returns a read-only view of Hostinfo. func (p *Hostinfo) View() HostinfoView { return HostinfoView{Đļ: p} } @@ -232,7 +232,7 @@ type HostinfoView struct { Đļ *Hostinfo } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v HostinfoView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -341,7 +341,7 @@ var _HostinfoViewNeedsRegeneration = Hostinfo(struct { Location *Location }{}) -// View returns a readonly view of NetInfo. +// View returns a read-only view of NetInfo. func (p *NetInfo) View() NetInfoView { return NetInfoView{Đļ: p} } @@ -357,7 +357,7 @@ type NetInfoView struct { Đļ *NetInfo } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v NetInfoView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -421,7 +421,7 @@ var _NetInfoViewNeedsRegeneration = NetInfo(struct { FirewallMode string }{}) -// View returns a readonly view of Login. +// View returns a read-only view of Login. func (p *Login) View() LoginView { return LoginView{Đļ: p} } @@ -437,7 +437,7 @@ type LoginView struct { Đļ *Login } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v LoginView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -482,7 +482,7 @@ var _LoginViewNeedsRegeneration = Login(struct { ProfilePicURL string }{}) -// View returns a readonly view of DNSConfig. +// View returns a read-only view of DNSConfig. func (p *DNSConfig) View() DNSConfigView { return DNSConfigView{Đļ: p} } @@ -498,7 +498,7 @@ type DNSConfigView struct { Đļ *DNSConfig } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v DNSConfigView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -563,7 +563,7 @@ var _DNSConfigViewNeedsRegeneration = DNSConfig(struct { TempCorpIssue13969 string }{}) -// View returns a readonly view of RegisterResponse. +// View returns a read-only view of RegisterResponse. func (p *RegisterResponse) View() RegisterResponseView { return RegisterResponseView{Đļ: p} } @@ -579,7 +579,7 @@ type RegisterResponseView struct { Đļ *RegisterResponse } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v RegisterResponseView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -629,7 +629,7 @@ var _RegisterResponseViewNeedsRegeneration = RegisterResponse(struct { Error string }{}) -// View returns a readonly view of RegisterResponseAuth. +// View returns a read-only view of RegisterResponseAuth. func (p *RegisterResponseAuth) View() RegisterResponseAuthView { return RegisterResponseAuthView{Đļ: p} } @@ -645,7 +645,7 @@ type RegisterResponseAuthView struct { Đļ *RegisterResponseAuth } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v RegisterResponseAuthView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -687,7 +687,7 @@ var _RegisterResponseAuthViewNeedsRegeneration = RegisterResponseAuth(struct { AuthKey string }{}) -// View returns a readonly view of RegisterRequest. +// View returns a read-only view of RegisterRequest. func (p *RegisterRequest) View() RegisterRequestView { return RegisterRequestView{Đļ: p} } @@ -703,7 +703,7 @@ type RegisterRequestView struct { Đļ *RegisterRequest } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v RegisterRequestView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -777,7 +777,7 @@ var _RegisterRequestViewNeedsRegeneration = RegisterRequest(struct { Tailnet string }{}) -// View returns a readonly view of DERPHomeParams. +// View returns a read-only view of DERPHomeParams. func (p *DERPHomeParams) View() DERPHomeParamsView { return DERPHomeParamsView{Đļ: p} } @@ -793,7 +793,7 @@ type DERPHomeParamsView struct { Đļ *DERPHomeParams } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v DERPHomeParamsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -831,7 +831,7 @@ var _DERPHomeParamsViewNeedsRegeneration = DERPHomeParams(struct { RegionScore map[int]float64 }{}) -// View returns a readonly view of DERPRegion. +// View returns a read-only view of DERPRegion. func (p *DERPRegion) View() DERPRegionView { return DERPRegionView{Đļ: p} } @@ -847,7 +847,7 @@ type DERPRegionView struct { Đļ *DERPRegion } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v DERPRegionView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -897,7 +897,7 @@ var _DERPRegionViewNeedsRegeneration = DERPRegion(struct { Nodes []*DERPNode }{}) -// View returns a readonly view of DERPMap. +// View returns a read-only view of DERPMap. func (p *DERPMap) View() DERPMapView { return DERPMapView{Đļ: p} } @@ -913,7 +913,7 @@ type DERPMapView struct { Đļ *DERPMap } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v DERPMapView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -958,7 +958,7 @@ var _DERPMapViewNeedsRegeneration = DERPMap(struct { OmitDefaultRegions bool }{}) -// View returns a readonly view of DERPNode. +// View returns a read-only view of DERPNode. func (p *DERPNode) View() DERPNodeView { return DERPNodeView{Đļ: p} } @@ -974,7 +974,7 @@ type DERPNodeView struct { Đļ *DERPNode } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v DERPNodeView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -1032,7 +1032,7 @@ var _DERPNodeViewNeedsRegeneration = DERPNode(struct { CanPort80 bool }{}) -// View returns a readonly view of SSHRule. +// View returns a read-only view of SSHRule. func (p *SSHRule) View() SSHRuleView { return SSHRuleView{Đļ: p} } @@ -1048,7 +1048,7 @@ type SSHRuleView struct { Đļ *SSHRule } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v SSHRuleView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -1098,7 +1098,7 @@ var _SSHRuleViewNeedsRegeneration = SSHRule(struct { AcceptEnv []string }{}) -// View returns a readonly view of SSHAction. +// View returns a read-only view of SSHAction. func (p *SSHAction) View() SSHActionView { return SSHActionView{Đļ: p} } @@ -1114,7 +1114,7 @@ type SSHActionView struct { Đļ *SSHAction } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v SSHActionView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -1170,7 +1170,7 @@ var _SSHActionViewNeedsRegeneration = SSHAction(struct { OnRecordingFailure *SSHRecorderFailureAction }{}) -// View returns a readonly view of SSHPrincipal. +// View returns a read-only view of SSHPrincipal. func (p *SSHPrincipal) View() SSHPrincipalView { return SSHPrincipalView{Đļ: p} } @@ -1186,7 +1186,7 @@ type SSHPrincipalView struct { Đļ *SSHPrincipal } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v SSHPrincipalView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -1232,7 +1232,7 @@ var _SSHPrincipalViewNeedsRegeneration = SSHPrincipal(struct { UnusedPubKeys []string }{}) -// View returns a readonly view of ControlDialPlan. +// View returns a read-only view of ControlDialPlan. func (p *ControlDialPlan) View() ControlDialPlanView { return ControlDialPlanView{Đļ: p} } @@ -1248,7 +1248,7 @@ type ControlDialPlanView struct { Đļ *ControlDialPlan } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v ControlDialPlanView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -1286,7 +1286,7 @@ var _ControlDialPlanViewNeedsRegeneration = ControlDialPlan(struct { Candidates []ControlIPCandidate }{}) -// View returns a readonly view of Location. +// View returns a read-only view of Location. func (p *Location) View() LocationView { return LocationView{Đļ: p} } @@ -1302,7 +1302,7 @@ type LocationView struct { Đļ *Location } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v LocationView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -1350,7 +1350,7 @@ var _LocationViewNeedsRegeneration = Location(struct { Priority int }{}) -// View returns a readonly view of UserProfile. +// View returns a read-only view of UserProfile. func (p *UserProfile) View() UserProfileView { return UserProfileView{Đļ: p} } @@ -1366,7 +1366,7 @@ type UserProfileView struct { Đļ *UserProfile } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v UserProfileView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/types/dnstype/dnstype_view.go b/types/dnstype/dnstype_view.go index c0e2b28ffb9b4..c77ff9a406106 100644 --- a/types/dnstype/dnstype_view.go +++ b/types/dnstype/dnstype_view.go @@ -15,7 +15,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=true -type=Resolver -// View returns a readonly view of Resolver. +// View returns a read-only view of Resolver. func (p *Resolver) View() ResolverView { return ResolverView{Đļ: p} } @@ -31,7 +31,7 @@ type ResolverView struct { Đļ *Resolver } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v ResolverView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/types/persist/persist_view.go b/types/persist/persist_view.go index 1d479b3bf10e7..ce600be3e0753 100644 --- a/types/persist/persist_view.go +++ b/types/persist/persist_view.go @@ -17,7 +17,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=false -type=Persist -// View returns a readonly view of Persist. +// View returns a read-only view of Persist. func (p *Persist) View() PersistView { return PersistView{Đļ: p} } @@ -33,7 +33,7 @@ type PersistView struct { Đļ *Persist } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v PersistView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/types/prefs/prefs.go b/types/prefs/prefs.go index 3bbd237fe5efe..4f79020770e2d 100644 --- a/types/prefs/prefs.go +++ b/types/prefs/prefs.go @@ -29,8 +29,8 @@ import ( var ( // ErrManaged is the error returned when attempting to modify a managed preference. ErrManaged = errors.New("cannot modify a managed preference") - // ErrReadOnly is the error returned when attempting to modify a readonly preference. - ErrReadOnly = errors.New("cannot modify a readonly preference") + // ErrReadOnly is the error returned when attempting to modify a read-only preference. + ErrReadOnly = errors.New("cannot modify a read-only preference") ) // metadata holds type-agnostic preference metadata. diff --git a/types/prefs/prefs_example/prefs_example_view.go b/types/prefs/prefs_example/prefs_example_view.go index 0256bd7e6d25b..9aaac6e9c3ed6 100644 --- a/types/prefs/prefs_example/prefs_example_view.go +++ b/types/prefs/prefs_example/prefs_example_view.go @@ -20,7 +20,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=false -type=Prefs,AutoUpdatePrefs,AppConnectorPrefs -// View returns a readonly view of Prefs. +// View returns a read-only view of Prefs. func (p *Prefs) View() PrefsView { return PrefsView{Đļ: p} } @@ -36,7 +36,7 @@ type PrefsView struct { Đļ *Prefs } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v PrefsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -132,7 +132,7 @@ var _PrefsViewNeedsRegeneration = Prefs(struct { Persist *persist.Persist }{}) -// View returns a readonly view of AutoUpdatePrefs. +// View returns a read-only view of AutoUpdatePrefs. func (p *AutoUpdatePrefs) View() AutoUpdatePrefsView { return AutoUpdatePrefsView{Đļ: p} } @@ -148,7 +148,7 @@ type AutoUpdatePrefsView struct { Đļ *AutoUpdatePrefs } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v AutoUpdatePrefsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -186,7 +186,7 @@ var _AutoUpdatePrefsViewNeedsRegeneration = AutoUpdatePrefs(struct { Apply prefs.Item[opt.Bool] }{}) -// View returns a readonly view of AppConnectorPrefs. +// View returns a read-only view of AppConnectorPrefs. func (p *AppConnectorPrefs) View() AppConnectorPrefsView { return AppConnectorPrefsView{Đļ: p} } @@ -202,7 +202,7 @@ type AppConnectorPrefsView struct { Đļ *AppConnectorPrefs } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v AppConnectorPrefsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/types/prefs/prefs_view_test.go b/types/prefs/prefs_view_test.go index ef9f09603ea71..f6cfc918d02c0 100644 --- a/types/prefs/prefs_view_test.go +++ b/types/prefs/prefs_view_test.go @@ -13,7 +13,7 @@ import ( //go:generate go run tailscale.com/cmd/cloner -clonefunc=false -type=TestPrefs,TestBundle,TestValueStruct,TestGenericStruct,TestPrefsGroup -tags=test -// View returns a readonly view of TestPrefs. +// View returns a read-only view of TestPrefs. func (p *TestPrefs) View() TestPrefsView { return TestPrefsView{Đļ: p} } @@ -29,7 +29,7 @@ type TestPrefsView struct { Đļ *TestPrefs } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v TestPrefsView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -117,7 +117,7 @@ var _TestPrefsViewNeedsRegeneration = TestPrefs(struct { Group TestPrefsGroup }{}) -// View returns a readonly view of TestBundle. +// View returns a read-only view of TestBundle. func (p *TestBundle) View() TestBundleView { return TestBundleView{Đļ: p} } @@ -133,7 +133,7 @@ type TestBundleView struct { Đļ *TestBundle } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v TestBundleView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -172,7 +172,7 @@ var _TestBundleViewNeedsRegeneration = TestBundle(struct { Nested *TestValueStruct }{}) -// View returns a readonly view of TestValueStruct. +// View returns a read-only view of TestValueStruct. func (p *TestValueStruct) View() TestValueStructView { return TestValueStructView{Đļ: p} } @@ -188,7 +188,7 @@ type TestValueStructView struct { Đļ *TestValueStruct } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v TestValueStructView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -225,7 +225,7 @@ var _TestValueStructViewNeedsRegeneration = TestValueStruct(struct { Value int }{}) -// View returns a readonly view of TestGenericStruct. +// View returns a read-only view of TestGenericStruct. func (p *TestGenericStruct[T]) View() TestGenericStructView[T] { return TestGenericStructView[T]{Đļ: p} } @@ -241,7 +241,7 @@ type TestGenericStructView[T ImmutableType] struct { Đļ *TestGenericStruct[T] } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v TestGenericStructView[T]) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with @@ -280,7 +280,7 @@ func _TestGenericStructViewNeedsRegeneration[T ImmutableType](TestGenericStruct[ }{}) } -// View returns a readonly view of TestPrefsGroup. +// View returns a read-only view of TestPrefsGroup. func (p *TestPrefsGroup) View() TestPrefsGroupView { return TestPrefsGroupView{Đļ: p} } @@ -296,7 +296,7 @@ type TestPrefsGroupView struct { Đļ *TestPrefsGroup } -// Valid reports whether underlying value is non-nil. +// Valid reports whether v's underlying value is non-nil. func (v TestPrefsGroupView) Valid() bool { return v.Đļ != nil } // AsStruct returns a clone of the underlying value which aliases no memory with diff --git a/types/prefs/struct_map.go b/types/prefs/struct_map.go index 2003eebe323fa..4d55da7a0b87d 100644 --- a/types/prefs/struct_map.go +++ b/types/prefs/struct_map.go @@ -83,7 +83,7 @@ type StructMapView[K MapKeyType, T views.ViewCloner[T, V], V views.StructView[T] Đļ *StructMap[K, T] } -// StructMapViewOf returns a readonly view of m. +// StructMapViewOf returns a read-only view of m. // It is used by [tailscale.com/cmd/viewer]. func StructMapViewOf[K MapKeyType, T views.ViewCloner[T, V], V views.StructView[T]](m *StructMap[K, T]) StructMapView[K, T, V] { return StructMapView[K, T, V]{m} From 66269dc934058cd41ddff7a5fd94c6425dbb28f8 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 14 Jan 2025 11:04:55 -0600 Subject: [PATCH 113/223] ipn/ipnlocal: allow Peer API access via either V4MasqAddr or V6MasqAddr when both are set This doesn't seem to have any immediate impact, but not allowing access via the IPv6 masquerade address when an IPv4 masquerade address is also set seems like a bug. Updates #cleanup Updates #14570 (found when working on it) Signed-off-by: Nick Khyl --- ipn/ipnlocal/peerapi.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index 7aa677640b81f..4d05489171e3e 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -233,11 +233,13 @@ func (h *peerAPIHandler) logf(format string, a ...any) { // isAddressValid reports whether addr is a valid destination address for this // node originating from the peer. func (h *peerAPIHandler) isAddressValid(addr netip.Addr) bool { - if v, ok := h.peerNode.SelfNodeV4MasqAddrForThisPeer().GetOk(); ok { - return v == addr + if !addr.IsValid() { + return false } - if v, ok := h.peerNode.SelfNodeV6MasqAddrForThisPeer().GetOk(); ok { - return v == addr + v4MasqAddr, hasMasqV4 := h.peerNode.SelfNodeV4MasqAddrForThisPeer().GetOk() + v6MasqAddr, hasMasqV6 := h.peerNode.SelfNodeV6MasqAddrForThisPeer().GetOk() + if hasMasqV4 || hasMasqV6 { + return addr == v4MasqAddr || addr == v6MasqAddr } pfx := netip.PrefixFrom(addr, addr.BitLen()) return views.SliceContains(h.selfNode.Addresses(), pfx) From 2fc4455e6dd9ab7f879d4e2f7cffc2be81f14077 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 14 Jan 2025 10:19:52 -0800 Subject: [PATCH 114/223] all: add Node.HomeDERP int, phase out "127.3.3.40:$region" hack [capver 111] This deprecates the old "DERP string" packing a DERP region ID into an IP:port of 127.3.3.40:$REGION_ID and just uses an integer, like PeerChange.DERPRegion does. We still support servers sending the old form; they're converted to the new form internally right when they're read off the network. Updates #14636 Change-Id: I9427ec071f02a2c6d75ccb0fcbf0ecff9f19f26f Signed-off-by: Brad Fitzpatrick --- control/controlbase/conn_test.go | 2 +- control/controlclient/map.go | 49 ++++++++--- control/controlclient/map_test.go | 83 +++++++++++++++---- ipn/ipnlocal/expiry.go | 2 +- ipn/ipnlocal/local.go | 10 +-- ipn/ipnlocal/local_test.go | 12 +-- tailcfg/tailcfg.go | 25 ++++-- tailcfg/tailcfg_clone.go | 3 +- tailcfg/tailcfg_test.go | 11 ++- tailcfg/tailcfg_view.go | 6 +- tstest/integration/testcontrol/testcontrol.go | 2 +- types/netmap/netmap.go | 9 +- types/netmap/netmap_test.go | 36 ++++---- types/netmap/nodemut.go | 3 +- wgengine/magicsock/endpoint.go | 4 +- wgengine/magicsock/magicsock.go | 5 +- wgengine/magicsock/magicsock_test.go | 2 +- wgengine/pendopen.go | 2 +- wgengine/wgcfg/nmcfg/nmcfg.go | 2 +- 19 files changed, 171 insertions(+), 97 deletions(-) diff --git a/control/controlbase/conn_test.go b/control/controlbase/conn_test.go index 8a0f46967e342..ed4642d3b179c 100644 --- a/control/controlbase/conn_test.go +++ b/control/controlbase/conn_test.go @@ -280,7 +280,7 @@ func TestConnMemoryOverhead(t *testing.T) { growthTotal := int64(ms.HeapAlloc) - int64(ms0.HeapAlloc) growthEach := float64(growthTotal) / float64(num) t.Logf("Alloced %v bytes, %.2f B/each", growthTotal, growthEach) - const max = 2000 + const max = 2048 if growthEach > max { t.Errorf("allocated more than expected; want max %v bytes/each", max) } diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 30c1da672b9dc..d5fd84c6d74f9 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -7,7 +7,6 @@ import ( "cmp" "context" "encoding/json" - "fmt" "maps" "net" "reflect" @@ -166,6 +165,7 @@ func (ms *mapSession) HandleNonKeepAliveMapResponse(ctx context.Context, resp *t // For responses that mutate the self node, check for updated nodeAttrs. if resp.Node != nil { + upgradeNode(resp.Node) if DevKnob.StripCaps() { resp.Node.Capabilities = nil resp.Node.CapMap = nil @@ -181,6 +181,13 @@ func (ms *mapSession) HandleNonKeepAliveMapResponse(ctx context.Context, resp *t ms.controlKnobs.UpdateFromNodeAttributes(resp.Node.CapMap) } + for _, p := range resp.Peers { + upgradeNode(p) + } + for _, p := range resp.PeersChanged { + upgradeNode(p) + } + // Call Node.InitDisplayNames on any changed nodes. initDisplayNames(cmp.Or(resp.Node.View(), ms.lastNode), resp) @@ -216,6 +223,26 @@ func (ms *mapSession) HandleNonKeepAliveMapResponse(ctx context.Context, resp *t return nil } +// upgradeNode upgrades Node fields from the server into the modern forms +// not using deprecated fields. +func upgradeNode(n *tailcfg.Node) { + if n == nil { + return + } + if n.LegacyDERPString != "" { + if n.HomeDERP == 0 { + ip, portStr, err := net.SplitHostPort(n.LegacyDERPString) + if ip == tailcfg.DerpMagicIP && err == nil { + port, err := strconv.Atoi(portStr) + if err == nil { + n.HomeDERP = port + } + } + } + n.LegacyDERPString = "" + } +} + func (ms *mapSession) tryHandleIncrementally(res *tailcfg.MapResponse) bool { if ms.controlKnobs != nil && ms.controlKnobs.DisableDeltaUpdates.Load() { return false @@ -443,7 +470,7 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s stats.changed++ mut := vp.AsStruct() if pc.DERPRegion != 0 { - mut.DERP = fmt.Sprintf("%s:%v", tailcfg.DerpMagicIP, pc.DERPRegion) + mut.HomeDERP = pc.DERPRegion patchDERPRegion.Add(1) } if pc.Cap != 0 { @@ -631,17 +658,13 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang if !views.SliceEqual(was.Endpoints(), views.SliceOf(n.Endpoints)) { pc().Endpoints = slices.Clone(n.Endpoints) } - case "DERP": - if was.DERP() != n.DERP { - ip, portStr, err := net.SplitHostPort(n.DERP) - if err != nil || ip != "127.3.3.40" { - return nil, false - } - port, err := strconv.Atoi(portStr) - if err != nil || port < 1 || port > 65535 { - return nil, false - } - pc().DERPRegion = port + case "LegacyDERPString": + if was.LegacyDERPString() != "" || n.LegacyDERPString != "" { + panic("unexpected; caller should've already called upgradeNode") + } + case "HomeDERP": + if was.HomeDERP() != n.HomeDERP { + pc().DERPRegion = n.HomeDERP } case "Hostinfo": if !was.Hostinfo().Valid() && !n.Hostinfo.Valid() { diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index ad8f7dd6e288b..9c8c0c3aa2f4c 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -50,9 +50,9 @@ func TestUpdatePeersStateFromResponse(t *testing.T) { n.LastSeen = &t } } - withDERP := func(d string) func(*tailcfg.Node) { + withDERP := func(regionID int) func(*tailcfg.Node) { return func(n *tailcfg.Node) { - n.DERP = d + n.HomeDERP = regionID } } withEP := func(ep string) func(*tailcfg.Node) { @@ -189,14 +189,14 @@ func TestUpdatePeersStateFromResponse(t *testing.T) { }, { name: "ep_change_derp", - prev: peers(n(1, "foo", withDERP("127.3.3.40:3"))), + prev: peers(n(1, "foo", withDERP(3))), mapRes: &tailcfg.MapResponse{ PeersChangedPatch: []*tailcfg.PeerChange{{ NodeID: 1, DERPRegion: 4, }}, }, - want: peers(n(1, "foo", withDERP("127.3.3.40:4"))), + want: peers(n(1, "foo", withDERP(4))), wantStats: updateStats{changed: 1}, }, { @@ -213,19 +213,19 @@ func TestUpdatePeersStateFromResponse(t *testing.T) { }, { name: "ep_change_udp_2", - prev: peers(n(1, "foo", withDERP("127.3.3.40:3"), withEP("1.2.3.4:111"))), + prev: peers(n(1, "foo", withDERP(3), withEP("1.2.3.4:111"))), mapRes: &tailcfg.MapResponse{ PeersChangedPatch: []*tailcfg.PeerChange{{ NodeID: 1, Endpoints: eps("1.2.3.4:56"), }}, }, - want: peers(n(1, "foo", withDERP("127.3.3.40:3"), withEP("1.2.3.4:56"))), + want: peers(n(1, "foo", withDERP(3), withEP("1.2.3.4:56"))), wantStats: updateStats{changed: 1}, }, { name: "ep_change_both", - prev: peers(n(1, "foo", withDERP("127.3.3.40:3"), withEP("1.2.3.4:111"))), + prev: peers(n(1, "foo", withDERP(3), withEP("1.2.3.4:111"))), mapRes: &tailcfg.MapResponse{ PeersChangedPatch: []*tailcfg.PeerChange{{ NodeID: 1, @@ -233,7 +233,7 @@ func TestUpdatePeersStateFromResponse(t *testing.T) { Endpoints: eps("1.2.3.4:56"), }}, }, - want: peers(n(1, "foo", withDERP("127.3.3.40:2"), withEP("1.2.3.4:56"))), + want: peers(n(1, "foo", withDERP(2), withEP("1.2.3.4:56"))), wantStats: updateStats{changed: 1}, }, { @@ -744,8 +744,8 @@ func TestPeerChangeDiff(t *testing.T) { }, { name: "patch-derp", - a: &tailcfg.Node{ID: 1, DERP: "127.3.3.40:1"}, - b: &tailcfg.Node{ID: 1, DERP: "127.3.3.40:2"}, + a: &tailcfg.Node{ID: 1, HomeDERP: 1}, + b: &tailcfg.Node{ID: 1, HomeDERP: 2}, want: &tailcfg.PeerChange{NodeID: 1, DERPRegion: 2}, }, { @@ -929,23 +929,23 @@ func TestPatchifyPeersChanged(t *testing.T) { mr0: &tailcfg.MapResponse{ Node: &tailcfg.Node{Name: "foo.bar.ts.net."}, Peers: []*tailcfg.Node{ - {ID: 1, DERP: "127.3.3.40:1", Hostinfo: hi}, - {ID: 2, DERP: "127.3.3.40:2", Hostinfo: hi}, - {ID: 3, DERP: "127.3.3.40:3", Hostinfo: hi}, + {ID: 1, HomeDERP: 1, Hostinfo: hi}, + {ID: 2, HomeDERP: 2, Hostinfo: hi}, + {ID: 3, HomeDERP: 3, Hostinfo: hi}, }, }, mr1: &tailcfg.MapResponse{ PeersChanged: []*tailcfg.Node{ - {ID: 1, DERP: "127.3.3.40:11", Hostinfo: hi}, + {ID: 1, HomeDERP: 11, Hostinfo: hi}, {ID: 2, StableID: "other-change", Hostinfo: hi}, - {ID: 3, DERP: "127.3.3.40:33", Hostinfo: hi}, - {ID: 4, DERP: "127.3.3.40:4", Hostinfo: hi}, + {ID: 3, HomeDERP: 33, Hostinfo: hi}, + {ID: 4, HomeDERP: 4, Hostinfo: hi}, }, }, want: &tailcfg.MapResponse{ PeersChanged: []*tailcfg.Node{ {ID: 2, StableID: "other-change", Hostinfo: hi}, - {ID: 4, DERP: "127.3.3.40:4", Hostinfo: hi}, + {ID: 4, HomeDERP: 4, Hostinfo: hi}, }, PeersChangedPatch: []*tailcfg.PeerChange{ {NodeID: 1, DERPRegion: 11}, @@ -1006,6 +1006,53 @@ func TestPatchifyPeersChanged(t *testing.T) { } } +func TestUpgradeNode(t *testing.T) { + tests := []struct { + name string + in *tailcfg.Node + want *tailcfg.Node + }{ + { + name: "nil", + in: nil, + want: nil, + }, + { + name: "empty", + in: new(tailcfg.Node), + want: new(tailcfg.Node), + }, + { + name: "derp-both", + in: &tailcfg.Node{HomeDERP: 1, LegacyDERPString: tailcfg.DerpMagicIP + ":2"}, + want: &tailcfg.Node{HomeDERP: 1}, + }, + { + name: "derp-str-only", + in: &tailcfg.Node{LegacyDERPString: tailcfg.DerpMagicIP + ":2"}, + want: &tailcfg.Node{HomeDERP: 2}, + }, + { + name: "derp-int-only", + in: &tailcfg.Node{HomeDERP: 2}, + want: &tailcfg.Node{HomeDERP: 2}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got *tailcfg.Node + if tt.in != nil { + got = ptr.To(*tt.in) // shallow clone + } + upgradeNode(got) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("wrong result (-want +got):\n%s", diff) + } + }) + } + +} + func BenchmarkMapSessionDelta(b *testing.B) { for _, size := range []int{10, 100, 1_000, 10_000} { b.Run(fmt.Sprintf("size_%d", size), func(b *testing.B) { @@ -1022,7 +1069,7 @@ func BenchmarkMapSessionDelta(b *testing.B) { res.Peers = append(res.Peers, &tailcfg.Node{ ID: tailcfg.NodeID(i + 2), Name: fmt.Sprintf("peer%d.bar.ts.net.", i), - DERP: "127.3.3.40:10", + HomeDERP: 10, Addresses: []netip.Prefix{netip.MustParsePrefix("100.100.2.3/32"), netip.MustParsePrefix("fd7a:115c:a1e0::123/128")}, AllowedIPs: []netip.Prefix{netip.MustParsePrefix("100.100.2.3/32"), netip.MustParsePrefix("fd7a:115c:a1e0::123/128")}, Endpoints: eps("192.168.1.2:345", "192.168.1.3:678"), diff --git a/ipn/ipnlocal/expiry.go b/ipn/ipnlocal/expiry.go index 04c10226d50a0..d1119981594da 100644 --- a/ipn/ipnlocal/expiry.go +++ b/ipn/ipnlocal/expiry.go @@ -116,7 +116,7 @@ func (em *expiryManager) flagExpiredPeers(netmap *netmap.NetworkMap, localNow ti // since we discover endpoints via DERP, and due to DERP return // path optimization. mut.Endpoints = nil - mut.DERP = "" + mut.HomeDERP = 0 // Defense-in-depth: break the node's public key as well, in // case something tries to communicate. diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index d33e2c9eef039..81a62045a9111 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -7381,15 +7381,7 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug } distances := make([]nodeDistance, 0, len(candidates)) for _, c := range candidates { - if c.DERP() != "" { - ipp, err := netip.ParseAddrPort(c.DERP()) - if err != nil { - continue - } - if ipp.Addr() != tailcfg.DerpMagicIPAddr { - continue - } - regionID := int(ipp.Port()) + if regionID := c.HomeDERP(); regionID != 0 { candidatesByRegion[regionID] = append(candidatesByRegion[regionID], c) continue } diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index f3ee24a6bd8aa..f9a967beada7d 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1007,8 +1007,8 @@ func TestUpdateNetmapDelta(t *testing.T) { wants := []*tailcfg.Node{ { - ID: 1, - DERP: "127.3.3.40:1", + ID: 1, + HomeDERP: 1, }, { ID: 2, @@ -2021,7 +2021,7 @@ func TestAutoExitNodeSetNetInfoCallback(t *testing.T) { netip.MustParsePrefix("100.64.1.1/32"), netip.MustParsePrefix("fe70::1/128"), }, - DERP: "127.3.3.40:2", + HomeDERP: 2, } defaultDERPMap := &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ @@ -2985,7 +2985,7 @@ func makePeer(id tailcfg.NodeID, opts ...peerOptFunc) tailcfg.NodeView { ID: id, StableID: tailcfg.StableNodeID(fmt.Sprintf("stable%d", id)), Name: fmt.Sprintf("peer%d", id), - DERP: fmt.Sprintf("127.3.3.40:%d", id), + HomeDERP: int(id), } for _, opt := range opts { opt(node) @@ -3001,13 +3001,13 @@ func withName(name string) peerOptFunc { func withDERP(region int) peerOptFunc { return func(n *tailcfg.Node) { - n.DERP = fmt.Sprintf("127.3.3.40:%d", region) + n.HomeDERP = region } } func withoutDERP() peerOptFunc { return func(n *tailcfg.Node) { - n.DERP = "" + n.HomeDERP = 0 } } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 1ede0bd9b08a5..76945ec10898e 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -153,7 +153,8 @@ type CapabilityVersion int // - 108: 2024-11-08: Client sends ServicesHash in Hostinfo, understands c2n GET /vip-services. // - 109: 2024-11-18: Client supports filtertype.Match.SrcCaps (issue #12542) // - 110: 2024-12-12: removed never-before-used Tailscale SSH public key support (#14373) -const CurrentCapabilityVersion CapabilityVersion = 110 +// - 111: 2025-01-14: Client supports a peer having Node.HomeDERP (issue #14636) +const CurrentCapabilityVersion CapabilityVersion = 111 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -346,15 +347,24 @@ type Node struct { AllowedIPs []netip.Prefix // range of IP addresses to route to this node Endpoints []netip.AddrPort `json:",omitempty"` // IP+port (public via STUN, and local LANs) - // DERP is this node's home DERP region ID integer, but shoved into an + // LegacyDERPString is this node's home LegacyDERPString region ID integer, but shoved into an // IP:port string for legacy reasons. The IP address is always "127.3.3.40" // (a loopback address (127) followed by the digits over the letters DERP on - // a QWERTY keyboard (3.3.40)). The "port number" is the home DERP region ID + // a QWERTY keyboard (3.3.40)). The "port number" is the home LegacyDERPString region ID // integer. // - // TODO(bradfitz): simplify this legacy mess; add a new HomeDERPRegionID int - // field behind a new capver bump. - DERP string `json:",omitempty"` // DERP-in-IP:port ("127.3.3.40:N") endpoint + // Deprecated: HomeDERP has replaced this, but old servers might still send + // this field. See tailscale/tailscale#14636. Do not use this field in code + // other than in the upgradeNode func, which canonicalizes it to HomeDERP + // if it arrives as a LegacyDERPString string on the wire. + LegacyDERPString string `json:"DERP,omitempty"` // DERP-in-IP:port ("127.3.3.40:N") endpoint + + // HomeDERP is the modern version of the DERP string field, with just an + // integer. The client advertises support for this as of capver 111. + // + // HomeDERP may be zero if not (yet) known, but ideally always be non-zero + // for magicsock connectivity to function normally. + HomeDERP int `json:",omitempty"` // DERP region ID of the node's home DERP Hostinfo HostinfoView Created time.Time @@ -2162,7 +2172,8 @@ func (n *Node) Equal(n2 *Node) bool { slicesx.EqualSameNil(n.AllowedIPs, n2.AllowedIPs) && slicesx.EqualSameNil(n.PrimaryRoutes, n2.PrimaryRoutes) && slicesx.EqualSameNil(n.Endpoints, n2.Endpoints) && - n.DERP == n2.DERP && + n.LegacyDERPString == n2.LegacyDERPString && + n.HomeDERP == n2.HomeDERP && n.Cap == n2.Cap && n.Hostinfo.Equal(n2.Hostinfo) && n.Created.Equal(n2.Created) && diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index d282719b7d182..42cef1598e8e5 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -99,7 +99,8 @@ var _NodeCloneNeedsRegeneration = Node(struct { Addresses []netip.Prefix AllowedIPs []netip.Prefix Endpoints []netip.AddrPort - DERP string + LegacyDERPString string + HomeDERP int Hostinfo HostinfoView Created time.Time Cap CapabilityVersion diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index b9a204eadf17a..560e28933eccd 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -367,7 +367,7 @@ func TestNodeEqual(t *testing.T) { nodeHandles := []string{ "ID", "StableID", "Name", "User", "Sharer", "Key", "KeyExpiry", "KeySignature", "Machine", "DiscoKey", - "Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo", + "Addresses", "AllowedIPs", "Endpoints", "LegacyDERPString", "HomeDERP", "Hostinfo", "Created", "Cap", "Tags", "PrimaryRoutes", "LastSeen", "Online", "MachineAuthorized", "Capabilities", "CapMap", @@ -530,8 +530,13 @@ func TestNodeEqual(t *testing.T) { true, }, { - &Node{DERP: "foo"}, - &Node{DERP: "bar"}, + &Node{LegacyDERPString: "foo"}, + &Node{LegacyDERPString: "bar"}, + false, + }, + { + &Node{HomeDERP: 1}, + &Node{HomeDERP: 2}, false, }, { diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index 8edd19c832092..3770f272f6bcb 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -139,7 +139,8 @@ func (v NodeView) DiscoKey() key.DiscoPublic { return v.Đļ.DiscoK func (v NodeView) Addresses() views.Slice[netip.Prefix] { return views.SliceOf(v.Đļ.Addresses) } func (v NodeView) AllowedIPs() views.Slice[netip.Prefix] { return views.SliceOf(v.Đļ.AllowedIPs) } func (v NodeView) Endpoints() views.Slice[netip.AddrPort] { return views.SliceOf(v.Đļ.Endpoints) } -func (v NodeView) DERP() string { return v.Đļ.DERP } +func (v NodeView) LegacyDERPString() string { return v.Đļ.LegacyDERPString } +func (v NodeView) HomeDERP() int { return v.Đļ.HomeDERP } func (v NodeView) Hostinfo() HostinfoView { return v.Đļ.Hostinfo } func (v NodeView) Created() time.Time { return v.Đļ.Created } func (v NodeView) Cap() CapabilityVersion { return v.Đļ.Cap } @@ -192,7 +193,8 @@ var _NodeViewNeedsRegeneration = Node(struct { Addresses []netip.Prefix AllowedIPs []netip.Prefix Endpoints []netip.AddrPort - DERP string + LegacyDERPString string + HomeDERP int Hostinfo HostinfoView Created time.Time Cap CapabilityVersion diff --git a/tstest/integration/testcontrol/testcontrol.go b/tstest/integration/testcontrol/testcontrol.go index 386359f19fa11..e127087a62629 100644 --- a/tstest/integration/testcontrol/testcontrol.go +++ b/tstest/integration/testcontrol/testcontrol.go @@ -805,7 +805,7 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey key.Machi node.Hostinfo = req.Hostinfo.View() if ni := node.Hostinfo.NetInfo(); ni.Valid() { if ni.PreferredDERP() != 0 { - node.DERP = fmt.Sprintf("127.3.3.40:%d", ni.PreferredDERP()) + node.HomeDERP = ni.PreferredDERP() } } } diff --git a/types/netmap/netmap.go b/types/netmap/netmap.go index b1ac612de9669..7662e145efa60 100644 --- a/types/netmap/netmap.go +++ b/types/netmap/netmap.go @@ -287,11 +287,8 @@ func printPeerConcise(buf *strings.Builder, p tailcfg.NodeView) { epStrs[i] = fmt.Sprintf("%21v", e+strings.Repeat(" ", spaces)) } - derp := p.DERP() - const derpPrefix = "127.3.3.40:" - if strings.HasPrefix(derp, derpPrefix) { - derp = "D" + derp[len(derpPrefix):] - } + derp := fmt.Sprintf("D%d", p.HomeDERP()) + var discoShort string if !p.DiscoKey().IsZero() { discoShort = p.DiscoKey().ShortString() + " " @@ -311,7 +308,7 @@ func printPeerConcise(buf *strings.Builder, p tailcfg.NodeView) { // nodeConciseEqual reports whether a and b are equal for the fields accessed by printPeerConcise. func nodeConciseEqual(a, b tailcfg.NodeView) bool { return a.Key() == b.Key() && - a.DERP() == b.DERP() && + a.HomeDERP() == b.HomeDERP() && a.DiscoKey() == b.DiscoKey() && views.SliceEqual(a.AllowedIPs(), b.AllowedIPs()) && views.SliceEqual(a.Endpoints(), b.Endpoints()) diff --git a/types/netmap/netmap_test.go b/types/netmap/netmap_test.go index e7e2d19575c44..40f504741bfea 100644 --- a/types/netmap/netmap_test.go +++ b/types/netmap/netmap_test.go @@ -63,12 +63,12 @@ func TestNetworkMapConcise(t *testing.T) { Peers: nodeViews([]*tailcfg.Node{ { Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, { Key: testNodeKey(3), - DERP: "127.3.3.40:4", + HomeDERP: 4, Endpoints: eps("10.2.0.100:12", "10.1.0.100:12345"), }, }), @@ -102,7 +102,7 @@ func TestConciseDiffFrom(t *testing.T) { Peers: nodeViews([]*tailcfg.Node{ { Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -112,7 +112,7 @@ func TestConciseDiffFrom(t *testing.T) { Peers: nodeViews([]*tailcfg.Node{ { Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -126,7 +126,7 @@ func TestConciseDiffFrom(t *testing.T) { Peers: nodeViews([]*tailcfg.Node{ { Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -136,7 +136,7 @@ func TestConciseDiffFrom(t *testing.T) { Peers: nodeViews([]*tailcfg.Node{ { Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -151,7 +151,7 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -162,19 +162,19 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 1, Key: testNodeKey(1), - DERP: "127.3.3.40:1", + HomeDERP: 1, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, { ID: 3, Key: testNodeKey(3), - DERP: "127.3.3.40:3", + HomeDERP: 3, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -189,19 +189,19 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 1, Key: testNodeKey(1), - DERP: "127.3.3.40:1", + HomeDERP: 1, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, { ID: 3, Key: testNodeKey(3), - DERP: "127.3.3.40:3", + HomeDERP: 3, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -212,7 +212,7 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "192.168.0.100:12354"), }, }), @@ -227,7 +227,7 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "1.1.1.1:1"), }, }), @@ -238,7 +238,7 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:12", "1.1.1.1:2"), }, }), @@ -253,7 +253,7 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:41641", "1.1.1.1:41641"), DiscoKey: testDiscoKey("f00f00f00f"), AllowedIPs: []netip.Prefix{netip.PrefixFrom(netaddr.IPv4(100, 102, 103, 104), 32)}, @@ -266,7 +266,7 @@ func TestConciseDiffFrom(t *testing.T) { { ID: 2, Key: testNodeKey(2), - DERP: "127.3.3.40:2", + HomeDERP: 2, Endpoints: eps("192.168.0.100:41641", "1.1.1.1:41641"), DiscoKey: testDiscoKey("ba4ba4ba4b"), AllowedIPs: []netip.Prefix{netip.PrefixFrom(netaddr.IPv4(100, 102, 103, 104), 32)}, diff --git a/types/netmap/nodemut.go b/types/netmap/nodemut.go index 46fbaefc640e0..6f116059e2bfa 100644 --- a/types/netmap/nodemut.go +++ b/types/netmap/nodemut.go @@ -5,7 +5,6 @@ package netmap import ( "cmp" - "fmt" "net/netip" "reflect" "slices" @@ -35,7 +34,7 @@ type NodeMutationDERPHome struct { } func (m NodeMutationDERPHome) Apply(n *tailcfg.Node) { - n.DERP = fmt.Sprintf("127.3.3.40:%v", m.DERPRegion) + n.HomeDERP = m.DERPRegion } // NodeMutation is a NodeMutation that says a node's endpoints have changed. diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index df4299b72a4cb..7780c7db6f1f9 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1359,7 +1359,7 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p }) de.resetLocked() } - if n.DERP() == "" { + if n.HomeDERP() == 0 { if de.derpAddr.IsValid() { de.debugUpdates.Add(EndpointChange{ When: time.Now(), @@ -1369,7 +1369,7 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p } de.derpAddr = netip.AddrPort{} } else { - newDerp, _ := netip.ParseAddrPort(n.DERP()) + newDerp := netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(n.HomeDERP())) if de.derpAddr != newDerp { de.debugUpdates.Add(EndpointChange{ When: time.Now(), diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 6a49f091ee370..98cb63b888b59 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2337,10 +2337,7 @@ func devPanicf(format string, a ...any) { func (c *Conn) logEndpointCreated(n tailcfg.NodeView) { c.logf("magicsock: created endpoint key=%s: disco=%s; %v", n.Key().ShortString(), n.DiscoKey().ShortString(), logger.ArgWriter(func(w *bufio.Writer) { - const derpPrefix = "127.3.3.40:" - if strings.HasPrefix(n.DERP(), derpPrefix) { - ipp, _ := netip.ParseAddrPort(n.DERP()) - regionID := int(ipp.Port()) + if regionID := n.HomeDERP(); regionID != 0 { code := c.derpRegionCodeLocked(regionID) if code != "" { code = "(" + code + ")" diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index d4c9f0cbb9a4d..090c1218f53b1 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -314,7 +314,7 @@ func meshStacks(logf logger.Logf, mutateNetmap func(idx int, nm *netmap.NetworkM Addresses: addrs, AllowedIPs: addrs, Endpoints: epFromTyped(eps[i]), - DERP: "127.3.3.40:1", + HomeDERP: 1, } nm.Peers = append(nm.Peers, peer.View()) } diff --git a/wgengine/pendopen.go b/wgengine/pendopen.go index f8e9198a516db..28d1f4f9d59e4 100644 --- a/wgengine/pendopen.go +++ b/wgengine/pendopen.go @@ -198,7 +198,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) { e.logf("open-conn-track: timeout opening %v; peer node %v running pre-0.100", flow, n.Key().ShortString()) return } - if n.DERP() == "" { + if n.HomeDERP() == 0 { e.logf("open-conn-track: timeout opening %v; peer node %v not connected to any DERP relay", flow, n.Key().ShortString()) return } diff --git a/wgengine/wgcfg/nmcfg/nmcfg.go b/wgengine/wgcfg/nmcfg/nmcfg.go index 97304aa415edb..45c235b4d5c58 100644 --- a/wgengine/wgcfg/nmcfg/nmcfg.go +++ b/wgengine/wgcfg/nmcfg/nmcfg.go @@ -85,7 +85,7 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags, skippedSubnets := new(bytes.Buffer) for _, peer := range nm.Peers { - if peer.DiscoKey().IsZero() && peer.DERP() == "" && !peer.IsWireGuardOnly() { + if peer.DiscoKey().IsZero() && peer.HomeDERP() == 0 && !peer.IsWireGuardOnly() { // Peer predates both DERP and active discovery, we cannot // communicate with it. logf("[v1] wgcfg: skipped peer %s, doesn't offer DERP or disco", peer.Key().ShortString()) From 27477983e333eda4cab540778d97fe64203c91ac Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 14 Jan 2025 12:36:09 -0800 Subject: [PATCH 115/223] control/controlclient: remove misleading TS_DEBUG_NETMAP, make it TS_DEBUG_MAP=2 (or more) Updates #cleanup Change-Id: Ic1edaed46b7b451ab58bb2303640225223eba9ce Signed-off-by: Brad Fitzpatrick --- control/controlclient/direct.go | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index dd361c4a20857..c436bc8b19926 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -650,7 +650,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new c.logf("RegisterReq sign error: %v", err) } } - if debugRegister() { + if DevKnob.DumpRegister() { j, _ := json.MarshalIndent(request, "", "\t") c.logf("RegisterRequest: %s", j) } @@ -691,7 +691,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new c.logf("error decoding RegisterResponse with server key %s and machine key %s: %v", serverKey, machinePrivKey.Public(), err) return regen, opt.URL, nil, fmt.Errorf("register request: %v", err) } - if debugRegister() { + if DevKnob.DumpRegister() { j, _ := json.MarshalIndent(resp, "", "\t") c.logf("RegisterResponse: %s", j) } @@ -877,7 +877,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap c.logf("[v1] PollNetMap: stream=%v ep=%v", isStreaming, epStrs) vlogf := logger.Discard - if DevKnob.DumpNetMaps() { + if DevKnob.DumpNetMapsVerbose() { // TODO(bradfitz): update this to use "[v2]" prefix perhaps? but we don't // want to upload it always. vlogf = c.logf @@ -1170,11 +1170,6 @@ func decode(res *http.Response, v any) error { return json.Unmarshal(msg, v) } -var ( - debugMap = envknob.RegisterBool("TS_DEBUG_MAP") - debugRegister = envknob.RegisterBool("TS_DEBUG_REGISTER") -) - var jsonEscapedZero = []byte(`\u0000`) // decodeMsg is responsible for uncompressing msg and unmarshaling into v. @@ -1183,7 +1178,7 @@ func (c *Direct) decodeMsg(compressedMsg []byte, v any) error { if err != nil { return err } - if debugMap() { + if DevKnob.DumpNetMaps() { var buf bytes.Buffer json.Indent(&buf, b, "", " ") log.Printf("MapResponse: %s", buf.Bytes()) @@ -1205,7 +1200,7 @@ func encode(v any) ([]byte, error) { if err != nil { return nil, err } - if debugMap() { + if DevKnob.DumpNetMaps() { if _, ok := v.(*tailcfg.MapRequest); ok { log.Printf("MapRequest: %s", b) } @@ -1253,18 +1248,23 @@ func loadServerPubKeys(ctx context.Context, httpc *http.Client, serverURL string var DevKnob = initDevKnob() type devKnobs struct { - DumpNetMaps func() bool - ForceProxyDNS func() bool - StripEndpoints func() bool // strip endpoints from control (only use disco messages) - StripCaps func() bool // strip all local node's control-provided capabilities + DumpRegister func() bool + DumpNetMaps func() bool + DumpNetMapsVerbose func() bool + ForceProxyDNS func() bool + StripEndpoints func() bool // strip endpoints from control (only use disco messages) + StripCaps func() bool // strip all local node's control-provided capabilities } func initDevKnob() devKnobs { + nm := envknob.RegisterInt("TS_DEBUG_MAP") return devKnobs{ - DumpNetMaps: envknob.RegisterBool("TS_DEBUG_NETMAP"), - ForceProxyDNS: envknob.RegisterBool("TS_DEBUG_PROXY_DNS"), - StripEndpoints: envknob.RegisterBool("TS_DEBUG_STRIP_ENDPOINTS"), - StripCaps: envknob.RegisterBool("TS_DEBUG_STRIP_CAPS"), + DumpNetMaps: func() bool { return nm() > 0 }, + DumpNetMapsVerbose: func() bool { return nm() > 1 }, + DumpRegister: envknob.RegisterBool("TS_DEBUG_REGISTER"), + ForceProxyDNS: envknob.RegisterBool("TS_DEBUG_PROXY_DNS"), + StripEndpoints: envknob.RegisterBool("TS_DEBUG_STRIP_ENDPOINTS"), + StripCaps: envknob.RegisterBool("TS_DEBUG_STRIP_CAPS"), } } From d818a58a7772698358a078684d57b27098b28dad Mon Sep 17 00:00:00 2001 From: Aaron Klotz Date: Mon, 13 Jan 2025 13:47:56 -0700 Subject: [PATCH 116/223] net/dns: ensure the Windows configurator does not touch the hosts file unless the configuration actually changed We build up maps of both the existing MagicDNS configuration in hosts and the desired MagicDNS configuration, compare the two, and only write out a new one if there are changes. The comparison doesn't need to be perfect, as the occasional false-positive is fine, but this should greatly reduce rewrites of the hosts file. I also changed the hosts updating code to remove the CRLF/LF conversion stuff, and use Fprintf instead of Frintln to let us write those inline. Updates #14428 Signed-off-by: Aaron Klotz --- net/dns/manager_windows.go | 64 ++++++++++++++++++++++++++------- net/dns/manager_windows_test.go | 52 +++++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 14 deletions(-) diff --git a/net/dns/manager_windows.go b/net/dns/manager_windows.go index 250a2557350dd..effdf23cae453 100644 --- a/net/dns/manager_windows.go +++ b/net/dns/manager_windows.go @@ -8,10 +8,12 @@ import ( "bytes" "errors" "fmt" + "maps" "net/netip" "os" "os/exec" "path/filepath" + "slices" "sort" "strings" "sync" @@ -140,9 +142,8 @@ func (m *windowsManager) setSplitDNS(resolvers []netip.Addr, domains []dnsname.F return m.nrptDB.WriteSplitDNSConfig(servers, domains) } -func setTailscaleHosts(prevHostsFile []byte, hosts []*HostEntry) ([]byte, error) { - b := bytes.ReplaceAll(prevHostsFile, []byte("\r\n"), []byte("\n")) - sc := bufio.NewScanner(bytes.NewReader(b)) +func setTailscaleHosts(logf logger.Logf, prevHostsFile []byte, hosts []*HostEntry) ([]byte, error) { + sc := bufio.NewScanner(bytes.NewReader(prevHostsFile)) const ( header = "# TailscaleHostsSectionStart" footer = "# TailscaleHostsSectionEnd" @@ -151,6 +152,32 @@ func setTailscaleHosts(prevHostsFile []byte, hosts []*HostEntry) ([]byte, error) "# This section contains MagicDNS entries for Tailscale.", "# Do not edit this section manually.", } + + prevEntries := make(map[netip.Addr][]string) + addPrevEntry := func(line string) { + if line == "" || line[0] == '#' { + return + } + + parts := strings.Split(line, " ") + if len(parts) < 1 { + return + } + + addr, err := netip.ParseAddr(parts[0]) + if err != nil { + logf("Parsing address from hosts: %v", err) + return + } + + prevEntries[addr] = parts[1:] + } + + nextEntries := make(map[netip.Addr][]string, len(hosts)) + for _, he := range hosts { + nextEntries[he.Addr] = he.Hosts + } + var out bytes.Buffer var inSection bool for sc.Scan() { @@ -164,26 +191,34 @@ func setTailscaleHosts(prevHostsFile []byte, hosts []*HostEntry) ([]byte, error) continue } if inSection { + addPrevEntry(line) continue } - fmt.Fprintln(&out, line) + fmt.Fprintf(&out, "%s\r\n", line) } if err := sc.Err(); err != nil { return nil, err } + + unchanged := maps.EqualFunc(prevEntries, nextEntries, func(a, b []string) bool { + return slices.Equal(a, b) + }) + if unchanged { + return nil, nil + } + if len(hosts) > 0 { - fmt.Fprintln(&out, header) + fmt.Fprintf(&out, "%s\r\n", header) for _, c := range comments { - fmt.Fprintln(&out, c) + fmt.Fprintf(&out, "%s\r\n", c) } - fmt.Fprintln(&out) + fmt.Fprintf(&out, "\r\n") for _, he := range hosts { - fmt.Fprintf(&out, "%s %s\n", he.Addr, strings.Join(he.Hosts, " ")) + fmt.Fprintf(&out, "%s %s\r\n", he.Addr, strings.Join(he.Hosts, " ")) } - fmt.Fprintln(&out) - fmt.Fprintln(&out, footer) + fmt.Fprintf(&out, "\r\n%s\r\n", footer) } - return bytes.ReplaceAll(out.Bytes(), []byte("\n"), []byte("\r\n")), nil + return out.Bytes(), nil } // setHosts sets the hosts file to contain the given host entries. @@ -197,10 +232,15 @@ func (m *windowsManager) setHosts(hosts []*HostEntry) error { if err != nil { return err } - outB, err := setTailscaleHosts(b, hosts) + outB, err := setTailscaleHosts(m.logf, b, hosts) if err != nil { return err } + if outB == nil { + // No change to hosts file, therefore no write necessary. + return nil + } + const fileMode = 0 // ignored on windows. // This can fail spuriously with an access denied error, so retry it a diff --git a/net/dns/manager_windows_test.go b/net/dns/manager_windows_test.go index 62c4dd9fbb740..edcf24ec04240 100644 --- a/net/dns/manager_windows_test.go +++ b/net/dns/manager_windows_test.go @@ -15,6 +15,7 @@ import ( "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" + "tailscale.com/types/logger" "tailscale.com/util/dnsname" "tailscale.com/util/winutil" "tailscale.com/util/winutil/gp" @@ -24,9 +25,56 @@ const testGPRuleID = "{7B1B6151-84E6-41A3-8967-62F7F7B45687}" func TestHostFileNewLines(t *testing.T) { in := []byte("#foo\r\n#bar\n#baz\n") - want := []byte("#foo\r\n#bar\r\n#baz\r\n") + want := []byte("#foo\r\n#bar\r\n#baz\r\n# TailscaleHostsSectionStart\r\n# This section contains MagicDNS entries for Tailscale.\r\n# Do not edit this section manually.\r\n\r\n192.168.1.1 aaron\r\n\r\n# TailscaleHostsSectionEnd\r\n") - got, err := setTailscaleHosts(in, nil) + he := []*HostEntry{ + &HostEntry{ + Addr: netip.MustParseAddr("192.168.1.1"), + Hosts: []string{"aaron"}, + }, + } + got, err := setTailscaleHosts(logger.Discard, in, he) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("got %q, want %q\n", got, want) + } +} + +func TestHostFileUnchanged(t *testing.T) { + in := []byte("#foo\r\n#bar\r\n#baz\r\n# TailscaleHostsSectionStart\r\n# This section contains MagicDNS entries for Tailscale.\r\n# Do not edit this section manually.\r\n\r\n192.168.1.1 aaron\r\n\r\n# TailscaleHostsSectionEnd\r\n") + + he := []*HostEntry{ + &HostEntry{ + Addr: netip.MustParseAddr("192.168.1.1"), + Hosts: []string{"aaron"}, + }, + } + got, err := setTailscaleHosts(logger.Discard, in, he) + if err != nil { + t.Fatal(err) + } + if got != nil { + t.Errorf("got %q, want nil\n", got) + } +} + +func TestHostFileChanged(t *testing.T) { + in := []byte("#foo\r\n#bar\r\n#baz\r\n# TailscaleHostsSectionStart\r\n# This section contains MagicDNS entries for Tailscale.\r\n# Do not edit this section manually.\r\n\r\n192.168.1.1 aaron1\r\n\r\n# TailscaleHostsSectionEnd\r\n") + want := []byte("#foo\r\n#bar\r\n#baz\r\n# TailscaleHostsSectionStart\r\n# This section contains MagicDNS entries for Tailscale.\r\n# Do not edit this section manually.\r\n\r\n192.168.1.1 aaron1\r\n192.168.1.2 aaron2\r\n\r\n# TailscaleHostsSectionEnd\r\n") + + he := []*HostEntry{ + &HostEntry{ + Addr: netip.MustParseAddr("192.168.1.1"), + Hosts: []string{"aaron1"}, + }, + &HostEntry{ + Addr: netip.MustParseAddr("192.168.1.2"), + Hosts: []string{"aaron2"}, + }, + } + got, err := setTailscaleHosts(logger.Discard, in, he) if err != nil { t.Fatal(err) } From d0ba91bdb22e2cfe8b0f35a37cc0b540842f0011 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Sat, 11 Jan 2025 17:58:27 -0600 Subject: [PATCH 117/223] ipn/ipnserver: use ipnauth.Actor instead of *ipnserver.actor whenever possible In preparation for adding test coverage for ipn/ipnserver.Server, we update it to use ipnauth.Actor instead of its concrete implementation where possible. Updates tailscale/corp#25804 Signed-off-by: Nick Khyl --- ipn/ipnserver/actor.go | 8 ++++---- ipn/ipnserver/server.go | 24 ++++++++++++++++-------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ipn/ipnserver/actor.go b/ipn/ipnserver/actor.go index 0e716009cc976..2df8986c3af68 100644 --- a/ipn/ipnserver/actor.go +++ b/ipn/ipnserver/actor.go @@ -112,11 +112,11 @@ func (a *actor) Username() (string, error) { } type actorOrError struct { - actor *actor + actor ipnauth.Actor err error } -func (a actorOrError) unwrap() (*actor, error) { +func (a actorOrError) unwrap() (ipnauth.Actor, error) { return a.actor, a.err } @@ -131,9 +131,9 @@ func contextWithActor(ctx context.Context, logf logger.Logf, c net.Conn) context return actorKey.WithValue(ctx, actorOrError{actor: actor, err: err}) } -// actorFromContext returns an [actor] associated with ctx, +// actorFromContext returns an [ipnauth.Actor] associated with ctx, // or an error if the context does not carry an actor's identity. -func actorFromContext(ctx context.Context) (*actor, error) { +func actorFromContext(ctx context.Context) (ipnauth.Actor, error) { return actorKey.Value(ctx).unwrap() } diff --git a/ipn/ipnserver/server.go b/ipn/ipnserver/server.go index 73b5e82abee76..574d1a55c18b1 100644 --- a/ipn/ipnserver/server.go +++ b/ipn/ipnserver/server.go @@ -22,6 +22,7 @@ import ( "tailscale.com/envknob" "tailscale.com/ipn" + "tailscale.com/ipn/ipnauth" "tailscale.com/ipn/ipnlocal" "tailscale.com/ipn/localapi" "tailscale.com/net/netmon" @@ -30,6 +31,7 @@ import ( "tailscale.com/util/mak" "tailscale.com/util/set" "tailscale.com/util/systemd" + "tailscale.com/util/testenv" ) // Server is an IPN backend and its set of 0 or more active localhost @@ -50,7 +52,7 @@ type Server struct { // lock order: mu, then LocalBackend.mu mu sync.Mutex lastUserID ipn.WindowsUserID // tracks last userid; on change, Reset state for paranoia - activeReqs map[*http.Request]*actor + activeReqs map[*http.Request]ipnauth.Actor backendWaiter waiterSet // of LocalBackend waiters zeroReqWaiter waiterSet // of blockUntilZeroConnections waiters } @@ -195,8 +197,12 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, "/localapi/") { lah := localapi.NewHandler(lb, s.logf, s.backendLogID) - lah.PermitRead, lah.PermitWrite = ci.Permissions(lb.OperatorUserID()) - lah.PermitCert = ci.CanFetchCerts() + if actor, ok := ci.(*actor); ok { + lah.PermitRead, lah.PermitWrite = actor.Permissions(lb.OperatorUserID()) + lah.PermitCert = actor.CanFetchCerts() + } else if testenv.InTest() { + lah.PermitRead, lah.PermitWrite = true, true + } lah.Actor = ci lah.ServeHTTP(w, r) return @@ -230,11 +236,11 @@ func (e inUseOtherUserError) Unwrap() error { return e.error } // The returned error, when non-nil, will be of type inUseOtherUserError. // // s.mu must be held. -func (s *Server) checkConnIdentityLocked(ci *actor) error { +func (s *Server) checkConnIdentityLocked(ci ipnauth.Actor) error { // If clients are already connected, verify they're the same user. // This mostly matters on Windows at the moment. if len(s.activeReqs) > 0 { - var active *actor + var active ipnauth.Actor for _, active = range s.activeReqs { break } @@ -251,7 +257,9 @@ func (s *Server) checkConnIdentityLocked(ci *actor) error { if username, err := active.Username(); err == nil { fmt.Fprintf(&b, " by %s", username) } - fmt.Fprintf(&b, ", pid %d", active.pid()) + if active, ok := active.(*actor); ok { + fmt.Fprintf(&b, ", pid %d", active.pid()) + } return inUseOtherUserError{errors.New(b.String())} } } @@ -267,7 +275,7 @@ func (s *Server) checkConnIdentityLocked(ci *actor) error { // // This is primarily used for the Windows GUI, to block until one user's done // controlling the tailscaled process. -func (s *Server) blockWhileIdentityInUse(ctx context.Context, actor *actor) error { +func (s *Server) blockWhileIdentityInUse(ctx context.Context, actor ipnauth.Actor) error { inUse := func() bool { s.mu.Lock() defer s.mu.Unlock() @@ -361,7 +369,7 @@ func (a *actor) CanFetchCerts() bool { // The returned error may be of type [inUseOtherUserError]. // // onDone must be called when the HTTP request is done. -func (s *Server) addActiveHTTPRequest(req *http.Request, actor *actor) (onDone func(), err error) { +func (s *Server) addActiveHTTPRequest(req *http.Request, actor ipnauth.Actor) (onDone func(), err error) { if actor == nil { return nil, errors.New("internal error: nil actor") } From c3c4c964898716511142e448f6e1006daace28f8 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 13 Jan 2025 17:37:29 -0600 Subject: [PATCH 118/223] ipn/{ipnauth,ipnlocal,ipnserver}, client/tailscale: make ipnserver.Server testable We update client/tailscale.LocalClient to allow specifying an optional Transport (http.RoundTripper) for LocalAPI HTTP requests, and implement one that injects an ipnauth.TestActor via request headers. We also add several functions and types to make testing an ipn/ipnserver.Server possible (or at least easier). We then use these updates to write basic tests for ipnserver.Server, ensuring it works on non-Windows platforms and correctly sets and unsets the LocalBackend's current user when a Windows user connects and disconnects. We intentionally omit tests for switching between different OS users and will add them in follow-up commits. Updates tailscale/corp#25804 Signed-off-by: Nick Khyl --- client/tailscale/localclient.go | 12 +- ipn/ipnauth/actor.go | 13 ++ ipn/ipnlocal/local.go | 9 + ipn/ipnserver/server_test.go | 358 ++++++++++++++++++++++++++++++++ 4 files changed, 389 insertions(+), 3 deletions(-) diff --git a/client/tailscale/localclient.go b/client/tailscale/localclient.go index 4e452f894d8e8..baa211d1fb5b0 100644 --- a/client/tailscale/localclient.go +++ b/client/tailscale/localclient.go @@ -62,6 +62,12 @@ type LocalClient struct { // machine's tailscaled or equivalent. If nil, a default is used. Dial func(ctx context.Context, network, addr string) (net.Conn, error) + // Transport optionally specified an alternate [http.RoundTripper] + // used to execute HTTP requests. If nil, a default [http.Transport] is used, + // potentially with custom dialing logic from [Dial]. + // It is primarily used for testing. + Transport http.RoundTripper + // Socket specifies an alternate path to the local Tailscale socket. // If empty, a platform-specific default is used. Socket string @@ -129,9 +135,9 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error) req.Header.Set("Tailscale-Cap", strconv.Itoa(int(tailcfg.CurrentCapabilityVersion))) lc.tsClientOnce.Do(func() { lc.tsClient = &http.Client{ - Transport: &http.Transport{ - DialContext: lc.dialer(), - }, + Transport: cmp.Or(lc.Transport, http.RoundTripper( + &http.Transport{DialContext: lc.dialer()}), + ), } }) if !lc.OmitAuth { diff --git a/ipn/ipnauth/actor.go b/ipn/ipnauth/actor.go index 1070172688a84..040d9b522e150 100644 --- a/ipn/ipnauth/actor.go +++ b/ipn/ipnauth/actor.go @@ -4,6 +4,7 @@ package ipnauth import ( + "encoding/json" "fmt" "tailscale.com/ipn" @@ -76,3 +77,15 @@ func (id ClientID) String() string { } return fmt.Sprint(id.v) } + +// MarshalJSON implements [json.Marshaler]. +// It is primarily used for testing. +func (id ClientID) MarshalJSON() ([]byte, error) { + return json.Marshal(id.v) +} + +// UnmarshalJSON implements [json.Unmarshaler]. +// It is primarily used for testing. +func (id *ClientID) UnmarshalJSON(b []byte) error { + return json.Unmarshal(b, &id.v) +} diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 81a62045a9111..576f01b6bbff8 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -3659,6 +3659,15 @@ func (b *LocalBackend) SetCurrentUser(actor ipnauth.Actor) (ipn.WindowsUserID, e return uid, nil } +// CurrentUserForTest returns the current user and the associated WindowsUserID. +// It is used for testing only, and will be removed along with the rest of the +// "current user" functionality as we progress on the multi-user improvements (tailscale/corp#18342). +func (b *LocalBackend) CurrentUserForTest() (ipn.WindowsUserID, ipnauth.Actor) { + b.mu.Lock() + defer b.mu.Unlock() + return b.pm.CurrentUserID(), b.currentUser +} + func (b *LocalBackend) CheckPrefs(p *ipn.Prefs) error { b.mu.Lock() defer b.mu.Unlock() diff --git a/ipn/ipnserver/server_test.go b/ipn/ipnserver/server_test.go index b7d5ea144c408..8a9324fab9898 100644 --- a/ipn/ipnserver/server_test.go +++ b/ipn/ipnserver/server_test.go @@ -5,8 +5,32 @@ package ipnserver import ( "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "runtime" "sync" + "sync/atomic" "testing" + + "tailscale.com/client/tailscale" + "tailscale.com/client/tailscale/apitype" + "tailscale.com/control/controlclient" + "tailscale.com/envknob" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnauth" + "tailscale.com/ipn/ipnlocal" + "tailscale.com/ipn/store/mem" + "tailscale.com/tsd" + "tailscale.com/tstest" + "tailscale.com/types/logger" + "tailscale.com/types/logid" + "tailscale.com/types/ptr" + "tailscale.com/util/mak" + "tailscale.com/wgengine" ) func TestWaiterSet(t *testing.T) { @@ -44,3 +68,337 @@ func TestWaiterSet(t *testing.T) { cleanup() wantLen(0, "at end") } + +func TestUserConnectDisconnectNonWindows(t *testing.T) { + enableLogging := false + if runtime.GOOS == "windows" { + setGOOSForTest(t, "linux") + } + + ctx := context.Background() + server := startDefaultTestIPNServer(t, ctx, enableLogging) + + // UserA connects and starts watching the IPN bus. + clientA := server.getClientAs("UserA") + watcherA, _ := clientA.WatchIPNBus(ctx, 0) + + // The concept of "current user" is only relevant on Windows + // and it should not be set on non-Windows platforms. + server.checkCurrentUser(nil) + + // Additionally, a different user should be able to connect and use the LocalAPI. + clientB := server.getClientAs("UserB") + if _, gotErr := clientB.Status(ctx); gotErr != nil { + t.Fatalf("Status(%q): want nil; got %v", clientB.User.Name, gotErr) + } + + // Watching the IPN bus should also work for UserB. + watcherB, _ := clientB.WatchIPNBus(ctx, 0) + + // And if we send a notification, both users should receive it. + wantErrMessage := "test error" + testNotify := ipn.Notify{ErrMessage: ptr.To(wantErrMessage)} + server.mustBackend().DebugNotify(testNotify) + + if n, err := watcherA.Next(); err != nil { + t.Fatalf("IPNBusWatcher.Next(%q): %v", clientA.User.Name, err) + } else if gotErrMessage := n.ErrMessage; gotErrMessage == nil || *gotErrMessage != wantErrMessage { + t.Fatalf("IPNBusWatcher.Next(%q): want %v; got %v", clientA.User.Name, wantErrMessage, gotErrMessage) + } + + if n, err := watcherB.Next(); err != nil { + t.Fatalf("IPNBusWatcher.Next(%q): %v", clientB.User.Name, err) + } else if gotErrMessage := n.ErrMessage; gotErrMessage == nil || *gotErrMessage != wantErrMessage { + t.Fatalf("IPNBusWatcher.Next(%q): want %v; got %v", clientB.User.Name, wantErrMessage, gotErrMessage) + } +} + +func TestUserConnectDisconnectOnWindows(t *testing.T) { + enableLogging := false + setGOOSForTest(t, "windows") + + ctx := context.Background() + server := startDefaultTestIPNServer(t, ctx, enableLogging) + + client := server.getClientAs("User") + _, cancelWatcher := client.WatchIPNBus(ctx, 0) + + // On Windows, however, the current user should be set to the user that connected. + server.checkCurrentUser(client.User) + + // Cancel the IPN bus watcher request and wait for the server to unblock. + cancelWatcher() + server.blockWhileInUse(ctx) + + // The current user should not be set after a disconnect, as no one is + // currently using the server. + server.checkCurrentUser(nil) +} + +func TestIPNAlreadyInUseOnWindows(t *testing.T) { + enableLogging := false + setGOOSForTest(t, "windows") + + ctx := context.Background() + server := startDefaultTestIPNServer(t, ctx, enableLogging) + + // UserA connects and starts watching the IPN bus. + clientA := server.getClientAs("UserA") + clientA.WatchIPNBus(ctx, 0) + + // While UserA is connected, UserB should not be able to connect. + clientB := server.getClientAs("UserB") + if _, gotErr := clientB.Status(ctx); gotErr == nil { + t.Fatalf("Status(%q): want error; got nil", clientB.User.Name) + } else if wantError := "401 Unauthorized: Tailscale already in use by UserA"; gotErr.Error() != wantError { + t.Fatalf("Status(%q): want %q; got %q", clientB.User.Name, wantError, gotErr.Error()) + } + + // Current user should still be UserA. + server.checkCurrentUser(clientA.User) +} + +func setGOOSForTest(tb testing.TB, goos string) { + tb.Helper() + envknob.Setenv("TS_DEBUG_FAKE_GOOS", goos) + tb.Cleanup(func() { envknob.Setenv("TS_DEBUG_FAKE_GOOS", "") }) +} + +func testLogger(tb testing.TB, enableLogging bool) logger.Logf { + tb.Helper() + if enableLogging { + return tstest.WhileTestRunningLogger(tb) + } + return logger.Discard +} + +// newTestIPNServer creates a new IPN server for testing, using the specified local backend. +func newTestIPNServer(tb testing.TB, lb *ipnlocal.LocalBackend, enableLogging bool) *Server { + tb.Helper() + server := New(testLogger(tb, enableLogging), logid.PublicID{}, lb.NetMon()) + server.lb.Store(lb) + return server +} + +type testIPNClient struct { + tb testing.TB + *tailscale.LocalClient + User *ipnauth.TestActor +} + +func (c *testIPNClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*tailscale.IPNBusWatcher, context.CancelFunc) { + c.tb.Helper() + ctx, cancelWatcher := context.WithCancel(ctx) + c.tb.Cleanup(cancelWatcher) + watcher, err := c.LocalClient.WatchIPNBus(ctx, mask) + if err != nil { + c.tb.Fatalf("WatchIPNBus(%q): %v", c.User.Name, err) + } + c.tb.Cleanup(func() { watcher.Close() }) + return watcher, cancelWatcher +} + +func pumpIPNBus(watcher *tailscale.IPNBusWatcher) { + for { + _, err := watcher.Next() + if err != nil { + break + } + } +} + +type testIPNServer struct { + tb testing.TB + *Server + clientID atomic.Int64 + getClient func(*ipnauth.TestActor) *tailscale.LocalClient + + actorsMu sync.Mutex + actors map[string]*ipnauth.TestActor +} + +func (s *testIPNServer) getClientAs(name string) *testIPNClient { + clientID := fmt.Sprintf("Client-%d", 1+s.clientID.Add(1)) + user := s.makeTestUser(name, clientID) + return &testIPNClient{ + tb: s.tb, + LocalClient: s.getClient(user), + User: user, + } +} + +func (s *testIPNServer) makeTestUser(name string, clientID string) *ipnauth.TestActor { + s.actorsMu.Lock() + defer s.actorsMu.Unlock() + actor := s.actors[name] + if actor == nil { + actor = &ipnauth.TestActor{Name: name} + if envknob.GOOS() == "windows" { + // Historically, as of 2025-01-13, IPN does not distinguish between + // different users on non-Windows devices. Therefore, the UID, which is + // an [ipn.WindowsUserID], should only be populated when the actual or + // fake GOOS is Windows. + actor.UID = ipn.WindowsUserID(fmt.Sprintf("S-1-5-21-1-0-0-%d", 1001+len(s.actors))) + } + mak.Set(&s.actors, name, actor) + s.tb.Cleanup(func() { delete(s.actors, name) }) + } + actor = ptr.To(*actor) + actor.CID = ipnauth.ClientIDFrom(clientID) + return actor +} + +func (s *testIPNServer) blockWhileInUse(ctx context.Context) error { + ready, cleanup := s.zeroReqWaiter.add(&s.mu, ctx) + <-ready + cleanup() + return ctx.Err() +} + +func (s *testIPNServer) checkCurrentUser(want *ipnauth.TestActor) { + s.tb.Helper() + var wantUID ipn.WindowsUserID + if want != nil { + wantUID = want.UID + } + gotUID, gotActor := s.mustBackend().CurrentUserForTest() + if gotUID != wantUID { + s.tb.Errorf("CurrentUser: got UID %q; want %q", gotUID, wantUID) + } + if gotActor, ok := gotActor.(*ipnauth.TestActor); ok != (want != nil) || (want != nil && *gotActor != *want) { + s.tb.Errorf("CurrentUser: got %v; want %v", gotActor, want) + } +} + +// startTestIPNServer starts a [httptest.Server] that hosts the specified IPN server for the +// duration of the test, using the specified base context for incoming requests. +// It returns a function that creates a [tailscale.LocalClient] as a given [ipnauth.TestActor]. +func startTestIPNServer(tb testing.TB, baseContext context.Context, server *Server) *testIPNServer { + tb.Helper() + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + actor, err := extractActorFromHeader(r.Header) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + tb.Errorf("extractActorFromHeader: %v", err) + return + } + ctx := newTestContextWithActor(r.Context(), actor) + server.serveHTTP(w, r.Clone(ctx)) + })) + ts.Config.Addr = "http://" + apitype.LocalAPIHost + ts.Config.BaseContext = func(_ net.Listener) context.Context { return baseContext } + ts.Config.ErrorLog = logger.StdLogger(logger.WithPrefix(server.logf, "ipnserver: ")) + ts.Start() + tb.Cleanup(ts.Close) + return &testIPNServer{ + tb: tb, + Server: server, + getClient: func(actor *ipnauth.TestActor) *tailscale.LocalClient { + return &tailscale.LocalClient{Transport: newTestRoundTripper(ts, actor)} + }, + } +} + +func startDefaultTestIPNServer(tb testing.TB, ctx context.Context, enableLogging bool) *testIPNServer { + tb.Helper() + lb := newLocalBackendWithTestControl(tb, newUnreachableControlClient, enableLogging) + ctx, stopServer := context.WithCancel(ctx) + tb.Cleanup(stopServer) + return startTestIPNServer(tb, ctx, newTestIPNServer(tb, lb, enableLogging)) +} + +type testRoundTripper struct { + transport http.RoundTripper + actor *ipnauth.TestActor +} + +// newTestRoundTripper creates a new [http.RoundTripper] that sends requests +// to the specified test server as the specified actor. +func newTestRoundTripper(ts *httptest.Server, actor *ipnauth.TestActor) *testRoundTripper { + return &testRoundTripper{ + transport: &http.Transport{DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + var std net.Dialer + return std.DialContext(ctx, network, ts.Listener.Addr().(*net.TCPAddr).String()) + }}, + actor: actor, + } +} + +const testActorHeaderName = "TS-Test-Actor" + +// RoundTrip implements [http.RoundTripper] by forwarding the request to the underlying transport +// and including the test actor's identity in the request headers. +func (rt *testRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + actorJSON, err := json.Marshal(&rt.actor) + if err != nil { + // An [http.RoundTripper] must always close the request body, including on error. + if r.Body != nil { + r.Body.Close() + } + return nil, err + } + + r = r.Clone(r.Context()) + r.Header.Set(testActorHeaderName, string(actorJSON)) + return rt.transport.RoundTrip(r) +} + +// extractActorFromHeader extracts a test actor from the specified request headers. +func extractActorFromHeader(h http.Header) (*ipnauth.TestActor, error) { + actorJSON := h.Get(testActorHeaderName) + if actorJSON == "" { + return nil, errors.New("missing Test-Actor header") + } + actor := &ipnauth.TestActor{} + if err := json.Unmarshal([]byte(actorJSON), &actor); err != nil { + return nil, fmt.Errorf("invalid Test-Actor header: %v", err) + } + return actor, nil +} + +type newControlClientFn func(tb testing.TB, opts controlclient.Options) controlclient.Client + +func newLocalBackendWithTestControl(tb testing.TB, newControl newControlClientFn, enableLogging bool) *ipnlocal.LocalBackend { + tb.Helper() + + sys := &tsd.System{} + store := &mem.Store{} + sys.Set(store) + + logf := testLogger(tb, enableLogging) + e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry()) + if err != nil { + tb.Fatalf("NewFakeUserspaceEngine: %v", err) + } + tb.Cleanup(e.Close) + sys.Set(e) + + b, err := ipnlocal.NewLocalBackend(logf, logid.PublicID{}, sys, 0) + if err != nil { + tb.Fatalf("NewLocalBackend: %v", err) + } + tb.Cleanup(b.Shutdown) + b.DisablePortMapperForTest() + + b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) { + return newControl(tb, opts), nil + }) + return b +} + +func newUnreachableControlClient(tb testing.TB, opts controlclient.Options) controlclient.Client { + tb.Helper() + opts.ServerURL = "https://127.0.0.1:1" + cc, err := controlclient.New(opts) + if err != nil { + tb.Fatal(err) + } + return cc +} + +// newTestContextWithActor returns a new context that carries the identity +// of the specified actor and can be used for testing. +// It can be retrieved with [actorFromContext]. +func newTestContextWithActor(ctx context.Context, actor ipnauth.Actor) context.Context { + return actorKey.WithValue(ctx, actorOrError{actor: actor}) +} From f33f5f99c00f187644bf807b5336e2008e0228b7 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 13 Jan 2025 17:42:12 -0600 Subject: [PATCH 119/223] ipn/{ipnlocal,ipnserver}: remove redundant (*LocalBackend).ResetForClientDisconnect In this commit, we add a failing test to verify that ipn/ipnserver.Server correctly sets and unsets the current user when two different users connect sequentially (A connects, A disconnects, B connects, B disconnects). We then fix the test by updating (*ipn/ipnserver.Server).addActiveHTTPRequest to avoid calling (*LocalBackend).ResetForClientDisconnect again after a new user has connected and been set as the current user with (*LocalBackend).SetCurrentUser(). Since ipn/ipnserver.Server does not allow simultaneous connections from different Windows users and relies on the LocalBackend's current user, and since we already reset the LocalBackend's state by calling ResetForClientDisconnect when the last active request completes (indicating the server is idle and can accept connections from any Windows user), it is unnecessary to track the last connected user on the ipnserver.Server side or call ResetForClientDisconnect again when the user changes. Additionally, the second call to ResetForClientDisconnect occurs after the new user has been set as the current user, resetting the correct state for the new user instead of the old state of the now-disconnected user, causing issues. Updates tailscale/corp#25804 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 26 ++++++++++++-------------- ipn/ipnserver/server.go | 23 +---------------------- ipn/ipnserver/server_test.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 576f01b6bbff8..c506f13766dac 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -3620,7 +3620,7 @@ func (b *LocalBackend) shouldUploadServices() bool { } // SetCurrentUser is used to implement support for multi-user systems (only -// Windows 2022-11-25). On such systems, the uid is used to determine which +// Windows 2022-11-25). On such systems, the actor is used to determine which // user's state should be used. The current user is maintained by active // connections open to the backend. // @@ -3634,11 +3634,8 @@ func (b *LocalBackend) shouldUploadServices() bool { // unattended mode. The user must disable unattended mode before the user can be // changed. // -// On non-multi-user systems, the user should be set to nil. -// -// SetCurrentUser returns the ipn.WindowsUserID associated with the user -// when successful. -func (b *LocalBackend) SetCurrentUser(actor ipnauth.Actor) (ipn.WindowsUserID, error) { +// On non-multi-user systems, the actor should be set to nil. +func (b *LocalBackend) SetCurrentUser(actor ipnauth.Actor) { var uid ipn.WindowsUserID if actor != nil { uid = actor.UserID() @@ -3647,16 +3644,17 @@ func (b *LocalBackend) SetCurrentUser(actor ipnauth.Actor) (ipn.WindowsUserID, e unlock := b.lockAndGetUnlock() defer unlock() - if b.pm.CurrentUserID() == uid { - return uid, nil + if actor != b.currentUser { + if c, ok := b.currentUser.(ipnauth.ActorCloser); ok { + c.Close() + } + b.currentUser = actor } - b.pm.SetCurrentUserID(uid) - if c, ok := b.currentUser.(ipnauth.ActorCloser); ok { - c.Close() + + if b.pm.CurrentUserID() != uid { + b.pm.SetCurrentUserID(uid) + b.resetForProfileChangeLockedOnEntry(unlock) } - b.currentUser = actor - b.resetForProfileChangeLockedOnEntry(unlock) - return uid, nil } // CurrentUserForTest returns the current user and the associated WindowsUserID. diff --git a/ipn/ipnserver/server.go b/ipn/ipnserver/server.go index 574d1a55c18b1..c0e99f5d88412 100644 --- a/ipn/ipnserver/server.go +++ b/ipn/ipnserver/server.go @@ -21,7 +21,6 @@ import ( "unicode" "tailscale.com/envknob" - "tailscale.com/ipn" "tailscale.com/ipn/ipnauth" "tailscale.com/ipn/ipnlocal" "tailscale.com/ipn/localapi" @@ -51,7 +50,6 @@ type Server struct { // mu guards the fields that follow. // lock order: mu, then LocalBackend.mu mu sync.Mutex - lastUserID ipn.WindowsUserID // tracks last userid; on change, Reset state for paranoia activeReqs map[*http.Request]ipnauth.Actor backendWaiter waiterSet // of LocalBackend waiters zeroReqWaiter waiterSet // of blockUntilZeroConnections waiters @@ -376,16 +374,6 @@ func (s *Server) addActiveHTTPRequest(req *http.Request, actor ipnauth.Actor) (o lb := s.mustBackend() - // If the connected user changes, reset the backend server state to make - // sure node keys don't leak between users. - var doReset bool - defer func() { - if doReset { - s.logf("identity changed; resetting server") - lb.ResetForClientDisconnect() - } - }() - s.mu.Lock() defer s.mu.Unlock() @@ -400,16 +388,7 @@ func (s *Server) addActiveHTTPRequest(req *http.Request, actor ipnauth.Actor) (o // Tell the LocalBackend about the identity we're now running as, // unless its the SYSTEM user. That user is not a real account and // doesn't have a home directory. - uid, err := lb.SetCurrentUser(actor) - if err != nil { - return nil, err - } - if s.lastUserID != uid { - if s.lastUserID != "" { - doReset = true - } - s.lastUserID = uid - } + lb.SetCurrentUser(actor) } } diff --git a/ipn/ipnserver/server_test.go b/ipn/ipnserver/server_test.go index 8a9324fab9898..7f6131328caa0 100644 --- a/ipn/ipnserver/server_test.go +++ b/ipn/ipnserver/server_test.go @@ -158,6 +158,35 @@ func TestIPNAlreadyInUseOnWindows(t *testing.T) { server.checkCurrentUser(clientA.User) } +func TestSequentialOSUserSwitchingOnWindows(t *testing.T) { + enableLogging := false + setGOOSForTest(t, "windows") + + ctx := context.Background() + server := startDefaultTestIPNServer(t, ctx, enableLogging) + + connectDisconnectAsUser := func(name string) { + // User connects and starts watching the IPN bus. + client := server.getClientAs(name) + watcher, cancelWatcher := client.WatchIPNBus(ctx, 0) + defer cancelWatcher() + go pumpIPNBus(watcher) + + // It should be the current user from the LocalBackend's perspective... + server.checkCurrentUser(client.User) + // until it disconnects. + cancelWatcher() + server.blockWhileInUse(ctx) + // Now, the current user should be unset. + server.checkCurrentUser(nil) + } + + // UserA logs in, uses Tailscale for a bit, then logs out. + connectDisconnectAsUser("UserA") + // Same for UserB. + connectDisconnectAsUser("UserB") +} + func setGOOSForTest(tb testing.TB, goos string) { tb.Helper() envknob.Setenv("TS_DEBUG_FAKE_GOOS", goos) From 6fac2903e1146164b2a587fd36d53a083ac7e98c Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 13 Jan 2025 18:20:09 -0600 Subject: [PATCH 120/223] ipn/ipnserver: fix race condition where LocalBackend is reset after a different user connects In this commit, we add a failing test to verify that ipn/ipnserver.Server correctly sets and unsets the current user when two different clients send requests concurrently (A sends request, B sends request, A's request completes, B's request completes). The expectation is that the user who wins the race becomes the current user from the LocalBackend's perspective, remaining in this state until they disconnect, after which a different user should be able to connect and use the LocalBackend. We then fix the second of two bugs in (*Server).addActiveHTTPRequest, where a race condition causes the LocalBackend's state to be reset after a new client connects, instead of after the last active request of the previous client completes and the server becomes idle. Fixes tailscale/corp#25804 Signed-off-by: Nick Khyl --- ipn/ipnserver/server.go | 15 ++++---- ipn/ipnserver/server_test.go | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/ipn/ipnserver/server.go b/ipn/ipnserver/server.go index c0e99f5d88412..a69e43067ea7b 100644 --- a/ipn/ipnserver/server.go +++ b/ipn/ipnserver/server.go @@ -394,11 +394,14 @@ func (s *Server) addActiveHTTPRequest(req *http.Request, actor ipnauth.Actor) (o onDone = func() { s.mu.Lock() + defer s.mu.Unlock() delete(s.activeReqs, req) - remain := len(s.activeReqs) - s.mu.Unlock() + if len(s.activeReqs) != 0 { + // The server is not idle yet. + return + } - if remain == 0 && s.resetOnZero { + if s.resetOnZero { if lb.InServerMode() { s.logf("client disconnected; staying alive in server mode") } else { @@ -408,11 +411,7 @@ func (s *Server) addActiveHTTPRequest(req *http.Request, actor ipnauth.Actor) (o } // Wake up callers waiting for the server to be idle: - if remain == 0 { - s.mu.Lock() - s.zeroReqWaiter.wakeAll() - s.mu.Unlock() - } + s.zeroReqWaiter.wakeAll() } return onDone, nil diff --git a/ipn/ipnserver/server_test.go b/ipn/ipnserver/server_test.go index 7f6131328caa0..97a616db88b7c 100644 --- a/ipn/ipnserver/server_test.go +++ b/ipn/ipnserver/server_test.go @@ -12,6 +12,7 @@ import ( "net/http" "net/http/httptest" "runtime" + "strconv" "sync" "sync/atomic" "testing" @@ -187,6 +188,72 @@ func TestSequentialOSUserSwitchingOnWindows(t *testing.T) { connectDisconnectAsUser("UserB") } +func TestConcurrentOSUserSwitchingOnWindows(t *testing.T) { + enableLogging := false + setGOOSForTest(t, "windows") + + ctx := context.Background() + server := startDefaultTestIPNServer(t, ctx, enableLogging) + + connectDisconnectAsUser := func(name string) { + // User connects and starts watching the IPN bus. + client := server.getClientAs(name) + watcher, cancelWatcher := client.WatchIPNBus(ctx, ipn.NotifyInitialState) + defer cancelWatcher() + + runtime.Gosched() + + // Get the current user from the LocalBackend's perspective + // as soon as we're connected. + gotUID, gotActor := server.mustBackend().CurrentUserForTest() + + // Wait for the first notification to arrive. + // It will either be the initial state we've requested via [ipn.NotifyInitialState], + // returned by an actual handler, or a "fake" notification sent by the server + // itself to indicate that it is being used by someone else. + n, err := watcher.Next() + if err != nil { + t.Fatal(err) + } + + // If our user lost the race and the IPN is in use by another user, + // we should just return. For the sake of this test, we're not + // interested in waiting for the server to become idle. + if n.State != nil && *n.State == ipn.InUseOtherUser { + return + } + + // Otherwise, our user should have been the current user since the time we connected. + if gotUID != client.User.UID { + t.Errorf("CurrentUser(Initial): got UID %q; want %q", gotUID, client.User.UID) + return + } + if gotActor, ok := gotActor.(*ipnauth.TestActor); !ok || *gotActor != *client.User { + t.Errorf("CurrentUser(Initial): got %v; want %v", gotActor, client.User) + return + } + + // And should still be the current user (as they're still connected)... + server.checkCurrentUser(client.User) + } + + numIterations := 10 + for range numIterations { + numGoRoutines := 100 + var wg sync.WaitGroup + wg.Add(numGoRoutines) + for i := range numGoRoutines { + // User logs in, uses Tailscale for a bit, then logs out + // in parallel with other users doing the same. + go func() { + defer wg.Done() + connectDisconnectAsUser("User-" + strconv.Itoa(i)) + }() + } + wg.Wait() + } +} + func setGOOSForTest(tb testing.TB, goos string) { tb.Helper() envknob.Setenv("TS_DEBUG_FAKE_GOOS", goos) From 2ac189800c4d2eefe182ee55222c61629ccd31da Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 14 Jan 2025 16:50:04 -0600 Subject: [PATCH 121/223] client/tailscale: fix typo in comment Updates #cleanup Signed-off-by: Nick Khyl --- client/tailscale/localclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/tailscale/localclient.go b/client/tailscale/localclient.go index baa211d1fb5b0..f440b19a8f424 100644 --- a/client/tailscale/localclient.go +++ b/client/tailscale/localclient.go @@ -62,7 +62,7 @@ type LocalClient struct { // machine's tailscaled or equivalent. If nil, a default is used. Dial func(ctx context.Context, network, addr string) (net.Conn, error) - // Transport optionally specified an alternate [http.RoundTripper] + // Transport optionally specifies an alternate [http.RoundTripper] // used to execute HTTP requests. If nil, a default [http.Transport] is used, // potentially with custom dialing logic from [Dial]. // It is primarily used for testing. From 6364b5f1e0f643cabefec3534e3b4e5b5847ae3b Mon Sep 17 00:00:00 2001 From: James Tucker Date: Fri, 10 Jan 2025 14:17:16 -0800 Subject: [PATCH 122/223] net/netmon: trim IPv6 endpoints in already routable subnets We have observed some clients with extremely large lists of IPv6 endpoints, in some cases from subnets where the machine also has the zero address for a whole /48 with then arbitrary addresses additionally assigned within that /48. It is in general unnecessary for reachability to report all of these addresses, typically only one will be necessary for reachability. We report two, to cover some other common cases such as some styles of IPv6 private address rotations. Updates tailscale/corp#25850 Signed-off-by: James Tucker --- net/netmon/state.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/net/netmon/state.go b/net/netmon/state.go index d9b360f5eee45..a612dd06d22b7 100644 --- a/net/netmon/state.go +++ b/net/netmon/state.go @@ -19,8 +19,14 @@ import ( "tailscale.com/net/netaddr" "tailscale.com/net/tsaddr" "tailscale.com/net/tshttpproxy" + "tailscale.com/util/mak" ) +// forceAllIPv6Endpoints is a debug knob that when set forces the client to +// report all IPv6 endpoints rather than trim endpoints that are siblings on the +// same interface and subnet. +var forceAllIPv6Endpoints = envknob.RegisterBool("TS_DEBUG_FORCE_ALL_IPV6_ENDPOINTS") + // LoginEndpointForProxyDetermination is the URL used for testing // which HTTP proxy the system should use. var LoginEndpointForProxyDetermination = "https://controlplane.tailscale.com/" @@ -65,6 +71,7 @@ func LocalAddresses() (regular, loopback []netip.Addr, err error) { if err != nil { return nil, nil, err } + var subnets map[netip.Addr]int for _, a := range addrs { switch v := a.(type) { case *net.IPNet: @@ -102,7 +109,15 @@ func LocalAddresses() (regular, loopback []netip.Addr, err error) { if ip.Is4() { regular4 = append(regular4, ip) } else { - regular6 = append(regular6, ip) + curMask, _ := netip.AddrFromSlice(v.IP.Mask(v.Mask)) + // Limit the number of addresses reported per subnet for + // IPv6, as we have seen some nodes with extremely large + // numbers of assigned addresses being carved out of + // same-subnet allocations. + if forceAllIPv6Endpoints() || subnets[curMask] < 2 { + regular6 = append(regular6, ip) + } + mak.Set(&subnets, curMask, subnets[curMask]+1) } } } From 7ecb69e32e003a15d093333980e1acaf0eacc8e2 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 14 Jan 2025 15:10:15 -0800 Subject: [PATCH 123/223] tailcfg,control/controlclient: treat nil AllowedIPs as Addresses [capver 112] Updates #14635 Change-Id: I21e2bd1ec4eb384eb7a3fc8379f0788a684893f3 Signed-off-by: Brad Fitzpatrick --- control/controlclient/map.go | 4 ++++ control/controlclient/map_test.go | 32 +++++++++++++++++++++++++++++++ tailcfg/tailcfg.go | 18 +++++++++++++---- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index d5fd84c6d74f9..13b11d6df431b 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -241,6 +241,10 @@ func upgradeNode(n *tailcfg.Node) { } n.LegacyDERPString = "" } + + if n.AllowedIPs == nil { + n.AllowedIPs = slices.Clone(n.Addresses) + } } func (ms *mapSession) tryHandleIncrementally(res *tailcfg.MapResponse) bool { diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index 9c8c0c3aa2f4c..09441d066a022 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -1007,10 +1007,16 @@ func TestPatchifyPeersChanged(t *testing.T) { } func TestUpgradeNode(t *testing.T) { + a1 := netip.MustParsePrefix("0.0.0.1/32") + a2 := netip.MustParsePrefix("0.0.0.2/32") + a3 := netip.MustParsePrefix("0.0.0.3/32") + a4 := netip.MustParsePrefix("0.0.0.4/32") + tests := []struct { name string in *tailcfg.Node want *tailcfg.Node + also func(t *testing.T, got *tailcfg.Node) // optional }{ { name: "nil", @@ -1037,6 +1043,29 @@ func TestUpgradeNode(t *testing.T) { in: &tailcfg.Node{HomeDERP: 2}, want: &tailcfg.Node{HomeDERP: 2}, }, + { + name: "implicit-allowed-ips-all-set", + in: &tailcfg.Node{Addresses: []netip.Prefix{a1, a2}, AllowedIPs: []netip.Prefix{a3, a4}}, + want: &tailcfg.Node{Addresses: []netip.Prefix{a1, a2}, AllowedIPs: []netip.Prefix{a3, a4}}, + }, + { + name: "implicit-allowed-ips-only-address-set", + in: &tailcfg.Node{Addresses: []netip.Prefix{a1, a2}}, + want: &tailcfg.Node{Addresses: []netip.Prefix{a1, a2}, AllowedIPs: []netip.Prefix{a1, a2}}, + also: func(t *testing.T, got *tailcfg.Node) { + if t.Failed() { + return + } + if &got.Addresses[0] == &got.AllowedIPs[0] { + t.Error("Addresses and AllowIPs alias the same memory") + } + }, + }, + { + name: "implicit-allowed-ips-set-empty-slice", + in: &tailcfg.Node{Addresses: []netip.Prefix{a1, a2}, AllowedIPs: []netip.Prefix{}}, + want: &tailcfg.Node{Addresses: []netip.Prefix{a1, a2}, AllowedIPs: []netip.Prefix{}}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1048,6 +1077,9 @@ func TestUpgradeNode(t *testing.T) { if diff := cmp.Diff(tt.want, got); diff != "" { t.Errorf("wrong result (-want +got):\n%s", diff) } + if tt.also != nil { + tt.also(t, got) + } }) } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 76945ec10898e..9b26e888388ce 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -154,7 +154,8 @@ type CapabilityVersion int // - 109: 2024-11-18: Client supports filtertype.Match.SrcCaps (issue #12542) // - 110: 2024-12-12: removed never-before-used Tailscale SSH public key support (#14373) // - 111: 2025-01-14: Client supports a peer having Node.HomeDERP (issue #14636) -const CurrentCapabilityVersion CapabilityVersion = 111 +// - 112: 2025-01-14: Client interprets AllowedIPs of nil as meaning same as Addresses +const CurrentCapabilityVersion CapabilityVersion = 112 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -343,9 +344,18 @@ type Node struct { KeySignature tkatype.MarshaledSignature `json:",omitempty"` Machine key.MachinePublic DiscoKey key.DiscoPublic - Addresses []netip.Prefix // IP addresses of this Node directly - AllowedIPs []netip.Prefix // range of IP addresses to route to this node - Endpoints []netip.AddrPort `json:",omitempty"` // IP+port (public via STUN, and local LANs) + + // Addresses are the IP addresses of this Node directly. + Addresses []netip.Prefix + + // AllowedIPs are the IP ranges to route to this node. + // + // As of CapabilityVersion 112, this may be nil (null or undefined) on the wire + // to mean the same as Addresses. Internally, it is always filled in with + // its possibly-implicit value. + AllowedIPs []netip.Prefix + + Endpoints []netip.AddrPort `json:",omitempty"` // IP+port (public via STUN, and local LANs) // LegacyDERPString is this node's home LegacyDERPString region ID integer, but shoved into an // IP:port string for legacy reasons. The IP address is always "127.3.3.40" From db05e83efc4c2e8e12fb8f16665827986839b381 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Tue, 7 Jan 2025 05:34:07 -0600 Subject: [PATCH 124/223] cmd/derper: support explicit configuration of mesh dial hosts The --mesh-with flag now supports the specification of hostname tuples like derp1a.tailscale.com/derp1a-vpc.tailscale.com, which instructs derp to mesh with host 'derp1a.tailscale.com' but dial TCP connections to 'derp1a-vpc.tailscale.com'. For backwards compatibility, --mesh-with still supports individual hostnames. The logic which attempts to auto-discover '[host]-vpc.tailscale.com' dial hosts has been removed. Updates tailscale/corp#25653 Signed-off-by: Percy Wegmann --- cmd/derper/derper.go | 2 +- cmd/derper/mesh.go | 57 +++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 6e24e0ab14b3d..46ff644b26ddd 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -63,7 +63,7 @@ var ( runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.") meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.") - meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list") + meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.") bootstrapDNS = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns") unpublishedDNS = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.") verifyClients = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.") diff --git a/cmd/derper/mesh.go b/cmd/derper/mesh.go index c4218dd9401f5..1d8e3ef93c8b3 100644 --- a/cmd/derper/mesh.go +++ b/cmd/derper/mesh.go @@ -10,7 +10,6 @@ import ( "log" "net" "strings" - "time" "tailscale.com/derp" "tailscale.com/derp/derphttp" @@ -25,15 +24,28 @@ func startMesh(s *derp.Server) error { if !s.HasMeshKey() { return errors.New("--mesh-with requires --mesh-psk-file") } - for _, host := range strings.Split(*meshWith, ",") { - if err := startMeshWithHost(s, host); err != nil { + for _, hostTuple := range strings.Split(*meshWith, ",") { + if err := startMeshWithHost(s, hostTuple); err != nil { return err } } return nil } -func startMeshWithHost(s *derp.Server, host string) error { +func startMeshWithHost(s *derp.Server, hostTuple string) error { + var host string + var dialHost string + hostParts := strings.Split(hostTuple, "/") + if len(hostParts) > 2 { + return fmt.Errorf("too many components in host tuple %q", hostTuple) + } + host = hostParts[0] + if len(hostParts) == 2 { + dialHost = hostParts[1] + } else { + dialHost = hostParts[0] + } + logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host)) netMon := netmon.NewStatic() // good enough for cmd/derper; no need for netns fanciness c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf, netMon) @@ -43,35 +55,20 @@ func startMeshWithHost(s *derp.Server, host string) error { c.MeshKey = s.MeshKey() c.WatchConnectionChanges = true - // For meshed peers within a region, connect via VPC addresses. - c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) { - host, port, err := net.SplitHostPort(addr) - if err != nil { - logf("failed to split %q: %v", addr, err) - return nil, err - } + logf("will dial %q for %q", dialHost, host) + if dialHost != host { var d net.Dialer - var r net.Resolver - if base, ok := strings.CutSuffix(host, ".tailscale.com"); ok && port == "443" { - subCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - vpcHost := base + "-vpc.tailscale.com" - ips, err := r.LookupIP(subCtx, "ip", vpcHost) + c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) { + _, port, err := net.SplitHostPort(addr) if err != nil { - logf("failed to resolve %v: %v", vpcHost, err) + logf("failed to split %q: %v", addr, err) + return nil, err } - if len(ips) > 0 { - vpcAddr := net.JoinHostPort(ips[0].String(), port) - c, err := d.DialContext(subCtx, network, vpcAddr) - if err == nil { - logf("connected to %v (%v) instead of %v", vpcHost, ips[0], base) - return c, nil - } - logf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err) - } - } - return d.DialContext(ctx, network, addr) - }) + dialAddr := net.JoinHostPort(dialHost, port) + logf("dialing %q instead of %q", dialAddr, addr) + return d.DialContext(ctx, network, dialAddr) + }) + } add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) } remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) } From beb951c74400f108544f555e17a31a2da662ef48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 10:53:38 -0700 Subject: [PATCH 125/223] .github: Bump actions/setup-go from 5.1.0 to 5.2.0 (#14391) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.1.0 to 5.2.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed...3041bf56c941b39c61721a86cd11f3bb1338122a) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ba21e8fe94a5b..6f551f14085d0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: # Install a more recent Go that understands modern go.mod content. - name: Install Go - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version-file: go.mod diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 6630e8de852ae..9f1f2b9d1db62 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version-file: go.mod cache: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d4c73ab7c7724..92ef57b50c169 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,7 +153,7 @@ jobs: uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install Go - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version-file: go.mod cache: false From 3431ab17202ed84ba5f9048c9f5afb9b7dfc63d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 10:54:34 -0700 Subject: [PATCH 126/223] .github: Bump github/codeql-action from 3.27.6 to 3.28.1 (#14618) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.27.6 to 3.28.1. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/aa578102511db1f4524ed59b8cc2bae4f6e88195...b6a472f63d85b9c78a3ac5e89422239fc15e9b3c) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6f551f14085d0..605f0939b2c84 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 + uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 + uses: github/codeql-action/autobuild@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 + uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 From fcf90260cef795fc4f4ce98d425da6c69070eecb Mon Sep 17 00:00:00 2001 From: Aaron Klotz Date: Mon, 13 Jan 2025 13:02:47 -0700 Subject: [PATCH 127/223] atomicfile: use ReplaceFile on Windows so that attributes and ACLs are preserved I moved the actual rename into separate, GOOS-specific files. On non-Windows, we do a simple os.Rename. On Windows, we first try ReplaceFile with a fallback to os.Rename if the target file does not exist. ReplaceFile is the recommended way to rename the file in this use case, as it preserves attributes and ACLs set on the target file. Updates #14428 Signed-off-by: Aaron Klotz --- atomicfile/atomicfile.go | 7 +- atomicfile/atomicfile_notwindows.go | 14 +++ atomicfile/atomicfile_windows.go | 33 ++++++ atomicfile/atomicfile_windows_test.go | 146 ++++++++++++++++++++++++++ atomicfile/mksyscall.go | 8 ++ atomicfile/zsyscall_windows.go | 52 +++++++++ cmd/derper/depaware.txt | 2 +- cmd/k8s-operator/depaware.txt | 2 +- cmd/tailscale/depaware.txt | 2 +- cmd/tailscaled/depaware.txt | 2 +- 10 files changed, 261 insertions(+), 7 deletions(-) create mode 100644 atomicfile/atomicfile_notwindows.go create mode 100644 atomicfile/atomicfile_windows.go create mode 100644 atomicfile/atomicfile_windows_test.go create mode 100644 atomicfile/mksyscall.go create mode 100644 atomicfile/zsyscall_windows.go diff --git a/atomicfile/atomicfile.go b/atomicfile/atomicfile.go index 5c18e85a896eb..b3c8c93da2af9 100644 --- a/atomicfile/atomicfile.go +++ b/atomicfile/atomicfile.go @@ -15,8 +15,9 @@ import ( ) // WriteFile writes data to filename+some suffix, then renames it into filename. -// The perm argument is ignored on Windows. If the target filename already -// exists but is not a regular file, WriteFile returns an error. +// The perm argument is ignored on Windows, but if the target filename already +// exists then the target file's attributes and ACLs are preserved. If the target +// filename already exists but is not a regular file, WriteFile returns an error. func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { fi, err := os.Stat(filename) if err == nil && !fi.Mode().IsRegular() { @@ -47,5 +48,5 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { if err := f.Close(); err != nil { return err } - return os.Rename(tmpName, filename) + return rename(tmpName, filename) } diff --git a/atomicfile/atomicfile_notwindows.go b/atomicfile/atomicfile_notwindows.go new file mode 100644 index 0000000000000..1ce2bb8acda7a --- /dev/null +++ b/atomicfile/atomicfile_notwindows.go @@ -0,0 +1,14 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !windows + +package atomicfile + +import ( + "os" +) + +func rename(srcFile, destFile string) error { + return os.Rename(srcFile, destFile) +} diff --git a/atomicfile/atomicfile_windows.go b/atomicfile/atomicfile_windows.go new file mode 100644 index 0000000000000..c67762df2b56c --- /dev/null +++ b/atomicfile/atomicfile_windows.go @@ -0,0 +1,33 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package atomicfile + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func rename(srcFile, destFile string) error { + // Use replaceFile when possible to preserve the original file's attributes and ACLs. + if err := replaceFile(destFile, srcFile); err == nil || err != windows.ERROR_FILE_NOT_FOUND { + return err + } + // destFile doesn't exist. Just do a normal rename. + return os.Rename(srcFile, destFile) +} + +func replaceFile(destFile, srcFile string) error { + destFile16, err := windows.UTF16PtrFromString(destFile) + if err != nil { + return err + } + + srcFile16, err := windows.UTF16PtrFromString(srcFile) + if err != nil { + return err + } + + return replaceFileW(destFile16, srcFile16, nil, 0, nil, nil) +} diff --git a/atomicfile/atomicfile_windows_test.go b/atomicfile/atomicfile_windows_test.go new file mode 100644 index 0000000000000..4dec1493e0224 --- /dev/null +++ b/atomicfile/atomicfile_windows_test.go @@ -0,0 +1,146 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package atomicfile + +import ( + "os" + "testing" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _SECURITY_RESOURCE_MANAGER_AUTHORITY = windows.SidIdentifierAuthority{[6]byte{0, 0, 0, 0, 0, 9}} + +// makeRandomSID generates a SID derived from a v4 GUID. +// This is basically the same algorithm used by browser sandboxes for generating +// random SIDs. +func makeRandomSID() (*windows.SID, error) { + guid, err := windows.GenerateGUID() + if err != nil { + return nil, err + } + + rids := *((*[4]uint32)(unsafe.Pointer(&guid))) + + var pSID *windows.SID + if err := windows.AllocateAndInitializeSid(&_SECURITY_RESOURCE_MANAGER_AUTHORITY, 4, rids[0], rids[1], rids[2], rids[3], 0, 0, 0, 0, &pSID); err != nil { + return nil, err + } + defer windows.FreeSid(pSID) + + // Make a copy that lives on the Go heap + return pSID.Copy() +} + +func getExistingFileSD(name string) (*windows.SECURITY_DESCRIPTOR, error) { + const infoFlags = windows.DACL_SECURITY_INFORMATION + return windows.GetNamedSecurityInfo(name, windows.SE_FILE_OBJECT, infoFlags) +} + +func getExistingFileDACL(name string) (*windows.ACL, error) { + sd, err := getExistingFileSD(name) + if err != nil { + return nil, err + } + + dacl, _, err := sd.DACL() + return dacl, err +} + +func addDenyACEForRandomSID(dacl *windows.ACL) (*windows.ACL, error) { + randomSID, err := makeRandomSID() + if err != nil { + return nil, err + } + + randomSIDTrustee := windows.TRUSTEE{nil, windows.NO_MULTIPLE_TRUSTEE, + windows.TRUSTEE_IS_SID, windows.TRUSTEE_IS_UNKNOWN, + windows.TrusteeValueFromSID(randomSID)} + + entries := []windows.EXPLICIT_ACCESS{ + { + windows.GENERIC_ALL, + windows.DENY_ACCESS, + windows.NO_INHERITANCE, + randomSIDTrustee, + }, + } + + return windows.ACLFromEntries(entries, dacl) +} + +func setExistingFileDACL(name string, dacl *windows.ACL) error { + return windows.SetNamedSecurityInfo(name, windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil) +} + +// makeOrigFileWithCustomDACL creates a new, temporary file with a custom +// DACL that we can check for later. It returns the name of the temporary +// file and the security descriptor for the file in SDDL format. +func makeOrigFileWithCustomDACL() (name, sddl string, err error) { + f, err := os.CreateTemp("", "foo*.tmp") + if err != nil { + return "", "", err + } + name = f.Name() + if err := f.Close(); err != nil { + return "", "", err + } + f = nil + defer func() { + if err != nil { + os.Remove(name) + } + }() + + dacl, err := getExistingFileDACL(name) + if err != nil { + return "", "", err + } + + // Add a harmless, deny-only ACE for a random SID that isn't used for anything + // (but that we can check for later). + dacl, err = addDenyACEForRandomSID(dacl) + if err != nil { + return "", "", err + } + + if err := setExistingFileDACL(name, dacl); err != nil { + return "", "", err + } + + sd, err := getExistingFileSD(name) + if err != nil { + return "", "", err + } + + return name, sd.String(), nil +} + +func TestPreserveSecurityInfo(t *testing.T) { + // Make a test file with a custom ACL. + origFileName, want, err := makeOrigFileWithCustomDACL() + if err != nil { + t.Fatalf("makeOrigFileWithCustomDACL returned %v", err) + } + t.Cleanup(func() { + os.Remove(origFileName) + }) + + if err := WriteFile(origFileName, []byte{}, 0); err != nil { + t.Fatalf("WriteFile returned %v", err) + } + + // We expect origFileName's security descriptor to be unchanged despite + // the WriteFile call. + sd, err := getExistingFileSD(origFileName) + if err != nil { + t.Fatalf("getExistingFileSD(%q) returned %v", origFileName, err) + } + + if got := sd.String(); got != want { + t.Errorf("security descriptor comparison failed: got %q, want %q", got, want) + } +} diff --git a/atomicfile/mksyscall.go b/atomicfile/mksyscall.go new file mode 100644 index 0000000000000..d8951a77c5ac6 --- /dev/null +++ b/atomicfile/mksyscall.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package atomicfile + +//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go mksyscall.go + +//sys replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) [int32(failretval)==0] = kernel32.ReplaceFileW diff --git a/atomicfile/zsyscall_windows.go b/atomicfile/zsyscall_windows.go new file mode 100644 index 0000000000000..f2f0b6d08cbb7 --- /dev/null +++ b/atomicfile/zsyscall_windows.go @@ -0,0 +1,52 @@ +// Code generated by 'go generate'; DO NOT EDIT. + +package atomicfile + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) + errERROR_EINVAL error = syscall.EINVAL +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return errERROR_EINVAL + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + + procReplaceFileW = modkernel32.NewProc("ReplaceFileW") +) + +func replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) { + r1, _, e1 := syscall.Syscall6(procReplaceFileW.Addr(), 6, uintptr(unsafe.Pointer(replaced)), uintptr(unsafe.Pointer(replacement)), uintptr(unsafe.Pointer(backup)), uintptr(flags), uintptr(exclude), uintptr(reserved)) + if int32(r1) == 0 { + err = errnoErr(e1) + } + return +} diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index d4b406d9dd67e..729122d796c6b 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -85,7 +85,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa google.golang.org/protobuf/runtime/protoimpl from github.com/prometheus/client_model/go+ google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ tailscale.com from tailscale.com/version - tailscale.com/atomicfile from tailscale.com/cmd/derper+ + đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/cmd/derper+ tailscale.com/client/tailscale from tailscale.com/derp tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale tailscale.com/derp from tailscale.com/cmd/derper+ diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index f757cda185c82..cb02038e3579d 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -643,7 +643,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ sigs.k8s.io/yaml/goyaml.v2 from sigs.k8s.io/yaml tailscale.com from tailscale.com/version tailscale.com/appc from tailscale.com/ipn/ipnlocal - tailscale.com/atomicfile from tailscale.com/ipn+ + đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/ipn+ tailscale.com/client/tailscale from tailscale.com/client/web+ tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ tailscale.com/client/web from tailscale.com/ipn/ipnlocal diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index e894e06742b05..9ccd6eebd2f4d 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -69,7 +69,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep software.sslmate.com/src/go-pkcs12 from tailscale.com/cmd/tailscale/cli software.sslmate.com/src/go-pkcs12/internal/rc2 from software.sslmate.com/src/go-pkcs12 tailscale.com from tailscale.com/version - tailscale.com/atomicfile from tailscale.com/cmd/tailscale/cli+ + đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/cmd/tailscale/cli+ tailscale.com/client/tailscale from tailscale.com/client/web+ tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ tailscale.com/client/web from tailscale.com/cmd/tailscale/cli diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 19254b6164800..8af347319ccb1 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -232,7 +232,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de gvisor.dev/gvisor/pkg/waiter from gvisor.dev/gvisor/pkg/context+ tailscale.com from tailscale.com/version tailscale.com/appc from tailscale.com/ipn/ipnlocal - tailscale.com/atomicfile from tailscale.com/ipn+ + đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/ipn+ LD tailscale.com/chirp from tailscale.com/cmd/tailscaled tailscale.com/client/tailscale from tailscale.com/client/web+ tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ From 1b303ee5baef3ddab40be4d1c2b8caa284fd811d Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Wed, 15 Jan 2025 13:32:13 -0800 Subject: [PATCH 128/223] ipn/ipnlocal: re-advertise appc routes on startup (#14609) There's at least one example of stored routes and advertised routes getting out of sync. I don't know how they got there yet, but this would backfill missing advertised routes on startup from stored routes. Also add logging in LocalBackend.AdvertiseRoute to record when new routes actually get put into prefs. Updates #14606 Signed-off-by: Andrew Lytvynov --- ipn/ipnlocal/local.go | 35 +++++++++++++++++++++++++--- ipn/ipnlocal/local_test.go | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index c506f13766dac..92d2f123fe5a9 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4319,6 +4319,33 @@ func (b *LocalBackend) reconfigAppConnectorLocked(nm *netmap.NetworkMap, prefs i b.appConnector.UpdateDomainsAndRoutes(domains, routes) } +func (b *LocalBackend) readvertiseAppConnectorRoutes() { + var domainRoutes map[string][]netip.Addr + b.mu.Lock() + if b.appConnector != nil { + domainRoutes = b.appConnector.DomainRoutes() + } + b.mu.Unlock() + if domainRoutes == nil { + return + } + + // Re-advertise the stored routes, in case stored state got out of + // sync with previously advertised routes in prefs. + var prefixes []netip.Prefix + for _, ips := range domainRoutes { + for _, ip := range ips { + prefixes = append(prefixes, netip.PrefixFrom(ip, ip.BitLen())) + } + } + // Note: AdvertiseRoute will trim routes that are already + // advertised, so if everything is already being advertised this is + // a noop. + if err := b.AdvertiseRoute(prefixes...); err != nil { + b.logf("error advertising stored app connector routes: %v", err) + } +} + // authReconfig pushes a new configuration into wgengine, if engine // updates are not currently blocked, based on the cached netmap and // user prefs. @@ -4397,6 +4424,7 @@ func (b *LocalBackend) authReconfig() { } b.initPeerAPIListener() + b.readvertiseAppConnectorRoutes() } // shouldUseOneCGNATRoute reports whether we should prefer to make one big @@ -7111,7 +7139,7 @@ var ErrDisallowedAutoRoute = errors.New("route is not allowed") // If the route is disallowed, ErrDisallowedAutoRoute is returned. func (b *LocalBackend) AdvertiseRoute(ipps ...netip.Prefix) error { finalRoutes := b.Prefs().AdvertiseRoutes().AsSlice() - newRoutes := false + var newRoutes []netip.Prefix for _, ipp := range ipps { if !allowedAutoRoute(ipp) { @@ -7127,13 +7155,14 @@ func (b *LocalBackend) AdvertiseRoute(ipps ...netip.Prefix) error { } finalRoutes = append(finalRoutes, ipp) - newRoutes = true + newRoutes = append(newRoutes, ipp) } - if !newRoutes { + if len(newRoutes) == 0 { return nil } + b.logf("advertising new app connector routes: %v", newRoutes) _, err := b.EditPrefs(&ipn.MaskedPrefs{ Prefs: ipn.Prefs{ AdvertiseRoutes: finalRoutes, diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index f9a967beada7d..5e8a3172caabb 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1501,6 +1501,53 @@ func TestReconfigureAppConnector(t *testing.T) { } } +func TestBackfillAppConnectorRoutes(t *testing.T) { + // Create backend with an empty app connector. + b := newTestBackend(t) + if err := b.Start(ipn.Options{}); err != nil { + t.Fatal(err) + } + if _, err := b.EditPrefs(&ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AppConnector: ipn.AppConnectorPrefs{Advertise: true}, + }, + AppConnectorSet: true, + }); err != nil { + t.Fatal(err) + } + b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs) + + // Smoke check that AdvertiseRoutes doesn't have the test IP. + ip := netip.MustParseAddr("1.2.3.4") + routes := b.Prefs().AdvertiseRoutes().AsSlice() + if slices.Contains(routes, netip.PrefixFrom(ip, ip.BitLen())) { + t.Fatalf("AdvertiseRoutes %v on a fresh backend already contains advertised route for %v", routes, ip) + } + + // Store the test IP in profile data, but not in Prefs.AdvertiseRoutes. + b.ControlKnobs().AppCStoreRoutes.Store(true) + if err := b.storeRouteInfo(&appc.RouteInfo{ + Domains: map[string][]netip.Addr{ + "example.com": {ip}, + }, + }); err != nil { + t.Fatal(err) + } + + // Mimic b.authReconfigure for the app connector bits. + b.mu.Lock() + b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs) + b.mu.Unlock() + b.readvertiseAppConnectorRoutes() + + // Check that Prefs.AdvertiseRoutes got backfilled with routes stored in + // profile data. + routes = b.Prefs().AdvertiseRoutes().AsSlice() + if !slices.Contains(routes, netip.PrefixFrom(ip, ip.BitLen())) { + t.Fatalf("AdvertiseRoutes %v was not backfilled from stored app connector routes with %v", routes, ip) + } +} + func resolversEqual(t *testing.T, a, b []*dnstype.Resolver) bool { if a == nil && b == nil { return true From f023c8603a8b519846b567052119739774e5ac57 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 14 Jan 2025 19:36:27 -0600 Subject: [PATCH 129/223] types/lazy: fix flaky TestDeferAfterDo This test verifies, among other things, that init functions cannot be deferred after (*DeferredFuncs).Do has already been called and that all subsequent calls to (*DeferredFuncs).Defer return false. However, the initial implementation of this check was racy: by the time (*DeferredFuncs).Do returned, not all goroutines that successfully deferred an init function may have incremented the atomic variable tracking the number of deferred functions. As a result, the variable's value could differ immediately after (*DeferredFuncs).Do returned and after all goroutines had completed execution (i.e., after wg.Wait()). In this PR, we replace the original racy check with a different one. Although this new check is also racy, it can only produce false negatives. This means that if the test fails, it indicates an actual bug rather than a flaky test. Fixes #14039 Signed-off-by: Nick Khyl --- types/lazy/deferred.go | 9 ++++++++- types/lazy/deferred_test.go | 32 ++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/types/lazy/deferred.go b/types/lazy/deferred.go index 964553cef6524..973082914c48c 100644 --- a/types/lazy/deferred.go +++ b/types/lazy/deferred.go @@ -22,7 +22,14 @@ type DeferredInit struct { // until the owner's [DeferredInit.Do] method is called // for the first time. // -// DeferredFuncs is safe for concurrent use. +// DeferredFuncs is safe for concurrent use. The execution +// order of functions deferred by different goroutines is +// unspecified and must not be relied upon. +// However, functions deferred by the same goroutine are +// executed in the same relative order they were deferred. +// Warning: this is the opposite of the behavior of Go's +// defer statement, which executes deferred functions in +// reverse order. type DeferredFuncs struct { m sync.Mutex funcs []func() error diff --git a/types/lazy/deferred_test.go b/types/lazy/deferred_test.go index 9de16c67a6067..98cacbfce7088 100644 --- a/types/lazy/deferred_test.go +++ b/types/lazy/deferred_test.go @@ -205,16 +205,38 @@ func TestDeferredErr(t *testing.T) { } } +// TestDeferAfterDo checks all of the following: +// - Deferring a function before [DeferredInit.Do] is called should always succeed. +// - All successfully deferred functions are executed by the time [DeferredInit.Do] completes. +// - No functions can be deferred after [DeferredInit.Do] is called, meaning: +// - [DeferredInit.Defer] should return false. +// - The deferred function should not be executed. +// +// This test is intentionally racy as it attempts to defer functions from multiple goroutines +// and then calls [DeferredInit.Do] without waiting for them to finish. Waiting would alter +// the observable behavior and render the test pointless. func TestDeferAfterDo(t *testing.T) { var di DeferredInit var deferred, called atomic.Int32 + // deferOnce defers a test function once and fails the test + // if [DeferredInit.Defer] returns true after [DeferredInit.Do] + // has already been called and any deferred functions have been executed. + // It's called concurrently by multiple goroutines. deferOnce := func() bool { + // canDefer is whether it's acceptable for Defer to return true. + // (but not it necessarily must return true) + // If its func has run before, it's definitely not okay for it to + // accept more Defer funcs. + canDefer := called.Load() == 0 ok := di.Defer(func() error { called.Add(1) return nil }) if ok { + if !canDefer { + t.Error("An init function was deferred after DeferredInit.Do() was already called") + } deferred.Add(1) } return ok @@ -242,19 +264,17 @@ func TestDeferAfterDo(t *testing.T) { if err := di.Do(); err != nil { t.Fatalf("DeferredInit.Do() failed: %v", err) } - wantDeferred, wantCalled := deferred.Load(), called.Load() + // The number of called funcs should remain unchanged after [DeferredInit.Do] returns. + wantCalled := called.Load() if deferOnce() { t.Error("An init func was deferred after DeferredInit.Do() returned") } // Wait for the goroutines deferring init funcs to exit. - // No funcs should be deferred after DeferredInit.Do() has returned, - // so the deferred and called counters should remain unchanged. + // No funcs should be called after DeferredInit.Do() has returned, + // and the number of called funcs should be equal to the number of deferred funcs. wg.Wait() - if gotDeferred := deferred.Load(); gotDeferred != wantDeferred { - t.Errorf("An init func was deferred after DeferredInit.Do() returned. Got %d, want %d", gotDeferred, wantDeferred) - } if gotCalled := called.Load(); gotCalled != wantCalled { t.Errorf("An init func was called after DeferredInit.Do() returned. Got %d, want %d", gotCalled, wantCalled) } From d8b00e39ef5ee560d0ff27d38d3c64d34c2f7d22 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 15 Jan 2025 13:43:36 -0800 Subject: [PATCH 130/223] cmd/tailscaled: add some more depchecker dep tests As we look to add github.com/prometheus/client_golang/prometheus to more parts of the codebase, lock in that we don't use it in tailscaled, primarily for binary size reasons. Updates #12614 Change-Id: I03c100d12a05019a22bdc23ce5c4df63d5a03ec6 Signed-off-by: Brad Fitzpatrick --- cmd/tailscaled/tailscaled_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/tailscaled/tailscaled_test.go b/cmd/tailscaled/tailscaled_test.go index 5045468d6543a..f36120f1300b5 100644 --- a/cmd/tailscaled/tailscaled_test.go +++ b/cmd/tailscaled/tailscaled_test.go @@ -29,8 +29,10 @@ func TestDeps(t *testing.T) { GOOS: "linux", GOARCH: "arm64", BadDeps: map[string]string{ - "testing": "do not use testing package in production code", - "gvisor.dev/gvisor/pkg/hostarch": "will crash on non-4K page sizes; see https://github.com/tailscale/tailscale/issues/8658", + "testing": "do not use testing package in production code", + "gvisor.dev/gvisor/pkg/hostarch": "will crash on non-4K page sizes; see https://github.com/tailscale/tailscale/issues/8658", + "google.golang.org/protobuf/proto": "unexpected", + "github.com/prometheus/client_golang/prometheus": "use tailscale.com/metrics in tailscaled", }, }.Check(t) } From 62fb85785710f9249e943eb8f248facf28eda6f7 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 15 Jan 2025 14:22:14 -0600 Subject: [PATCH 131/223] ipn/ipnserver: fix TestConcurrentOSUserSwitchingOnWindows I made a last-minute change in #14626 to split a single loop that created 1_000 concurrent connections into an inner and outer loop that create 100 concurrent connections 10 times. This introduced a race because the last user's connection may still be active (from the server's perspective) when a new outer iteration begins. Since every new client gets a unique ClientID, but we reuse usernames and UIDs, the server may let a user in (as the UID matches, which is fine), but the test might then fail due to a ClientID mismatch: server_test.go:232: CurrentUser(Initial): got &{S-1-5-21-1-0-0-1001 User-4 Client-2 false false}; want &{S-1-5-21-1-0-0-1001 User-4 Client-114 false false} In this PR, we update (*testIPNServer).blockWhileInUse to check whether the server is currently busy and wait until it frees up. We then call blockWhileInUse at the end of each outer iteration so that the server is always in a known idle state at the beginning of the inner loop. We also check that the current user is not set when the server is idle. Updates tailscale/corp#25804 Updates #14655 (found when working on it) Signed-off-by: Nick Khyl --- ipn/ipnserver/server_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ipn/ipnserver/server_test.go b/ipn/ipnserver/server_test.go index 97a616db88b7c..e77901c354e00 100644 --- a/ipn/ipnserver/server_test.go +++ b/ipn/ipnserver/server_test.go @@ -251,6 +251,12 @@ func TestConcurrentOSUserSwitchingOnWindows(t *testing.T) { }() } wg.Wait() + + if err := server.blockWhileInUse(ctx); err != nil { + t.Fatalf("blockWhileInUse: %v", err) + } + + server.checkCurrentUser(nil) } } @@ -346,7 +352,14 @@ func (s *testIPNServer) makeTestUser(name string, clientID string) *ipnauth.Test func (s *testIPNServer) blockWhileInUse(ctx context.Context) error { ready, cleanup := s.zeroReqWaiter.add(&s.mu, ctx) - <-ready + + s.mu.Lock() + busy := len(s.activeReqs) != 0 + s.mu.Unlock() + + if busy { + <-ready + } cleanup() return ctx.Err() } From 0481042738b6320fd328cfcd4998bdaae1c93534 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 15 Jan 2025 16:03:21 -0600 Subject: [PATCH 132/223] ipn/ipnserver: fix a deadlock in (*Server).blockWhileIdentityInUse If the server was in use at the time of the initial check, but disconnected and was removed from the activeReqs map by the time we registered a waiter, the ready channel will never be closed, resulting in a deadlock. To avoid this, we check whether the server is still busy after registering the wait. Fixes #14655 Signed-off-by: Nick Khyl --- ipn/ipnserver/server.go | 13 +++++++++- ipn/ipnserver/server_test.go | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ipn/ipnserver/server.go b/ipn/ipnserver/server.go index a69e43067ea7b..3d9c9e3d4180c 100644 --- a/ipn/ipnserver/server.go +++ b/ipn/ipnserver/server.go @@ -283,7 +283,18 @@ func (s *Server) blockWhileIdentityInUse(ctx context.Context, actor ipnauth.Acto for inUse() { // Check whenever the connection count drops down to zero. ready, cleanup := s.zeroReqWaiter.add(&s.mu, ctx) - <-ready + if inUse() { + // If the server was in use at the time of the initial check, + // but disconnected and was removed from the activeReqs map + // by the time we registered a waiter, the ready channel + // will never be closed, resulting in a deadlock. To avoid + // this, we can check again after registering the waiter. + // + // This method is planned for complete removal as part of the + // multi-user improvements in tailscale/corp#18342, + // and this approach should be fine as a temporary solution. + <-ready + } cleanup() if err := ctx.Err(); err != nil { return err diff --git a/ipn/ipnserver/server_test.go b/ipn/ipnserver/server_test.go index e77901c354e00..e56ae8dab7d4a 100644 --- a/ipn/ipnserver/server_test.go +++ b/ipn/ipnserver/server_test.go @@ -260,6 +260,52 @@ func TestConcurrentOSUserSwitchingOnWindows(t *testing.T) { } } +func TestBlockWhileIdentityInUse(t *testing.T) { + enableLogging := false + setGOOSForTest(t, "windows") + + ctx := context.Background() + server := startDefaultTestIPNServer(t, ctx, enableLogging) + + // connectWaitDisconnectAsUser connects as a user with the specified name + // and keeps the IPN bus watcher alive until the context is canceled. + // It returns a channel that is closed when done. + connectWaitDisconnectAsUser := func(ctx context.Context, name string) <-chan struct{} { + client := server.getClientAs(name) + watcher, cancelWatcher := client.WatchIPNBus(ctx, 0) + + done := make(chan struct{}) + go func() { + defer cancelWatcher() + defer close(done) + for { + _, err := watcher.Next() + if err != nil { + // There's either an error or the request has been canceled. + break + } + } + }() + return done + } + + for range 100 { + // Connect as UserA, and keep the connection alive + // until disconnectUserA is called. + userAContext, disconnectUserA := context.WithCancel(ctx) + userADone := connectWaitDisconnectAsUser(userAContext, "UserA") + disconnectUserA() + // Check if userB can connect. Calling it directly increases + // the likelihood of triggering a deadlock due to a race condition + // in blockWhileIdentityInUse. But the issue also occurs during + // the normal execution path when UserB connects to the IPN server + // while UserA is disconnecting. + userB := server.makeTestUser("UserB", "ClientB") + server.blockWhileIdentityInUse(ctx, userB) + <-userADone + } +} + func setGOOSForTest(tb testing.TB, goos string) { tb.Helper() envknob.Setenv("TS_DEBUG_FAKE_GOOS", goos) From 84b0379dd5869bad00fb3c2d4f4cfe59c8204559 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 15 Jan 2025 15:47:26 -0800 Subject: [PATCH 133/223] prober: remove per-packet DERP pub key copying overheads (#14658) Updates tailscale/corp#25883 Signed-off-by: Jordan Whited --- prober/derp.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/prober/derp.go b/prober/derp.go index 870460d964a70..e811d41ff68f9 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -925,6 +925,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT destinationAddrBytes := destinationAddr.AsSlice() scratch := make([]byte, 4) + toPubDERPKey := toc.SelfPublicKey() for { n, err := dev.Read(bufs, sizes, tunStartOffset) if err != nil { @@ -953,7 +954,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT copy(pkt[12:16], pkt[16:20]) copy(pkt[16:20], scratch) - if err := fromc.Send(toc.SelfPublicKey(), pkt); err != nil { + if err := fromc.Send(toPubDERPKey, pkt); err != nil { tunReadErrC <- err return } @@ -971,6 +972,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT buf := make([]byte, mtu+tunStartOffset) bufs := make([][]byte, 1) + fromDERPPubKey := fromc.SelfPublicKey() for { m, err := toc.Recv() if err != nil { @@ -979,7 +981,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT } switch v := m.(type) { case derp.ReceivedPacket: - if v.Source != fromc.SelfPublicKey() { + if v.Source != fromDERPPubKey { recvErrC <- fmt.Errorf("got data packet from unexpected source, %v", v.Source) return } From 00bd906797fe35d53db8fda24a36ba2bc3f34852 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 15 Jan 2025 16:28:49 -0800 Subject: [PATCH 134/223] prober: remove DERP pub key copying overheads in qd and non-tun measures (#14659) Updates tailscale/corp#25883 Signed-off-by: Jordan Whited --- prober/derp.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/prober/derp.go b/prober/derp.go index e811d41ff68f9..995a69626fc14 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -431,6 +431,7 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. t := time.NewTicker(time.Second / time.Duration(packetsPerSecond)) defer t.Stop() + toDERPPubKey := toc.SelfPublicKey() seq := uint64(0) for { select { @@ -446,7 +447,7 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. txRecordsMu.Unlock() binary.BigEndian.PutUint64(pkt, seq) seq++ - if err := fromc.Send(toc.SelfPublicKey(), pkt); err != nil { + if err := fromc.Send(toDERPPubKey, pkt); err != nil { sendErrC <- fmt.Errorf("sending packet %w", err) return } @@ -460,6 +461,7 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. go func() { defer wg.Done() defer close(recvFinishedC) // to break out of 'select' below. + fromDERPPubKey := fromc.SelfPublicKey() for { m, err := toc.Recv() if err != nil { @@ -469,7 +471,7 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. switch v := m.(type) { case derp.ReceivedPacket: now := time.Now() - if v.Source != fromc.SelfPublicKey() { + if v.Source != fromDERPPubKey { recvFinishedC <- fmt.Errorf("got data packet from unexpected source, %v", v.Source) return } @@ -767,9 +769,10 @@ func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc // Send the packets. sendc := make(chan error, 1) go func() { + toDERPPubKey := toc.SelfPublicKey() for idx, pkt := range pkts { inFlight.AcquireContext(ctx) - if err := fromc.Send(toc.SelfPublicKey(), pkt); err != nil { + if err := fromc.Send(toDERPPubKey, pkt); err != nil { sendc <- fmt.Errorf("sending packet %d: %w", idx, err) return } @@ -781,6 +784,7 @@ func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc go func() { defer close(recvc) // to break out of 'select' below. idx := 0 + fromDERPPubKey := fromc.SelfPublicKey() for { m, err := toc.Recv() if err != nil { @@ -790,7 +794,7 @@ func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc switch v := m.(type) { case derp.ReceivedPacket: inFlight.Release() - if v.Source != fromc.SelfPublicKey() { + if v.Source != fromDERPPubKey { recvc <- fmt.Errorf("got data packet %d from unexpected source, %v", idx, v.Source) return } @@ -925,7 +929,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT destinationAddrBytes := destinationAddr.AsSlice() scratch := make([]byte, 4) - toPubDERPKey := toc.SelfPublicKey() + toDERPPubKey := toc.SelfPublicKey() for { n, err := dev.Read(bufs, sizes, tunStartOffset) if err != nil { @@ -954,7 +958,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT copy(pkt[12:16], pkt[16:20]) copy(pkt[16:20], scratch) - if err := fromc.Send(toPubDERPKey, pkt); err != nil { + if err := fromc.Send(toDERPPubKey, pkt); err != nil { tunReadErrC <- err return } From 2d1f6f18cc4f8ed1cb09aa6bee0a922219ba6aa6 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Thu, 16 Jan 2025 11:15:36 +0000 Subject: [PATCH 135/223] cmd/k8s-operator: require namespace config (#14648) Most users should not run into this because it's set in the helm chart and the deploy manifest, but if namespace is not set we get confusing authz errors because the kube client tries to fetch some namespaced resources as though they're cluster-scoped and reports permission denied. Try to detect namespace from the default projected volume, and otherwise fatal. Fixes #cleanup Change-Id: I64b34191e440b61204b9ad30bbfa117abbbe09c3 Signed-off-by: Tom Proctor --- cmd/k8s-operator/operator.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index b2483908229d1..7f8f94673e837 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -88,6 +88,15 @@ func main() { zlog := kzap.NewRaw(opts...).Sugar() logf.SetLogger(zapr.NewLogger(zlog.Desugar())) + if tsNamespace == "" { + const namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + b, err := os.ReadFile(namespaceFile) + if err != nil { + zlog.Fatalf("Could not get operator namespace from OPERATOR_NAMESPACE environment variable or default projected volume: %v", err) + } + tsNamespace = strings.TrimSpace(string(b)) + } + // The operator can run either as a plain operator or it can // additionally act as api-server proxy // https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy. From 7d73a38b40a57e39080eb8b74aed08c69c0fb562 Mon Sep 17 00:00:00 2001 From: Aaron Klotz Date: Fri, 10 Jan 2025 14:42:11 -0700 Subject: [PATCH 136/223] net/dns: only populate OSConfig.Hosts when MagicDNS is enabled Previously we were doing this unconditionally. Updates #14428 Signed-off-by: Aaron Klotz --- net/dns/manager.go | 4 ++- net/dns/manager_test.go | 70 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/net/dns/manager.go b/net/dns/manager.go index 5ac2f69fc1a04..ebf91811a0243 100644 --- a/net/dns/manager.go +++ b/net/dns/manager.go @@ -246,8 +246,10 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig // the OS. rcfg.Hosts = cfg.Hosts routes := map[dnsname.FQDN][]*dnstype.Resolver{} // assigned conditionally to rcfg.Routes below. + var propagateHostsToOS bool for suffix, resolvers := range cfg.Routes { if len(resolvers) == 0 { + propagateHostsToOS = true rcfg.LocalDomains = append(rcfg.LocalDomains, suffix) } else { routes[suffix] = resolvers @@ -256,7 +258,7 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig // Similarly, the OS always gets search paths. ocfg.SearchDomains = cfg.SearchDomains - if m.goos == "windows" { + if propagateHostsToOS && m.goos == "windows" { ocfg.Hosts = compileHostEntries(cfg) } diff --git a/net/dns/manager_test.go b/net/dns/manager_test.go index 366e08bbf8644..2bdbc72e26093 100644 --- a/net/dns/manager_test.go +++ b/net/dns/manager_test.go @@ -836,6 +836,76 @@ func TestManager(t *testing.T) { }, goos: "darwin", }, + { + name: "populate-hosts-magicdns", + in: Config{ + Routes: upstreams( + "corp.com", "2.2.2.2", + "ts.com", ""), + Hosts: hosts( + "dave.ts.com.", "1.2.3.4", + "bradfitz.ts.com.", "2.3.4.5"), + SearchDomains: fqdns("ts.com", "universe.tf"), + }, + split: true, + os: OSConfig{ + Hosts: []*HostEntry{ + { + Addr: netip.MustParseAddr("2.3.4.5"), + Hosts: []string{ + "bradfitz.ts.com.", + "bradfitz", + }, + }, + { + Addr: netip.MustParseAddr("1.2.3.4"), + Hosts: []string{ + "dave.ts.com.", + "dave", + }, + }, + }, + Nameservers: mustIPs("100.100.100.100"), + SearchDomains: fqdns("ts.com", "universe.tf"), + MatchDomains: fqdns("corp.com", "ts.com"), + }, + rs: resolver.Config{ + Routes: upstreams("corp.com.", "2.2.2.2"), + Hosts: hosts( + "dave.ts.com.", "1.2.3.4", + "bradfitz.ts.com.", "2.3.4.5"), + LocalDomains: fqdns("ts.com."), + }, + goos: "windows", + }, + { + // Regression test for https://github.com/tailscale/tailscale/issues/14428 + name: "nopopulate-hosts-nomagicdns", + in: Config{ + Routes: upstreams( + "corp.com", "2.2.2.2", + "ts.com", "1.1.1.1"), + Hosts: hosts( + "dave.ts.com.", "1.2.3.4", + "bradfitz.ts.com.", "2.3.4.5"), + SearchDomains: fqdns("ts.com", "universe.tf"), + }, + split: true, + os: OSConfig{ + Nameservers: mustIPs("100.100.100.100"), + SearchDomains: fqdns("ts.com", "universe.tf"), + MatchDomains: fqdns("corp.com", "ts.com"), + }, + rs: resolver.Config{ + Routes: upstreams( + "corp.com.", "2.2.2.2", + "ts.com", "1.1.1.1"), + Hosts: hosts( + "dave.ts.com.", "1.2.3.4", + "bradfitz.ts.com.", "2.3.4.5"), + }, + goos: "windows", + }, } trIP := cmp.Transformer("ipStr", func(ip netip.Addr) string { return ip.String() }) From de5683f7c61098337fe2825e2febe98b6809b291 Mon Sep 17 00:00:00 2001 From: Mario Minardi Date: Thu, 16 Jan 2025 12:21:33 -0700 Subject: [PATCH 137/223] derp: change packets_dropped metric to also have reason and kind labels (#14651) Metrics currently exist for dropped packets by reason, and total received packets by kind (e.g., `disco` or `other`), but relating these two together to gleam information about the drop rate for specific reasons on a per-kind basis is not currently possible. Change `derp_packets_dropped` to use a `metrics.MultiLabelMap` to track both the `reason` and `kind` in the same metric to allow for this desired level of granularity. Drop metrics that this makes unnecessary (namely `packetsDroppedReason` and `packetsDroppedType`). Updates https://github.com/tailscale/corp/issues/25489 Signed-off-by: Mario Minardi --- derp/derp_server.go | 224 +++++++++++++++++++++----------------- derp/dropreason_string.go | 33 ------ 2 files changed, 126 insertions(+), 131 deletions(-) delete mode 100644 derp/dropreason_string.go diff --git a/derp/derp_server.go b/derp/derp_server.go index 8066b7f19ef43..08fd280a99a3f 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -112,6 +112,14 @@ const ( disableFighters ) +// packetKind is the kind of packet being sent through DERP +type packetKind string + +const ( + packetKindDisco packetKind = "disco" + packetKindOther packetKind = "other" +) + type align64 [0]atomic.Int64 // for side effect of its 64-bit alignment // Server is a DERP server. @@ -131,44 +139,37 @@ type Server struct { debug bool // Counters: - packetsSent, bytesSent expvar.Int - packetsRecv, bytesRecv expvar.Int - packetsRecvByKind metrics.LabelMap - packetsRecvDisco *expvar.Int - packetsRecvOther *expvar.Int - _ align64 - packetsDropped expvar.Int - packetsDroppedReason metrics.LabelMap - packetsDroppedReasonCounters []*expvar.Int // indexed by dropReason - packetsDroppedType metrics.LabelMap - packetsDroppedTypeDisco *expvar.Int - packetsDroppedTypeOther *expvar.Int - _ align64 - packetsForwardedOut expvar.Int - packetsForwardedIn expvar.Int - peerGoneDisconnectedFrames expvar.Int // number of peer disconnected frames sent - peerGoneNotHereFrames expvar.Int // number of peer not here frames sent - gotPing expvar.Int // number of ping frames from client - sentPong expvar.Int // number of pong frames enqueued to client - accepts expvar.Int - curClients expvar.Int - curClientsNotIdeal expvar.Int - curHomeClients expvar.Int // ones with preferred - dupClientKeys expvar.Int // current number of public keys we have 2+ connections for - dupClientConns expvar.Int // current number of connections sharing a public key - dupClientConnTotal expvar.Int // total number of accepted connections when a dup key existed - unknownFrames expvar.Int - homeMovesIn expvar.Int // established clients announce home server moves in - homeMovesOut expvar.Int // established clients announce home server moves out - multiForwarderCreated expvar.Int - multiForwarderDeleted expvar.Int - removePktForwardOther expvar.Int - sclientWriteTimeouts expvar.Int - avgQueueDuration *uint64 // In milliseconds; accessed atomically - tcpRtt metrics.LabelMap // histogram - meshUpdateBatchSize *metrics.Histogram - meshUpdateLoopCount *metrics.Histogram - bufferedWriteFrames *metrics.Histogram // how many sendLoop frames (or groups of related frames) get written per flush + packetsSent, bytesSent expvar.Int + packetsRecv, bytesRecv expvar.Int + packetsRecvByKind metrics.LabelMap + packetsRecvDisco *expvar.Int + packetsRecvOther *expvar.Int + _ align64 + packetsForwardedOut expvar.Int + packetsForwardedIn expvar.Int + peerGoneDisconnectedFrames expvar.Int // number of peer disconnected frames sent + peerGoneNotHereFrames expvar.Int // number of peer not here frames sent + gotPing expvar.Int // number of ping frames from client + sentPong expvar.Int // number of pong frames enqueued to client + accepts expvar.Int + curClients expvar.Int + curClientsNotIdeal expvar.Int + curHomeClients expvar.Int // ones with preferred + dupClientKeys expvar.Int // current number of public keys we have 2+ connections for + dupClientConns expvar.Int // current number of connections sharing a public key + dupClientConnTotal expvar.Int // total number of accepted connections when a dup key existed + unknownFrames expvar.Int + homeMovesIn expvar.Int // established clients announce home server moves in + homeMovesOut expvar.Int // established clients announce home server moves out + multiForwarderCreated expvar.Int + multiForwarderDeleted expvar.Int + removePktForwardOther expvar.Int + sclientWriteTimeouts expvar.Int + avgQueueDuration *uint64 // In milliseconds; accessed atomically + tcpRtt metrics.LabelMap // histogram + meshUpdateBatchSize *metrics.Histogram + meshUpdateLoopCount *metrics.Histogram + bufferedWriteFrames *metrics.Histogram // how many sendLoop frames (or groups of related frames) get written per flush // verifyClientsLocalTailscaled only accepts client connections to the DERP // server if the clientKey is a known peer in the network, as specified by a @@ -351,6 +352,11 @@ type Conn interface { SetWriteDeadline(time.Time) error } +var packetsDropped = metrics.NewMultiLabelMap[dropReasonKindLabels]( + "derp_packets_dropped", + "counter", + "DERP packets dropped by reason and by kind") + // NewServer returns a new DERP server. It doesn't listen on its own. // Connections are given to it via Server.Accept. func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server { @@ -358,61 +364,81 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server { runtime.ReadMemStats(&ms) s := &Server{ - debug: envknob.Bool("DERP_DEBUG_LOGS"), - privateKey: privateKey, - publicKey: privateKey.Public(), - logf: logf, - limitedLogf: logger.RateLimitedFn(logf, 30*time.Second, 5, 100), - packetsRecvByKind: metrics.LabelMap{Label: "kind"}, - packetsDroppedReason: metrics.LabelMap{Label: "reason"}, - packetsDroppedType: metrics.LabelMap{Label: "type"}, - clients: map[key.NodePublic]*clientSet{}, - clientsMesh: map[key.NodePublic]PacketForwarder{}, - netConns: map[Conn]chan struct{}{}, - memSys0: ms.Sys, - watchers: set.Set[*sclient]{}, - peerGoneWatchers: map[key.NodePublic]set.HandleSet[func(key.NodePublic)]{}, - avgQueueDuration: new(uint64), - tcpRtt: metrics.LabelMap{Label: "le"}, - meshUpdateBatchSize: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000}), - meshUpdateLoopCount: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100}), - bufferedWriteFrames: metrics.NewHistogram([]float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 50, 100}), - keyOfAddr: map[netip.AddrPort]key.NodePublic{}, - clock: tstime.StdClock{}, + debug: envknob.Bool("DERP_DEBUG_LOGS"), + privateKey: privateKey, + publicKey: privateKey.Public(), + logf: logf, + limitedLogf: logger.RateLimitedFn(logf, 30*time.Second, 5, 100), + packetsRecvByKind: metrics.LabelMap{Label: "kind"}, + clients: map[key.NodePublic]*clientSet{}, + clientsMesh: map[key.NodePublic]PacketForwarder{}, + netConns: map[Conn]chan struct{}{}, + memSys0: ms.Sys, + watchers: set.Set[*sclient]{}, + peerGoneWatchers: map[key.NodePublic]set.HandleSet[func(key.NodePublic)]{}, + avgQueueDuration: new(uint64), + tcpRtt: metrics.LabelMap{Label: "le"}, + meshUpdateBatchSize: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000}), + meshUpdateLoopCount: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100}), + bufferedWriteFrames: metrics.NewHistogram([]float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 50, 100}), + keyOfAddr: map[netip.AddrPort]key.NodePublic{}, + clock: tstime.StdClock{}, } s.initMetacert() - s.packetsRecvDisco = s.packetsRecvByKind.Get("disco") - s.packetsRecvOther = s.packetsRecvByKind.Get("other") + s.packetsRecvDisco = s.packetsRecvByKind.Get(string(packetKindDisco)) + s.packetsRecvOther = s.packetsRecvByKind.Get(string(packetKindOther)) - s.packetsDroppedReasonCounters = s.genPacketsDroppedReasonCounters() - - s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco") - s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other") + genPacketsDroppedCounters() s.perClientSendQueueDepth = getPerClientSendQueueDepth() return s } -func (s *Server) genPacketsDroppedReasonCounters() []*expvar.Int { - getMetric := s.packetsDroppedReason.Get - ret := []*expvar.Int{ - dropReasonUnknownDest: getMetric("unknown_dest"), - dropReasonUnknownDestOnFwd: getMetric("unknown_dest_on_fwd"), - dropReasonGoneDisconnected: getMetric("gone_disconnected"), - dropReasonQueueHead: getMetric("queue_head"), - dropReasonQueueTail: getMetric("queue_tail"), - dropReasonWriteError: getMetric("write_error"), - dropReasonDupClient: getMetric("dup_client"), +func genPacketsDroppedCounters() { + initMetrics := func(reason dropReason) { + packetsDropped.Add(dropReasonKindLabels{ + Kind: string(packetKindDisco), + Reason: string(reason), + }, 0) + packetsDropped.Add(dropReasonKindLabels{ + Kind: string(packetKindOther), + Reason: string(reason), + }, 0) + } + getMetrics := func(reason dropReason) []expvar.Var { + return []expvar.Var{ + packetsDropped.Get(dropReasonKindLabels{ + Kind: string(packetKindDisco), + Reason: string(reason), + }), + packetsDropped.Get(dropReasonKindLabels{ + Kind: string(packetKindOther), + Reason: string(reason), + }), + } } - if len(ret) != int(numDropReasons) { - panic("dropReason metrics out of sync") + + dropReasons := []dropReason{ + dropReasonUnknownDest, + dropReasonUnknownDestOnFwd, + dropReasonGoneDisconnected, + dropReasonQueueHead, + dropReasonQueueTail, + dropReasonWriteError, + dropReasonDupClient, } - for i := range numDropReasons { - if ret[i] == nil { + + for _, dr := range dropReasons { + initMetrics(dr) + m := getMetrics(dr) + if len(m) != 2 { + panic("dropReason metrics out of sync") + } + + if m[0] == nil || m[1] == nil { panic("dropReason metrics out of sync") } } - return ret } // SetMesh sets the pre-shared key that regional DERP servers used to mesh @@ -1152,31 +1178,36 @@ func (c *sclient) debugLogf(format string, v ...any) { } } -// dropReason is why we dropped a DERP frame. -type dropReason int +type dropReasonKindLabels struct { + Reason string // metric label corresponding to a given dropReason + Kind string // either `disco` or `other` +} -//go:generate go run tailscale.com/cmd/addlicense -file dropreason_string.go go run golang.org/x/tools/cmd/stringer -type=dropReason -trimprefix=dropReason +// dropReason is why we dropped a DERP frame. +type dropReason string const ( - dropReasonUnknownDest dropReason = iota // unknown destination pubkey - dropReasonUnknownDestOnFwd // unknown destination pubkey on a derp-forwarded packet - dropReasonGoneDisconnected // destination tailscaled disconnected before we could send - dropReasonQueueHead // destination queue is full, dropped packet at queue head - dropReasonQueueTail // destination queue is full, dropped packet at queue tail - dropReasonWriteError // OS write() failed - dropReasonDupClient // the public key is connected 2+ times (active/active, fighting) - numDropReasons // unused; keep last + dropReasonUnknownDest dropReason = "unknown_dest" // unknown destination pubkey + dropReasonUnknownDestOnFwd dropReason = "unknown_dest_on_fwd" // unknown destination pubkey on a derp-forwarded packet + dropReasonGoneDisconnected dropReason = "gone_disconnected" // destination tailscaled disconnected before we could send + dropReasonQueueHead dropReason = "queue_head" // destination queue is full, dropped packet at queue head + dropReasonQueueTail dropReason = "queue_tail" // destination queue is full, dropped packet at queue tail + dropReasonWriteError dropReason = "write_error" // OS write() failed + dropReasonDupClient dropReason = "dup_client" // the public key is connected 2+ times (active/active, fighting) ) func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, reason dropReason) { - s.packetsDropped.Add(1) - s.packetsDroppedReasonCounters[reason].Add(1) + labels := dropReasonKindLabels{ + Reason: string(reason), + } looksDisco := disco.LooksLikeDiscoWrapper(packetBytes) if looksDisco { - s.packetsDroppedTypeDisco.Add(1) + labels.Kind = string(packetKindDisco) } else { - s.packetsDroppedTypeOther.Add(1) + labels.Kind = string(packetKindOther) } + packetsDropped.Add(labels, 1) + if verboseDropKeys[dstKey] { // Preformat the log string prior to calling limitedLogf. The // limiter acts based on the format string, and we want to @@ -2095,9 +2126,6 @@ func (s *Server) ExpVar() expvar.Var { m.Set("accepts", &s.accepts) m.Set("bytes_received", &s.bytesRecv) m.Set("bytes_sent", &s.bytesSent) - m.Set("packets_dropped", &s.packetsDropped) - m.Set("counter_packets_dropped_reason", &s.packetsDroppedReason) - m.Set("counter_packets_dropped_type", &s.packetsDroppedType) m.Set("counter_packets_received_kind", &s.packetsRecvByKind) m.Set("packets_sent", &s.packetsSent) m.Set("packets_received", &s.packetsRecv) diff --git a/derp/dropreason_string.go b/derp/dropreason_string.go deleted file mode 100644 index 3ad0728194b73..0000000000000 --- a/derp/dropreason_string.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -// Code generated by "stringer -type=dropReason -trimprefix=dropReason"; DO NOT EDIT. - -package derp - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[dropReasonUnknownDest-0] - _ = x[dropReasonUnknownDestOnFwd-1] - _ = x[dropReasonGoneDisconnected-2] - _ = x[dropReasonQueueHead-3] - _ = x[dropReasonQueueTail-4] - _ = x[dropReasonWriteError-5] - _ = x[dropReasonDupClient-6] - _ = x[numDropReasons-7] -} - -const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneDisconnectedQueueHeadQueueTailWriteErrorDupClientnumDropReasons" - -var _dropReason_index = [...]uint8{0, 11, 27, 43, 52, 61, 71, 80, 94} - -func (i dropReason) String() string { - if i < 0 || i >= dropReason(len(_dropReason_index)-1) { - return "dropReason(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _dropReason_name[_dropReason_index[i]:_dropReason_index[i+1]] -} From d912a49be6cca5252612e52e20fdbce6a89486ec Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 16 Jan 2025 16:04:35 -0800 Subject: [PATCH 138/223] net/tstun: add logging to aid developers missing Start calls Since 5297bd2cff8ed03679, tstun.Wrapper has required its Start method to be called for it to function. Failure to do so just results in weird hangs and I've wasted too much time multiple times now debugging. Hopefully this prevents more lost time. Updates tailscale/corp#24454 Change-Id: I87f4539f7be7dc154627f8835a37a8db88c31be0 Signed-off-by: Brad Fitzpatrick --- net/tstun/wrap.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/net/tstun/wrap.go b/net/tstun/wrap.go index deb8bc0944a37..e4ff36b49322b 100644 --- a/net/tstun/wrap.go +++ b/net/tstun/wrap.go @@ -904,9 +904,23 @@ func (t *Wrapper) IdleDuration() time.Duration { return mono.Since(t.lastActivityAtomic.LoadAtomic()) } +func (t *Wrapper) awaitStart() { + for { + select { + case <-t.startCh: + return + case <-time.After(1 * time.Second): + // Multiple times while remixing tailscaled I (Brad) have forgotten + // to call Start and then wasted far too much time debugging. + // I do not wish that debugging on anyone else. Hopefully this'll help: + t.logf("tstun: awaiting Wrapper.Start call") + } + } +} + func (t *Wrapper) Read(buffs [][]byte, sizes []int, offset int) (int, error) { if !t.started.Load() { - <-t.startCh + t.awaitStart() } // packet from OS read and sent to WG res, ok := <-t.vectorOutbound From 97a44d6453e83c966cfe109df77f9863830344ff Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 17 Jan 2025 05:37:53 +0000 Subject: [PATCH 139/223] go.{mod,sum},cmd/{k8s-operator,derper,stund}/depaware.txt: bump kube deps (#14601) Updates kube deps and mkctr, regenerates kube yamls with the updated tooling. Updates#cleanup Signed-off-by: Irbe Krumina --- cmd/derper/depaware.txt | 4 +- cmd/k8s-operator/connector_test.go | 36 ++-- cmd/k8s-operator/depaware.txt | 167 +++++++++++++++-- .../deploy/crds/tailscale.com_connectors.yaml | 2 +- .../deploy/crds/tailscale.com_dnsconfigs.yaml | 2 +- .../crds/tailscale.com_proxyclasses.yaml | 82 +++++++-- .../crds/tailscale.com_proxygroups.yaml | 2 +- .../deploy/crds/tailscale.com_recorders.yaml | 74 ++++++-- .../deploy/manifests/operator.yaml | 162 +++++++++++++---- cmd/k8s-operator/egress-eps_test.go | 4 +- .../egress-services-readiness_test.go | 14 +- cmd/k8s-operator/egress-services_test.go | 6 +- cmd/k8s-operator/ingress_test.go | 44 ++--- cmd/k8s-operator/nameserver_test.go | 10 +- cmd/k8s-operator/operator_test.go | 141 ++++++++------- cmd/k8s-operator/proxyclass_test.go | 16 +- cmd/k8s-operator/proxygroup_test.go | 24 +-- cmd/k8s-operator/testutils_test.go | 10 +- cmd/k8s-operator/tsrecorder_test.go | 18 +- cmd/stund/depaware.txt | 3 +- go.mod | 64 +++---- go.sum | 169 +++++++++--------- 22 files changed, 690 insertions(+), 364 deletions(-) diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 729122d796c6b..498677a49f8da 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -35,11 +35,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa L github.com/mdlayher/netlink/nltest from github.com/google/nftables L đŸ’Ŗ github.com/mdlayher/socket from github.com/mdlayher/netlink đŸ’Ŗ github.com/mitchellh/go-ps from tailscale.com/safesocket + github.com/munnerz/goautoneg from github.com/prometheus/common/expfmt đŸ’Ŗ github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb/promvarz github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ - github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+ LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs @@ -264,7 +264,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa html/template from tailscale.com/cmd/derper io from bufio+ io/fs from crypto/x509+ - io/ioutil from github.com/mitchellh/go-ps+ + L io/ioutil from github.com/mitchellh/go-ps+ iter from maps+ log from expvar+ log/internal from log diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 242f1f99fdd90..f32fe3282020c 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -79,8 +79,8 @@ func TestConnector(t *testing.T) { subnetRoutes: "10.40.0.0/14", app: kubetypes.AppConnector, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Connector status should get updated with the IP/hostname info when available. const hostname = "foo.tailnetxyz.ts.net" @@ -106,7 +106,7 @@ func TestConnector(t *testing.T) { opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Remove a route. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -114,7 +114,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Remove the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -122,7 +122,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Re-add the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -132,7 +132,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -175,8 +175,8 @@ func TestConnector(t *testing.T) { hostname: "test-connector", app: kubetypes.AppConnector, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Add an exit node. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -184,7 +184,7 @@ func TestConnector(t *testing.T) { }) opts.isExitNode = true expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -261,8 +261,8 @@ func TestConnectorWithProxyClass(t *testing.T) { subnetRoutes: "10.40.0.0/14", app: kubetypes.AppConnector, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 2. Update Connector to specify a ProxyClass. ProxyClass is not yet // ready, so its configuration is NOT applied to the Connector @@ -271,7 +271,7 @@ func TestConnectorWithProxyClass(t *testing.T) { conn.Spec.ProxyClass = "custom-metadata" }) expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 3. ProxyClass is set to Ready by proxy-class reconciler. Connector // get reconciled and configuration from the ProxyClass is applied to @@ -286,7 +286,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = pc.Name expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 4. Connector.spec.proxyClass field is unset, Connector gets // reconciled and configuration from the ProxyClass is removed from the @@ -296,7 +296,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) } func TestConnectorWithAppConnector(t *testing.T) { @@ -351,8 +351,8 @@ func TestConnectorWithAppConnector(t *testing.T) { app: kubetypes.AppConnector, isAppConnector: true, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // Connector's ready condition should be set to true cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer") @@ -364,7 +364,7 @@ func TestConnectorWithAppConnector(t *testing.T) { Reason: reasonConnectorCreated, Message: reasonConnectorCreated, }} - expectEqual(t, fc, cn, nil) + expectEqual(t, fc, cn) // 2. Connector with invalid app connector routes has status set to invalid mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -379,7 +379,7 @@ func TestConnectorWithAppConnector(t *testing.T) { Reason: reasonConnectorInvalid, Message: "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5", }} - expectEqual(t, fc, cn, nil) + expectEqual(t, fc, cn) // 3. Connector with valid app connnector routes becomes ready mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index cb02038e3579d..80c9f0c060187 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -94,7 +94,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/evanphx/json-patch/v5 from sigs.k8s.io/controller-runtime/pkg/client github.com/evanphx/json-patch/v5/internal/json from github.com/evanphx/json-patch/v5 đŸ’Ŗ github.com/fsnotify/fsnotify from sigs.k8s.io/controller-runtime/pkg/certwatcher - github.com/fxamacker/cbor/v2 from tailscale.com/tka + github.com/fxamacker/cbor/v2 from tailscale.com/tka+ github.com/gaissmai/bart from tailscale.com/net/ipset+ github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json/internal/jsonflags+ @@ -110,11 +110,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/go-openapi/jsonpointer from github.com/go-openapi/jsonreference github.com/go-openapi/jsonreference from k8s.io/kube-openapi/pkg/internal+ github.com/go-openapi/jsonreference/internal from github.com/go-openapi/jsonreference - github.com/go-openapi/swag from github.com/go-openapi/jsonpointer+ + đŸ’Ŗ github.com/go-openapi/swag from github.com/go-openapi/jsonpointer+ L đŸ’Ŗ github.com/godbus/dbus/v5 from tailscale.com/net/dns đŸ’Ŗ github.com/gogo/protobuf/proto from k8s.io/api/admission/v1+ github.com/gogo/protobuf/sortkeys from k8s.io/api/admission/v1+ - github.com/golang/groupcache/lru from k8s.io/client-go/tools/record+ + github.com/golang/groupcache/lru from tailscale.com/net/dnscache github.com/golang/protobuf/proto from k8s.io/client-go/discovery+ github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+ github.com/google/gnostic-models/compiler from github.com/google/gnostic-models/openapiv2+ @@ -140,7 +140,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L đŸ’Ŗ github.com/illarion/gonotify/v2 from tailscale.com/net/dns - github.com/imdario/mergo from k8s.io/client-go/tools/clientcmd L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/net/tstun L github.com/insomniacslk/dhcp/iana from github.com/insomniacslk/dhcp/dhcpv4 L github.com/insomniacslk/dhcp/interfaces from github.com/insomniacslk/dhcp/dhcpv4 @@ -171,7 +170,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ đŸ’Ŗ github.com/mitchellh/go-ps from tailscale.com/safesocket github.com/modern-go/concurrent from github.com/json-iterator/go đŸ’Ŗ github.com/modern-go/reflect2 from github.com/json-iterator/go - github.com/munnerz/goautoneg from k8s.io/kube-openapi/pkg/handler3 + github.com/munnerz/goautoneg from k8s.io/kube-openapi/pkg/handler3+ github.com/opencontainers/go-digest from github.com/distribution/reference L github.com/pierrec/lz4/v4 from github.com/u-root/uio/uio L github.com/pierrec/lz4/v4/internal/lz4block from github.com/pierrec/lz4/v4+ @@ -186,7 +185,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/prometheus/client_golang/prometheus/promhttp from sigs.k8s.io/controller-runtime/pkg/metrics/server+ github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ - github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+ LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs @@ -250,6 +248,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ google.golang.org/protobuf/internal/descopts from google.golang.org/protobuf/internal/filedesc+ google.golang.org/protobuf/internal/detrand from google.golang.org/protobuf/internal/descfmt+ google.golang.org/protobuf/internal/editiondefaults from google.golang.org/protobuf/internal/filedesc+ + google.golang.org/protobuf/internal/editionssupport from google.golang.org/protobuf/reflect/protodesc google.golang.org/protobuf/internal/encoding/defval from google.golang.org/protobuf/internal/encoding/tag+ google.golang.org/protobuf/internal/encoding/messageset from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/encoding/tag from google.golang.org/protobuf/internal/impl @@ -275,8 +274,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ google.golang.org/protobuf/types/gofeaturespb from google.golang.org/protobuf/reflect/protodesc google.golang.org/protobuf/types/known/anypb from github.com/google/gnostic-models/compiler+ google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ + gopkg.in/evanphx/json-patch.v4 from k8s.io/client-go/testing gopkg.in/inf.v0 from k8s.io/apimachinery/pkg/api/resource - gopkg.in/yaml.v2 from k8s.io/kube-openapi/pkg/util/proto+ gopkg.in/yaml.v3 from github.com/go-openapi/swag+ gvisor.dev/gvisor/pkg/atomicbitops from gvisor.dev/gvisor/pkg/buffer+ gvisor.dev/gvisor/pkg/bits from gvisor.dev/gvisor/pkg/buffer @@ -345,6 +344,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/api/certificates/v1alpha1 from k8s.io/client-go/applyconfigurations/certificates/v1alpha1+ k8s.io/api/certificates/v1beta1 from k8s.io/client-go/applyconfigurations/certificates/v1beta1+ k8s.io/api/coordination/v1 from k8s.io/client-go/applyconfigurations/coordination/v1+ + k8s.io/api/coordination/v1alpha2 from k8s.io/client-go/applyconfigurations/coordination/v1alpha2+ k8s.io/api/coordination/v1beta1 from k8s.io/client-go/applyconfigurations/coordination/v1beta1+ k8s.io/api/core/v1 from k8s.io/api/apps/v1+ k8s.io/api/discovery/v1 from k8s.io/client-go/applyconfigurations/discovery/v1+ @@ -367,7 +367,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/api/rbac/v1 from k8s.io/client-go/applyconfigurations/rbac/v1+ k8s.io/api/rbac/v1alpha1 from k8s.io/client-go/applyconfigurations/rbac/v1alpha1+ k8s.io/api/rbac/v1beta1 from k8s.io/client-go/applyconfigurations/rbac/v1beta1+ - k8s.io/api/resource/v1alpha2 from k8s.io/client-go/applyconfigurations/resource/v1alpha2+ + k8s.io/api/resource/v1alpha3 from k8s.io/client-go/applyconfigurations/resource/v1alpha3+ + k8s.io/api/resource/v1beta1 from k8s.io/client-go/applyconfigurations/resource/v1beta1+ k8s.io/api/scheduling/v1 from k8s.io/client-go/applyconfigurations/scheduling/v1+ k8s.io/api/scheduling/v1alpha1 from k8s.io/client-go/applyconfigurations/scheduling/v1alpha1+ k8s.io/api/scheduling/v1beta1 from k8s.io/client-go/applyconfigurations/scheduling/v1beta1+ @@ -380,10 +381,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/apimachinery/pkg/api/equality from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+ k8s.io/apimachinery/pkg/api/errors from k8s.io/apimachinery/pkg/util/managedfields/internal+ k8s.io/apimachinery/pkg/api/meta from k8s.io/apimachinery/pkg/api/validation+ + k8s.io/apimachinery/pkg/api/meta/testrestmapper from k8s.io/client-go/testing k8s.io/apimachinery/pkg/api/resource from k8s.io/api/autoscaling/v1+ k8s.io/apimachinery/pkg/api/validation from k8s.io/apimachinery/pkg/util/managedfields/internal+ đŸ’Ŗ k8s.io/apimachinery/pkg/apis/meta/internalversion from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+ k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme from k8s.io/client-go/metadata + k8s.io/apimachinery/pkg/apis/meta/internalversion/validation from k8s.io/client-go/util/watchlist đŸ’Ŗ k8s.io/apimachinery/pkg/apis/meta/v1 from k8s.io/api/admission/v1+ k8s.io/apimachinery/pkg/apis/meta/v1/unstructured from k8s.io/apimachinery/pkg/runtime/serializer/versioning+ k8s.io/apimachinery/pkg/apis/meta/v1/validation from k8s.io/apimachinery/pkg/api/validation+ @@ -395,6 +398,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/apimachinery/pkg/runtime from k8s.io/api/admission/v1+ k8s.io/apimachinery/pkg/runtime/schema from k8s.io/api/admission/v1+ k8s.io/apimachinery/pkg/runtime/serializer from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+ + k8s.io/apimachinery/pkg/runtime/serializer/cbor from k8s.io/client-go/dynamic+ + k8s.io/apimachinery/pkg/runtime/serializer/cbor/direct from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+ + k8s.io/apimachinery/pkg/runtime/serializer/cbor/internal/modes from k8s.io/apimachinery/pkg/runtime/serializer/cbor+ k8s.io/apimachinery/pkg/runtime/serializer/json from k8s.io/apimachinery/pkg/runtime/serializer+ k8s.io/apimachinery/pkg/runtime/serializer/protobuf from k8s.io/apimachinery/pkg/runtime/serializer k8s.io/apimachinery/pkg/runtime/serializer/recognizer from k8s.io/apimachinery/pkg/runtime/serializer+ @@ -446,6 +452,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/applyconfigurations/certificates/v1alpha1 from k8s.io/client-go/kubernetes/typed/certificates/v1alpha1 k8s.io/client-go/applyconfigurations/certificates/v1beta1 from k8s.io/client-go/kubernetes/typed/certificates/v1beta1 k8s.io/client-go/applyconfigurations/coordination/v1 from k8s.io/client-go/kubernetes/typed/coordination/v1 + k8s.io/client-go/applyconfigurations/coordination/v1alpha2 from k8s.io/client-go/kubernetes/typed/coordination/v1alpha2 k8s.io/client-go/applyconfigurations/coordination/v1beta1 from k8s.io/client-go/kubernetes/typed/coordination/v1beta1 k8s.io/client-go/applyconfigurations/core/v1 from k8s.io/client-go/applyconfigurations/apps/v1+ k8s.io/client-go/applyconfigurations/discovery/v1 from k8s.io/client-go/kubernetes/typed/discovery/v1 @@ -470,7 +477,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/applyconfigurations/rbac/v1 from k8s.io/client-go/kubernetes/typed/rbac/v1 k8s.io/client-go/applyconfigurations/rbac/v1alpha1 from k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 k8s.io/client-go/applyconfigurations/rbac/v1beta1 from k8s.io/client-go/kubernetes/typed/rbac/v1beta1 - k8s.io/client-go/applyconfigurations/resource/v1alpha2 from k8s.io/client-go/kubernetes/typed/resource/v1alpha2 + k8s.io/client-go/applyconfigurations/resource/v1alpha3 from k8s.io/client-go/kubernetes/typed/resource/v1alpha3 + k8s.io/client-go/applyconfigurations/resource/v1beta1 from k8s.io/client-go/kubernetes/typed/resource/v1beta1 k8s.io/client-go/applyconfigurations/scheduling/v1 from k8s.io/client-go/kubernetes/typed/scheduling/v1 k8s.io/client-go/applyconfigurations/scheduling/v1alpha1 from k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 k8s.io/client-go/applyconfigurations/scheduling/v1beta1 from k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 @@ -480,8 +488,80 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1 from k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1 k8s.io/client-go/discovery from k8s.io/client-go/applyconfigurations/meta/v1+ k8s.io/client-go/dynamic from sigs.k8s.io/controller-runtime/pkg/cache/internal+ - k8s.io/client-go/features from k8s.io/client-go/tools/cache - k8s.io/client-go/kubernetes from k8s.io/client-go/tools/leaderelection/resourcelock + k8s.io/client-go/features from k8s.io/client-go/tools/cache+ + k8s.io/client-go/gentype from k8s.io/client-go/kubernetes/typed/admissionregistration/v1+ + k8s.io/client-go/informers from k8s.io/client-go/tools/leaderelection + k8s.io/client-go/informers/admissionregistration from k8s.io/client-go/informers + k8s.io/client-go/informers/admissionregistration/v1 from k8s.io/client-go/informers/admissionregistration + k8s.io/client-go/informers/admissionregistration/v1alpha1 from k8s.io/client-go/informers/admissionregistration + k8s.io/client-go/informers/admissionregistration/v1beta1 from k8s.io/client-go/informers/admissionregistration + k8s.io/client-go/informers/apiserverinternal from k8s.io/client-go/informers + k8s.io/client-go/informers/apiserverinternal/v1alpha1 from k8s.io/client-go/informers/apiserverinternal + k8s.io/client-go/informers/apps from k8s.io/client-go/informers + k8s.io/client-go/informers/apps/v1 from k8s.io/client-go/informers/apps + k8s.io/client-go/informers/apps/v1beta1 from k8s.io/client-go/informers/apps + k8s.io/client-go/informers/apps/v1beta2 from k8s.io/client-go/informers/apps + k8s.io/client-go/informers/autoscaling from k8s.io/client-go/informers + k8s.io/client-go/informers/autoscaling/v1 from k8s.io/client-go/informers/autoscaling + k8s.io/client-go/informers/autoscaling/v2 from k8s.io/client-go/informers/autoscaling + k8s.io/client-go/informers/autoscaling/v2beta1 from k8s.io/client-go/informers/autoscaling + k8s.io/client-go/informers/autoscaling/v2beta2 from k8s.io/client-go/informers/autoscaling + k8s.io/client-go/informers/batch from k8s.io/client-go/informers + k8s.io/client-go/informers/batch/v1 from k8s.io/client-go/informers/batch + k8s.io/client-go/informers/batch/v1beta1 from k8s.io/client-go/informers/batch + k8s.io/client-go/informers/certificates from k8s.io/client-go/informers + k8s.io/client-go/informers/certificates/v1 from k8s.io/client-go/informers/certificates + k8s.io/client-go/informers/certificates/v1alpha1 from k8s.io/client-go/informers/certificates + k8s.io/client-go/informers/certificates/v1beta1 from k8s.io/client-go/informers/certificates + k8s.io/client-go/informers/coordination from k8s.io/client-go/informers + k8s.io/client-go/informers/coordination/v1 from k8s.io/client-go/informers/coordination + k8s.io/client-go/informers/coordination/v1alpha2 from k8s.io/client-go/informers/coordination + k8s.io/client-go/informers/coordination/v1beta1 from k8s.io/client-go/informers/coordination + k8s.io/client-go/informers/core from k8s.io/client-go/informers + k8s.io/client-go/informers/core/v1 from k8s.io/client-go/informers/core + k8s.io/client-go/informers/discovery from k8s.io/client-go/informers + k8s.io/client-go/informers/discovery/v1 from k8s.io/client-go/informers/discovery + k8s.io/client-go/informers/discovery/v1beta1 from k8s.io/client-go/informers/discovery + k8s.io/client-go/informers/events from k8s.io/client-go/informers + k8s.io/client-go/informers/events/v1 from k8s.io/client-go/informers/events + k8s.io/client-go/informers/events/v1beta1 from k8s.io/client-go/informers/events + k8s.io/client-go/informers/extensions from k8s.io/client-go/informers + k8s.io/client-go/informers/extensions/v1beta1 from k8s.io/client-go/informers/extensions + k8s.io/client-go/informers/flowcontrol from k8s.io/client-go/informers + k8s.io/client-go/informers/flowcontrol/v1 from k8s.io/client-go/informers/flowcontrol + k8s.io/client-go/informers/flowcontrol/v1beta1 from k8s.io/client-go/informers/flowcontrol + k8s.io/client-go/informers/flowcontrol/v1beta2 from k8s.io/client-go/informers/flowcontrol + k8s.io/client-go/informers/flowcontrol/v1beta3 from k8s.io/client-go/informers/flowcontrol + k8s.io/client-go/informers/internalinterfaces from k8s.io/client-go/informers+ + k8s.io/client-go/informers/networking from k8s.io/client-go/informers + k8s.io/client-go/informers/networking/v1 from k8s.io/client-go/informers/networking + k8s.io/client-go/informers/networking/v1alpha1 from k8s.io/client-go/informers/networking + k8s.io/client-go/informers/networking/v1beta1 from k8s.io/client-go/informers/networking + k8s.io/client-go/informers/node from k8s.io/client-go/informers + k8s.io/client-go/informers/node/v1 from k8s.io/client-go/informers/node + k8s.io/client-go/informers/node/v1alpha1 from k8s.io/client-go/informers/node + k8s.io/client-go/informers/node/v1beta1 from k8s.io/client-go/informers/node + k8s.io/client-go/informers/policy from k8s.io/client-go/informers + k8s.io/client-go/informers/policy/v1 from k8s.io/client-go/informers/policy + k8s.io/client-go/informers/policy/v1beta1 from k8s.io/client-go/informers/policy + k8s.io/client-go/informers/rbac from k8s.io/client-go/informers + k8s.io/client-go/informers/rbac/v1 from k8s.io/client-go/informers/rbac + k8s.io/client-go/informers/rbac/v1alpha1 from k8s.io/client-go/informers/rbac + k8s.io/client-go/informers/rbac/v1beta1 from k8s.io/client-go/informers/rbac + k8s.io/client-go/informers/resource from k8s.io/client-go/informers + k8s.io/client-go/informers/resource/v1alpha3 from k8s.io/client-go/informers/resource + k8s.io/client-go/informers/resource/v1beta1 from k8s.io/client-go/informers/resource + k8s.io/client-go/informers/scheduling from k8s.io/client-go/informers + k8s.io/client-go/informers/scheduling/v1 from k8s.io/client-go/informers/scheduling + k8s.io/client-go/informers/scheduling/v1alpha1 from k8s.io/client-go/informers/scheduling + k8s.io/client-go/informers/scheduling/v1beta1 from k8s.io/client-go/informers/scheduling + k8s.io/client-go/informers/storage from k8s.io/client-go/informers + k8s.io/client-go/informers/storage/v1 from k8s.io/client-go/informers/storage + k8s.io/client-go/informers/storage/v1alpha1 from k8s.io/client-go/informers/storage + k8s.io/client-go/informers/storage/v1beta1 from k8s.io/client-go/informers/storage + k8s.io/client-go/informers/storagemigration from k8s.io/client-go/informers + k8s.io/client-go/informers/storagemigration/v1alpha1 from k8s.io/client-go/informers/storagemigration + k8s.io/client-go/kubernetes from k8s.io/client-go/tools/leaderelection/resourcelock+ k8s.io/client-go/kubernetes/scheme from k8s.io/client-go/discovery+ k8s.io/client-go/kubernetes/typed/admissionregistration/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes @@ -505,6 +585,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/kubernetes/typed/certificates/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/certificates/v1beta1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/coordination/v1 from k8s.io/client-go/kubernetes+ + k8s.io/client-go/kubernetes/typed/coordination/v1alpha2 from k8s.io/client-go/kubernetes+ k8s.io/client-go/kubernetes/typed/coordination/v1beta1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/core/v1 from k8s.io/client-go/kubernetes+ k8s.io/client-go/kubernetes/typed/discovery/v1 from k8s.io/client-go/kubernetes @@ -527,7 +608,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/kubernetes/typed/rbac/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/rbac/v1beta1 from k8s.io/client-go/kubernetes - k8s.io/client-go/kubernetes/typed/resource/v1alpha2 from k8s.io/client-go/kubernetes + k8s.io/client-go/kubernetes/typed/resource/v1alpha3 from k8s.io/client-go/kubernetes + k8s.io/client-go/kubernetes/typed/resource/v1beta1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/scheduling/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 from k8s.io/client-go/kubernetes @@ -535,6 +617,56 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/kubernetes/typed/storage/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/storage/v1beta1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1 from k8s.io/client-go/kubernetes + k8s.io/client-go/listers from k8s.io/client-go/listers/admissionregistration/v1+ + k8s.io/client-go/listers/admissionregistration/v1 from k8s.io/client-go/informers/admissionregistration/v1 + k8s.io/client-go/listers/admissionregistration/v1alpha1 from k8s.io/client-go/informers/admissionregistration/v1alpha1 + k8s.io/client-go/listers/admissionregistration/v1beta1 from k8s.io/client-go/informers/admissionregistration/v1beta1 + k8s.io/client-go/listers/apiserverinternal/v1alpha1 from k8s.io/client-go/informers/apiserverinternal/v1alpha1 + k8s.io/client-go/listers/apps/v1 from k8s.io/client-go/informers/apps/v1 + k8s.io/client-go/listers/apps/v1beta1 from k8s.io/client-go/informers/apps/v1beta1 + k8s.io/client-go/listers/apps/v1beta2 from k8s.io/client-go/informers/apps/v1beta2 + k8s.io/client-go/listers/autoscaling/v1 from k8s.io/client-go/informers/autoscaling/v1 + k8s.io/client-go/listers/autoscaling/v2 from k8s.io/client-go/informers/autoscaling/v2 + k8s.io/client-go/listers/autoscaling/v2beta1 from k8s.io/client-go/informers/autoscaling/v2beta1 + k8s.io/client-go/listers/autoscaling/v2beta2 from k8s.io/client-go/informers/autoscaling/v2beta2 + k8s.io/client-go/listers/batch/v1 from k8s.io/client-go/informers/batch/v1 + k8s.io/client-go/listers/batch/v1beta1 from k8s.io/client-go/informers/batch/v1beta1 + k8s.io/client-go/listers/certificates/v1 from k8s.io/client-go/informers/certificates/v1 + k8s.io/client-go/listers/certificates/v1alpha1 from k8s.io/client-go/informers/certificates/v1alpha1 + k8s.io/client-go/listers/certificates/v1beta1 from k8s.io/client-go/informers/certificates/v1beta1 + k8s.io/client-go/listers/coordination/v1 from k8s.io/client-go/informers/coordination/v1 + k8s.io/client-go/listers/coordination/v1alpha2 from k8s.io/client-go/informers/coordination/v1alpha2 + k8s.io/client-go/listers/coordination/v1beta1 from k8s.io/client-go/informers/coordination/v1beta1 + k8s.io/client-go/listers/core/v1 from k8s.io/client-go/informers/core/v1 + k8s.io/client-go/listers/discovery/v1 from k8s.io/client-go/informers/discovery/v1 + k8s.io/client-go/listers/discovery/v1beta1 from k8s.io/client-go/informers/discovery/v1beta1 + k8s.io/client-go/listers/events/v1 from k8s.io/client-go/informers/events/v1 + k8s.io/client-go/listers/events/v1beta1 from k8s.io/client-go/informers/events/v1beta1 + k8s.io/client-go/listers/extensions/v1beta1 from k8s.io/client-go/informers/extensions/v1beta1 + k8s.io/client-go/listers/flowcontrol/v1 from k8s.io/client-go/informers/flowcontrol/v1 + k8s.io/client-go/listers/flowcontrol/v1beta1 from k8s.io/client-go/informers/flowcontrol/v1beta1 + k8s.io/client-go/listers/flowcontrol/v1beta2 from k8s.io/client-go/informers/flowcontrol/v1beta2 + k8s.io/client-go/listers/flowcontrol/v1beta3 from k8s.io/client-go/informers/flowcontrol/v1beta3 + k8s.io/client-go/listers/networking/v1 from k8s.io/client-go/informers/networking/v1 + k8s.io/client-go/listers/networking/v1alpha1 from k8s.io/client-go/informers/networking/v1alpha1 + k8s.io/client-go/listers/networking/v1beta1 from k8s.io/client-go/informers/networking/v1beta1 + k8s.io/client-go/listers/node/v1 from k8s.io/client-go/informers/node/v1 + k8s.io/client-go/listers/node/v1alpha1 from k8s.io/client-go/informers/node/v1alpha1 + k8s.io/client-go/listers/node/v1beta1 from k8s.io/client-go/informers/node/v1beta1 + k8s.io/client-go/listers/policy/v1 from k8s.io/client-go/informers/policy/v1 + k8s.io/client-go/listers/policy/v1beta1 from k8s.io/client-go/informers/policy/v1beta1 + k8s.io/client-go/listers/rbac/v1 from k8s.io/client-go/informers/rbac/v1 + k8s.io/client-go/listers/rbac/v1alpha1 from k8s.io/client-go/informers/rbac/v1alpha1 + k8s.io/client-go/listers/rbac/v1beta1 from k8s.io/client-go/informers/rbac/v1beta1 + k8s.io/client-go/listers/resource/v1alpha3 from k8s.io/client-go/informers/resource/v1alpha3 + k8s.io/client-go/listers/resource/v1beta1 from k8s.io/client-go/informers/resource/v1beta1 + k8s.io/client-go/listers/scheduling/v1 from k8s.io/client-go/informers/scheduling/v1 + k8s.io/client-go/listers/scheduling/v1alpha1 from k8s.io/client-go/informers/scheduling/v1alpha1 + k8s.io/client-go/listers/scheduling/v1beta1 from k8s.io/client-go/informers/scheduling/v1beta1 + k8s.io/client-go/listers/storage/v1 from k8s.io/client-go/informers/storage/v1 + k8s.io/client-go/listers/storage/v1alpha1 from k8s.io/client-go/informers/storage/v1alpha1 + k8s.io/client-go/listers/storage/v1beta1 from k8s.io/client-go/informers/storage/v1beta1 + k8s.io/client-go/listers/storagemigration/v1alpha1 from k8s.io/client-go/informers/storagemigration/v1alpha1 k8s.io/client-go/metadata from sigs.k8s.io/controller-runtime/pkg/cache/internal+ k8s.io/client-go/openapi from k8s.io/client-go/discovery k8s.io/client-go/pkg/apis/clientauthentication from k8s.io/client-go/pkg/apis/clientauthentication/install+ @@ -546,6 +678,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/rest from k8s.io/client-go/discovery+ k8s.io/client-go/rest/watch from k8s.io/client-go/rest k8s.io/client-go/restmapper from sigs.k8s.io/controller-runtime/pkg/client/apiutil + k8s.io/client-go/testing from k8s.io/client-go/gentype k8s.io/client-go/tools/auth from k8s.io/client-go/tools/clientcmd k8s.io/client-go/tools/cache from sigs.k8s.io/controller-runtime/pkg/cache+ k8s.io/client-go/tools/cache/synctrack from k8s.io/client-go/tools/cache @@ -562,11 +695,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/tools/record/util from k8s.io/client-go/tools/record k8s.io/client-go/tools/reference from k8s.io/client-go/kubernetes/typed/core/v1+ k8s.io/client-go/transport from k8s.io/client-go/plugin/pkg/client/auth/exec+ + k8s.io/client-go/util/apply from k8s.io/client-go/dynamic+ k8s.io/client-go/util/cert from k8s.io/client-go/rest+ k8s.io/client-go/util/connrotation from k8s.io/client-go/plugin/pkg/client/auth/exec+ + k8s.io/client-go/util/consistencydetector from k8s.io/client-go/dynamic+ k8s.io/client-go/util/flowcontrol from k8s.io/client-go/kubernetes+ k8s.io/client-go/util/homedir from k8s.io/client-go/tools/clientcmd k8s.io/client-go/util/keyutil from k8s.io/client-go/util/cert + k8s.io/client-go/util/watchlist from k8s.io/client-go/dynamic+ k8s.io/client-go/util/workqueue from k8s.io/client-go/transport+ k8s.io/klog/v2 from k8s.io/apimachinery/pkg/api/meta+ k8s.io/klog/v2/internal/buffer from k8s.io/klog/v2 @@ -587,11 +723,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/utils/buffer from k8s.io/client-go/tools/cache k8s.io/utils/clock from k8s.io/apimachinery/pkg/util/cache+ k8s.io/utils/clock/testing from k8s.io/client-go/util/flowcontrol + k8s.io/utils/internal/third_party/forked/golang/golang-lru from k8s.io/utils/lru k8s.io/utils/internal/third_party/forked/golang/net from k8s.io/utils/net + k8s.io/utils/lru from k8s.io/client-go/tools/record k8s.io/utils/net from k8s.io/apimachinery/pkg/util/net+ k8s.io/utils/pointer from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+ k8s.io/utils/ptr from k8s.io/client-go/tools/cache+ - k8s.io/utils/strings/slices from k8s.io/apimachinery/pkg/labels k8s.io/utils/trace from k8s.io/client-go/tools/cache sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator sigs.k8s.io/controller-runtime/pkg/cache from sigs.k8s.io/controller-runtime/pkg/cluster+ @@ -624,12 +761,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ sigs.k8s.io/controller-runtime/pkg/metrics from sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics+ sigs.k8s.io/controller-runtime/pkg/metrics/server from sigs.k8s.io/controller-runtime/pkg/manager sigs.k8s.io/controller-runtime/pkg/predicate from sigs.k8s.io/controller-runtime/pkg/builder+ - sigs.k8s.io/controller-runtime/pkg/ratelimiter from sigs.k8s.io/controller-runtime/pkg/controller+ sigs.k8s.io/controller-runtime/pkg/reconcile from sigs.k8s.io/controller-runtime/pkg/builder+ sigs.k8s.io/controller-runtime/pkg/recorder from sigs.k8s.io/controller-runtime/pkg/leaderelection+ sigs.k8s.io/controller-runtime/pkg/source from sigs.k8s.io/controller-runtime/pkg/builder+ sigs.k8s.io/controller-runtime/pkg/webhook from sigs.k8s.io/controller-runtime/pkg/manager sigs.k8s.io/controller-runtime/pkg/webhook/admission from sigs.k8s.io/controller-runtime/pkg/builder+ + sigs.k8s.io/controller-runtime/pkg/webhook/admission/metrics from sigs.k8s.io/controller-runtime/pkg/webhook/admission sigs.k8s.io/controller-runtime/pkg/webhook/conversion from sigs.k8s.io/controller-runtime/pkg/builder sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics from sigs.k8s.io/controller-runtime/pkg/webhook+ sigs.k8s.io/json from k8s.io/apimachinery/pkg/runtime/serializer/json+ @@ -640,7 +777,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ sigs.k8s.io/structured-merge-diff/v4/typed from k8s.io/apimachinery/pkg/util/managedfields+ sigs.k8s.io/structured-merge-diff/v4/value from k8s.io/apimachinery/pkg/runtime+ sigs.k8s.io/yaml from k8s.io/apimachinery/pkg/runtime/serializer/json+ - sigs.k8s.io/yaml/goyaml.v2 from sigs.k8s.io/yaml + sigs.k8s.io/yaml/goyaml.v2 from sigs.k8s.io/yaml+ tailscale.com from tailscale.com/version tailscale.com/appc from tailscale.com/ipn/ipnlocal đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/ipn+ diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml index 4434c12835ba1..1917e31de36d1 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: connectors.tailscale.com spec: group: tailscale.com diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml index 13aee9b9e9ebf..242debd2719bd 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: dnsconfigs.tailscale.com spec: group: tailscale.com diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 2e53d5ee801fe..a620c3887ae43 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: proxyclasses.tailscale.com spec: group: tailscale.com @@ -428,7 +428,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -443,7 +443,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -600,7 +600,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -615,7 +615,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -773,7 +773,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -788,7 +788,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -945,7 +945,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -960,7 +960,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -1174,6 +1174,32 @@ spec: Note that this field cannot be set when spec.os.name is windows. type: integer format: int64 + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string seLinuxOptions: description: |- The SELinux context to be applied to all containers. @@ -1222,18 +1248,28 @@ spec: type: string supplementalGroups: description: |- - A list of groups applied to the first process run in each container, in addition - to the container's primary GID, the fsGroup (if specified), and group memberships - defined in the container image for the uid of the container process. If unspecified, - no additional groups are added to any container. Note that group memberships - defined in the container image for the uid of the container process are still effective, - even if they are not included in this list. + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. Note that this field cannot be set when spec.os.name is windows. type: array items: type: integer format: int64 x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string sysctls: description: |- Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported @@ -1389,6 +1425,12 @@ spec: the Pod where this field is used. It makes that resource available inside a container. type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map @@ -1493,7 +1535,7 @@ spec: procMount: description: |- procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for + The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. @@ -1713,6 +1755,12 @@ spec: the Pod where this field is used. It makes that resource available inside a container. type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map @@ -1817,7 +1865,7 @@ spec: procMount: description: |- procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for + The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index d6a4fe7415dd1..86e74e441bc52 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: proxygroups.tailscale.com spec: group: tailscale.com diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml index 5b22297d8b774..22bbed810cbbb 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: recorders.tailscale.com spec: group: tailscale.com @@ -372,7 +372,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -387,7 +387,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -544,7 +544,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -559,7 +559,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -717,7 +717,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -732,7 +732,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -889,7 +889,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -904,7 +904,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -1066,6 +1066,12 @@ spec: the Pod where this field is used. It makes that resource available inside a container. type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map @@ -1165,7 +1171,7 @@ spec: procMount: description: |- procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for + The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. @@ -1401,6 +1407,32 @@ spec: Note that this field cannot be set when spec.os.name is windows. type: integer format: int64 + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string seLinuxOptions: description: |- The SELinux context to be applied to all containers. @@ -1449,18 +1481,28 @@ spec: type: string supplementalGroups: description: |- - A list of groups applied to the first process run in each container, in addition - to the container's primary GID, the fsGroup (if specified), and group memberships - defined in the container image for the uid of the container process. If unspecified, - no additional groups are added to any container. Note that group memberships - defined in the container image for the uid of the container process are still effective, - even if they are not included in this list. + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. Note that this field cannot be set when spec.os.name is windows. type: array items: type: integer format: int64 x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string sysctls: description: |- Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 0026ffef57c26..def5716f6ce5f 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -31,7 +31,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: connectors.tailscale.com spec: group: tailscale.com @@ -294,7 +294,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: dnsconfigs.tailscale.com spec: group: tailscale.com @@ -476,7 +476,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: proxyclasses.tailscale.com spec: group: tailscale.com @@ -886,7 +886,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -901,7 +901,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1062,7 +1062,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1077,7 +1077,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1231,7 +1231,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1246,7 +1246,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1407,7 +1407,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1422,7 +1422,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1641,6 +1641,32 @@ spec: Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string seLinuxOptions: description: |- The SELinux context to be applied to all containers. @@ -1689,18 +1715,28 @@ spec: type: object supplementalGroups: description: |- - A list of groups applied to the first process run in each container, in addition - to the container's primary GID, the fsGroup (if specified), and group memberships - defined in the container image for the uid of the container process. If unspecified, - no additional groups are added to any container. Note that group memberships - defined in the container image for the uid of the container process are still effective, - even if they are not included in this list. + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. Note that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer type: array x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string sysctls: description: |- Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported @@ -1851,6 +1887,12 @@ spec: the Pod where this field is used. It makes that resource available inside a container. type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string required: - name type: object @@ -1959,7 +2001,7 @@ spec: procMount: description: |- procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for + The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. @@ -2175,6 +2217,12 @@ spec: the Pod where this field is used. It makes that resource available inside a container. type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string required: - name type: object @@ -2283,7 +2331,7 @@ spec: procMount: description: |- procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for + The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. @@ -2717,7 +2765,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: proxygroups.tailscale.com spec: group: tailscale.com @@ -2927,7 +2975,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab + controller-gen.kubebuilder.io/version: v0.17.0 name: recorders.tailscale.com spec: group: tailscale.com @@ -3281,7 +3329,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3296,7 +3344,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3457,7 +3505,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3472,7 +3520,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3626,7 +3674,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3641,7 +3689,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3802,7 +3850,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3817,7 +3865,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3979,6 +4027,12 @@ spec: the Pod where this field is used. It makes that resource available inside a container. type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string required: - name type: object @@ -4082,7 +4136,7 @@ spec: procMount: description: |- procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for + The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. @@ -4319,6 +4373,32 @@ spec: Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string seLinuxOptions: description: |- The SELinux context to be applied to all containers. @@ -4367,18 +4447,28 @@ spec: type: object supplementalGroups: description: |- - A list of groups applied to the first process run in each container, in addition - to the container's primary GID, the fsGroup (if specified), and group memberships - defined in the container image for the uid of the container process. If unspecified, - no additional groups are added to any container. Note that group memberships - defined in the container image for the uid of the container process are still effective, - even if they are not included in this list. + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. Note that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer type: array x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string sysctls: description: |- Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported diff --git a/cmd/k8s-operator/egress-eps_test.go b/cmd/k8s-operator/egress-eps_test.go index a64f3e4e1bb50..bd81071cb5e4f 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/cmd/k8s-operator/egress-eps_test.go @@ -112,7 +112,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { Terminating: pointer.ToBool(false), }, }) - expectEqual(t, fc, eps, nil) + expectEqual(t, fc, eps) }) t.Run("status_does_not_match_pod_ip", func(t *testing.T) { _, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1 @@ -122,7 +122,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { }) expectReconciled(t, er, "operator-ns", "foo") eps.Endpoints = []discoveryv1.Endpoint{} - expectEqual(t, fc, eps, nil) + expectEqual(t, fc, eps) }) } diff --git a/cmd/k8s-operator/egress-services-readiness_test.go b/cmd/k8s-operator/egress-services-readiness_test.go index 052eb1a493801..ce947329ddfb8 100644 --- a/cmd/k8s-operator/egress-services-readiness_test.go +++ b/cmd/k8s-operator/egress-services-readiness_test.go @@ -67,24 +67,24 @@ func TestEgressServiceReadiness(t *testing.T) { setClusterNotReady(egressSvc, cl, zl.Sugar()) t.Run("endpointslice_does_not_exist", func(t *testing.T) { expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc, nil) // not ready + expectEqual(t, fc, egressSvc) // not ready }) t.Run("proxy_group_does_not_exist", func(t *testing.T) { mustCreate(t, fc, eps) expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc, nil) // still not ready + expectEqual(t, fc, egressSvc) // still not ready }) t.Run("proxy_group_not_ready", func(t *testing.T) { mustCreate(t, fc, pg) expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc, nil) // still not ready + expectEqual(t, fc, egressSvc) // still not ready }) t.Run("no_ready_replicas", func(t *testing.T) { setPGReady(pg, cl, zl.Sugar()) mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { p.Status = pg.Status }) - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) for i := range pgReplicas(pg) { p := pod(pg, i) mustCreate(t, fc, p) @@ -94,7 +94,7 @@ func TestEgressServiceReadiness(t *testing.T) { } expectReconciled(t, rec, "dev", "my-app") setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg)) - expectEqual(t, fc, egressSvc, nil) // still not ready + expectEqual(t, fc, egressSvc) // still not ready }) t.Run("one_ready_replica", func(t *testing.T) { setEndpointForReplica(pg, 0, eps) @@ -103,7 +103,7 @@ func TestEgressServiceReadiness(t *testing.T) { }) setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1) expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc, nil) // partially ready + expectEqual(t, fc, egressSvc) // partially ready }) t.Run("all_replicas_ready", func(t *testing.T) { for i := range pgReplicas(pg) { @@ -114,7 +114,7 @@ func TestEgressServiceReadiness(t *testing.T) { }) setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg)) expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc, nil) // ready + expectEqual(t, fc, egressSvc) // ready }) } diff --git a/cmd/k8s-operator/egress-services_test.go b/cmd/k8s-operator/egress-services_test.go index 06fe977ecc130..ab0008ca0af25 100644 --- a/cmd/k8s-operator/egress-services_test.go +++ b/cmd/k8s-operator/egress-services_test.go @@ -96,7 +96,7 @@ func TestTailscaleEgressServices(t *testing.T) { expectReconciled(t, esr, "default", "test") // Service should have EgressSvcValid condition set to Unknown. svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)} - expectEqual(t, fc, svc, nil) + expectEqual(t, fc, svc) }) t.Run("proxy_group_ready", func(t *testing.T) { @@ -162,7 +162,7 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc) clusterSvc := mustGetClusterIPSvc(t, fc, name) // Verify that an EndpointSlice has been created. - expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil) + expectEqual(t, fc, endpointSlice(name, svc, clusterSvc)) // Verify that ConfigMap contains configuration for the new egress service. mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm) r := svcConfiguredReason(svc, true, zl.Sugar()) @@ -174,7 +174,7 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco } svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"} svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name) - expectEqual(t, fc, svc, nil) + expectEqual(t, fc, svc) } diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index 955258cc3b1ed..74eddff56e273 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -103,9 +103,9 @@ func TestTailscaleIngress(t *testing.T) { } opts.serveConfig = serveConfig - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 2. Ingress status gets updated with ingress proxy's MagicDNS name // once that becomes available. @@ -120,7 +120,7 @@ func TestTailscaleIngress(t *testing.T) { {Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}}, }, } - expectEqual(t, fc, ing, nil) + expectEqual(t, fc, ing) // 3. Resources get created for Ingress that should allow forwarding // cluster traffic @@ -129,7 +129,7 @@ func TestTailscaleIngress(t *testing.T) { }) opts.shouldEnableForwardingClusterTrafficViaIngress = true expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 4. Resources get cleaned up when Ingress class is unset mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { @@ -229,9 +229,9 @@ func TestTailscaleIngressHostname(t *testing.T) { } opts.serveConfig = serveConfig - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 2. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint set mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { @@ -243,7 +243,7 @@ func TestTailscaleIngressHostname(t *testing.T) { expectReconciled(t, ingR, "default", "test") ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer") - expectEqual(t, fc, ing, nil) + expectEqual(t, fc, ing) // 3. Ingress proxy with capability version >= 110 advertises HTTPS endpoint mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { @@ -259,7 +259,7 @@ func TestTailscaleIngressHostname(t *testing.T) { {Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}}, }, } - expectEqual(t, fc, ing, nil) + expectEqual(t, fc, ing) // 4. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint ready mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { @@ -271,7 +271,7 @@ func TestTailscaleIngressHostname(t *testing.T) { }) expectReconciled(t, ingR, "default", "test") ing.Status.LoadBalancer.Ingress = nil - expectEqual(t, fc, ing, nil) + expectEqual(t, fc, ing) // 5. Ingress proxy's state has https_endpoints set, but its capver is not matching Pod UID (downgrade) mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { @@ -287,7 +287,7 @@ func TestTailscaleIngressHostname(t *testing.T) { }, } expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, ing, nil) + expectEqual(t, fc, ing) } func TestTailscaleIngressWithProxyClass(t *testing.T) { @@ -383,9 +383,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { } opts.serveConfig = serveConfig - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet // ready, so proxy resource configuration does not change. @@ -393,7 +393,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata") }) expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get // reconciled and configuration from the ProxyClass is applied to the @@ -408,7 +408,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = pc.Name - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 4. tailscale.com/proxy-class label is removed from the Ingress, the // Ingress gets reconciled and the custom ProxyClass configuration is @@ -418,7 +418,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = "" - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) } func TestTailscaleIngressWithServiceMonitor(t *testing.T) { @@ -526,20 +526,20 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqual(t, fc, expectedMetricsService(opts)) // 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) { pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true, Labels: tsapi.Labels{"foo": "bar"}} }) expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqual(t, fc, expectedMetricsService(opts)) // 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created mustCreate(t, fc, crd) expectReconciled(t, ingR, "default", "test") opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"} - expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqual(t, fc, expectedMetricsService(opts)) expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) // 4. Update ServiceMonitor CRD and reconcile- ServiceMonitor should get updated @@ -549,7 +549,7 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { expectReconciled(t, ingR, "default", "test") opts.serviceMonitorLabels = nil opts.resourceVersion = "2" - expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqual(t, fc, expectedMetricsService(opts)) expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) // 5. Disable metrics - metrics resources should get deleted. diff --git a/cmd/k8s-operator/nameserver_test.go b/cmd/k8s-operator/nameserver_test.go index 695710212e57b..cec95b84ee719 100644 --- a/cmd/k8s-operator/nameserver_test.go +++ b/cmd/k8s-operator/nameserver_test.go @@ -69,7 +69,7 @@ func TestNameserverReconciler(t *testing.T) { wantsDeploy.Namespace = "tailscale" labels := nameserverResourceLabels("test", "tailscale") wantsDeploy.ObjectMeta.Labels = labels - expectEqual(t, fc, wantsDeploy, nil) + expectEqual(t, fc, wantsDeploy) // Verify that DNSConfig advertizes the nameserver's Service IP address, // has the ready status condition and tailscale finalizer. @@ -88,7 +88,7 @@ func TestNameserverReconciler(t *testing.T) { Message: reasonNameserverCreated, LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, }) - expectEqual(t, fc, dnsCfg, nil) + expectEqual(t, fc, dnsCfg) // // Verify that nameserver image gets updated to match DNSConfig spec. mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { @@ -96,7 +96,7 @@ func TestNameserverReconciler(t *testing.T) { }) expectReconciled(t, nr, "", "test") wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" - expectEqual(t, fc, wantsDeploy, nil) + expectEqual(t, fc, wantsDeploy) // Verify that when another actor sets ConfigMap data, it does not get // overwritten by nameserver reconciler. @@ -114,7 +114,7 @@ func TestNameserverReconciler(t *testing.T) { TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, Data: map[string]string{"records.json": string(bs)}, } - expectEqual(t, fc, wantCm, nil) + expectEqual(t, fc, wantCm) // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, // the nameserver image defaults to tailscale/k8s-nameserver:unstable. @@ -123,5 +123,5 @@ func TestNameserverReconciler(t *testing.T) { }) expectReconciled(t, nr, "", "test") wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable" - expectEqual(t, fc, wantsDeploy, nil) + expectEqual(t, fc, wantsDeploy) } diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 1998fe3bcc36d..2fa14e33b8ecb 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -106,7 +106,7 @@ func TestLoadBalancerClass(t *testing.T) { }}, }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) // Delete the misconfiguration so the proxy starts getting created on the // next reconcile. @@ -128,9 +128,9 @@ func TestLoadBalancerClass(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) want.Annotations = nil want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"} @@ -143,7 +143,7 @@ func TestLoadBalancerClass(t *testing.T) { Message: "no Tailscale hostname known yet, waiting for proxy pod to finish auth", }}, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -169,7 +169,7 @@ func TestLoadBalancerClass(t *testing.T) { }, }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) // Turn the service back into a ClusterIP service, which should make the // operator clean up. @@ -206,7 +206,7 @@ func TestLoadBalancerClass(t *testing.T) { Type: corev1.ServiceTypeClusterIP, }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) } func TestTailnetTargetFQDNAnnotation(t *testing.T) { @@ -266,9 +266,9 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { app: kubetypes.AppEgressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -288,10 +288,10 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, want) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) // Change the tailscale-target-fqdn annotation which should update the // StatefulSet @@ -378,9 +378,9 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { app: kubetypes.AppEgressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -400,10 +400,10 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, want) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) // Change the tailscale-target-ip annotation which should update the // StatefulSet @@ -501,7 +501,7 @@ func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) { }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) } func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) { @@ -572,7 +572,7 @@ func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) { }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) } func TestAnnotations(t *testing.T) { @@ -629,9 +629,9 @@ func TestAnnotations(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -650,7 +650,7 @@ func TestAnnotations(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) // Turn the service back into a ClusterIP service, which should make the // operator clean up. @@ -678,7 +678,7 @@ func TestAnnotations(t *testing.T) { Type: corev1.ServiceTypeClusterIP, }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) } func TestAnnotationIntoLB(t *testing.T) { @@ -735,9 +735,9 @@ func TestAnnotationIntoLB(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, since it would have normally happened at @@ -769,7 +769,7 @@ func TestAnnotationIntoLB(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) // Remove Tailscale's annotation, and at the same time convert the service // into a tailscale LoadBalancer. @@ -780,8 +780,8 @@ func TestAnnotationIntoLB(t *testing.T) { }) expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) // ... but the service should have a LoadBalancer status. want = &corev1.Service{ @@ -810,7 +810,7 @@ func TestAnnotationIntoLB(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) } func TestLBIntoAnnotation(t *testing.T) { @@ -865,9 +865,9 @@ func TestLBIntoAnnotation(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -907,7 +907,7 @@ func TestLBIntoAnnotation(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) // Turn the service back into a ClusterIP service, but also add the // tailscale annotation. @@ -926,8 +926,8 @@ func TestLBIntoAnnotation(t *testing.T) { }) expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) want = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -947,7 +947,7 @@ func TestLBIntoAnnotation(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) } func TestCustomHostname(t *testing.T) { @@ -1005,9 +1005,9 @@ func TestCustomHostname(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, o), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, o)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -1027,7 +1027,7 @@ func TestCustomHostname(t *testing.T) { Conditions: proxyCreatedCondition(clock), }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) // Turn the service back into a ClusterIP service, which should make the // operator clean up. @@ -1058,7 +1058,7 @@ func TestCustomHostname(t *testing.T) { Type: corev1.ServiceTypeClusterIP, }, } - expectEqual(t, fc, want, nil) + expectEqual(t, fc, want) } func TestCustomPriorityClassName(t *testing.T) { @@ -1118,7 +1118,7 @@ func TestCustomPriorityClassName(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) } func TestProxyClassForService(t *testing.T) { @@ -1186,9 +1186,9 @@ func TestProxyClassForService(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 2. The Service gets updated with tailscale.com/proxy-class label // pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not @@ -1197,8 +1197,8 @@ func TestProxyClassForService(t *testing.T) { mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata") }) expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSecret(t, fc, opts)) // 3. ProxyClass is set to Ready, the Service gets reconciled by the // services-reconciler and the customization from the ProxyClass is @@ -1213,7 +1213,7 @@ func TestProxyClassForService(t *testing.T) { }) opts.proxyClass = pc.Name expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t)) // 4. tailscale.com/proxy-class label is removed from the Service, the @@ -1224,7 +1224,7 @@ func TestProxyClassForService(t *testing.T) { }) opts.proxyClass = "" expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) } func TestDefaultLoadBalancer(t *testing.T) { @@ -1270,7 +1270,7 @@ func TestDefaultLoadBalancer(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) o := configOpts{ stsName: shortName, secretName: fullName, @@ -1280,8 +1280,7 @@ func TestDefaultLoadBalancer(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) - + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) } func TestProxyFirewallMode(t *testing.T) { @@ -1337,7 +1336,7 @@ func TestProxyFirewallMode(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) } func TestTailscaledConfigfileHash(t *testing.T) { @@ -1393,7 +1392,7 @@ func TestTailscaledConfigfileHash(t *testing.T) { confFileHash: "848bff4b5ba83ac999e6984c8464e597156daba961ae045e7dbaef606d54ab5e", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), nil) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // 2. Hostname gets changed, configfile is updated and a new hash value // is produced. @@ -1403,7 +1402,7 @@ func TestTailscaledConfigfileHash(t *testing.T) { o.hostname = "another-test" o.confFileHash = "d4cc13f09f55f4f6775689004f9a466723325b84d2b590692796bfe22aeaa389" expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, o), nil) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func Test_isMagicDNSName(t *testing.T) { tests := []struct { @@ -1681,9 +1680,9 @@ func Test_authKeyRemoval(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 2. Apply update to the Secret that imitates the proxy setting device_id. s := expectedSecret(t, fc, opts) @@ -1695,7 +1694,7 @@ func Test_authKeyRemoval(t *testing.T) { expectReconciled(t, sr, "default", "test") opts.shouldRemoveAuthKey = true opts.secretExtraData = map[string][]byte{"device_id": []byte("dkkdi4CNTRL")} - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts)) } func Test_externalNameService(t *testing.T) { @@ -1755,9 +1754,9 @@ func Test_externalNameService(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSecret(t, fc, opts), nil) - expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) // 2. Change the ExternalName and verify that changes get propagated. mustUpdate(t, sr, "default", "test", func(s *corev1.Service) { @@ -1765,7 +1764,7 @@ func Test_externalNameService(t *testing.T) { }) expectReconciled(t, sr, "default", "test") opts.clusterTargetDNS = "bar.com" - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) } func Test_metricsResourceCreation(t *testing.T) { @@ -1835,7 +1834,7 @@ func Test_metricsResourceCreation(t *testing.T) { }) expectReconciled(t, sr, "default", "test") opts.enableMetrics = true - expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqual(t, fc, expectedMetricsService(opts)) // 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) { @@ -1855,7 +1854,7 @@ func Test_metricsResourceCreation(t *testing.T) { expectReconciled(t, sr, "default", "test") opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"} opts.resourceVersion = "2" - expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqual(t, fc, expectedMetricsService(opts)) expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) // 5. Disable metrics- expect metrics Service to be deleted diff --git a/cmd/k8s-operator/proxyclass_test.go b/cmd/k8s-operator/proxyclass_test.go index 78828107a09e0..48290eea782b5 100644 --- a/cmd/k8s-operator/proxyclass_test.go +++ b/cmd/k8s-operator/proxyclass_test.go @@ -78,7 +78,7 @@ func TestProxyClass(t *testing.T) { LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, }) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) // 2. A ProxyClass resource with invalid labels gets its status updated to Invalid with an error message. pc.Spec.StatefulSet.Labels["foo"] = "?!someVal" @@ -88,7 +88,7 @@ func TestProxyClass(t *testing.T) { expectReconciled(t, pcr, "", "test") msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')` tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')" expectEvents(t, fr, []string{expectedEvent}) @@ -102,7 +102,7 @@ func TestProxyClass(t *testing.T) { expectReconciled(t, pcr, "", "test") msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase` tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase` expectEvents(t, fr, []string{expectedEvent}) @@ -121,7 +121,7 @@ func TestProxyClass(t *testing.T) { expectReconciled(t, pcr, "", "test") msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase` tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase` expectEvents(t, fr, []string{expectedEvent}) @@ -145,7 +145,7 @@ func TestProxyClass(t *testing.T) { expectReconciled(t, pcr, "", "test") msg = `ProxyClass is not valid: spec.metrics.serviceMonitor: Invalid value: "enable": ProxyClass defines that a ServiceMonitor custom resource should be created, but "servicemonitors.monitoring.coreos.com" CRD was not found` tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) expectedEvent = "Warning ProxyClassInvalid " + msg expectEvents(t, fr, []string{expectedEvent}) @@ -154,7 +154,7 @@ func TestProxyClass(t *testing.T) { mustCreate(t, fc, crd) expectReconciled(t, pcr, "", "test") tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar()) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) // 7. A ProxyClass with invalid ServiceMonitor labels gets its status updated to Invalid with an error message. pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar!"} @@ -164,7 +164,7 @@ func TestProxyClass(t *testing.T) { expectReconciled(t, pcr, "", "test") msg = `ProxyClass is not valid: .spec.metrics.serviceMonitor.labels: Invalid value: "bar!": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')` tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) // 8. A ProxyClass with valid ServiceMonitor labels gets its status updated to Valid. pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar", "xyz1234": "abc567", "empty": "", "onechar": "a"} @@ -173,7 +173,7 @@ func TestProxyClass(t *testing.T) { }) expectReconciled(t, pcr, "", "test") tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar()) - expectEqual(t, fc, pc, nil) + expectEqual(t, fc, pc) } func TestValidateProxyClass(t *testing.T) { diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 96ffefbed1d7d..c920c90d1b69b 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -96,7 +96,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, false, "") }) @@ -117,7 +117,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, "") if expected := 1; reconciler.egressProxyGroups.Len() != expected { t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) @@ -153,7 +153,7 @@ func TestProxyGroup(t *testing.T) { }, } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, initialCfgHash) }) @@ -164,7 +164,7 @@ func TestProxyGroup(t *testing.T) { }) expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, initialCfgHash) addNodeIDToStateSecrets(t, fc, pg) @@ -174,7 +174,7 @@ func TestProxyGroup(t *testing.T) { Hostname: "hostname-nodeid-2", TailnetIPs: []string{"1.2.3.4", "::1"}, }) - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, initialCfgHash) }) @@ -187,7 +187,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, initialCfgHash) }) @@ -201,7 +201,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) - expectEqual(t, fc, pg, nil) + expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74") }) @@ -211,7 +211,7 @@ func TestProxyGroup(t *testing.T) { p.Spec = pc.Spec }) expectReconciled(t, reconciler, "", pg.Name) - expectEqual(t, fc, expectedMetricsService(opts), nil) + expectEqual(t, fc, expectedMetricsService(opts)) }) t.Run("enable_service_monitor_no_crd", func(t *testing.T) { pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true} @@ -389,10 +389,10 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox } if shouldExist { - expectEqual(t, fc, role, nil) - expectEqual(t, fc, roleBinding, nil) - expectEqual(t, fc, serviceAccount, nil) - expectEqual(t, fc, statefulSet, nil) + expectEqual(t, fc, role) + expectEqual(t, fc, roleBinding) + expectEqual(t, fc, serviceAccount) + expectEqual(t, fc, statefulSet, removeResourceReqs) } else { expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name) expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name) diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 277bd16dfbc47..240a7df15ef23 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -618,7 +618,7 @@ func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client, // modify func to ensure that they are removed from the cluster object and the // object passed as 'want'. If no such modifications are needed, you can pass // nil in place of the modify function. -func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifier func(O)) { +func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifiers ...func(O)) { t.Helper() got := O(new(T)) if err := client.Get(context.Background(), types.NamespacedName{ @@ -632,7 +632,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want // so just remove it from both got and want. got.SetResourceVersion("") want.SetResourceVersion("") - if modifier != nil { + for _, modifier := range modifiers { modifier(want) modifier(got) } @@ -799,6 +799,12 @@ func removeHashAnnotation(sts *appsv1.StatefulSet) { } } +func removeResourceReqs(sts *appsv1.StatefulSet) { + if sts != nil { + sts.Spec.Template.Spec.Resources = nil + } +} + func removeTargetPortsFromSvc(svc *corev1.Service) { newPorts := make([]corev1.ServicePort, 0) for _, p := range svc.Spec.Ports { diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go index bd73e8fb9ec26..4de1089a9fed0 100644 --- a/cmd/k8s-operator/tsrecorder_test.go +++ b/cmd/k8s-operator/tsrecorder_test.go @@ -57,7 +57,7 @@ func TestRecorder(t *testing.T) { msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible" tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr, nil) + expectEqual(t, fc, tsr) if expected := 0; reconciler.recorders.Len() != expected { t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) } @@ -76,7 +76,7 @@ func TestRecorder(t *testing.T) { expectReconciled(t, reconciler, "", tsr.Name) tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr, nil) + expectEqual(t, fc, tsr) if expected := 1; reconciler.recorders.Len() != expected { t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) } @@ -112,7 +112,7 @@ func TestRecorder(t *testing.T) { URL: "https://test-0.example.ts.net", }, } - expectEqual(t, fc, tsr, nil) + expectEqual(t, fc, tsr) }) t.Run("delete the Recorder and observe cleanup", func(t *testing.T) { @@ -145,12 +145,12 @@ func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recor statefulSet := tsrStatefulSet(tsr, tsNamespace) if shouldExist { - expectEqual(t, fc, auth, nil) - expectEqual(t, fc, state, nil) - expectEqual(t, fc, role, nil) - expectEqual(t, fc, roleBinding, nil) - expectEqual(t, fc, serviceAccount, nil) - expectEqual(t, fc, statefulSet, nil) + expectEqual(t, fc, auth) + expectEqual(t, fc, state) + expectEqual(t, fc, role) + expectEqual(t, fc, roleBinding) + expectEqual(t, fc, serviceAccount) + expectEqual(t, fc, statefulSet, removeResourceReqs) } else { expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name) expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name) diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index 9599f6a01b1e4..52d649a1d6c39 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -8,11 +8,11 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/munnerz/goautoneg from github.com/prometheus/common/expfmt đŸ’Ŗ github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb/promvarz github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ - github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+ LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs @@ -155,7 +155,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar html from net/http/pprof+ io from bufio+ io/fs from crypto/x509+ - io/ioutil from google.golang.org/protobuf/internal/impl iter from maps+ log from expvar+ log/internal from log diff --git a/go.mod b/go.mod index 79374eb9c198a..92dd1bf6501ed 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/evanw/esbuild v0.19.11 github.com/fogleman/gg v1.3.0 github.com/frankban/quicktest v1.14.6 - github.com/fxamacker/cbor/v2 v2.6.0 + github.com/fxamacker/cbor/v2 v2.7.0 github.com/gaissmai/bart v0.11.1 github.com/go-json-experiment/json v0.0.0-20250103232110-6a9a0fde9288 github.com/go-logr/zapr v1.3.0 @@ -68,7 +68,7 @@ require ( github.com/pkg/sftp v1.13.6 github.com/prometheus-community/pro-bing v0.4.0 github.com/prometheus/client_golang v1.19.1 - github.com/prometheus/common v0.48.0 + github.com/prometheus/common v0.55.0 github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff github.com/safchain/ethtool v0.3.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e @@ -79,7 +79,7 @@ require ( github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a - github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10 + github.com/tailscale/mkctr v0.0.0-20250110151924-54977352e4a6 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 @@ -109,12 +109,12 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 honnef.co/go/tools v0.5.1 - k8s.io/api v0.30.3 - k8s.io/apimachinery v0.30.3 - k8s.io/apiserver v0.30.3 - k8s.io/client-go v0.30.3 - sigs.k8s.io/controller-runtime v0.18.4 - sigs.k8s.io/controller-tools v0.15.1-0.20240618033008-7824932b0cab + k8s.io/api v0.32.0 + k8s.io/apimachinery v0.32.0 + k8s.io/apiserver v0.32.0 + k8s.io/client-go v0.32.0 + sigs.k8s.io/controller-runtime v0.19.4 + sigs.k8s.io/controller-tools v0.17.0 sigs.k8s.io/yaml v1.4.0 software.sslmate.com/src/go-pkcs12 v0.4.0 ) @@ -143,12 +143,11 @@ require ( github.com/ghostiam/protogetter v0.3.5 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect - github.com/gobuffalo/flect v1.0.2 // indirect + github.com/gobuffalo/flect v1.0.3 // indirect github.com/goccy/go-yaml v1.12.0 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golangci/plugin-module-register v0.1.1 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect - github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/jjti/go-spancheck v0.5.3 // indirect github.com/karamaru-alpha/copyloopvar v1.0.8 // indirect @@ -159,12 +158,14 @@ require ( github.com/ykadowak/zerologlint v0.1.5 // indirect go-simpler.org/musttag v0.9.0 // indirect go-simpler.org/sloglint v0.5.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect - go.opentelemetry.io/otel v1.32.0 // indirect - go.opentelemetry.io/otel/metric v1.32.0 // indirect - go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect ) require ( @@ -210,25 +211,24 @@ require ( github.com/breml/errchkjson v0.3.6 // indirect github.com/butuzov/ireturn v0.3.0 // indirect github.com/cavaliergopher/cpio v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/curioswitch/go-reassign v0.2.0 // indirect github.com/daixiang0/gci v0.12.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect - github.com/docker/cli v27.3.1+incompatible // indirect + github.com/docker/cli v27.4.1+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v27.3.1+incompatible // indirect + github.com/docker/docker v27.4.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/emicklei/go-restful/v3 v3.11.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/ettle/strcase v0.2.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect - github.com/fatih/color v1.17.0 // indirect + github.com/fatih/color v1.18.0 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/firefart/nonamedreturns v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 @@ -238,9 +238,9 @@ require ( github.com/go-git/go-billy/v5 v5.6.1 // indirect github.com/go-git/go-git/v5 v5.13.1 // indirect github.com/go-logr/logr v1.4.2 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/swag v0.22.7 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.1.0 // indirect github.com/go-toolsmith/astequal v1.2.0 // indirect @@ -328,14 +328,14 @@ require ( github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.4.8 // indirect - github.com/prometheus/client_model v0.5.0 - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_model v0.6.1 + github.com/prometheus/procfs v0.15.1 // indirect github.com/quasilyte/go-ruleguard v0.4.2 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/ryancurrah/gomodguard v1.3.1 // indirect github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect @@ -387,19 +387,19 @@ require ( golang.org/x/image v0.23.0 // indirect golang.org/x/text v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.0 // indirect - k8s.io/apiextensions-apiserver v0.30.3 + k8s.io/apiextensions-apiserver v0.32.0 k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 mvdan.cc/gofumpt v0.6.0 // indirect mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect ) diff --git a/go.sum b/go.sum index 28315ad1eff46..0354c33648139 100644 --- a/go.sum +++ b/go.sum @@ -211,13 +211,13 @@ github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI github.com/ccojocar/zxcvbn-go v1.0.2 h1:na/czXU8RrhXO4EZme6eQJLR4PzcGsahsBOAwU6I3Vg= github.com/ccojocar/zxcvbn-go v1.0.2/go.mod h1:g1qkXtUSvHP8lhHp5GrSmTz6uWALGRMQdw6Qnz/hi60= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4= github.com/charithe/durationcheck v0.0.10/go.mod h1:bCWXb7gYRysD1CU3C+u4ceO49LoGOY1C1L6uouGNreQ= github.com/chavacava/garif v0.1.0 h1:2JHa3hbYf5D9dsgseMKAmc/MZ109otzgNFk5s87H9Pc= @@ -237,8 +237,8 @@ github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NA github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= -github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6 h1:8h5+bWd7R6AYUslN6c6iuZWTKsKxUFDlpnmilO6R2n0= github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= @@ -275,12 +275,12 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/cli v27.3.1+incompatible h1:qEGdFBF3Xu6SCvCYhc7CzaQTlBmqDuzxPDpigSyeKQQ= -github.com/docker/cli v27.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= +github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= -github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.4.1+incompatible h1:ZJvcY7gfwHn1JF48PfbyXg7Jyt9ZCWDW+GGXOIxEwp4= +github.com/docker/docker v27.4.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -309,8 +309,8 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/evanw/esbuild v0.19.11 h1:mbPO1VJ/df//jjUd+p/nRLYCpizXxXb2w/zZMShxa2k= github.com/evanw/esbuild v0.19.11/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -323,8 +323,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= -github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/gaissmai/bart v0.11.1 h1:5Uv5XwsaFBRo4E5VBcb9TzY8B7zxFf+U7isDxqOrRfc= @@ -365,12 +365,12 @@ github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= -github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= @@ -381,7 +381,8 @@ github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= @@ -405,8 +406,8 @@ github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsM github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U= github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= -github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= -github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= +github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4= +github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM= @@ -508,8 +509,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/rpmpack v0.5.0 h1:L16KZ3QvkFGpYhmp23iQip+mx1X39foEsqszjMNBm8A= github.com/google/rpmpack v0.5.0/go.mod h1:uqVAUVQLq8UY2hCDfmJ/+rtO3aw7qyhc90rCVEabEfI= @@ -545,8 +546,8 @@ github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoISdUv3PPQgHY= github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -738,10 +739,10 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= -github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -790,21 +791,21 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= -github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff h1:X1Tly81aZ22DA1fxBdfvR3iw8+yFoUBUHMEd+AX/ZXI= github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff/go.mod h1:FvE8dtQ1Ww63IlyKBn1V4s+zMwF9kHkVNkQBR1pM4CU= github.com/quasilyte/go-ruleguard v0.4.2 h1:htXcXDK6/rO12kiTHKfHuqR4kr3Y4M0J0rOL6CH/BYs= @@ -820,8 +821,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.1 h1:fH+fUg+ngsQO0ruZXXHnA/2aNllWA1whly4a6UvyzGE= github.com/ryancurrah/gomodguard v1.3.1/go.mod h1:DGFHzEhi6iJ0oIDfMuo3TgrS+L9gZvrEfmjjuelnRU0= @@ -926,8 +927,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPx github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8= github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw= github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8= -github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10 h1:ZB47BgnHcEHQJODkDubs5ZiNeJxMhcgzefV3lykRwVQ= -github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10/go.mod h1:iDx/0Rr9VV/KanSUDpJ6I/ROf0sQ7OqljXc/esl0UIA= +github.com/tailscale/mkctr v0.0.0-20250110151924-54977352e4a6 h1:9SuADtKJAGQkIpnpg5znEJ86QaxacN25pHkiEXTDjzg= +github.com/tailscale/mkctr v0.0.0-20250110151924-54977352e4a6/go.mod h1:qTslktI+Qh9hXo7ZP8xLkl5V8AxUMfxG0xLtkCFLxnw= github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 h1:uFsXVBE9Qr4ZoF094vE6iYTLDl0qCiKzYXlL6UeWObU= github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0= github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc h1:24heQPtnFR+yfntqhI3oAu9i27nEojcQ4NuBQOo5ZFA= @@ -1017,22 +1018,24 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -1384,11 +1387,11 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 h1:nz5NESFLZbJGPFxDT/HCn+V1mZ8JGNoY4nUpmW/Y2eg= -google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac h1:OZkkudMUu9LVQMCoRUbI/1p5VCo9BOrlvkqMvWtqa6s= -google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1401,8 +1404,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= -google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1415,8 +1418,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1424,6 +1427,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= @@ -1461,22 +1466,22 @@ honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I= honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= -k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= -k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= -k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= -k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= -k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/apiserver v0.30.3 h1:QZJndA9k2MjFqpnyYv/PH+9PE0SHhx3hBho4X0vE65g= -k8s.io/apiserver v0.30.3/go.mod h1:6Oa88y1CZqnzetd2JdepO0UXzQX4ZnOekx2/PtEjrOg= -k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= -k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= +k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= +k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= +k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= +k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= +k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= +k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apiserver v0.32.0 h1:VJ89ZvQZ8p1sLeiWdRJpRD6oLozNZD2+qVSLi+ft5Qs= +k8s.io/apiserver v0.32.0/go.mod h1:HFh+dM1/BE/Hm4bS4nTXHVfN6Z6tFIZPi649n83b4Ag= +k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= +k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo= mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA= mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14 h1:zCr3iRRgdk5eIikZNDphGcM6KGVTx3Yu+/Uu9Es254w= @@ -1484,14 +1489,14 @@ mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14/go.mod h1:ZzZjEpJDOmx8TdVU6u rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= -sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= -sigs.k8s.io/controller-tools v0.15.1-0.20240618033008-7824932b0cab h1:Fq4VD28nejtsijBNTeRRy9Tt3FVwq+o6NB7fIxja8uY= -sigs.k8s.io/controller-tools v0.15.1-0.20240618033008-7824932b0cab/go.mod h1:egedX5jq2KrZ3A2zaOz3e2DSsh5BhFyyjvNcBRIQel8= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/controller-runtime v0.19.4 h1:SUmheabttt0nx8uJtoII4oIP27BVVvAKFvdvGFwV/Qo= +sigs.k8s.io/controller-runtime v0.19.4/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/controller-tools v0.17.0 h1:KaEQZbhrdY6J3zLBHplt+0aKUp8PeIttlhtF2UDo6bI= +sigs.k8s.io/controller-tools v0.17.0/go.mod h1:SKoWY8rwGWDzHtfnhmOwljn6fViG0JF7/xmnxpklgjo= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= From c79b736a856b63adc76610a82e7080fc3a468f29 Mon Sep 17 00:00:00 2001 From: Andrea Gottardo Date: Fri, 17 Jan 2025 14:52:47 -0800 Subject: [PATCH 140/223] ipnlocal: allow overriding os.Hostname() via syspolicy (#14676) Updates tailscale/corp#25936 This defines a new syspolicy 'Hostname' and allows an IT administrator to override the value we normally read from os.Hostname(). This is particularly useful on Android and iOS devices, where the hostname we get from the OS is really just the device model (a platform restriction to prevent fingerprinting). If we don't implement this, all devices on the customer's side will look like `google-pixel-7a-1`, `google-pixel-7a-2`, `google-pixel-7a-3`, etc. and it is not feasible for the customer to use the API or worse the admin console to manually fix these names. Apply code review comment by @nickkhyl Signed-off-by: Andrea Gottardo Co-authored-by: Nick Khyl <1761190+nickkhyl@users.noreply.github.com> --- ipn/ipnlocal/local.go | 31 +++++++++++++++++++++++++++++++ util/syspolicy/policy_keys.go | 6 ++++++ 2 files changed, 37 insertions(+) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 92d2f123fe5a9..c59df833d19ec 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1703,6 +1703,37 @@ func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID anyChange = true } + const sentinel = "HostnameDefaultValue" + hostnameFromPolicy, _ := syspolicy.GetString(syspolicy.Hostname, sentinel) + switch hostnameFromPolicy { + case sentinel: + // An empty string for this policy value means that the admin wants to delete + // the hostname stored in the ipn.Prefs. To make that work, we need to + // distinguish between an empty string and a policy that was not set. + // We cannot do that with the current implementation of syspolicy.GetString. + // It currently does not return an error if a policy was not configured. + // Instead, it returns the default value provided as the second argument. + // This behavior makes it impossible to distinguish between a policy that + // was not set and a policy that was set to an empty default value. + // Checking for sentinel here is a workaround to distinguish between + // the two cases. If we get it, we do nothing because the policy was not set. + // + // TODO(angott,nickkhyl): clean up this behavior once syspolicy.GetString starts + // properly returning errors. + case "": + // The policy was set to an empty string, which means the admin intends + // to clear the hostname stored in preferences. + prefs.Hostname = "" + anyChange = true + default: + // The policy was set to a non-empty string, which means the admin wants + // to override the hostname stored in preferences. + if prefs.Hostname != hostnameFromPolicy { + prefs.Hostname = hostnameFromPolicy + anyChange = true + } + } + if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" { exitNodeID := tailcfg.StableNodeID(exitNodeIDStr) if shouldAutoExitNode() && lastSuggestedExitNode != "" { diff --git a/util/syspolicy/policy_keys.go b/util/syspolicy/policy_keys.go index bb9a5d6cc5934..35a36130e38d2 100644 --- a/util/syspolicy/policy_keys.go +++ b/util/syspolicy/policy_keys.go @@ -123,6 +123,11 @@ const ( // Example: "CN=Tailscale Inc Test Root CA,OU=Tailscale Inc Test Certificate Authority,O=Tailscale Inc,ST=ON,C=CA" MachineCertificateSubject Key = "MachineCertificateSubject" + // Hostname is the hostname of the device that is running Tailscale. + // When this policy is set, it overrides the hostname that the client + // would otherwise obtain from the OS, e.g. by calling os.Hostname(). + Hostname Key = "Hostname" + // Keys with a string array value. // AllowedSuggestedExitNodes's string array value is a list of exit node IDs that restricts which exit nodes are considered when generating suggestions for exit nodes. AllowedSuggestedExitNodes Key = "AllowedSuggestedExitNodes" @@ -148,6 +153,7 @@ var implicitDefinitions = []*setting.Definition{ setting.NewDefinition(ExitNodeID, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(ExitNodeIP, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(FlushDNSOnSessionUnlock, setting.DeviceSetting, setting.BooleanValue), + setting.NewDefinition(Hostname, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(LogSCMInteractions, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(LogTarget, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(MachineCertificateSubject, setting.DeviceSetting, setting.StringValue), From 6c30840cac13f184474654c90b7b0cc314069b25 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Sun, 19 Jan 2025 19:00:21 +0000 Subject: [PATCH 141/223] ipn: [serve] warn that foreground funnel won't work if shields are up (#14685) We throw error early with a warning if users attempt to enable background funnel for a node that does not allow incoming connections (shields up), but if it done in foreground mode, we just silently fail (the funnel command succeeds, but the connections are not allowed). This change makes sure that we also error early in foreground mode. Updates tailscale/tailscale#11049 Signed-off-by: Irbe Krumina --- ipn/serve.go | 20 ++++++----- ipn/serve_test.go | 85 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 8 deletions(-) diff --git a/ipn/serve.go b/ipn/serve.go index b7effa874c136..176c6d984fbd4 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -63,12 +63,12 @@ type ServeConfig struct { // traffic is allowed, from trusted ingress peers. AllowFunnel map[HostPort]bool `json:",omitempty"` - // Foreground is a map of an IPN Bus session ID to an alternate foreground - // serve config that's valid for the life of that WatchIPNBus session ID. - // This. This allows the config to specify ephemeral configs that are - // used in the CLI's foreground mode to ensure ungraceful shutdowns - // of either the client or the LocalBackend does not expose ports - // that users are not aware of. + // Foreground is a map of an IPN Bus session ID to an alternate foreground serve config that's valid for the + // life of that WatchIPNBus session ID. This allows the config to specify ephemeral configs that are used + // in the CLI's foreground mode to ensure ungraceful shutdowns of either the client or the LocalBackend does not + // expose ports that users are not aware of. In practice this contains any serve config set via 'tailscale + // serve' command run without the '--bg' flag. ServeConfig contained by Foreground is not expected itself to contain + // another Foreground block. Foreground map[string]*ServeConfig `json:",omitempty"` // ETag is the checksum of the serve config that's populated @@ -389,8 +389,7 @@ func (sc *ServeConfig) RemoveTCPForwarding(port uint16) { // View version of ServeConfig.IsFunnelOn. func (v ServeConfigView) IsFunnelOn() bool { return v.Đļ.IsFunnelOn() } -// IsFunnelOn reports whether if ServeConfig is currently allowing funnel -// traffic for any host:port. +// IsFunnelOn reports whether any funnel endpoint is currently enabled for this node. func (sc *ServeConfig) IsFunnelOn() bool { if sc == nil { return false @@ -400,6 +399,11 @@ func (sc *ServeConfig) IsFunnelOn() bool { return true } } + for _, conf := range sc.Foreground { + if conf.IsFunnelOn() { + return true + } + } return false } diff --git a/ipn/serve_test.go b/ipn/serve_test.go index e9d8e8f322075..ae1d56eef6b09 100644 --- a/ipn/serve_test.go +++ b/ipn/serve_test.go @@ -182,3 +182,88 @@ func TestExpandProxyTargetDev(t *testing.T) { }) } } + +func TestIsFunnelOn(t *testing.T) { + tests := []struct { + name string + sc *ServeConfig + want bool + }{ + { + name: "nil_config", + }, + { + name: "empty_config", + sc: &ServeConfig{}, + }, + { + name: "funnel_enabled_in_background", + sc: &ServeConfig{ + AllowFunnel: map[HostPort]bool{ + "tailnet.xyz:443": true, + }, + }, + want: true, + }, + { + name: "funnel_disabled_in_background", + sc: &ServeConfig{ + AllowFunnel: map[HostPort]bool{ + "tailnet.xyz:443": false, + }, + }, + }, + { + name: "funnel_enabled_in_foreground", + sc: &ServeConfig{ + Foreground: map[string]*ServeConfig{ + "abc123": { + AllowFunnel: map[HostPort]bool{ + "tailnet.xyz:443": true, + }, + }, + }, + }, + want: true, + }, + { + name: "funnel_disabled_in_both", + sc: &ServeConfig{ + AllowFunnel: map[HostPort]bool{ + "tailnet.xyz:443": false, + }, + Foreground: map[string]*ServeConfig{ + "abc123": { + AllowFunnel: map[HostPort]bool{ + "tailnet.xyz:8443": false, + }, + }, + }, + }, + }, + { + name: "funnel_enabled_in_both", + sc: &ServeConfig{ + AllowFunnel: map[HostPort]bool{ + "tailnet.xyz:443": true, + }, + Foreground: map[string]*ServeConfig{ + "abc123": { + AllowFunnel: map[HostPort]bool{ + "tailnet.xyz:8443": true, + }, + }, + }, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.sc.IsFunnelOn(); got != tt.want { + t.Errorf("ServeConfig.IsFunnelOn() = %v, want %v", got, tt.want) + } + }) + } +} From 6e3c746942b5f5c65ef813713f861449b0c7f54e Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Mon, 20 Jan 2025 12:31:26 -0500 Subject: [PATCH 142/223] derp: add bytes dropped metric (#14698) Add bytes dropped counter metric by reason and kind. Fixes tailscale/corp#25918 Signed-off-by: Mike O'Driscoll --- derp/derp_server.go | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/derp/derp_server.go b/derp/derp_server.go index 08fd280a99a3f..983b5dc002c83 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -357,6 +357,12 @@ var packetsDropped = metrics.NewMultiLabelMap[dropReasonKindLabels]( "counter", "DERP packets dropped by reason and by kind") +var bytesDropped = metrics.NewMultiLabelMap[dropReasonKindLabels]( + "derp_bytes_dropped", + "counter", + "DERP bytes dropped by reason and by kind", +) + // NewServer returns a new DERP server. It doesn't listen on its own. // Connections are given to it via Server.Accept. func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server { @@ -388,13 +394,13 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server { s.packetsRecvDisco = s.packetsRecvByKind.Get(string(packetKindDisco)) s.packetsRecvOther = s.packetsRecvByKind.Get(string(packetKindOther)) - genPacketsDroppedCounters() + genDroppedCounters() s.perClientSendQueueDepth = getPerClientSendQueueDepth() return s } -func genPacketsDroppedCounters() { +func genDroppedCounters() { initMetrics := func(reason dropReason) { packetsDropped.Add(dropReasonKindLabels{ Kind: string(packetKindDisco), @@ -404,6 +410,14 @@ func genPacketsDroppedCounters() { Kind: string(packetKindOther), Reason: string(reason), }, 0) + bytesDropped.Add(dropReasonKindLabels{ + Kind: string(packetKindDisco), + Reason: string(reason), + }, 0) + bytesDropped.Add(dropReasonKindLabels{ + Kind: string(packetKindOther), + Reason: string(reason), + }, 0) } getMetrics := func(reason dropReason) []expvar.Var { return []expvar.Var{ @@ -415,6 +429,14 @@ func genPacketsDroppedCounters() { Kind: string(packetKindOther), Reason: string(reason), }), + bytesDropped.Get(dropReasonKindLabels{ + Kind: string(packetKindDisco), + Reason: string(reason), + }), + bytesDropped.Get(dropReasonKindLabels{ + Kind: string(packetKindOther), + Reason: string(reason), + }), } } @@ -431,12 +453,14 @@ func genPacketsDroppedCounters() { for _, dr := range dropReasons { initMetrics(dr) m := getMetrics(dr) - if len(m) != 2 { + if len(m) != 4 { panic("dropReason metrics out of sync") } - if m[0] == nil || m[1] == nil { - panic("dropReason metrics out of sync") + for _, v := range m { + if v == nil { + panic("dropReason metrics out of sync") + } } } } @@ -1207,6 +1231,7 @@ func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, r labels.Kind = string(packetKindOther) } packetsDropped.Add(labels, 1) + bytesDropped.Add(labels, int64(len(packetBytes))) if verboseDropKeys[dstKey] { // Preformat the log string prior to calling limitedLogf. The From 174af763eb36d31a75ff397a1ed2b1789f41b16b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 10:57:18 -0700 Subject: [PATCH 143/223] .github: Bump actions/upload-artifact from 4.4.3 to 4.6.0 (#14697) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.4.3 to 4.6.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882...65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92ef57b50c169..20f215cd0ed42 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -467,7 +467,7 @@ jobs: run: | echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV - name: upload crash - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 if: steps.run.outcome != 'success' && steps.build.outcome == 'success' with: name: artifacts From 33e62a31bdd9c0e919adbc9c6dbc42aba65e7f23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:18:42 -0700 Subject: [PATCH 144/223] .github: Bump peter-evans/create-pull-request from 7.0.5 to 7.0.6 (#14695) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 7.0.5 to 7.0.6. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/5e914681df9dc83aa4e4905692ca88beb2f9e91f...67ccf781d68cd99b580ae25a5c18a1cc84ffff1f) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/update-flake.yml | 2 +- .github/workflows/update-webclient-prebuilt.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index f79248c1ed4e9..151ed6bab4b9c 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -36,7 +36,7 @@ jobs: private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }} - name: Send pull request - uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5 + uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6 with: token: ${{ steps.generate-token.outputs.token }} author: Flakes Updater diff --git a/.github/workflows/update-webclient-prebuilt.yml b/.github/workflows/update-webclient-prebuilt.yml index a0ae95cd77ba4..11665460be8ec 100644 --- a/.github/workflows/update-webclient-prebuilt.yml +++ b/.github/workflows/update-webclient-prebuilt.yml @@ -35,7 +35,7 @@ jobs: - name: Send pull request id: pull-request - uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5 + uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6 with: token: ${{ steps.generate-token.outputs.token }} author: OSS Updater From 682c06a0e7921e979ef8f24e57a14078eb5dd115 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 12:48:50 -0700 Subject: [PATCH 145/223] .github: Bump golangci/golangci-lint-action from 6.1.0 to 6.2.0 (#14696) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.1.0 to 6.2.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/aaa42aa0628b4ae2578232a66b541047968fac86...ec5d18412c0aeab7936cb16880d708ba2a64e1ae) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/golangci-lint.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 9f1f2b9d1db62..ad135f784adec 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -31,8 +31,7 @@ jobs: cache: false - name: golangci-lint - # Note: this is the 'v6.1.0' tag as of 2024-08-21 - uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 + uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 with: version: v1.60 From 70c7b0d77f134d33943a220a4f63949266c83373 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 13:05:37 -0700 Subject: [PATCH 146/223] build(deps): bump nanoid from 3.3.4 to 3.3.8 in /cmd/tsconnect (#14352) Bumps [nanoid](https://github.com/ai/nanoid) from 3.3.4 to 3.3.8. - [Release notes](https://github.com/ai/nanoid/releases) - [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md) - [Commits](https://github.com/ai/nanoid/compare/3.3.4...3.3.8) --- updated-dependencies: - dependency-name: nanoid dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cmd/tsconnect/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tsconnect/yarn.lock b/cmd/tsconnect/yarn.lock index 663a1244ebf69..811eddeb7f5c1 100644 --- a/cmd/tsconnect/yarn.lock +++ b/cmd/tsconnect/yarn.lock @@ -349,9 +349,9 @@ minimist@^1.2.6: integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== nanoid@^3.3.4: - version "3.3.4" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" - integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" From 69a985fb1edfae8c887e42711d5df814ce7353a7 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Tue, 21 Jan 2025 05:17:27 +0000 Subject: [PATCH 147/223] ipn/ipnlocal,tailcfg: communicate to control whether funnel is enabled (#14688) Adds a new Hostinfo.IngressEnabled bool field that holds whether funnel is currently enabled for the node. Triggers control update when this value changes. Bumps capver so that control can distinguish the new field being false vs non-existant in previous clients. This is part of a fix for an issue where nodes with any AllowFunnel block set in their serve config are being displayed as if actively routing funnel traffic in the admin panel. Updates tailscale/tailscale#11572 Updates tailscale/corp#25931 Signed-off-by: Irbe Krumina --- ipn/ipnlocal/local.go | 42 ++++++++++- ipn/ipnlocal/local_test.go | 151 +++++++++++++++++++++++++++++++++++++ tailcfg/tailcfg.go | 4 +- tailcfg/tailcfg_clone.go | 1 + tailcfg/tailcfg_test.go | 21 ++++++ tailcfg/tailcfg_view.go | 2 + 6 files changed, 216 insertions(+), 5 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index c59df833d19ec..214d3a4e478a1 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -3988,6 +3988,12 @@ func (b *LocalBackend) wantIngressLocked() bool { return b.serveConfig.Valid() && b.serveConfig.HasAllowFunnel() } +// hasIngressEnabledLocked reports whether the node has any funnel endpoint enabled. This bool is sent to control (in +// Hostinfo.IngressEnabled) to determine whether 'Funnel' badge should be displayed on this node in the admin panel. +func (b *LocalBackend) hasIngressEnabledLocked() bool { + return b.serveConfig.Valid() && b.serveConfig.IsFunnelOn() +} + // setPrefsLockedOnEntry requires b.mu be held to call it, but it // unlocks b.mu when done. newp ownership passes to this function. // It returns a read-only copy of the new prefs. @@ -5086,7 +5092,12 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip // if this is accidentally false, then control may not configure DNS // properly. This exists as an optimization to control to program fewer DNS // records that have ingress enabled but are not actually being used. + // TODO(irbekrm): once control knows that if hostinfo.IngressEnabled is true, + // then wireIngress can be considered true, don't send wireIngress in that case. hi.WireIngress = b.wantIngressLocked() + // The Hostinfo.IngressEnabled field is used to communicate to control whether + // the funnel is actually enabled. + hi.IngressEnabled = b.hasIngressEnabledLocked() hi.AppConnector.Set(prefs.AppConnector().Advertise) } @@ -6009,14 +6020,37 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn. b.updateServeTCPPortNetMapAddrListenersLocked(servePorts) } } - // Kick off a Hostinfo update to control if WireIngress changed. - if wire := b.wantIngressLocked(); b.hostinfo != nil && b.hostinfo.WireIngress != wire { + + // Update funnel info in hostinfo and kick off control update if needed. + b.updateIngressLocked() + b.setTCPPortsIntercepted(handlePorts) +} + +// updateIngressLocked updates the hostinfo.WireIngress and hostinfo.IngressEnabled fields and kicks off a Hostinfo +// update if the values have changed. +// TODO(irbekrm): once control knows that if hostinfo.IngressEnabled is true, then wireIngress can be considered true, +// we can stop sending hostinfo.WireIngress in that case. +// +// b.mu must be held. +func (b *LocalBackend) updateIngressLocked() { + if b.hostinfo == nil { + return + } + hostInfoChanged := false + if wire := b.wantIngressLocked(); b.hostinfo.WireIngress != wire { b.logf("Hostinfo.WireIngress changed to %v", wire) b.hostinfo.WireIngress = wire + hostInfoChanged = true + } + if ie := b.hasIngressEnabledLocked(); b.hostinfo.IngressEnabled != ie { + b.logf("Hostinfo.IngressEnabled changed to %v", ie) + b.hostinfo.IngressEnabled = ie + hostInfoChanged = true + } + // Kick off a Hostinfo update to control if ingress status has changed. + if hostInfoChanged { b.goTracker.Go(b.doSetHostinfoFilterServices) } - - b.setTCPPortsIntercepted(handlePorts) } // setServeProxyHandlersLocked ensures there is an http proxy handler for each diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 5e8a3172caabb..348bdcab37e68 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -4838,3 +4838,154 @@ func TestUpdatePrefsOnSysPolicyChange(t *testing.T) { }) } } + +func TestUpdateIngressLocked(t *testing.T) { + tests := []struct { + name string + hi *tailcfg.Hostinfo + sc *ipn.ServeConfig + wantIngress bool + wantWireIngress bool + wantControlUpdate bool + }{ + { + name: "no_hostinfo_no_serve_config", + hi: nil, + }, + { + name: "empty_hostinfo_no_serve_config", + hi: &tailcfg.Hostinfo{}, + }, + { + name: "empty_hostinfo_funnel_enabled", + hi: &tailcfg.Hostinfo{}, + sc: &ipn.ServeConfig{ + AllowFunnel: map[ipn.HostPort]bool{ + "tailnet.xyz:443": true, + }, + }, + wantIngress: true, + wantWireIngress: true, + wantControlUpdate: true, + }, + { + name: "empty_hostinfo_funnel_disabled", + hi: &tailcfg.Hostinfo{}, + sc: &ipn.ServeConfig{ + AllowFunnel: map[ipn.HostPort]bool{ + "tailnet.xyz:443": false, + }, + }, + wantWireIngress: true, // true if there is any AllowFunnel block + wantControlUpdate: true, + }, + { + name: "empty_hostinfo_no_funnel", + hi: &tailcfg.Hostinfo{}, + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTPS: true}, + }, + }, + }, + { + name: "funnel_enabled_no_change", + hi: &tailcfg.Hostinfo{ + IngressEnabled: true, + WireIngress: true, + }, + sc: &ipn.ServeConfig{ + AllowFunnel: map[ipn.HostPort]bool{ + "tailnet.xyz:443": true, + }, + }, + wantIngress: true, + wantWireIngress: true, + }, + { + name: "funnel_disabled_no_change", + hi: &tailcfg.Hostinfo{ + WireIngress: true, + }, + sc: &ipn.ServeConfig{ + AllowFunnel: map[ipn.HostPort]bool{ + "tailnet.xyz:443": false, + }, + }, + wantWireIngress: true, // true if there is any AllowFunnel block + }, + { + name: "funnel_changes_to_disabled", + hi: &tailcfg.Hostinfo{ + IngressEnabled: true, + WireIngress: true, + }, + sc: &ipn.ServeConfig{ + AllowFunnel: map[ipn.HostPort]bool{ + "tailnet.xyz:443": false, + }, + }, + wantWireIngress: true, // true if there is any AllowFunnel block + wantControlUpdate: true, + }, + { + name: "funnel_changes_to_enabled", + hi: &tailcfg.Hostinfo{ + WireIngress: true, + }, + sc: &ipn.ServeConfig{ + AllowFunnel: map[ipn.HostPort]bool{ + "tailnet.xyz:443": true, + }, + }, + wantWireIngress: true, + wantIngress: true, + wantControlUpdate: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := newTestLocalBackend(t) + b.hostinfo = tt.hi + b.serveConfig = tt.sc.View() + allDone := make(chan bool, 1) + defer b.goTracker.AddDoneCallback(func() { + b.mu.Lock() + defer b.mu.Unlock() + if b.goTracker.RunningGoroutines() > 0 { + return + } + select { + case allDone <- true: + default: + } + })() + + was := b.goTracker.StartedGoroutines() + b.updateIngressLocked() + + if tt.hi != nil { + if tt.hi.IngressEnabled != tt.wantIngress { + t.Errorf("IngressEnabled = %v, want %v", tt.hi.IngressEnabled, tt.wantIngress) + } + if tt.hi.WireIngress != tt.wantWireIngress { + t.Errorf("WireIngress = %v, want %v", tt.hi.WireIngress, tt.wantWireIngress) + } + } + + startedGoroutine := b.goTracker.StartedGoroutines() != was + if startedGoroutine != tt.wantControlUpdate { + t.Errorf("control update triggered = %v, want %v", startedGoroutine, tt.wantControlUpdate) + } + + if startedGoroutine { + select { + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for goroutine to finish") + case <-allDone: + } + } + }) + } +} diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 9b26e888388ce..937f619e67430 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -155,7 +155,8 @@ type CapabilityVersion int // - 110: 2024-12-12: removed never-before-used Tailscale SSH public key support (#14373) // - 111: 2025-01-14: Client supports a peer having Node.HomeDERP (issue #14636) // - 112: 2025-01-14: Client interprets AllowedIPs of nil as meaning same as Addresses -const CurrentCapabilityVersion CapabilityVersion = 112 +// - 113: 2025-01-20: Client communicates to control whether funnel is enabled by sending Hostinfo.IngressEnabled (#14688) +const CurrentCapabilityVersion CapabilityVersion = 113 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -869,6 +870,7 @@ type Hostinfo struct { ShareeNode bool `json:",omitempty"` // indicates this node exists in netmap because it's owned by a shared-to user NoLogsNoSupport bool `json:",omitempty"` // indicates that the user has opted out of sending logs and support WireIngress bool `json:",omitempty"` // indicates that the node wants the option to receive ingress connections + IngressEnabled bool `json:",omitempty"` // if the node has any funnel endpoint enabled AllowsUpdate bool `json:",omitempty"` // indicates that the node has opted-in to admin-console-drive remote updates Machine string `json:",omitempty"` // the current host's machine type (uname -m) GoArch string `json:",omitempty"` // GOARCH value (of the built binary) diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index 42cef1598e8e5..f7126ca418d9e 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -166,6 +166,7 @@ var _HostinfoCloneNeedsRegeneration = Hostinfo(struct { ShareeNode bool NoLogsNoSupport bool WireIngress bool + IngressEnabled bool AllowsUpdate bool Machine string GoArch string diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index 560e28933eccd..da5873847f35a 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -51,6 +51,7 @@ func TestHostinfoEqual(t *testing.T) { "ShareeNode", "NoLogsNoSupport", "WireIngress", + "IngressEnabled", "AllowsUpdate", "Machine", "GoArch", @@ -251,6 +252,26 @@ func TestHostinfoEqual(t *testing.T) { &Hostinfo{}, false, }, + { + &Hostinfo{IngressEnabled: true}, + &Hostinfo{}, + false, + }, + { + &Hostinfo{IngressEnabled: true}, + &Hostinfo{IngressEnabled: true}, + true, + }, + { + &Hostinfo{IngressEnabled: false}, + &Hostinfo{}, + true, + }, + { + &Hostinfo{IngressEnabled: false}, + &Hostinfo{IngressEnabled: true}, + false, + }, } for i, tt := range tests { got := tt.a.Equal(tt.b) diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index 3770f272f6bcb..55c244fbff805 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -283,6 +283,7 @@ func (v HostinfoView) ShieldsUp() bool { return v.Đļ.Shie func (v HostinfoView) ShareeNode() bool { return v.Đļ.ShareeNode } func (v HostinfoView) NoLogsNoSupport() bool { return v.Đļ.NoLogsNoSupport } func (v HostinfoView) WireIngress() bool { return v.Đļ.WireIngress } +func (v HostinfoView) IngressEnabled() bool { return v.Đļ.IngressEnabled } func (v HostinfoView) AllowsUpdate() bool { return v.Đļ.AllowsUpdate } func (v HostinfoView) Machine() string { return v.Đļ.Machine } func (v HostinfoView) GoArch() string { return v.Đļ.GoArch } @@ -324,6 +325,7 @@ var _HostinfoViewNeedsRegeneration = Hostinfo(struct { ShareeNode bool NoLogsNoSupport bool WireIngress bool + IngressEnabled bool AllowsUpdate bool Machine string GoArch string From 817ba1c300ad8378bb87f14e2c1709e428a54372 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Tue, 21 Jan 2025 05:21:03 +0000 Subject: [PATCH 148/223] cmd/{k8s-operator,containerboot},kube/kubetypes: parse Ingresses for ingress ProxyGroup (#14583) cmd/k8s-operator: add logic to parse L7 Ingresses in HA mode - Wrap the Tailscale API client used by the Kubernetes Operator into a client that knows how to manage VIPServices. - Create/Delete VIPServices and update serve config for L7 Ingresses for ProxyGroup. - Ensure that ingress ProxyGroup proxies mount serve config from a shared ConfigMap. Updates tailscale/corp#24795 Signed-off-by: Irbe Krumina --- cmd/containerboot/serve.go | 7 + cmd/k8s-operator/ingress-for-pg.go | 567 ++++++++++++++++++++++++ cmd/k8s-operator/ingress-for-pg_test.go | 337 ++++++++++++++ cmd/k8s-operator/ingress.go | 166 +++---- cmd/k8s-operator/operator.go | 103 +++-- cmd/k8s-operator/proxygroup.go | 11 +- cmd/k8s-operator/proxygroup_specs.go | 52 ++- cmd/k8s-operator/proxygroup_test.go | 31 +- cmd/k8s-operator/sts.go | 4 +- cmd/k8s-operator/testutils_test.go | 50 +++ cmd/k8s-operator/tsclient.go | 185 ++++++++ kube/kubetypes/types.go | 5 +- 12 files changed, 1391 insertions(+), 127 deletions(-) create mode 100644 cmd/k8s-operator/ingress-for-pg.go create mode 100644 cmd/k8s-operator/ingress-for-pg_test.go create mode 100644 cmd/k8s-operator/tsclient.go diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 1729e65b5594c..aad22820b0737 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -65,6 +65,10 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan if err != nil { log.Fatalf("serve proxy: failed to read serve config: %v", err) } + if sc == nil { + log.Printf("serve proxy: no serve config at %q, skipping", path) + continue + } if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) { continue } @@ -131,6 +135,9 @@ func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) { } j, err := os.ReadFile(path) if err != nil { + if os.IsNotExist(err) { + return nil, nil + } return nil, err } // Serve config can be provided by users as well as the Kubernetes Operator (for its proxies). User-provided diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go new file mode 100644 index 0000000000000..4dcaf7c6d1a86 --- /dev/null +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -0,0 +1,567 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "reflect" + "slices" + "strings" + "sync" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/client/tailscale" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/util/clientmetric" + "tailscale.com/util/dnsname" + "tailscale.com/util/mak" + "tailscale.com/util/set" +) + +const ( + serveConfigKey = "serve-config.json" + VIPSvcOwnerRef = "tailscale.com/k8s-operator:owned-by:%s" + // FinalizerNamePG is the finalizer used by the IngressPGReconciler + FinalizerNamePG = "tailscale.com/ingress-pg-finalizer" +) + +var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount) + +// IngressPGReconciler is a controller that reconciles Tailscale Ingresses should be exposed on an ingress ProxyGroup +// (in HA mode). +type IngressPGReconciler struct { + client.Client + + recorder record.EventRecorder + logger *zap.SugaredLogger + tsClient tsClient + tsnetServer tsnetServer + tsNamespace string + lc localClient + defaultTags []string + + mu sync.Mutex // protects following + // managedIngresses is a set of all ingress resources that we're currently + // managing. This is only used for metrics. + managedIngresses set.Slice[types.UID] +} + +// Reconcile reconciles Ingresses that should be exposed over Tailscale in HA mode (on a ProxyGroup). It looks at all +// Ingresses with tailscale.com/proxy-group annotation. For each such Ingress, it ensures that a VIPService named after +// the hostname of the Ingress exists and is up to date. It also ensures that the serve config for the ingress +// ProxyGroup is updated to route traffic for the VIPService to the Ingress's backend Services. +// When an Ingress is deleted or unexposed, the VIPService and the associated serve config are cleaned up. +// Ingress hostname change also results in the VIPService for the previous hostname being cleaned up and a new VIPService +// being created for the new hostname. +func (a *IngressPGReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + logger := a.logger.With("Ingress", req.NamespacedName) + logger.Debugf("starting reconcile") + defer logger.Debugf("reconcile finished") + + ing := new(networkingv1.Ingress) + err = a.Get(ctx, req.NamespacedName, ing) + if apierrors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + logger.Debugf("Ingress not found, assuming it was deleted") + return res, nil + } else if err != nil { + return res, fmt.Errorf("failed to get Ingress: %w", err) + } + + // hostname is the name of the VIPService that will be created for this Ingress as well as the first label in + // the MagicDNS name of the Ingress. + hostname := hostnameForIngress(ing) + logger = logger.With("hostname", hostname) + + if !ing.DeletionTimestamp.IsZero() || !a.shouldExpose(ing) { + return res, a.maybeCleanup(ctx, hostname, ing, logger) + } + + if err := a.maybeProvision(ctx, hostname, ing, logger); err != nil { + return res, fmt.Errorf("failed to provision: %w", err) + } + return res, nil +} + +// maybeProvision ensures that the VIPService and serve config for the Ingress are created or updated. +func (a *IngressPGReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error { + if err := validateIngressClass(ctx, a.Client); err != nil { + logger.Infof("error validating tailscale IngressClass: %v.", err) + return nil + } + + // Get and validate ProxyGroup readiness + pgName := ing.Annotations[AnnotationProxyGroup] + if pgName == "" { + logger.Infof("[unexpected] no ProxyGroup annotation, skipping VIPService provisioning") + return nil + } + pg := &tsapi.ProxyGroup{} + if err := a.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil { + if apierrors.IsNotFound(err) { + logger.Infof("ProxyGroup %q does not exist", pgName) + return nil + } + return fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) + } + if !tsoperator.ProxyGroupIsReady(pg) { + // TODO(irbekrm): we need to reconcile ProxyGroup Ingresses on ProxyGroup changes to not miss the status update + // in this case. + logger.Infof("ProxyGroup %q is not ready", pgName) + return nil + } + + // Validate Ingress configuration + if err := a.validateIngress(ing, pg); err != nil { + logger.Infof("invalid Ingress configuration: %v", err) + a.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error()) + return nil + } + + if !IsHTTPSEnabledOnTailnet(a.tsnetServer) { + a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work") + } + + logger = logger.With("proxy-group", pg) + + if !slices.Contains(ing.Finalizers, FinalizerNamePG) { + // This log line is printed exactly once during initial provisioning, + // because once the finalizer is in place this block gets skipped. So, + // this is a nice place to tell the operator that the high level, + // multi-reconcile operation is underway. + logger.Infof("exposing Ingress over tailscale") + ing.Finalizers = append(ing.Finalizers, FinalizerNamePG) + if err := a.Update(ctx, ing); err != nil { + return fmt.Errorf("failed to add finalizer: %w", err) + } + a.mu.Lock() + a.managedIngresses.Add(ing.UID) + gaugePGIngressResources.Set(int64(a.managedIngresses.Len())) + a.mu.Unlock() + } + + // 1. Ensure that if Ingress' hostname has changed, any VIPService resources corresponding to the old hostname + // are cleaned up. + // In practice, this function will ensure that any VIPServices that are associated with the provided ProxyGroup + // and no longer owned by an Ingress are cleaned up. This is fine- it is not expensive and ensures that in edge + // cases (a single update changed both hostname and removed ProxyGroup annotation) the VIPService is more likely + // to be (eventually) removed. + if err := a.maybeCleanupProxyGroup(ctx, pgName, logger); err != nil { + return fmt.Errorf("failed to cleanup VIPService resources for ProxyGroup: %w", err) + } + + // 2. Ensure that there isn't a VIPService with the same hostname already created and not owned by this Ingress. + // TODO(irbekrm): perhaps in future we could have record names being stored on VIPServices. I am not certain if + // there might not be edge cases (custom domains, etc?) where attempting to determine the DNS name of the + // VIPService in this way won't be incorrect. + tcd, err := a.tailnetCertDomain(ctx) + if err != nil { + return fmt.Errorf("error determining DNS name base: %w", err) + } + dnsName := hostname + "." + tcd + existingVIPSvc, err := a.tsClient.getVIPServiceByName(ctx, hostname) + // TODO(irbekrm): here and when creating the VIPService, verify if the error is not terminal (and therefore + // should not be reconciled). For example, if the hostname is already a hostname of a Tailscale node, the GET + // here will fail. + if err != nil { + errResp := &tailscale.ErrResponse{} + if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound { + return fmt.Errorf("error getting VIPService %q: %w", hostname, err) + } + } + if existingVIPSvc != nil && !isVIPServiceForIngress(existingVIPSvc, ing) { + logger.Infof("VIPService %q for MagicDNS name %q already exists, but is not owned by this Ingress. Please delete it manually and recreate this Ingress to proceed or create an Ingress for a different MagicDNS name", hostname, dnsName) + a.recorder.Event(ing, corev1.EventTypeWarning, "ConflictingVIPServiceExists", fmt.Sprintf("VIPService %q for MagicDNS name %q already exists, but is not owned by this Ingress. Please delete it manually to proceed or create an Ingress for a different MagicDNS name", hostname, dnsName)) + return nil + } + + // 3. Ensure that the serve config for the ProxyGroup contains the VIPService + cm, cfg, err := a.proxyGroupServeConfig(ctx, pgName) + if err != nil { + return fmt.Errorf("error getting ingress serve config: %w", err) + } + if cm == nil { + logger.Infof("no ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.") + return nil + } + ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName)) + handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, dnsName, logger) + if err != nil { + return fmt.Errorf("failed to get handlers for ingress: %w", err) + } + ingCfg := &ipn.ServiceConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + HTTPS: true, + }, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + ep: { + Handlers: handlers, + }, + }, + } + var gotCfg *ipn.ServiceConfig + if cfg != nil && cfg.Services != nil { + gotCfg = cfg.Services[hostname] + } + if !reflect.DeepEqual(gotCfg, ingCfg) { + logger.Infof("Updating serve config") + mak.Set(&cfg.Services, hostname, ingCfg) + cfgBytes, err := json.Marshal(cfg) + if err != nil { + return fmt.Errorf("error marshaling serve config: %w", err) + } + mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) + if err := a.Update(ctx, cm); err != nil { + return fmt.Errorf("error updating serve config: %w", err) + } + } + + // 4. Ensure that the VIPService exists and is up to date. + tags := a.defaultTags + if tstr, ok := ing.Annotations[AnnotationTags]; ok { + tags = strings.Split(tstr, ",") + } + + vipSvc := &VIPService{ + Name: hostname, + Tags: tags, + Ports: []string{"443"}, // always 443 for Ingress + Comment: fmt.Sprintf(VIPSvcOwnerRef, ing.UID), + } + if existingVIPSvc != nil { + vipSvc.Addrs = existingVIPSvc.Addrs + } + if existingVIPSvc == nil || !reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) { + logger.Infof("Ensuring VIPService %q exists and is up to date", hostname) + if err := a.tsClient.createOrUpdateVIPServiceByName(ctx, vipSvc); err != nil { + logger.Infof("error creating VIPService: %v", err) + return fmt.Errorf("error creating VIPService: %w", err) + } + } + + // 5. Update Ingress status + oldStatus := ing.Status.DeepCopy() + // TODO(irbekrm): once we have ingress ProxyGroup, we can determine if instances are ready to route traffic to the VIPService + ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{ + { + Hostname: dnsName, + Ports: []networkingv1.IngressPortStatus{ + { + Protocol: "TCP", + Port: 443, + }, + }, + }, + } + if apiequality.Semantic.DeepEqual(oldStatus, ing.Status) { + return nil + } + if err := a.Status().Update(ctx, ing); err != nil { + return fmt.Errorf("failed to update Ingress status: %w", err) + } + return nil +} + +// maybeCleanupProxyGroup ensures that if an Ingress hostname has changed, any VIPService resources created for the +// Ingress' ProxyGroup corresponding to the old hostname are cleaned up. A run of this function will ensure that any +// VIPServices that are associated with the provided ProxyGroup and no longer owned by an Ingress are cleaned up. +func (a *IngressPGReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger) error { + // Get serve config for the ProxyGroup + cm, cfg, err := a.proxyGroupServeConfig(ctx, proxyGroupName) + if err != nil { + return fmt.Errorf("getting serve config: %w", err) + } + if cfg == nil { + return nil // ProxyGroup does not have any VIPServices + } + + ingList := &networkingv1.IngressList{} + if err := a.List(ctx, ingList); err != nil { + return fmt.Errorf("listing Ingresses: %w", err) + } + serveConfigChanged := false + // For each VIPService in serve config... + for vipHostname := range cfg.Services { + // ...check if there is currently an Ingress with this hostname + found := false + for _, i := range ingList.Items { + ingressHostname := hostnameForIngress(&i) + if ingressHostname == vipHostname { + found = true + break + } + } + + if !found { + logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipHostname) + svc, err := a.getVIPService(ctx, vipHostname, logger) + if err != nil { + errResp := &tailscale.ErrResponse{} + if errors.As(err, &errResp) && errResp.Status == http.StatusNotFound { + delete(cfg.Services, vipHostname) + serveConfigChanged = true + continue + } + return err + } + if isVIPServiceForAnyIngress(svc) { + logger.Infof("cleaning up orphaned VIPService %q", vipHostname) + if err := a.tsClient.deleteVIPServiceByName(ctx, vipHostname); err != nil { + errResp := &tailscale.ErrResponse{} + if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound { + return fmt.Errorf("deleting VIPService %q: %w", vipHostname, err) + } + } + } + delete(cfg.Services, vipHostname) + serveConfigChanged = true + } + } + + if serveConfigChanged { + cfgBytes, err := json.Marshal(cfg) + if err != nil { + return fmt.Errorf("marshaling serve config: %w", err) + } + mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) + if err := a.Update(ctx, cm); err != nil { + return fmt.Errorf("updating serve config: %w", err) + } + } + return nil +} + +// maybeCleanup ensures that any resources, such as a VIPService created for this Ingress, are cleaned up when the +// Ingress is being deleted or is unexposed. +func (a *IngressPGReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error { + logger.Debugf("Ensuring any resources for Ingress are cleaned up") + ix := slices.Index(ing.Finalizers, FinalizerNamePG) + if ix < 0 { + logger.Debugf("no finalizer, nothing to do") + a.mu.Lock() + defer a.mu.Unlock() + a.managedIngresses.Remove(ing.UID) + gaugePGIngressResources.Set(int64(a.managedIngresses.Len())) + return nil + } + + // 1. Check if there is a VIPService created for this Ingress. + pg := ing.Annotations[AnnotationProxyGroup] + cm, cfg, err := a.proxyGroupServeConfig(ctx, pg) + if err != nil { + return fmt.Errorf("error getting ProxyGroup serve config: %w", err) + } + // VIPService is always first added to serve config and only then created in the Tailscale API, so if it is not + // found in the serve config, we can assume that there is no VIPService. TODO(irbekrm): once we have ingress + // ProxyGroup, we will probably add currently exposed VIPServices to its status. At that point, we can use the + // status rather than checking the serve config each time. + if cfg == nil || cfg.Services == nil || cfg.Services[hostname] == nil { + return nil + } + logger.Infof("Ensuring that VIPService %q configuration is cleaned up", hostname) + + // 2. Delete the VIPService. + if err := a.deleteVIPServiceIfExists(ctx, hostname, ing, logger); err != nil { + return fmt.Errorf("error deleting VIPService: %w", err) + } + + // 3. Remove the VIPService from the serve config for the ProxyGroup. + logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg) + delete(cfg.Services, hostname) + cfgBytes, err := json.Marshal(cfg) + if err != nil { + return fmt.Errorf("error marshaling serve config: %w", err) + } + mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) + if err := a.Update(ctx, cm); err != nil { + return fmt.Errorf("error updating ConfigMap %q: %w", cm.Name, err) + } + + if err := a.deleteFinalizer(ctx, ing, logger); err != nil { + return fmt.Errorf("failed to remove finalizer: %w", err) + } + a.mu.Lock() + defer a.mu.Unlock() + a.managedIngresses.Remove(ing.UID) + gaugePGIngressResources.Set(int64(a.managedIngresses.Len())) + return nil +} + +func (a *IngressPGReconciler) deleteFinalizer(ctx context.Context, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error { + found := false + ing.Finalizers = slices.DeleteFunc(ing.Finalizers, func(f string) bool { + found = true + return f == FinalizerNamePG + }) + if !found { + return nil + } + logger.Debug("ensure %q finalizer is removed", FinalizerNamePG) + + if err := a.Update(ctx, ing); err != nil { + return fmt.Errorf("failed to remove finalizer %q: %w", FinalizerNamePG, err) + } + return nil +} + +func pgIngressCMName(pg string) string { + return fmt.Sprintf("%s-ingress-config", pg) +} + +func (a *IngressPGReconciler) proxyGroupServeConfig(ctx context.Context, pg string) (cm *corev1.ConfigMap, cfg *ipn.ServeConfig, err error) { + name := pgIngressCMName(pg) + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: a.tsNamespace, + }, + } + if err := a.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil && !apierrors.IsNotFound(err) { + return nil, nil, fmt.Errorf("error retrieving ingress serve config ConfigMap %s: %v", name, err) + } + if apierrors.IsNotFound(err) { + return nil, nil, nil + } + cfg = &ipn.ServeConfig{} + if len(cm.BinaryData[serveConfigKey]) != 0 { + if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil { + return nil, nil, fmt.Errorf("error unmarshaling ingress serve config %v: %w", cm.BinaryData[serveConfigKey], err) + } + } + return cm, cfg, nil +} + +type localClient interface { + StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) +} + +// tailnetCertDomain returns the base domain (TCD) of the current tailnet. +func (a *IngressPGReconciler) tailnetCertDomain(ctx context.Context) (string, error) { + st, err := a.lc.StatusWithoutPeers(ctx) + if err != nil { + return "", fmt.Errorf("error getting tailscale status: %w", err) + } + return st.CurrentTailnet.MagicDNSSuffix, nil +} + +// shouldExpose returns true if the Ingress should be exposed over Tailscale in HA mode (on a ProxyGroup) +func (a *IngressPGReconciler) shouldExpose(ing *networkingv1.Ingress) bool { + isTSIngress := ing != nil && + ing.Spec.IngressClassName != nil && + *ing.Spec.IngressClassName == tailscaleIngressClassName + pgAnnot := ing.Annotations[AnnotationProxyGroup] + return isTSIngress && pgAnnot != "" +} + +func (a *IngressPGReconciler) getVIPService(ctx context.Context, hostname string, logger *zap.SugaredLogger) (*VIPService, error) { + svc, err := a.tsClient.getVIPServiceByName(ctx, hostname) + if err != nil { + errResp := &tailscale.ErrResponse{} + if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound { + logger.Infof("error getting VIPService %q: %v", hostname, err) + return nil, fmt.Errorf("error getting VIPService %q: %w", hostname, err) + } + } + return svc, nil +} + +func isVIPServiceForIngress(svc *VIPService, ing *networkingv1.Ingress) bool { + if svc == nil || ing == nil { + return false + } + return strings.EqualFold(svc.Comment, fmt.Sprintf(VIPSvcOwnerRef, ing.UID)) +} + +func isVIPServiceForAnyIngress(svc *VIPService) bool { + if svc == nil { + return false + } + return strings.HasPrefix(svc.Comment, "tailscale.com/k8s-operator:owned-by:") +} + +// validateIngress validates that the Ingress is properly configured. +// Currently validates: +// - Any tags provided via tailscale.com/tags annotation are valid Tailscale ACL tags +// - The derived hostname is a valid DNS label +// - The referenced ProxyGroup exists and is of type 'ingress' +// - Ingress' TLS block is invalid +func (a *IngressPGReconciler) validateIngress(ing *networkingv1.Ingress, pg *tsapi.ProxyGroup) error { + var errs []error + + // Validate tags if present + if tstr, ok := ing.Annotations[AnnotationTags]; ok { + tags := strings.Split(tstr, ",") + for _, tag := range tags { + tag = strings.TrimSpace(tag) + if err := tailcfg.CheckTag(tag); err != nil { + errs = append(errs, fmt.Errorf("tailscale.com/tags annotation contains invalid tag %q: %w", tag, err)) + } + } + } + + // Validate TLS configuration + if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { + errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS)) + } + + // Validate that the hostname will be a valid DNS label + hostname := hostnameForIngress(ing) + if err := dnsname.ValidLabel(hostname); err != nil { + errs = append(errs, fmt.Errorf("invalid hostname %q: %w. Ensure that the hostname is a valid DNS label", hostname, err)) + } + + // Validate ProxyGroup type + if pg.Spec.Type != tsapi.ProxyGroupTypeIngress { + errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q", + pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress)) + } + + // Validate ProxyGroup readiness + if !tsoperator.ProxyGroupIsReady(pg) { + errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name)) + } + + return errors.Join(errs...) +} + +// deleteVIPServiceIfExists attempts to delete the VIPService if it exists and is owned by the given Ingress. +func (a *IngressPGReconciler) deleteVIPServiceIfExists(ctx context.Context, name string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error { + svc, err := a.getVIPService(ctx, name, logger) + if err != nil { + return fmt.Errorf("error getting VIPService: %w", err) + } + + // isVIPServiceForIngress handles nil svc, so we don't need to check it here + if !isVIPServiceForIngress(svc, ing) { + return nil + } + + logger.Infof("Deleting VIPService %q", name) + if err = a.tsClient.deleteVIPServiceByName(ctx, name); err != nil { + return fmt.Errorf("error deleting VIPService: %w", err) + } + return nil +} diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go new file mode 100644 index 0000000000000..2cd340962f993 --- /dev/null +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -0,0 +1,337 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "encoding/json" + "testing" + + "slices" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/types/ptr" +) + +func TestIngressPGReconciler(t *testing.T) { + tsIngressClass := &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, + Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}, + } + + // Pre-create the ProxyGroup + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg", + Generation: 1, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + } + + // Pre-create the ConfigMap for the ProxyGroup + pgConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg-ingress-config", + Namespace: "operator-ns", + }, + BinaryData: map[string][]byte{ + "serve-config.json": []byte(`{"Services":{}}`), + }, + } + + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, pgConfigMap, tsIngressClass). + WithStatusSubresource(pg). + Build() + mustUpdateStatus(t, fc, "", pg.Name, func(pg *tsapi.ProxyGroup) { + pg.Status.Conditions = []metav1.Condition{ + { + Type: string(tsapi.ProxyGroupReady), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + }, + } + }) + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + lc := &fakeLocalClient{ + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{ + MagicDNSSuffix: "ts.net", + }, + }, + } + ingPGR := &IngressPGReconciler{ + Client: fc, + tsClient: ft, + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + tsNamespace: "operator-ns", + logger: zl.Sugar(), + recorder: record.NewFakeRecorder(10), + lc: lc, + } + + // Test 1: Default tags + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"my-svc.tailnetxyz.ts.net"}}, + }, + }, + } + mustCreate(t, fc, ing) + + // Verify initial reconciliation + expectReconciled(t, ingPGR, "default", "test-ingress") + + // Get and verify the ConfigMap was updated + cm := &corev1.ConfigMap{} + if err := fc.Get(context.Background(), types.NamespacedName{ + Name: "test-pg-ingress-config", + Namespace: "operator-ns", + }, cm); err != nil { + t.Fatalf("getting ConfigMap: %v", err) + } + + cfg := &ipn.ServeConfig{} + if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil { + t.Fatalf("unmarshaling serve config: %v", err) + } + + if cfg.Services["my-svc"] == nil { + t.Error("expected serve config to contain VIPService configuration") + } + + // Verify VIPService uses default tags + vipSvc, err := ft.getVIPServiceByName(context.Background(), "my-svc") + if err != nil { + t.Fatalf("getting VIPService: %v", err) + } + if vipSvc == nil { + t.Fatal("VIPService not created") + } + wantTags := []string{"tag:k8s"} // default tags + if !slices.Equal(vipSvc.Tags, wantTags) { + t.Errorf("incorrect VIPService tags: got %v, want %v", vipSvc.Tags, wantTags) + } + + // Test 2: Custom tags + mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { + ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test" + }) + expectReconciled(t, ingPGR, "default", "test-ingress") + + // Verify VIPService uses custom tags + vipSvc, err = ft.getVIPServiceByName(context.Background(), "my-svc") + if err != nil { + t.Fatalf("getting VIPService: %v", err) + } + if vipSvc == nil { + t.Fatal("VIPService not created") + } + wantTags = []string{"tag:custom", "tag:test"} // custom tags only + gotTags := slices.Clone(vipSvc.Tags) + slices.Sort(gotTags) + slices.Sort(wantTags) + if !slices.Equal(gotTags, wantTags) { + t.Errorf("incorrect VIPService tags: got %v, want %v", gotTags, wantTags) + } + + // Delete the Ingress and verify cleanup + if err := fc.Delete(context.Background(), ing); err != nil { + t.Fatalf("deleting Ingress: %v", err) + } + + expectReconciled(t, ingPGR, "default", "test-ingress") + + // Verify the ConfigMap was cleaned up + cm = &corev1.ConfigMap{} + if err := fc.Get(context.Background(), types.NamespacedName{ + Name: "test-pg-ingress-config", + Namespace: "operator-ns", + }, cm); err != nil { + t.Fatalf("getting ConfigMap: %v", err) + } + + cfg = &ipn.ServeConfig{} + if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil { + t.Fatalf("unmarshaling serve config: %v", err) + } + + if len(cfg.Services) > 0 { + t.Error("serve config not cleaned up") + } +} + +func TestValidateIngress(t *testing.T) { + baseIngress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + }, + } + + readyProxyGroup := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg", + Generation: 1, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + Status: tsapi.ProxyGroupStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ProxyGroupReady), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + }, + }, + }, + } + + tests := []struct { + name string + ing *networkingv1.Ingress + pg *tsapi.ProxyGroup + wantErr string + }{ + { + name: "valid_ingress_with_hostname", + ing: &networkingv1.Ingress{ + ObjectMeta: baseIngress.ObjectMeta, + Spec: networkingv1.IngressSpec{ + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"test.example.com"}}, + }, + }, + }, + pg: readyProxyGroup, + }, + { + name: "valid_ingress_with_default_hostname", + ing: baseIngress, + pg: readyProxyGroup, + }, + { + name: "invalid_tags", + ing: &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: baseIngress.Name, + Namespace: baseIngress.Namespace, + Annotations: map[string]string{ + AnnotationTags: "tag:invalid!", + }, + }, + }, + pg: readyProxyGroup, + wantErr: "tailscale.com/tags annotation contains invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes", + }, + { + name: "multiple_TLS_entries", + ing: &networkingv1.Ingress{ + ObjectMeta: baseIngress.ObjectMeta, + Spec: networkingv1.IngressSpec{ + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"test1.example.com"}}, + {Hosts: []string{"test2.example.com"}}, + }, + }, + }, + pg: readyProxyGroup, + wantErr: "Ingress contains invalid TLS block [{[test1.example.com] } {[test2.example.com] }]: only a single TLS entry with a single host is allowed", + }, + { + name: "multiple_hosts_in_TLS_entry", + ing: &networkingv1.Ingress{ + ObjectMeta: baseIngress.ObjectMeta, + Spec: networkingv1.IngressSpec{ + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"test1.example.com", "test2.example.com"}}, + }, + }, + }, + pg: readyProxyGroup, + wantErr: "Ingress contains invalid TLS block [{[test1.example.com test2.example.com] }]: only a single TLS entry with a single host is allowed", + }, + { + name: "wrong_proxy_group_type", + ing: baseIngress, + pg: &tsapi.ProxyGroup{ + ObjectMeta: readyProxyGroup.ObjectMeta, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupType("foo"), + }, + Status: readyProxyGroup.Status, + }, + wantErr: "ProxyGroup \"test-pg\" is of type \"foo\" but must be of type \"ingress\"", + }, + { + name: "proxy_group_not_ready", + ing: baseIngress, + pg: &tsapi.ProxyGroup{ + ObjectMeta: readyProxyGroup.ObjectMeta, + Spec: readyProxyGroup.Spec, + Status: tsapi.ProxyGroupStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ProxyGroupReady), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + }, + }, + }, + }, + wantErr: "ProxyGroup \"test-pg\" is not ready", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &IngressPGReconciler{} + err := r.validateIngress(tt.ing, tt.pg) + if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) { + t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 3eb47dfb00ad3..7cadaecc406c8 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -26,6 +26,7 @@ import ( "tailscale.com/kube/kubetypes" "tailscale.com/types/opt" "tailscale.com/util/clientmetric" + "tailscale.com/util/mak" "tailscale.com/util/set" ) @@ -58,7 +59,7 @@ var ( ) func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { - logger := a.logger.With("ingress-ns", req.Namespace, "ingress-name", req.Name) + logger := a.logger.With("Ingress", req.NamespacedName) logger.Debugf("starting reconcile") defer logger.Debugf("reconcile finished") @@ -128,9 +129,8 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare // This function adds a finalizer to ing, ensuring that we can handle orderly // deprovisioning later. func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error { - if err := a.validateIngressClass(ctx); err != nil { + if err := validateIngressClass(ctx, a.Client); err != nil { logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err) - } if !slices.Contains(ing.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, @@ -159,7 +159,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga gaugeIngressResources.Set(int64(a.managedIngresses.Len())) a.mu.Unlock() - if !a.ssr.IsHTTPSEnabledOnTailnet() { + if !IsHTTPSEnabledOnTailnet(a.ssr.tsnetServer) { a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work") } @@ -185,73 +185,16 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga } web := sc.Web[magic443] - addIngressBackend := func(b *networkingv1.IngressBackend, path string) { - if b == nil { - return - } - if b.Service == nil { - a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path) - return - } - var svc corev1.Service - if err := a.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil { - a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err) - return - } - if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" { - a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path) - return - } - var port int32 - if b.Service.Port.Name != "" { - for _, p := range svc.Spec.Ports { - if p.Name == b.Service.Port.Name { - port = p.Port - break - } - } - } else { - port = b.Service.Port.Number - } - if port == 0 { - a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path) - return - } - proto := "http://" - if port == 443 || b.Service.Port.Name == "https" { - proto = "https+insecure://" - } - web.Handlers[path] = &ipn.HTTPHandler{ - Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path, - } - } - addIngressBackend(ing.Spec.DefaultBackend, "/") var tlsHost string // hostname or FQDN or empty if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { tlsHost = ing.Spec.TLS[0].Hosts[0] } - for _, rule := range ing.Spec.Rules { - // Host is optional, but if it's present it must match the TLS host - // otherwise we ignore the rule. - if rule.Host != "" && rule.Host != tlsHost { - a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host) - continue - } - for _, p := range rule.HTTP.Paths { - // Send a warning if folks use Exact path type - to make - // it easier for us to support Exact path type matching - // in the future if needed. - // https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types - if *p.PathType == networkingv1.PathTypeExact { - msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future." - logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg)) - a.recorder.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg) - } - addIngressBackend(&p.Backend, p.Path) - } + handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, tlsHost, logger) + if err != nil { + return fmt.Errorf("failed to get handlers for ingress: %w", err) } - + web.Handlers = handlers if len(web.Handlers) == 0 { logger.Warn("Ingress contains no valid backends") a.recorder.Eventf(ing, corev1.EventTypeWarning, "NoValidBackends", "no valid backends") @@ -263,10 +206,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga if tstr, ok := ing.Annotations[AnnotationTags]; ok { tags = strings.Split(tstr, ",") } - hostname := ing.Namespace + "-" + ing.Name + "-ingress" - if tlsHost != "" { - hostname, _, _ = strings.Cut(tlsHost, ".") - } + hostname := hostnameForIngress(ing) sts := &tailscaleSTSConfig{ Hostname: hostname, @@ -322,28 +262,106 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { return ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == tailscaleIngressClassName + *ing.Spec.IngressClassName == tailscaleIngressClassName && + ing.Annotations[AnnotationProxyGroup] == "" } // validateIngressClass attempts to validate that 'tailscale' IngressClass // included in Tailscale installation manifests exists and has not been modified // to attempt to enable features that we do not support. -func (a *IngressReconciler) validateIngressClass(ctx context.Context) error { +func validateIngressClass(ctx context.Context, cl client.Client) error { ic := &networkingv1.IngressClass{ ObjectMeta: metav1.ObjectMeta{ Name: tailscaleIngressClassName, }, } - if err := a.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) { - return errors.New("Tailscale IngressClass not found in cluster. Latest installation manifests include a tailscale IngressClass - please update") + if err := cl.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) { + return errors.New("'tailscale' IngressClass not found in cluster.") } else if err != nil { return fmt.Errorf("error retrieving 'tailscale' IngressClass: %w", err) } if ic.Spec.Controller != tailscaleIngressControllerName { - return fmt.Errorf("Tailscale Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName) + return fmt.Errorf("'tailscale' Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName) } if ic.GetAnnotations()[ingressClassDefaultAnnotation] != "" { return fmt.Errorf("%s annotation is set on 'tailscale' IngressClass, but Tailscale Ingress controller does not support default Ingress class. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ingressClassDefaultAnnotation) } return nil } + +func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl client.Client, rec record.EventRecorder, tlsHost string, logger *zap.SugaredLogger) (handlers map[string]*ipn.HTTPHandler, err error) { + addIngressBackend := func(b *networkingv1.IngressBackend, path string) { + if b == nil { + return + } + if b.Service == nil { + rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path) + return + } + var svc corev1.Service + if err := cl.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil { + rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err) + return + } + if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" { + rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path) + return + } + var port int32 + if b.Service.Port.Name != "" { + for _, p := range svc.Spec.Ports { + if p.Name == b.Service.Port.Name { + port = p.Port + break + } + } + } else { + port = b.Service.Port.Number + } + if port == 0 { + rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path) + return + } + proto := "http://" + if port == 443 || b.Service.Port.Name == "https" { + proto = "https+insecure://" + } + mak.Set(&handlers, path, &ipn.HTTPHandler{ + Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path, + }) + } + addIngressBackend(ing.Spec.DefaultBackend, "/") + for _, rule := range ing.Spec.Rules { + // Host is optional, but if it's present it must match the TLS host + // otherwise we ignore the rule. + if rule.Host != "" && rule.Host != tlsHost { + rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host) + continue + } + for _, p := range rule.HTTP.Paths { + // Send a warning if folks use Exact path type - to make + // it easier for us to support Exact path type matching + // in the future if needed. + // https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types + if *p.PathType == networkingv1.PathTypeExact { + msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future." + logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg)) + rec.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg) + } + addIngressBackend(&p.Backend, p.Path) + } + } + return handlers, nil +} + +// hostnameForIngress returns the hostname for an Ingress resource. +// If the Ingress has TLS configured with a host, it returns the first component of that host. +// Otherwise, it returns a hostname derived from the Ingress name and namespace. +func hostnameForIngress(ing *networkingv1.Ingress) string { + if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { + h := ing.Spec.TLS[0].Hosts[0] + hostname, _, _ := strings.Cut(h, ".") + return hostname + } + return ing.Namespace + "-" + ing.Name + "-ingress" +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 7f8f94673e837..6368698d87638 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -18,7 +18,6 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "golang.org/x/oauth2/clientcredentials" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -107,14 +106,14 @@ func main() { hostinfo.SetApp(kubetypes.AppAPIServerProxy) } - s, tsClient := initTSNet(zlog) + s, tsc := initTSNet(zlog) defer s.Close() restConfig := config.GetConfigOrDie() maybeLaunchAPIServerProxy(zlog, restConfig, s, mode) rOpts := reconcilerOpts{ log: zlog, tsServer: s, - tsClient: tsClient, + tsClient: tsc, tailscaleNamespace: tsNamespace, restConfig: restConfig, proxyImage: image, @@ -130,7 +129,7 @@ func main() { // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the // CLIENT_ID_FILE and CLIENT_SECRET_FILE environment variables to authenticate // with Tailscale. -func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) { +func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { var ( clientIDPath = defaultEnv("CLIENT_ID_FILE", "") clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "") @@ -142,23 +141,10 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) { if clientIDPath == "" || clientSecretPath == "" { startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set") } - clientID, err := os.ReadFile(clientIDPath) + tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath) if err != nil { - startlog.Fatalf("reading client ID %q: %v", clientIDPath, err) + startlog.Fatalf("error creating Tailscale client: %v", err) } - clientSecret, err := os.ReadFile(clientSecretPath) - if err != nil { - startlog.Fatalf("reading client secret %q: %v", clientSecretPath, err) - } - credentials := clientcredentials.Config{ - ClientID: string(clientID), - ClientSecret: string(clientSecret), - TokenURL: "https://login.tailscale.com/api/v2/oauth/token", - } - tsClient := tailscale.NewClient("-", nil) - tsClient.UserAgent = "tailscale-k8s-operator" - tsClient.HTTPClient = credentials.Client(context.Background()) - s := &tsnet.Server{ Hostname: hostname, Logf: zlog.Named("tailscaled").Debugf, @@ -211,7 +197,7 @@ waitOnline: }, }, } - authkey, _, err := tsClient.CreateKey(ctx, caps) + authkey, _, err := tsc.CreateKey(ctx, caps) if err != nil { startlog.Fatalf("creating operator authkey: %v", err) } @@ -235,7 +221,7 @@ waitOnline: } time.Sleep(time.Second) } - return s, tsClient + return s, tsc } // runReconcilers starts the controller-runtime manager and registers the @@ -343,6 +329,27 @@ func runReconcilers(opts reconcilerOpts) { if err != nil { startlog.Fatalf("could not create ingress reconciler: %v", err) } + lc, err := opts.tsServer.LocalClient() + if err != nil { + startlog.Fatalf("could not get local client: %v", err) + } + err = builder. + ControllerManagedBy(mgr). + For(&networkingv1.Ingress{}). + Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))). + Complete(&IngressPGReconciler{ + recorder: eventRecorder, + tsClient: opts.tsClient, + tsnetServer: opts.tsServer, + defaultTags: strings.Split(opts.proxyTags, ","), + Client: mgr.GetClient(), + logger: opts.log.Named("ingress-pg-reconciler"), + lc: lc, + tsNamespace: opts.tailscaleNamespace, + }) + if err != nil { + startlog.Fatalf("could not create ingress-pg-reconciler: %v", err) + } connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector")) // If a ProxyClassChanges, enqueue all Connectors that have @@ -514,6 +521,7 @@ func runReconcilers(opts reconcilerOpts) { err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter). + Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter). Watches(&corev1.Secret{}, ownedByProxyGroupFilter). Watches(&rbacv1.Role{}, ownedByProxyGroupFilter). @@ -545,7 +553,7 @@ func runReconcilers(opts reconcilerOpts) { type reconcilerOpts struct { log *zap.SugaredLogger tsServer *tsnet.Server - tsClient *tailscale.Client + tsClient tsClient tailscaleNamespace string // namespace in which operator resources will be deployed restConfig *rest.Config // config for connecting to the kube API server proxyImage string // : @@ -670,12 +678,6 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c } } -type tsClient interface { - CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error) - Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) - DeleteDevice(ctx context.Context, nodeStableID string) error -} - func isManagedResource(o client.Object) bool { ls := o.GetLabels() return ls[LabelManaged] == "true" @@ -811,6 +813,10 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { return nil } + if hasProxyGroupAnnotation(&ing) { + // We don't want to reconcile backend Services for Ingresses for ProxyGroups. + continue + } if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() { reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) } @@ -1094,3 +1100,44 @@ func indexEgressServices(o client.Object) []string { } return []string{o.GetAnnotations()[AnnotationProxyGroup]} } + +// serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service +// associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation, +// the associated Ingress gets reconciled. +func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + ingList := networkingv1.IngressList{} + if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil { + logger.Debugf("error listing Ingresses: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + for _, ing := range ingList.Items { + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { + continue + } + if !hasProxyGroupAnnotation(&ing) { + continue + } + if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + } + for _, rule := range ing.Spec.Rules { + if rule.HTTP == nil { + continue + } + for _, path := range rule.HTTP.Paths { + if path.Backend.Service != nil && path.Backend.Service.Name == o.GetName() { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + } + } + } + } + return reqs + } +} + +func hasProxyGroupAnnotation(obj client.Object) bool { + ing := obj.(*networkingv1.Ingress) + return ing.Annotations[AnnotationProxyGroup] != "" +} diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index a4befa039a820..f6de31727311e 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -258,7 +258,16 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.Labels = cm.ObjectMeta.Labels existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning ConfigMap: %w", err) + return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err) + } + } + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { + cm := pgIngressCM(pg, r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { + existing.ObjectMeta.Labels = cm.ObjectMeta.Labels + existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences + }); err != nil { + return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode) diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index dc58b9f0e6ff0..556a2ed7690b4 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -56,6 +56,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string } tmpl.Spec.ServiceAccountName = pg.Name tmpl.Spec.InitContainers[0].Image = image + proxyConfigVolName := pgEgressCMName(pg.Name) + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { + proxyConfigVolName = pgIngressCMName(pg.Name) + } tmpl.Spec.Volumes = func() []corev1.Volume { var volumes []corev1.Volume for i := range pgReplicas(pg) { @@ -69,18 +73,16 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string }) } - if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - volumes = append(volumes, corev1.Volume{ - Name: pgEgressCMName(pg.Name), - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: pgEgressCMName(pg.Name), - }, + volumes = append(volumes, corev1.Volume{ + Name: proxyConfigVolName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: proxyConfigVolName, }, }, - }) - } + }, + }) return volumes }() @@ -102,13 +104,11 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string }) } - if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - mounts = append(mounts, corev1.VolumeMount{ - Name: pgEgressCMName(pg.Name), - MountPath: "/etc/proxies", - ReadOnly: true, - }) - } + mounts = append(mounts, corev1.VolumeMount{ + Name: proxyConfigVolName, + MountPath: "/etc/proxies", + ReadOnly: true, + }) return mounts }() @@ -154,11 +154,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Value: kubetypes.AppProxyGroupEgress, }, ) - } else { + } else { // ingress envs = append(envs, corev1.EnvVar{ Name: "TS_INTERNAL_APP", Value: kubetypes.AppProxyGroupIngress, - }) + }, + corev1.EnvVar{ + Name: "TS_SERVE_CONFIG", + Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey), + }) } return append(c.Env, envs...) }() @@ -264,6 +268,16 @@ func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { }, } } +func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgIngressCMName(pg.Name), + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + } +} func pgSecretLabels(pgName, typ string) map[string]string { return pgLabels(pgName, map[string]string{ diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index c920c90d1b69b..e7c85d3871d5d 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -332,7 +332,8 @@ func TestProxyGroupTypes(t *testing.T) { UID: "test-ingress-uid", }, Spec: tsapi.ProxyGroupSpec{ - Type: tsapi.ProxyGroupTypeIngress, + Type: tsapi.ProxyGroupTypeIngress, + Replicas: ptr.To[int32](0), }, } if err := fc.Create(context.Background(), pg); err != nil { @@ -347,6 +348,34 @@ func TestProxyGroupTypes(t *testing.T) { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) + verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json") + + // Verify ConfigMap volume mount + cmName := fmt.Sprintf("%s-ingress-config", pg.Name) + expectedVolume := corev1.Volume{ + Name: cmName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmName, + }, + }, + }, + } + + expectedVolumeMount := corev1.VolumeMount{ + Name: cmName, + MountPath: "/etc/proxies", + ReadOnly: true, + } + + if diff := cmp.Diff([]corev1.Volume{expectedVolume}, sts.Spec.Template.Spec.Volumes); diff != "" { + t.Errorf("unexpected volumes (-want +got):\n%s", diff) + } + + if diff := cmp.Diff([]corev1.VolumeMount{expectedVolumeMount}, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" { + t.Errorf("unexpected volume mounts (-want +got):\n%s", diff) + } }) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index c2b9250589936..fce6bfdd73b69 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -172,8 +172,8 @@ func (sts tailscaleSTSReconciler) validate() error { } // IsHTTPSEnabledOnTailnet reports whether HTTPS is enabled on the tailnet. -func (a *tailscaleSTSReconciler) IsHTTPSEnabledOnTailnet() bool { - return len(a.tsnetServer.CertDomains()) > 0 +func IsHTTPSEnabledOnTailnet(tsnetServer tsnetServer) bool { + return len(tsnetServer.CertDomains()) > 0 } // Provision ensures that the StatefulSet for the given service is running and diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 240a7df15ef23..160f24ec90fa1 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "net/netip" "reflect" "strings" @@ -29,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/client/tailscale" "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/types/ptr" "tailscale.com/util/mak" @@ -737,6 +739,7 @@ type fakeTSClient struct { sync.Mutex keyRequests []tailscale.KeyCapabilities deleted []string + vipServices map[string]*VIPService } type fakeTSNetServer struct { certDomains []string @@ -842,3 +845,50 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) { } } } + +func (c *fakeTSClient) getVIPServiceByName(ctx context.Context, name string) (*VIPService, error) { + c.Lock() + defer c.Unlock() + if c.vipServices == nil { + return nil, &tailscale.ErrResponse{Status: http.StatusNotFound} + } + svc, ok := c.vipServices[name] + if !ok { + return nil, &tailscale.ErrResponse{Status: http.StatusNotFound} + } + return svc, nil +} + +func (c *fakeTSClient) createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error { + c.Lock() + defer c.Unlock() + if c.vipServices == nil { + c.vipServices = make(map[string]*VIPService) + } + c.vipServices[svc.Name] = svc + return nil +} + +func (c *fakeTSClient) deleteVIPServiceByName(ctx context.Context, name string) error { + c.Lock() + defer c.Unlock() + if c.vipServices != nil { + delete(c.vipServices, name) + } + return nil +} + +type fakeLocalClient struct { + status *ipnstate.Status +} + +func (f *fakeLocalClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { + if f.status == nil { + return &ipnstate.Status{ + Self: &ipnstate.PeerStatus{ + DNSName: "test-node.test.ts.net.", + }, + }, nil + } + return f.status, nil +} diff --git a/cmd/k8s-operator/tsclient.go b/cmd/k8s-operator/tsclient.go new file mode 100644 index 0000000000000..5352629de476a --- /dev/null +++ b/cmd/k8s-operator/tsclient.go @@ -0,0 +1,185 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + + "golang.org/x/oauth2/clientcredentials" + "tailscale.com/client/tailscale" + "tailscale.com/util/httpm" +) + +// defaultTailnet is a value that can be used in Tailscale API calls instead of tailnet name to indicate that the API +// call should be performed on the default tailnet for the provided credentials. +const ( + defaultTailnet = "-" + defaultBaseURL = "https://api.tailscale.com" +) + +func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (tsClient, error) { + clientID, err := os.ReadFile(clientIDPath) + if err != nil { + return nil, fmt.Errorf("error reading client ID %q: %w", clientIDPath, err) + } + clientSecret, err := os.ReadFile(clientSecretPath) + if err != nil { + return nil, fmt.Errorf("reading client secret %q: %w", clientSecretPath, err) + } + credentials := clientcredentials.Config{ + ClientID: string(clientID), + ClientSecret: string(clientSecret), + TokenURL: "https://login.tailscale.com/api/v2/oauth/token", + } + c := tailscale.NewClient(defaultTailnet, nil) + c.UserAgent = "tailscale-k8s-operator" + c.HTTPClient = credentials.Client(ctx) + tsc := &tsClientImpl{ + Client: c, + baseURL: defaultBaseURL, + tailnet: defaultTailnet, + } + return tsc, nil +} + +type tsClient interface { + CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error) + Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) + DeleteDevice(ctx context.Context, nodeStableID string) error + getVIPServiceByName(ctx context.Context, name string) (*VIPService, error) + createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error + deleteVIPServiceByName(ctx context.Context, name string) error +} + +type tsClientImpl struct { + *tailscale.Client + baseURL string + tailnet string +} + +// VIPService is a Tailscale VIPService with Tailscale API JSON representation. +type VIPService struct { + // Name is the leftmost label of the DNS name of the VIP service. + // Name is required. + Name string `json:"name,omitempty"` + // Addrs are the IP addresses of the VIP Service. There are two addresses: + // the first is IPv4 and the second is IPv6. + // When creating a new VIP Service, the IP addresses are optional: if no + // addresses are specified then they will be selected. If an IPv4 address is + // specified at index 0, then that address will attempt to be used. An IPv6 + // address can not be specified upon creation. + Addrs []string `json:"addrs,omitempty"` + // Comment is an optional text string for display in the admin panel. + Comment string `json:"comment,omitempty"` + // Ports are the ports of a VIPService that will be configured via Tailscale serve config. + // If set, any node wishing to advertise this VIPService must have this port configured via Tailscale serve. + Ports []string `json:"ports,omitempty"` + // Tags are optional ACL tags that will be applied to the VIPService. + Tags []string `json:"tags,omitempty"` +} + +// GetVIPServiceByName retrieves a VIPService by its name. It returns 404 if the VIPService is not found. +func (c *tsClientImpl) getVIPServiceByName(ctx context.Context, name string) (*VIPService, error) { + path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(name)) + req, err := http.NewRequestWithContext(ctx, httpm.GET, path, nil) + if err != nil { + return nil, fmt.Errorf("error creating new HTTP request: %w", err) + } + b, resp, err := c.sendRequest(req) + if err != nil { + return nil, fmt.Errorf("error making Tailsale API request: %w", err) + } + // If status code was not successful, return the error. + // TODO: Change the check for the StatusCode to include other 2XX success codes. + if resp.StatusCode != http.StatusOK { + return nil, handleErrorResponse(b, resp) + } + svc := &VIPService{} + if err := json.Unmarshal(b, svc); err != nil { + return nil, err + } + return svc, nil +} + +// CreateOrUpdateVIPServiceByName creates or updates a VIPService by its name. Caller must ensure that, if the +// VIPService already exists, the VIPService is fetched first to ensure that any auto-allocated IP addresses are not +// lost during the update. If the VIPService was created without any IP addresses explicitly set (so that they were +// auto-allocated by Tailscale) any subsequent request to this function that does not set any IP addresses will error. +func (c *tsClientImpl) createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error { + data, err := json.Marshal(svc) + if err != nil { + return err + } + path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(svc.Name)) + req, err := http.NewRequestWithContext(ctx, httpm.PUT, path, bytes.NewBuffer(data)) + if err != nil { + return fmt.Errorf("error creating new HTTP request: %w", err) + } + b, resp, err := c.sendRequest(req) + if err != nil { + return fmt.Errorf("error making Tailscale API request: %w", err) + } + // If status code was not successful, return the error. + // TODO: Change the check for the StatusCode to include other 2XX success codes. + if resp.StatusCode != http.StatusOK { + return handleErrorResponse(b, resp) + } + return nil +} + +// DeleteVIPServiceByName deletes a VIPService by its name. It returns an error if the VIPService +// does not exist or if the deletion fails. +func (c *tsClientImpl) deleteVIPServiceByName(ctx context.Context, name string) error { + path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(name)) + req, err := http.NewRequestWithContext(ctx, httpm.DELETE, path, nil) + if err != nil { + return fmt.Errorf("error creating new HTTP request: %w", err) + } + b, resp, err := c.sendRequest(req) + if err != nil { + return fmt.Errorf("error making Tailscale API request: %w", err) + } + // If status code was not successful, return the error. + if resp.StatusCode != http.StatusOK { + return handleErrorResponse(b, resp) + } + return nil +} + +// sendRequest add the authentication key to the request and sends it. It +// receives the response and reads up to 10MB of it. +func (c *tsClientImpl) sendRequest(req *http.Request) ([]byte, *http.Response, error) { + resp, err := c.Do(req) + if err != nil { + return nil, resp, fmt.Errorf("error actually doing request: %w", err) + } + defer resp.Body.Close() + + // Read response + b, err := io.ReadAll(resp.Body) + if err != nil { + err = fmt.Errorf("error reading response body: %v", err) + } + return b, resp, err +} + +// handleErrorResponse decodes the error message from the server and returns +// an ErrResponse from it. +func handleErrorResponse(b []byte, resp *http.Response) error { + var errResp tailscale.ErrResponse + if err := json.Unmarshal(b, &errResp); err != nil { + return err + } + errResp.Status = resp.StatusCode + return errResp +} diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 3c97d8c7da2c5..afc4890188ea8 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -15,8 +15,9 @@ const ( AppProxyGroupIngress = "k8s-operator-proxygroup-ingress" // Clientmetrics for Tailscale Kubernetes Operator components - MetricIngressProxyCount = "k8s_ingress_proxies" // L3 - MetricIngressResourceCount = "k8s_ingress_resources" // L7 + MetricIngressProxyCount = "k8s_ingress_proxies" // L3 + MetricIngressResourceCount = "k8s_ingress_resources" // L7 + MetricIngressPGResourceCount = "k8s_ingress_pg_resources" // L7 on ProxyGroup MetricEgressProxyCount = "k8s_egress_proxies" MetricConnectorResourceCount = "k8s_connector_resources" MetricConnectorWithSubnetRouterCount = "k8s_connector_subnetrouter_resources" From bcc262269f058923fe7e88c577aabaf212bafde1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 22:24:13 -0700 Subject: [PATCH 149/223] build(deps): bump braces from 3.0.2 to 3.0.3 in /cmd/tsconnect (#12468) Bumps [braces](https://github.com/micromatch/braces) from 3.0.2 to 3.0.3. - [Changelog](https://github.com/micromatch/braces/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/braces/compare/3.0.2...3.0.3) --- updated-dependencies: - dependency-name: braces dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cmd/tsconnect/yarn.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/tsconnect/yarn.lock b/cmd/tsconnect/yarn.lock index 811eddeb7f5c1..d9d9db32f66a0 100644 --- a/cmd/tsconnect/yarn.lock +++ b/cmd/tsconnect/yarn.lock @@ -90,11 +90,11 @@ binary-extensions@^2.0.0: integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== braces@^3.0.2, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" camelcase-css@^2.0.1: version "2.0.1" @@ -231,10 +231,10 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" From 51adaec35a3e4d25df88d81e6264584e151bd33d Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 21 Jan 2025 08:02:24 -0800 Subject: [PATCH 150/223] Revert "ipn/ipnlocal: re-advertise appc routes on startup (#14609)" This reverts commit 1b303ee5baef3ddab40be4d1c2 (#14609). It caused a deadlock; see tailscale/corp#25965 Updates tailscale/corp#25965 Updates #13680 Updates #14606 --- ipn/ipnlocal/local.go | 35 +++------------------------- ipn/ipnlocal/local_test.go | 47 -------------------------------------- 2 files changed, 3 insertions(+), 79 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 214d3a4e478a1..bb84012fd2625 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4356,33 +4356,6 @@ func (b *LocalBackend) reconfigAppConnectorLocked(nm *netmap.NetworkMap, prefs i b.appConnector.UpdateDomainsAndRoutes(domains, routes) } -func (b *LocalBackend) readvertiseAppConnectorRoutes() { - var domainRoutes map[string][]netip.Addr - b.mu.Lock() - if b.appConnector != nil { - domainRoutes = b.appConnector.DomainRoutes() - } - b.mu.Unlock() - if domainRoutes == nil { - return - } - - // Re-advertise the stored routes, in case stored state got out of - // sync with previously advertised routes in prefs. - var prefixes []netip.Prefix - for _, ips := range domainRoutes { - for _, ip := range ips { - prefixes = append(prefixes, netip.PrefixFrom(ip, ip.BitLen())) - } - } - // Note: AdvertiseRoute will trim routes that are already - // advertised, so if everything is already being advertised this is - // a noop. - if err := b.AdvertiseRoute(prefixes...); err != nil { - b.logf("error advertising stored app connector routes: %v", err) - } -} - // authReconfig pushes a new configuration into wgengine, if engine // updates are not currently blocked, based on the cached netmap and // user prefs. @@ -4461,7 +4434,6 @@ func (b *LocalBackend) authReconfig() { } b.initPeerAPIListener() - b.readvertiseAppConnectorRoutes() } // shouldUseOneCGNATRoute reports whether we should prefer to make one big @@ -7204,7 +7176,7 @@ var ErrDisallowedAutoRoute = errors.New("route is not allowed") // If the route is disallowed, ErrDisallowedAutoRoute is returned. func (b *LocalBackend) AdvertiseRoute(ipps ...netip.Prefix) error { finalRoutes := b.Prefs().AdvertiseRoutes().AsSlice() - var newRoutes []netip.Prefix + newRoutes := false for _, ipp := range ipps { if !allowedAutoRoute(ipp) { @@ -7220,14 +7192,13 @@ func (b *LocalBackend) AdvertiseRoute(ipps ...netip.Prefix) error { } finalRoutes = append(finalRoutes, ipp) - newRoutes = append(newRoutes, ipp) + newRoutes = true } - if len(newRoutes) == 0 { + if !newRoutes { return nil } - b.logf("advertising new app connector routes: %v", newRoutes) _, err := b.EditPrefs(&ipn.MaskedPrefs{ Prefs: ipn.Prefs{ AdvertiseRoutes: finalRoutes, diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 348bdcab37e68..415791c60ab68 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1501,53 +1501,6 @@ func TestReconfigureAppConnector(t *testing.T) { } } -func TestBackfillAppConnectorRoutes(t *testing.T) { - // Create backend with an empty app connector. - b := newTestBackend(t) - if err := b.Start(ipn.Options{}); err != nil { - t.Fatal(err) - } - if _, err := b.EditPrefs(&ipn.MaskedPrefs{ - Prefs: ipn.Prefs{ - AppConnector: ipn.AppConnectorPrefs{Advertise: true}, - }, - AppConnectorSet: true, - }); err != nil { - t.Fatal(err) - } - b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs) - - // Smoke check that AdvertiseRoutes doesn't have the test IP. - ip := netip.MustParseAddr("1.2.3.4") - routes := b.Prefs().AdvertiseRoutes().AsSlice() - if slices.Contains(routes, netip.PrefixFrom(ip, ip.BitLen())) { - t.Fatalf("AdvertiseRoutes %v on a fresh backend already contains advertised route for %v", routes, ip) - } - - // Store the test IP in profile data, but not in Prefs.AdvertiseRoutes. - b.ControlKnobs().AppCStoreRoutes.Store(true) - if err := b.storeRouteInfo(&appc.RouteInfo{ - Domains: map[string][]netip.Addr{ - "example.com": {ip}, - }, - }); err != nil { - t.Fatal(err) - } - - // Mimic b.authReconfigure for the app connector bits. - b.mu.Lock() - b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs) - b.mu.Unlock() - b.readvertiseAppConnectorRoutes() - - // Check that Prefs.AdvertiseRoutes got backfilled with routes stored in - // profile data. - routes = b.Prefs().AdvertiseRoutes().AsSlice() - if !slices.Contains(routes, netip.PrefixFrom(ip, ip.BitLen())) { - t.Fatalf("AdvertiseRoutes %v was not backfilled from stored app connector routes with %v", routes, ip) - } -} - func resolversEqual(t *testing.T, a, b []*dnstype.Resolver) bool { if a == nil && b == nil { return true From 7f3c1932b54fb6af2d8d1e367e0e456ff7fa40fd Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 21 Jan 2025 10:23:58 -0800 Subject: [PATCH 151/223] tsnet: fix panic on race between listener.Close and incoming packet I saw this panic while writing a new test for #14715: panic: send on closed channel goroutine 826 [running]: tailscale.com/tsnet.(*listener).handle(0x1400031a500, {0x1035fbb00, 0x14000b82300}) /Users/bradfitz/src/tailscale.com/tsnet/tsnet.go:1317 +0xac tailscale.com/wgengine/netstack.(*Impl).acceptTCP(0x14000204700, 0x14000882100) /Users/bradfitz/src/tailscale.com/wgengine/netstack/netstack.go:1320 +0x6dc created by gvisor.dev/gvisor/pkg/tcpip/transport/tcp.(*Forwarder).HandlePacket in goroutine 807 /Users/bradfitz/go/pkg/mod/gvisor.dev/gvisor@v0.0.0-20240722211153-64c016c92987/pkg/tcpip/transport/tcp/forwarder.go:98 +0x32c FAIL tailscale.com/tsnet 0.927s Updates #14715 Change-Id: I9924e0a6c2b801d46ee44eb8eeea0da2f9ea17c4 Signed-off-by: Brad Fitzpatrick --- tsnet/tsnet.go | 25 ++++++++++++++----------- tsnet/tsnet_test.go | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 5f1d8073ae5b9..fd894c38a4f6a 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -1180,7 +1180,8 @@ func (s *Server) listen(network, addr string, lnOn listenOn) (net.Listener, erro keys: keys, addr: addr, - conn: make(chan net.Conn), + closedc: make(chan struct{}), + conn: make(chan net.Conn), } s.mu.Lock() for _, key := range keys { @@ -1243,11 +1244,12 @@ type listenKey struct { } type listener struct { - s *Server - keys []listenKey - addr string - conn chan net.Conn - closed bool // guarded by s.mu + s *Server + keys []listenKey + addr string + conn chan net.Conn // unbuffered, never closed + closedc chan struct{} // closed on [listener.Close] + closed bool // guarded by s.mu } func (ln *listener) Accept() (net.Conn, error) { @@ -1277,21 +1279,22 @@ func (ln *listener) closeLocked() error { delete(ln.s.listeners, key) } } - close(ln.conn) + close(ln.closedc) ln.closed = true return nil } func (ln *listener) handle(c net.Conn) { - t := time.NewTimer(time.Second) - defer t.Stop() select { case ln.conn <- c: - case <-t.C: + return + case <-ln.closedc: + case <-ln.s.shutdownCtx.Done(): + case <-time.After(time.Second): // TODO(bradfitz): this isn't ideal. Think about how // we how we want to do pushback. - c.Close() } + c.Close() } // Server returns the tsnet Server associated with the listener. diff --git a/tsnet/tsnet_test.go b/tsnet/tsnet_test.go index 14d600817ad70..c2f27d0f3df94 100644 --- a/tsnet/tsnet_test.go +++ b/tsnet/tsnet_test.go @@ -494,6 +494,25 @@ func TestListenerCleanup(t *testing.T) { if err := ln.Close(); !errors.Is(err, net.ErrClosed) { t.Fatalf("second ln.Close error: %v, want net.ErrClosed", err) } + + // Verify that handling a connection from gVisor (from a packet arriving) + // after a listener closed doesn't panic (previously: sending on a closed + // channel) or hang. + c := &closeTrackConn{} + ln.(*listener).handle(c) + if !c.closed { + t.Errorf("c.closed = false, want true") + } +} + +type closeTrackConn struct { + net.Conn + closed bool +} + +func (wc *closeTrackConn) Close() error { + wc.closed = true + return nil } // tests https://github.com/tailscale/tailscale/issues/6973 -- that we can start a tsnet server, From 27299426389e7fb7eab54994596cae09c0e11ec9 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Sat, 18 Jan 2025 20:41:36 -0600 Subject: [PATCH 152/223] prober: fix nil pointer access in tcp-in-tcp probes If unable to accept a connection from the bandwidth probe listener, return from the goroutine immediately since the accepted connection will be nil. Updates tailscale/corp#25958 Signed-off-by: Percy Wegmann --- prober/derp.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prober/derp.go b/prober/derp.go index 995a69626fc14..05cc8f05cbfe2 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -1048,6 +1048,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT readConn, err := l.Accept() if err != nil { readFinishedC <- err + return } defer readConn.Close() deadline, ok := ctx.Deadline() @@ -1055,6 +1056,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT // Don't try reading past our context's deadline. if err := readConn.SetReadDeadline(deadline); err != nil { readFinishedC <- fmt.Errorf("unable to set read deadline: %w", err) + return } } n, err := io.CopyN(io.Discard, readConn, size) From b50d32059f1b33311dbba96a57c82d33a28f0e1f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 21 Jan 2025 09:50:45 -0800 Subject: [PATCH 153/223] tsnet: block in Server.Dial until backend is Running Updates #14715 Change-Id: I8c91e94fd1c6278c7f94a6b890274ed8a01e6f25 Signed-off-by: Brad Fitzpatrick --- tsnet/tsnet.go | 32 ++++++++++++++++++++++++++++++++ tsnet/tsnet_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index fd894c38a4f6a..b769e719cbabd 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -169,9 +169,41 @@ func (s *Server) Dial(ctx context.Context, network, address string) (net.Conn, e if err := s.Start(); err != nil { return nil, err } + if err := s.awaitRunning(ctx); err != nil { + return nil, err + } return s.dialer.UserDial(ctx, network, address) } +// awaitRunning waits until the backend is in state Running. +// If the backend is in state Starting, it blocks until it reaches +// a terminal state (such as Stopped, NeedsMachineAuth) +// or the context expires. +func (s *Server) awaitRunning(ctx context.Context) error { + st := s.lb.State() + for { + if err := ctx.Err(); err != nil { + return err + } + switch st { + case ipn.Running: + return nil + case ipn.NeedsLogin, ipn.Starting: + // Even after LocalBackend.Start, the state machine is still briefly + // in the "NeedsLogin" state. So treat that as also "Starting" and + // wait for us to get out of that state. + s.lb.WatchNotifications(ctx, ipn.NotifyInitialState, nil, func(n *ipn.Notify) (keepGoing bool) { + if n.State != nil { + st = *n.State + } + return st == ipn.NeedsLogin || st == ipn.Starting + }) + default: + return fmt.Errorf("tsnet: backend in state %v", st) + } + } +} + // HTTPClient returns an HTTP client that is configured to connect over Tailscale. // // This is useful if you need to have your tsnet services connect to other devices on diff --git a/tsnet/tsnet_test.go b/tsnet/tsnet_test.go index c2f27d0f3df94..552e8dbee390a 100644 --- a/tsnet/tsnet_test.go +++ b/tsnet/tsnet_test.go @@ -232,6 +232,46 @@ func startServer(t *testing.T, ctx context.Context, controlURL, hostname string) return s, status.TailscaleIPs[0], status.Self.PublicKey } +func TestDialBlocks(t *testing.T) { + tstest.ResourceCheck(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + controlURL, _ := startControl(t) + + // Make one tsnet that blocks until it's up. + s1, _, _ := startServer(t, ctx, controlURL, "s1") + + ln, err := s1.Listen("tcp", ":8080") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + // Then make another tsnet node that will only be woken up + // upon the first dial. + tmp := filepath.Join(t.TempDir(), "s2") + os.MkdirAll(tmp, 0755) + s2 := &Server{ + Dir: tmp, + ControlURL: controlURL, + Hostname: "s2", + Store: new(mem.Store), + Ephemeral: true, + getCertForTesting: testCertRoot.getCert, + } + if *verboseNodes { + s2.Logf = log.Printf + } + t.Cleanup(func() { s2.Close() }) + + c, err := s2.Dial(ctx, "tcp", "s1:8080") + if err != nil { + t.Fatal(err) + } + defer c.Close() +} + func TestConn(t *testing.T) { tstest.ResourceCheck(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) From 8b9d5fd6bc9fd1bd13ec77903f2c23d1189c1a7d Mon Sep 17 00:00:00 2001 From: James Tucker Date: Tue, 21 Jan 2025 10:34:28 -0800 Subject: [PATCH 154/223] go.mod: bump github.com/inetaf/tcpproxy Updates tailscale/corp#25169 Signed-off-by: James Tucker --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 92dd1bf6501ed..4265953a4023d 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/goreleaser/nfpm/v2 v2.33.1 github.com/hdevalence/ed25519consensus v0.2.0 github.com/illarion/gonotify/v2 v2.0.3 - github.com/inetaf/tcpproxy v0.0.0-20240214030015-3ce58045626c + github.com/inetaf/tcpproxy v0.0.0-20250121183218-48c7e53d7ac4 github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 github.com/jellydator/ttlcache/v3 v3.1.0 github.com/jsimonetti/rtnetlink v1.4.0 diff --git a/go.sum b/go.sum index 0354c33648139..2623cb6e93667 100644 --- a/go.sum +++ b/go.sum @@ -572,8 +572,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/inetaf/tcpproxy v0.0.0-20240214030015-3ce58045626c h1:gYfYE403/nlrGNYj6BEOs9ucLCAGB9gstlSk92DttTg= -github.com/inetaf/tcpproxy v0.0.0-20240214030015-3ce58045626c/go.mod h1:Di7LXRyUcnvAcLicFhtM9/MlZl/TNgRSDHORM2c6CMI= +github.com/inetaf/tcpproxy v0.0.0-20250121183218-48c7e53d7ac4 h1:5u/LhBmv8Y+BhTTADTuh8ma0DcZ3zzx+GINbMeMG9nM= +github.com/inetaf/tcpproxy v0.0.0-20250121183218-48c7e53d7ac4/go.mod h1:Di7LXRyUcnvAcLicFhtM9/MlZl/TNgRSDHORM2c6CMI= github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 h1:9K06NfxkBh25x56yVhWWlKFE8YpicaSfHwoV8SFbueA= github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= From e12b2a7267afbd8189c7834b840d2fcdb8786d64 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 21 Jan 2025 15:42:12 -0800 Subject: [PATCH 155/223] cmd/tailscale/cli: clean up how optional commands get registered Both @agottardo and I tripped over this today. Updates #cleanup Change-Id: I64380a03bfc952b9887b1512dbcadf26499ff1cd Signed-off-by: Brad Fitzpatrick --- cmd/tailscale/cli/advertise.go | 26 ++++++++++++-------------- cmd/tailscale/cli/cli.go | 10 ++++++++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmd/tailscale/cli/advertise.go b/cmd/tailscale/cli/advertise.go index c9474c4274dd2..00b5024f0d3b8 100644 --- a/cmd/tailscale/cli/advertise.go +++ b/cmd/tailscale/cli/advertise.go @@ -21,23 +21,21 @@ var advertiseArgs struct { // TODO(naman): This flag may move to set.go or serve_v2.go after the WIPCode // envknob is not needed. -var advertiseCmd = &ffcli.Command{ - Name: "advertise", - ShortUsage: "tailscale advertise --services=", - ShortHelp: "Advertise this node as a destination for a service", - Exec: runAdvertise, - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("advertise") - fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")") - return fs - })(), -} - -func maybeAdvertiseCmd() []*ffcli.Command { +func advertiseCmd() *ffcli.Command { if !envknob.UseWIPCode() { return nil } - return []*ffcli.Command{advertiseCmd} + return &ffcli.Command{ + Name: "advertise", + ShortUsage: "tailscale advertise --services=", + ShortHelp: "Advertise this node as a destination for a service", + Exec: runAdvertise, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("advertise") + fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")") + return fs + })(), + } } func runAdvertise(ctx context.Context, args []string) error { diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index 542a2e4644c3a..645859038a1f3 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -25,6 +25,7 @@ import ( "tailscale.com/cmd/tailscale/cli/ffcomplete" "tailscale.com/envknob" "tailscale.com/paths" + "tailscale.com/util/slicesx" "tailscale.com/version/distro" ) @@ -182,7 +183,7 @@ For help on subcommands, add --help after: "tailscale status --help". This CLI is still under active development. Commands and flags will change in the future. `), - Subcommands: append([]*ffcli.Command{ + Subcommands: nonNilCmds( upCmd, downCmd, setCmd, @@ -214,7 +215,8 @@ change in the future. debugCmd, driveCmd, idTokenCmd, - }, maybeAdvertiseCmd()...), + advertiseCmd(), + ), FlagSet: rootfs, Exec: func(ctx context.Context, args []string) error { if len(args) > 0 { @@ -239,6 +241,10 @@ change in the future. return rootCmd } +func nonNilCmds(cmds ...*ffcli.Command) []*ffcli.Command { + return slicesx.Filter(cmds[:0], cmds, func(c *ffcli.Command) bool { return c != nil }) +} + func fatalf(format string, a ...any) { if Fatalf != nil { Fatalf(format, a...) From 150cd30b1d28613b50cebde9f18595ef78a2a803 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 21 Jan 2025 15:30:55 -0800 Subject: [PATCH 156/223] ipn/ipnlocal: also use LetsEncrypt-baked-in roots for cert validation We previously baked in the LetsEncrypt x509 root CA for our tlsdial package. This moves that out into a new "bakedroots" package and is now also shared by ipn/ipnlocal's cert validation code (validCertPEM) that decides whether it's time to fetch a new cert. Otherwise, a machine without LetsEncrypt roots locally in its system roots is unable to use tailscale cert/serve and fetch certs. Fixes #14690 Change-Id: Ic88b3bdaabe25d56b9ff07ada56a27e3f11d7159 Signed-off-by: Brad Fitzpatrick --- cmd/derper/depaware.txt | 1 + cmd/k8s-operator/depaware.txt | 1 + cmd/tailscale/depaware.txt | 1 + cmd/tailscaled/depaware.txt | 1 + ipn/ipnlocal/cert.go | 19 ++++- net/bakedroots/bakedroots.go | 122 ++++++++++++++++++++++++++++++ net/bakedroots/bakedroots_test.go | 15 ++++ net/tlsdial/tlsdial.go | 86 +-------------------- net/tlsdial/tlsdial_test.go | 28 +------ 9 files changed, 164 insertions(+), 110 deletions(-) create mode 100644 net/bakedroots/bakedroots.go create mode 100644 net/bakedroots/bakedroots_test.go diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 498677a49f8da..3a730dd997437 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -99,6 +99,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa tailscale.com/ipn/ipnstate from tailscale.com/client/tailscale+ tailscale.com/kube/kubetypes from tailscale.com/envknob tailscale.com/metrics from tailscale.com/cmd/derper+ + tailscale.com/net/bakedroots from tailscale.com/net/tlsdial tailscale.com/net/dnscache from tailscale.com/derp/derphttp tailscale.com/net/ktimeout from tailscale.com/cmd/derper tailscale.com/net/netaddr from tailscale.com/ipn+ diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 80c9f0c060187..a27e1761d83c5 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -835,6 +835,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/logtail/backoff from tailscale.com/control/controlclient+ tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+ tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ tailscale.com/net/connstats from tailscale.com/net/tstun+ tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+ diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 9ccd6eebd2f4d..774d97d8ef373 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -97,6 +97,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/kube/kubetypes from tailscale.com/envknob tailscale.com/licenses from tailscale.com/client/web+ tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/net/bakedroots from tailscale.com/net/tlsdial tailscale.com/net/captivedetection from tailscale.com/net/netcheck tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback tailscale.com/net/dnscache from tailscale.com/control/controlhttp+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 8af347319ccb1..1fc1b8d70060a 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -286,6 +286,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/logtail/backoff from tailscale.com/cmd/tailscaled+ tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+ tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ tailscale.com/net/connstats from tailscale.com/net/tstun+ tailscale.com/net/dns from tailscale.com/cmd/tailscaled+ diff --git a/ipn/ipnlocal/cert.go b/ipn/ipnlocal/cert.go index d87374bbbcd61..0d92c7cf809b4 100644 --- a/ipn/ipnlocal/cert.go +++ b/ipn/ipnlocal/cert.go @@ -40,6 +40,7 @@ import ( "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store" "tailscale.com/ipn/store/mem" + "tailscale.com/net/bakedroots" "tailscale.com/types/logger" "tailscale.com/util/testenv" "tailscale.com/version" @@ -665,7 +666,7 @@ func acmeClient(cs certStore) (*acme.Client, error) { // validCertPEM reports whether the given certificate is valid for domain at now. // // If roots != nil, it is used instead of the system root pool. This is meant -// to support testing, and production code should pass roots == nil. +// to support testing; production code should pass roots == nil. func validCertPEM(domain string, keyPEM, certPEM []byte, roots *x509.CertPool, now time.Time) bool { if len(keyPEM) == 0 || len(certPEM) == 0 { return false @@ -688,15 +689,29 @@ func validCertPEM(domain string, keyPEM, certPEM []byte, roots *x509.CertPool, n intermediates.AddCert(cert) } } + return validateLeaf(leaf, intermediates, domain, now, roots) +} + +// validateLeaf is a helper for [validCertPEM]. +// +// If called with roots == nil, it will use the system root pool as well as the +// baked-in roots. If non-nil, only those roots are used. +func validateLeaf(leaf *x509.Certificate, intermediates *x509.CertPool, domain string, now time.Time, roots *x509.CertPool) bool { if leaf == nil { return false } - _, err = leaf.Verify(x509.VerifyOptions{ + _, err := leaf.Verify(x509.VerifyOptions{ DNSName: domain, CurrentTime: now, Roots: roots, Intermediates: intermediates, }) + if err != nil && roots == nil { + // If validation failed and they specified nil for roots (meaning to use + // the system roots), then give it another chance to validate using the + // binary's baked-in roots (LetsEncrypt). See tailscale/tailscale#14690. + return validateLeaf(leaf, intermediates, domain, now, bakedroots.Get()) + } return err == nil } diff --git a/net/bakedroots/bakedroots.go b/net/bakedroots/bakedroots.go new file mode 100644 index 0000000000000..f7e4fa21e5799 --- /dev/null +++ b/net/bakedroots/bakedroots.go @@ -0,0 +1,122 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package bakedroots contains WebPKI CA roots we bake into the tailscaled binary, +// lest the system's CA roots be missing them (or entirely empty). +package bakedroots + +import ( + "crypto/x509" + "sync" + + "tailscale.com/util/testenv" +) + +// Get returns the baked-in roots. +// +// As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 root. +func Get() *x509.CertPool { + roots.once.Do(func() { roots.parsePEM([]byte(letsEncryptX1)) }) + return roots.p +} + +// testingTB is a subset of testing.TB needed +// to verify the caller isn't in a parallel test. +type testingTB interface { + // Setenv panics if it's in a parallel test. + Setenv(k, v string) +} + +// ResetForTest resets the cached roots for testing, +// optionally setting them to caPEM if non-nil. +func ResetForTest(tb testingTB, caPEM []byte) { + if !testenv.InTest() { + panic("not in test") + } + tb.Setenv("ASSERT_NOT_PARALLEL_TEST", "1") // panics if tb's Parallel was called + + roots = rootsOnce{} + if caPEM != nil { + roots.once.Do(func() { roots.parsePEM(caPEM) }) + } +} + +var roots rootsOnce + +type rootsOnce struct { + once sync.Once + p *x509.CertPool +} + +func (r *rootsOnce) parsePEM(caPEM []byte) { + p := x509.NewCertPool() + if !p.AppendCertsFromPEM(caPEM) { + panic("bogus PEM") + } + r.p = p +} + +/* +letsEncryptX1 is the LetsEncrypt X1 root: + +Certificate: + + Data: + Version: 3 (0x2) + Serial Number: + 82:10:cf:b0:d2:40:e3:59:44:63:e0:bb:63:82:8b:00 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C = US, O = Internet Security Research Group, CN = ISRG Root X1 + Validity + Not Before: Jun 4 11:04:38 2015 GMT + Not After : Jun 4 11:04:38 2035 GMT + Subject: C = US, O = Internet Security Research Group, CN = ISRG Root X1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + +We bake it into the binary as a fallback verification root, +in case the system we're running on doesn't have it. +(Tailscale runs on some ancient devices.) + +To test that this code is working on Debian/Ubuntu: + +$ sudo mv /usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt{,.old} +$ sudo update-ca-certificates + +Then restart tailscaled. To also test dnsfallback's use of it, nuke +your /etc/resolv.conf and it should still start & run fine. +*/ +const letsEncryptX1 = ` +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 +WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu +ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY +MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc +h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ +0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U +A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW +T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH +B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC +B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv +KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn +OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn +jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw +qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI +rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq +hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL +ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ +3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK +NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 +ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur +TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC +jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc +oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq +4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA +mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d +emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- +` diff --git a/net/bakedroots/bakedroots_test.go b/net/bakedroots/bakedroots_test.go new file mode 100644 index 0000000000000..9aa4366c8e267 --- /dev/null +++ b/net/bakedroots/bakedroots_test.go @@ -0,0 +1,15 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package bakedroots + +import "testing" + +func TestBakedInRoots(t *testing.T) { + ResetForTest(t, nil) + p := Get() + got := p.Subjects() + if len(got) != 1 { + t.Errorf("subjects = %v; want 1", len(got)) + } +} diff --git a/net/tlsdial/tlsdial.go b/net/tlsdial/tlsdial.go index 2a109c790632d..2af87bd0240e1 100644 --- a/net/tlsdial/tlsdial.go +++ b/net/tlsdial/tlsdial.go @@ -27,6 +27,7 @@ import ( "tailscale.com/envknob" "tailscale.com/health" "tailscale.com/hostinfo" + "tailscale.com/net/bakedroots" "tailscale.com/net/tlsdial/blockblame" ) @@ -154,7 +155,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // Always verify with our baked-in Let's Encrypt certificate, // so we can log an informational message. This is useful for // detecting SSL MiTM. - opts.Roots = bakedInRoots() + opts.Roots = bakedroots.Get() _, bakedErr := cs.PeerCertificates[0].Verify(opts) if debug() { log.Printf("tlsdial(bake %q): %v", host, bakedErr) @@ -233,7 +234,7 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) { if errSys == nil { return nil } - opts.Roots = bakedInRoots() + opts.Roots = bakedroots.Get() _, err := certs[0].Verify(opts) if debug() { log.Printf("tlsdial(bake %q/%q): %v", c.ServerName, certDNSName, err) @@ -260,84 +261,3 @@ func NewTransport() *http.Transport { }, } } - -/* -letsEncryptX1 is the LetsEncrypt X1 root: - -Certificate: - - Data: - Version: 3 (0x2) - Serial Number: - 82:10:cf:b0:d2:40:e3:59:44:63:e0:bb:63:82:8b:00 - Signature Algorithm: sha256WithRSAEncryption - Issuer: C = US, O = Internet Security Research Group, CN = ISRG Root X1 - Validity - Not Before: Jun 4 11:04:38 2015 GMT - Not After : Jun 4 11:04:38 2035 GMT - Subject: C = US, O = Internet Security Research Group, CN = ISRG Root X1 - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - RSA Public-Key: (4096 bit) - -We bake it into the binary as a fallback verification root, -in case the system we're running on doesn't have it. -(Tailscale runs on some ancient devices.) - -To test that this code is working on Debian/Ubuntu: - -$ sudo mv /usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt{,.old} -$ sudo update-ca-certificates - -Then restart tailscaled. To also test dnsfallback's use of it, nuke -your /etc/resolv.conf and it should still start & run fine. -*/ -const letsEncryptX1 = ` ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw -TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh -cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 -WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu -ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY -MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc -h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ -0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U -A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW -T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH -B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC -B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv -KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn -OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn -jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw -qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI -rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV -HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq -hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL -ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ -3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK -NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 -ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur -TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC -jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc -oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq -4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA -mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d -emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= ------END CERTIFICATE----- -` - -var bakedInRootsOnce struct { - sync.Once - p *x509.CertPool -} - -func bakedInRoots() *x509.CertPool { - bakedInRootsOnce.Do(func() { - p := x509.NewCertPool() - if !p.AppendCertsFromPEM([]byte(letsEncryptX1)) { - panic("bogus PEM") - } - bakedInRootsOnce.p = p - }) - return bakedInRootsOnce.p -} diff --git a/net/tlsdial/tlsdial_test.go b/net/tlsdial/tlsdial_test.go index 26814ebbd8dc0..6723b82e0d1c9 100644 --- a/net/tlsdial/tlsdial_test.go +++ b/net/tlsdial/tlsdial_test.go @@ -4,37 +4,22 @@ package tlsdial import ( - "crypto/x509" "io" "net" "net/http" "os" "os/exec" "path/filepath" - "reflect" "runtime" "sync/atomic" "testing" "tailscale.com/health" + "tailscale.com/net/bakedroots" ) -func resetOnce() { - rv := reflect.ValueOf(&bakedInRootsOnce).Elem() - rv.Set(reflect.Zero(rv.Type())) -} - -func TestBakedInRoots(t *testing.T) { - resetOnce() - p := bakedInRoots() - got := p.Subjects() - if len(got) != 1 { - t.Errorf("subjects = %v; want 1", len(got)) - } -} - func TestFallbackRootWorks(t *testing.T) { - defer resetOnce() + defer bakedroots.ResetForTest(t, nil) const debug = false if runtime.GOOS != "linux" { @@ -69,14 +54,7 @@ func TestFallbackRootWorks(t *testing.T) { if err != nil { t.Fatal(err) } - resetOnce() - bakedInRootsOnce.Do(func() { - p := x509.NewCertPool() - if !p.AppendCertsFromPEM(caPEM) { - t.Fatal("failed to add") - } - bakedInRootsOnce.p = p - }) + bakedroots.ResetForTest(t, caPEM) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { From 042ed6bf693da7061c37bf62b5d823a7b35ae9b5 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 21 Jan 2025 17:19:42 -0800 Subject: [PATCH 157/223] net/bakedroots: add LetsEncrypt ISRG Root X2 Updates #14690 Change-Id: Ib85e318d48450fc6534f7b0c1d4cc4335de7c0ff Signed-off-by: Brad Fitzpatrick --- net/bakedroots/bakedroots.go | 29 ++++++++++++++++++++++++++++- net/bakedroots/bakedroots_test.go | 23 ++++++++++++++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/net/bakedroots/bakedroots.go b/net/bakedroots/bakedroots.go index f7e4fa21e5799..42e70c0dd2abb 100644 --- a/net/bakedroots/bakedroots.go +++ b/net/bakedroots/bakedroots.go @@ -16,7 +16,12 @@ import ( // // As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 root. func Get() *x509.CertPool { - roots.once.Do(func() { roots.parsePEM([]byte(letsEncryptX1)) }) + roots.once.Do(func() { + roots.parsePEM(append( + []byte(letsEncryptX1), + letsEncryptX2..., + )) + }) return roots.p } @@ -120,3 +125,25 @@ mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= -----END CERTIFICATE----- ` + +// letsEncryptX2 is the ISRG Root X2. +// +// Subject: O = Internet Security Research Group, CN = ISRG Root X2 +// Key type: ECDSA P-384 +// Validity: until 2035-09-04 (generated 2020-09-04) +const letsEncryptX2 = ` +-----BEGIN CERTIFICATE----- +MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw +CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg +R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 +MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT +ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw +EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW ++1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9 +ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T +AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI +zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW +tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 +/q4AaOeMSQ+2b1tbFfLn +-----END CERTIFICATE----- +` diff --git a/net/bakedroots/bakedroots_test.go b/net/bakedroots/bakedroots_test.go index 9aa4366c8e267..8ba502a7827e0 100644 --- a/net/bakedroots/bakedroots_test.go +++ b/net/bakedroots/bakedroots_test.go @@ -3,13 +3,30 @@ package bakedroots -import "testing" +import ( + "slices" + "testing" +) func TestBakedInRoots(t *testing.T) { ResetForTest(t, nil) p := Get() got := p.Subjects() - if len(got) != 1 { - t.Errorf("subjects = %v; want 1", len(got)) + if len(got) != 2 { + t.Errorf("subjects = %v; want 2", len(got)) + } + + // TODO(bradfitz): is there a way to easily make this test prettier without + // writing a DER decoder? I'm not seeing how. + var name []string + for _, der := range got { + name = append(name, string(der)) + } + want := []string{ + "0O1\v0\t\x06\x03U\x04\x06\x13\x02US1)0'\x06\x03U\x04\n\x13 Internet Security Research Group1\x150\x13\x06\x03U\x04\x03\x13\fISRG Root X1", + "0O1\v0\t\x06\x03U\x04\x06\x13\x02US1)0'\x06\x03U\x04\n\x13 Internet Security Research Group1\x150\x13\x06\x03U\x04\x03\x13\fISRG Root X2", + } + if !slices.Equal(name, want) { + t.Errorf("subjects = %q; want %q", name, want) } } From cb3b1a1dcf84b467ec25821efd2faca0cb3af93f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 22 Jan 2025 06:20:14 -0800 Subject: [PATCH 158/223] tsweb: add missing debug pprof endpoints Updates tailscale/corp#26016 Change-Id: I47a5671e881cc092d83c1e992e2271f90afcae7e Signed-off-by: Brad Fitzpatrick --- tsweb/debug.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tsweb/debug.go b/tsweb/debug.go index 6db3f25cf06d5..9e6ce4df416ed 100644 --- a/tsweb/debug.go +++ b/tsweb/debug.go @@ -52,15 +52,15 @@ func Debugger(mux *http.ServeMux) *DebugHandler { ret.KV("Version", version.Long()) ret.Handle("vars", "Metrics (Go)", expvar.Handler()) ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(promvarz.Handler)) + + // pprof.Index serves everything that runtime/pprof.Lookup finds: + // goroutine, threadcreate, heap, allocs, block, mutex ret.Handle("pprof/", "pprof (index)", http.HandlerFunc(pprof.Index)) - // the CPU profile handler is special because it responds - // streamily, unlike every other pprof handler. This means it's - // not made available through pprof.Index the way all the other - // pprof types are, you have to register the CPU profile handler - // separately. Use HandleSilent for that to not pollute the human - // debug list with a link that produces streaming line noise if - // you click it. + // But register the other ones from net/http/pprof directly: + ret.HandleSilent("pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) ret.HandleSilent("pprof/profile", http.HandlerFunc(pprof.Profile)) + ret.HandleSilent("pprof/symbol", http.HandlerFunc(pprof.Symbol)) + ret.HandleSilent("pprof/trace", http.HandlerFunc(pprof.Trace)) ret.URL("/debug/pprof/goroutine?debug=1", "Goroutines (collapsed)") ret.URL("/debug/pprof/goroutine?debug=2", "Goroutines (full)") ret.Handle("gc", "force GC", http.HandlerFunc(gcHandler)) From 8c8750f1b3e69aa3ca5ac0ebd15f3b406818c5d2 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Mon, 20 Jan 2025 12:02:53 -0500 Subject: [PATCH 159/223] ipn/ipnlocal: Support TCP and Web VIP services This commit intend to provide support for TCP and Web VIP services and also allow user to use Tun for VIP services if they want to. The commit includes: 1.Setting TCP intercept function for VIP Services. 2.Update netstack to send packet written from WG to netStack handler for VIP service. 3.Return correct TCP hander for VIP services when netstack acceptTCP. This commit also includes unit tests for if the local backend setServeConfig would set correct TCP intercept function and test if a hander gets returned when getting TCPHandlerForDst. The shouldProcessInbound check is not unit tested since the test result just depends on mocked functions. There should be an integration test to cover shouldProcessInbound and if the returned TCP handler actually does what the serveConfig says. Updates tailscale/corp#24604 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- ipn/ipnlocal/local.go | 105 +++++++++++++-- ipn/ipnlocal/local_test.go | 215 ++++++++++++++++++++++++++++++ ipn/ipnlocal/serve.go | 123 ++++++++++++++++- ipn/ipnlocal/serve_test.go | 197 +++++++++++++++++++++++++++ ipn/serve.go | 54 +++++++- tailcfg/tailcfg.go | 16 +++ types/netmap/IPServiceMappings.go | 19 +++ types/netmap/netmap.go | 48 +++++++ wgengine/netstack/netstack.go | 27 +++- 9 files changed, 783 insertions(+), 21 deletions(-) create mode 100644 types/netmap/IPServiceMappings.go diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index bb84012fd2625..05f56fcbdc80c 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -228,10 +228,11 @@ type LocalBackend struct { // is never called. getTCPHandlerForFunnelFlow func(srcAddr netip.AddrPort, dstPort uint16) (handler func(net.Conn)) - filterAtomic atomic.Pointer[filter.Filter] - containsViaIPFuncAtomic syncs.AtomicValue[func(netip.Addr) bool] - shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool] - numClientStatusCalls atomic.Uint32 + filterAtomic atomic.Pointer[filter.Filter] + containsViaIPFuncAtomic syncs.AtomicValue[func(netip.Addr) bool] + shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool] + shouldInterceptVIPServicesTCPPortAtomic syncs.AtomicValue[func(netip.AddrPort) bool] + numClientStatusCalls atomic.Uint32 // goTracker accounts for all goroutines started by LocalBacked, primarily // for testing and graceful shutdown purposes. @@ -317,8 +318,9 @@ type LocalBackend struct { offlineAutoUpdateCancel func() // ServeConfig fields. (also guarded by mu) - lastServeConfJSON mem.RO // last JSON that was parsed into serveConfig - serveConfig ipn.ServeConfigView // or !Valid if none + lastServeConfJSON mem.RO // last JSON that was parsed into serveConfig + serveConfig ipn.ServeConfigView // or !Valid if none + ipVIPServiceMap netmap.IPServiceMappings // map of VIPService IPs to their corresponding service names webClient webClient webClientListeners map[netip.AddrPort]*localListener // listeners for local web client traffic @@ -523,6 +525,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo b.e.SetJailedFilter(noneFilter) b.setTCPPortsIntercepted(nil) + b.setVIPServicesTCPPortsIntercepted(nil) b.statusChanged = sync.NewCond(&b.statusLock) b.e.SetStatusCallback(b.setWgengineStatus) @@ -3362,10 +3365,7 @@ func (b *LocalBackend) clearMachineKeyLocked() error { return nil } -// setTCPPortsIntercepted populates b.shouldInterceptTCPPortAtomic with an -// efficient func for ShouldInterceptTCPPort to use, which is called on every -// incoming packet. -func (b *LocalBackend) setTCPPortsIntercepted(ports []uint16) { +func generateInterceptTCPPortFunc(ports []uint16) func(uint16) bool { slices.Sort(ports) ports = slices.Compact(ports) var f func(uint16) bool @@ -3396,7 +3396,61 @@ func (b *LocalBackend) setTCPPortsIntercepted(ports []uint16) { } } } - b.shouldInterceptTCPPortAtomic.Store(f) + return f +} + +// setTCPPortsIntercepted populates b.shouldInterceptTCPPortAtomic with an +// efficient func for ShouldInterceptTCPPort to use, which is called on every +// incoming packet. +func (b *LocalBackend) setTCPPortsIntercepted(ports []uint16) { + b.shouldInterceptTCPPortAtomic.Store(generateInterceptTCPPortFunc(ports)) +} + +func generateInterceptVIPServicesTCPPortFunc(svcAddrPorts map[netip.Addr]func(uint16) bool) func(netip.AddrPort) bool { + return func(ap netip.AddrPort) bool { + if f, ok := svcAddrPorts[ap.Addr()]; ok { + return f(ap.Port()) + } + return false + } +} + +// setVIPServicesTCPPortsIntercepted populates b.shouldInterceptVIPServicesTCPPortAtomic with an +// efficient func for ShouldInterceptTCPPort to use, which is called on every incoming packet. +func (b *LocalBackend) setVIPServicesTCPPortsIntercepted(svcPorts map[string][]uint16) { + b.mu.Lock() + defer b.mu.Unlock() + b.setVIPServicesTCPPortsInterceptedLocked(svcPorts) +} + +func (b *LocalBackend) setVIPServicesTCPPortsInterceptedLocked(svcPorts map[string][]uint16) { + if len(svcPorts) == 0 { + b.shouldInterceptVIPServicesTCPPortAtomic.Store(func(netip.AddrPort) bool { return false }) + return + } + nm := b.netMap + if nm == nil { + b.logf("can't set intercept function for Service TCP Ports, netMap is nil") + return + } + vipServiceIPMap := nm.GetVIPServiceIPMap() + if len(vipServiceIPMap) == 0 { + // No approved VIP Services + return + } + + svcAddrPorts := make(map[netip.Addr]func(uint16) bool) + // Only set the intercept function if the service has been assigned a VIP. + for svcName, ports := range svcPorts { + if addrs, ok := vipServiceIPMap[svcName]; ok { + interceptFn := generateInterceptTCPPortFunc(ports) + for _, addr := range addrs { + svcAddrPorts[addr] = interceptFn + } + } + } + + b.shouldInterceptVIPServicesTCPPortAtomic.Store(generateInterceptVIPServicesTCPPortFunc(svcAddrPorts)) } // setAtomicValuesFromPrefsLocked populates sshAtomicBool, containsViaIPFuncAtomic, @@ -3409,6 +3463,7 @@ func (b *LocalBackend) setAtomicValuesFromPrefsLocked(p ipn.PrefsView) { if !p.Valid() { b.containsViaIPFuncAtomic.Store(ipset.FalseContainsIPFunc()) b.setTCPPortsIntercepted(nil) + b.setVIPServicesTCPPortsInterceptedLocked(nil) b.lastServeConfJSON = mem.B(nil) b.serveConfig = ipn.ServeConfigView{} } else { @@ -4159,6 +4214,11 @@ func (b *LocalBackend) TCPHandlerForDst(src, dst netip.AddrPort) (handler func(c } } + // TODO(corp#26001): Get handler for VIP services and Local IPs using + // the same function. + if handler := b.tcpHandlerForVIPService(dst, src); handler != nil { + return handler, opts + } // Then handle external connections to the local IP. if !b.isLocalIP(dst.Addr()) { return nil, nil @@ -5676,6 +5736,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { netns.SetDisableBindConnToInterface(nm.HasCap(tailcfg.CapabilityDebugDisableBindConnToInterface)) b.setTCPPortsInterceptedFromNetmapAndPrefsLocked(b.pm.CurrentPrefs()) + b.ipVIPServiceMap = nm.GetIPVIPServiceMap() if nm == nil { b.nodeByAddr = nil @@ -5962,6 +6023,7 @@ func (b *LocalBackend) reloadServeConfigLocked(prefs ipn.PrefsView) { // b.mu must be held. func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn.PrefsView) { handlePorts := make([]uint16, 0, 4) + vipServicesPorts := make(map[string][]uint16) if prefs.Valid() && prefs.RunSSH() && envknob.CanSSHD() { handlePorts = append(handlePorts, 22) @@ -5985,6 +6047,20 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn. } handlePorts = append(handlePorts, servePorts...) + for svc, cfg := range b.serveConfig.Services().All() { + servicePorts := make([]uint16, 0, 3) + for port := range cfg.TCP().All() { + if port > 0 { + servicePorts = append(servicePorts, uint16(port)) + } + } + if _, ok := vipServicesPorts[svc]; !ok { + vipServicesPorts[svc] = servicePorts + } else { + vipServicesPorts[svc] = append(vipServicesPorts[svc], servicePorts...) + } + } + b.setServeProxyHandlersLocked() // don't listen on netmap addresses if we're in userspace mode @@ -5996,6 +6072,7 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn. // Update funnel info in hostinfo and kick off control update if needed. b.updateIngressLocked() b.setTCPPortsIntercepted(handlePorts) + b.setVIPServicesTCPPortsInterceptedLocked(vipServicesPorts) } // updateIngressLocked updates the hostinfo.WireIngress and hostinfo.IngressEnabled fields and kicks off a Hostinfo @@ -6854,6 +6931,12 @@ func (b *LocalBackend) ShouldInterceptTCPPort(port uint16) bool { return b.shouldInterceptTCPPortAtomic.Load()(port) } +// ShouldInterceptVIPServiceTCPPort reports whether the given TCP port number +// to a VIP service should be intercepted by Tailscaled and handled in-process. +func (b *LocalBackend) ShouldInterceptVIPServiceTCPPort(ap netip.AddrPort) bool { + return b.shouldInterceptVIPServicesTCPPortAtomic.Load()(ap) +} + // SwitchProfile switches to the profile with the given id. // It will restart the backend on success. // If the profile is not known, it returns an errProfileNotFound. diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 415791c60ab68..f851bb0f8a018 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -2615,6 +2615,150 @@ func TestOnTailnetDefaultAutoUpdate(t *testing.T) { func TestTCPHandlerForDst(t *testing.T) { b := newTestBackend(t) + tests := []struct { + desc string + dst string + intercept bool + }{ + { + desc: "intercept port 80 (Web UI) on quad100 IPv4", + dst: "100.100.100.100:80", + intercept: true, + }, + { + desc: "intercept port 80 (Web UI) on quad100 IPv6", + dst: "[fd7a:115c:a1e0::53]:80", + intercept: true, + }, + { + desc: "don't intercept port 80 on local ip", + dst: "100.100.103.100:80", + intercept: false, + }, + { + desc: "intercept port 8080 (Taildrive) on quad100 IPv4", + dst: "[fd7a:115c:a1e0::53]:8080", + intercept: true, + }, + { + desc: "don't intercept port 8080 on local ip", + dst: "100.100.103.100:8080", + intercept: false, + }, + { + desc: "don't intercept port 9080 on quad100 IPv4", + dst: "100.100.100.100:9080", + intercept: false, + }, + { + desc: "don't intercept port 9080 on quad100 IPv6", + dst: "[fd7a:115c:a1e0::53]:9080", + intercept: false, + }, + { + desc: "don't intercept port 9080 on local ip", + dst: "100.100.103.100:9080", + intercept: false, + }, + } + for _, tt := range tests { + t.Run(tt.dst, func(t *testing.T) { + t.Log(tt.desc) + src := netip.MustParseAddrPort("100.100.102.100:51234") + h, _ := b.TCPHandlerForDst(src, netip.MustParseAddrPort(tt.dst)) + if !tt.intercept && h != nil { + t.Error("intercepted traffic we shouldn't have") + } else if tt.intercept && h == nil { + t.Error("failed to intercept traffic we should have") + } + }) + } +} + +func TestTCPHandlerForDstWithVIPService(t *testing.T) { + b := newTestBackend(t) + svcIPMap := tailcfg.ServiceIPMappings{ + "svc:foo": []netip.Addr{ + netip.MustParseAddr("100.101.101.101"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:6565:6565"), + }, + "svc:bar": []netip.Addr{ + netip.MustParseAddr("100.99.99.99"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:626b:628b"), + }, + "svc:baz": []netip.Addr{ + netip.MustParseAddr("100.133.133.133"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:8585:8585"), + }, + } + svcIPMapJSON, err := json.Marshal(svcIPMap) + if err != nil { + t.Fatal(err) + } + b.setNetMapLocked( + &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + Name: "example.ts.net", + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{tailcfg.RawMessage(svcIPMapJSON)}, + }, + }).View(), + UserProfiles: map[tailcfg.UserID]tailcfg.UserProfile{ + tailcfg.UserID(1): { + LoginName: "someone@example.com", + DisplayName: "Some One", + ProfilePicURL: "https://example.com/photo.jpg", + }, + }, + }, + ) + + err = b.setServeConfigLocked( + &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 882: {HTTP: true}, + 883: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.example.ts.net:882": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:3000"}, + }, + }, + "foo.example.ts.net:883": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Text: "test"}, + }, + }, + }, + }, + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 990: {TCPForward: "127.0.0.1:8443"}, + 991: {TCPForward: "127.0.0.1:5432", TerminateTLS: "bar.test.ts.net"}, + }, + }, + "svc:qux": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 600: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "qux.example.ts.net:600": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Text: "qux"}, + }, + }, + }, + }, + }, + }, + "", + ) + if err != nil { + t.Fatal(err) + } tests := []struct { desc string @@ -2666,6 +2810,77 @@ func TestTCPHandlerForDst(t *testing.T) { dst: "100.100.103.100:9080", intercept: false, }, + // VIP service destinations + { + desc: "intercept port 882 (HTTP) on service foo IPv4", + dst: "100.101.101.101:882", + intercept: true, + }, + { + desc: "intercept port 882 (HTTP) on service foo IPv6", + dst: "[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:882", + intercept: true, + }, + { + desc: "intercept port 883 (HTTPS) on service foo IPv4", + dst: "100.101.101.101:883", + intercept: true, + }, + { + desc: "intercept port 883 (HTTPS) on service foo IPv6", + dst: "[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:883", + intercept: true, + }, + { + desc: "intercept port 990 (TCPForward) on service bar IPv4", + dst: "100.99.99.99:990", + intercept: true, + }, + { + desc: "intercept port 990 (TCPForward) on service bar IPv6", + dst: "[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:990", + intercept: true, + }, + { + desc: "intercept port 991 (TCPForward with TerminateTLS) on service bar IPv4", + dst: "100.99.99.99:990", + intercept: true, + }, + { + desc: "intercept port 991 (TCPForward with TerminateTLS) on service bar IPv6", + dst: "[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:990", + intercept: true, + }, + { + desc: "don't intercept port 4444 on service foo IPv4", + dst: "100.101.101.101:4444", + intercept: false, + }, + { + desc: "don't intercept port 4444 on service foo IPv6", + dst: "[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:4444", + intercept: false, + }, + { + desc: "don't intercept port 600 on unknown service IPv4", + dst: "100.22.22.22:883", + intercept: false, + }, + { + desc: "don't intercept port 600 on unknown service IPv6", + dst: "[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:883", + intercept: false, + }, + { + desc: "don't intercept port 600 (HTTPS) on service baz IPv4", + dst: "100.133.133.133:600", + intercept: false, + }, + { + desc: "don't intercept port 600 (HTTPS) on service baz IPv6", + dst: "[fd7a:115c:a1e0:ab12:4843:cd96:8585:8585]:600", + intercept: false, + }, } for _, tt := range tests { diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index c144fa5299a8f..c20172a425900 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -54,8 +54,9 @@ var ErrETagMismatch = errors.New("etag mismatch") var serveHTTPContextKey ctxkey.Key[*serveHTTPContext] type serveHTTPContext struct { - SrcAddr netip.AddrPort - DestPort uint16 + SrcAddr netip.AddrPort + ForVIPService bool + DestPort uint16 // provides funnel-specific context, nil if not funneled Funnel *funnelFlow @@ -275,6 +276,12 @@ func (b *LocalBackend) setServeConfigLocked(config *ipn.ServeConfig, etag string return errors.New("can't reconfigure tailscaled when using a config file; config file is locked") } + if config != nil { + if err := config.CheckValidServicesConfig(); err != nil { + return err + } + } + nm := b.netMap if nm == nil { return errors.New("netMap is nil") @@ -432,6 +439,105 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target handler(c) } +// tcpHandlerForVIPService returns a handler for a TCP connection to a VIP service +// that is being served via the ipn.ServeConfig. It returns nil if the destination +// address is not a VIP service or if the VIP service does not have a TCP handler set. +func (b *LocalBackend) tcpHandlerForVIPService(dstAddr, srcAddr netip.AddrPort) (handler func(net.Conn) error) { + b.mu.Lock() + sc := b.serveConfig + ipVIPServiceMap := b.ipVIPServiceMap + b.mu.Unlock() + + if !sc.Valid() { + return nil + } + + dport := dstAddr.Port() + + dstSvc, ok := ipVIPServiceMap[dstAddr.Addr()] + if !ok { + return nil + } + + tcph, ok := sc.FindServiceTCP(dstSvc, dstAddr.Port()) + if !ok { + b.logf("The destination service doesn't have a TCP handler set.") + return nil + } + + if tcph.HTTPS() || tcph.HTTP() { + hs := &http.Server{ + Handler: http.HandlerFunc(b.serveWebHandler), + BaseContext: func(_ net.Listener) context.Context { + return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{ + SrcAddr: srcAddr, + ForVIPService: true, + DestPort: dport, + }) + }, + } + if tcph.HTTPS() { + // TODO(kevinliang10): just leaving this TLS cert creation as if we don't have other + // hostnames, but for services this getTLSServeCetForPort will need a version that also take + // in the hostname. How to store the TLS cert is still being discussed. + hs.TLSConfig = &tls.Config{ + GetCertificate: b.getTLSServeCertForPort(dport, true), + } + return func(c net.Conn) error { + return hs.ServeTLS(netutil.NewOneConnListener(c, nil), "", "") + } + } + + return func(c net.Conn) error { + return hs.Serve(netutil.NewOneConnListener(c, nil)) + } + } + + if backDst := tcph.TCPForward(); backDst != "" { + return func(conn net.Conn) error { + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + backConn, err := b.dialer.SystemDial(ctx, "tcp", backDst) + cancel() + if err != nil { + b.logf("localbackend: failed to TCP proxy port %v (from %v) to %s: %v", dport, srcAddr, backDst, err) + return nil + } + defer backConn.Close() + if sni := tcph.TerminateTLS(); sni != "" { + conn = tls.Server(conn, &tls.Config{ + GetCertificate: func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pair, err := b.GetCertPEM(ctx, sni) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(pair.CertPEM, pair.KeyPEM) + if err != nil { + return nil, err + } + return &cert, nil + }, + }) + } + + errc := make(chan error, 1) + go func() { + _, err := io.Copy(backConn, conn) + errc <- err + }() + go func() { + _, err := io.Copy(conn, backConn) + errc <- err + }() + return <-errc + } + } + + return nil +} + // tcpHandlerForServe returns a handler for a TCP connection to be served via // the ipn.ServeConfig. The funnelFlow can be nil if this is not a funneled // connection. @@ -462,7 +568,7 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort, } if tcph.HTTPS() { hs.TLSConfig = &tls.Config{ - GetCertificate: b.getTLSServeCertForPort(dport), + GetCertificate: b.getTLSServeCertForPort(dport, false), } return func(c net.Conn) error { return hs.ServeTLS(netutil.NewOneConnListener(c, nil), "", "") @@ -542,7 +648,7 @@ func (b *LocalBackend) getServeHandler(r *http.Request) (_ ipn.HTTPHandlerView, b.logf("[unexpected] localbackend: no serveHTTPContext in request") return z, "", false } - wsc, ok := b.webServerConfig(hostname, sctx.DestPort) + wsc, ok := b.webServerConfig(hostname, sctx.ForVIPService, sctx.DestPort) if !ok { return z, "", false } @@ -900,7 +1006,7 @@ func allNumeric(s string) bool { return s != "" } -func (b *LocalBackend) webServerConfig(hostname string, port uint16) (c ipn.WebServerConfigView, ok bool) { +func (b *LocalBackend) webServerConfig(hostname string, forVIPService bool, port uint16) (c ipn.WebServerConfigView, ok bool) { key := ipn.HostPort(fmt.Sprintf("%s:%v", hostname, port)) b.mu.Lock() @@ -909,15 +1015,18 @@ func (b *LocalBackend) webServerConfig(hostname string, port uint16) (c ipn.WebS if !b.serveConfig.Valid() { return c, false } + if forVIPService { + return b.serveConfig.FindServiceWeb(key) + } return b.serveConfig.FindWeb(key) } -func (b *LocalBackend) getTLSServeCertForPort(port uint16) func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { +func (b *LocalBackend) getTLSServeCertForPort(port uint16, forVIPService bool) func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { return func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { if hi == nil || hi.ServerName == "" { return nil, errors.New("no SNI ServerName") } - _, ok := b.webServerConfig(hi.ServerName, port) + _, ok := b.webServerConfig(hi.ServerName, forVIPService, port) if !ok { return nil, errors.New("no webserver configured for name/port") } diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index 73e66c2b9db16..f2ea8e5cd6dbf 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -296,6 +296,203 @@ func TestServeConfigForeground(t *testing.T) { } } +// TestServeConfigServices tests the side effects of setting the +// Services field in a ServeConfig. The Services field is a map +// of all services the current service host is serving. Unlike what we +// serve for node itself, there is no foreground and no local handlers +// for the services. So the only things we need to test are if the +// services configured are valid and if they correctly set intercept +// functions for netStack. +func TestServeConfigServices(t *testing.T) { + b := newTestBackend(t) + svcIPMap := tailcfg.ServiceIPMappings{ + "svc:foo": []netip.Addr{ + netip.MustParseAddr("100.101.101.101"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:6565:6565"), + }, + "svc:bar": []netip.Addr{ + netip.MustParseAddr("100.99.99.99"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:626b:628b"), + }, + } + svcIPMapJSON, err := json.Marshal(svcIPMap) + if err != nil { + t.Fatal(err) + } + + b.netMap = &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + Name: "example.ts.net", + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{tailcfg.RawMessage(svcIPMapJSON)}, + }, + }).View(), + UserProfiles: map[tailcfg.UserID]tailcfg.UserProfile{ + tailcfg.UserID(1): { + LoginName: "someone@example.com", + DisplayName: "Some One", + ProfilePicURL: "https://example.com/photo.jpg", + }, + }, + } + + tests := []struct { + name string + conf *ipn.ServeConfig + expectedErr error + packetDstAddrPort []netip.AddrPort + intercepted bool + }{ + { + name: "no-services", + conf: &ipn.ServeConfig{}, + packetDstAddrPort: []netip.AddrPort{ + netip.MustParseAddrPort("100.101.101.101:443"), + }, + intercepted: false, + }, + { + name: "one-incorrectly-configured-service", + conf: &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Tun: true, + }, + }, + }, + expectedErr: ipn.ErrServiceConfigHasBothTCPAndTun, + }, + { + // one correctly configured service with packet should be intercepted + name: "one-service-intercept-packet", + conf: &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 81: {HTTPS: true}, + }, + }, + }, + }, + packetDstAddrPort: []netip.AddrPort{ + netip.MustParseAddrPort("100.101.101.101:80"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:80"), + }, + intercepted: true, + }, + { + // one correctly configured service with packet should not be intercepted + name: "one-service-not-intercept-packet", + conf: &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 81: {HTTPS: true}, + }, + }, + }, + }, + packetDstAddrPort: []netip.AddrPort{ + netip.MustParseAddrPort("100.99.99.99:80"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80"), + netip.MustParseAddrPort("100.101.101.101:82"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:82"), + }, + intercepted: false, + }, + { + //multiple correctly configured service with packet should be intercepted + name: "multiple-service-intercept-packet", + conf: &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 81: {HTTPS: true}, + }, + }, + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 81: {HTTPS: true}, + 82: {HTTPS: true}, + }, + }, + }, + }, + packetDstAddrPort: []netip.AddrPort{ + netip.MustParseAddrPort("100.99.99.99:80"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80"), + netip.MustParseAddrPort("100.101.101.101:81"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:81"), + }, + intercepted: true, + }, + { + // multiple correctly configured service with packet should not be intercepted + name: "multiple-service-not-intercept-packet", + conf: &ipn.ServeConfig{ + Services: map[string]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 81: {HTTPS: true}, + }, + }, + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 81: {HTTPS: true}, + 82: {HTTPS: true}, + }, + }, + }, + }, + packetDstAddrPort: []netip.AddrPort{ + // ips in capmap but port is not hosting service + netip.MustParseAddrPort("100.99.99.99:77"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:77"), + netip.MustParseAddrPort("100.101.101.101:85"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:85"), + // ips not in capmap + netip.MustParseAddrPort("100.102.102.102:80"), + netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:6666:6666]:80"), + }, + intercepted: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := b.SetServeConfig(tt.conf, "") + if err != nil && tt.expectedErr != nil { + if !errors.Is(err, tt.expectedErr) { + t.Fatalf("expected error %v,\n got %v", tt.expectedErr, err) + } + return + } + if err != nil { + t.Fatal(err) + } + for _, addrPort := range tt.packetDstAddrPort { + if tt.intercepted != b.ShouldInterceptVIPServiceTCPPort(addrPort) { + if tt.intercepted { + t.Fatalf("expected packet to be intercepted") + } else { + t.Fatalf("expected packet not to be intercepted") + } + } + } + }) + } + +} + func TestServeConfigETag(t *testing.T) { b := newTestBackend(t) diff --git a/ipn/serve.go b/ipn/serve.go index 176c6d984fbd4..472b327a369ff 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -55,8 +55,8 @@ type ServeConfig struct { // keyed by mount point ("/", "/foo", etc) Web map[HostPort]*WebServerConfig `json:",omitempty"` - // Services maps from service name to a ServiceConfig. Which describes the - // L3, L4, and L7 forwarding information for the service. + // Services maps from service name (in the form "svc:dns-label") to a ServiceConfig. + // Which describes the L3, L4, and L7 forwarding information for the service. Services map[string]*ServiceConfig `json:",omitempty"` // AllowFunnel is the set of SNI:port values for which funnel @@ -607,7 +607,32 @@ func (v ServeConfigView) Webs() iter.Seq2[HostPort, WebServerConfigView] { } } } + for _, service := range v.Services().All() { + for k, v := range service.Web().All() { + if !yield(k, v) { + return + } + } + } + } +} + +// FindServiceTCP return the TCPPortHandlerView for the given service name and port. +func (v ServeConfigView) FindServiceTCP(svcName string, port uint16) (res TCPPortHandlerView, ok bool) { + svcCfg, ok := v.Services().GetOk(svcName) + if !ok { + return res, ok } + return svcCfg.TCP().GetOk(port) +} + +func (v ServeConfigView) FindServiceWeb(hp HostPort) (res WebServerConfigView, ok bool) { + for _, service := range v.Services().All() { + if res, ok := service.Web().GetOk(hp); ok { + return res, ok + } + } + return res, ok } // FindTCP returns the first TCP that matches with the given port. It @@ -662,6 +687,17 @@ func (v ServeConfigView) HasFunnelForTarget(target HostPort) bool { return false } +// CheckValidServicesConfig reports whether the ServeConfig has +// invalid service configurations. +func (sc *ServeConfig) CheckValidServicesConfig() error { + for svcName, service := range sc.Services { + if err := service.checkValidConfig(); err != nil { + return fmt.Errorf("invalid service configuration for %q: %w", svcName, err) + } + } + return nil +} + // ServicePortRange returns the list of tailcfg.ProtoPortRange that represents // the proto/ports pairs that are being served by the service. // @@ -699,3 +735,17 @@ func (v ServiceConfigView) ServicePortRange() []tailcfg.ProtoPortRange { } return ranges } + +// ErrServiceConfigHasBothTCPAndTun signals that a service +// in Tun mode cannot also has TCP or Web handlers set. +var ErrServiceConfigHasBothTCPAndTun = errors.New("the VIP Service configuration can not set TUN at the same time as TCP or Web") + +// checkValidConfig checks if the service configuration is valid. +// Currently, the only invalid configuration is when the service is in Tun mode +// and has TCP or Web handlers. +func (v *ServiceConfig) checkValidConfig() error { + if v.Tun && (len(v.TCP) > 0 || len(v.Web) > 0) { + return ErrServiceConfigHasBothTCPAndTun + } + return nil +} diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 937f619e67430..b69139d340005 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2997,3 +2997,19 @@ const LBHeader = "Ts-Lb" // correspond to those IPs. Any services that don't correspond to a service // this client is hosting can be ignored. type ServiceIPMappings map[string][]netip.Addr + +// IPServiceMappings maps IP addresses to service names. This is the inverse of +// [ServiceIPMappings], and is used to inform clients which services is an VIP +// address associated with. This is set to b.ipVIPServiceMap every time the +// netmap is updated. This is used to reduce the cost for looking up the service +// name for the dst IP address in the netStack packet processing workflow. +// +// This is of the form: +// +// { +// "100.65.32.1": "svc:samba", +// "fd7a:115c:a1e0::1234": "svc:samba", +// "100.102.42.3": "svc:web", +// "fd7a:115c:a1e0::abcd": "svc:web", +// } +type IPServiceMappings map[netip.Addr]string diff --git a/types/netmap/IPServiceMappings.go b/types/netmap/IPServiceMappings.go new file mode 100644 index 0000000000000..0cd207fb865e5 --- /dev/null +++ b/types/netmap/IPServiceMappings.go @@ -0,0 +1,19 @@ +package netmap + +import "net/netip" + +// IPServiceMappings maps IP addresses to service names. This is the inverse of +// [ServiceIPMappings], and is used to inform clients which services is an VIP +// address associated with. This is set to b.ipVIPServiceMap every time the +// netmap is updated. This is used to reduce the cost for looking up the service +// name for the dst IP address in the netStack packet processing workflow. +// +// This is of the form: +// +// { +// "100.65.32.1": "svc:samba", +// "fd7a:115c:a1e0::1234": "svc:samba", +// "100.102.42.3": "svc:web", +// "fd7a:115c:a1e0::abcd": "svc:web", +// } +type IPServiceMappings map[netip.Addr]string diff --git a/types/netmap/netmap.go b/types/netmap/netmap.go index 7662e145efa60..c9f909b1a810d 100644 --- a/types/netmap/netmap.go +++ b/types/netmap/netmap.go @@ -101,6 +101,54 @@ func (nm *NetworkMap) GetAddresses() views.Slice[netip.Prefix] { return nm.SelfNode.Addresses() } +// GetVIPServiceIPMap returns a map of service names to the slice of +// VIP addresses that correspond to the service. The service names are +// with the prefix "svc:". +// +// TODO(corp##25997): cache the result of decoding the capmap so that +// we don't have to decode it multiple times after each netmap update. +func (nm *NetworkMap) GetVIPServiceIPMap() tailcfg.ServiceIPMappings { + if nm == nil { + return nil + } + if !nm.SelfNode.Valid() { + return nil + } + + ipMaps, err := tailcfg.UnmarshalNodeCapJSON[tailcfg.ServiceIPMappings](nm.SelfNode.CapMap().AsMap(), tailcfg.NodeAttrServiceHost) + if len(ipMaps) != 1 || err != nil { + return nil + } + + return ipMaps[0] +} + +// GetIPVIPServiceMap returns a map of VIP addresses to the service +// names that has the VIP address. The service names are with the +// prefix "svc:". +func (nm *NetworkMap) GetIPVIPServiceMap() IPServiceMappings { + var res IPServiceMappings + if nm == nil { + return res + } + + if !nm.SelfNode.Valid() { + return res + } + + serviceIPMap := nm.GetVIPServiceIPMap() + if serviceIPMap == nil { + return res + } + res = make(IPServiceMappings) + for svc, addrs := range serviceIPMap { + for _, addr := range addrs { + res[addr] = svc + } + } + return res +} + // AnyPeersAdvertiseRoutes reports whether any peer is advertising non-exit node routes. func (nm *NetworkMap) AnyPeersAdvertiseRoutes() bool { for _, p := range nm.Peers { diff --git a/wgengine/netstack/netstack.go b/wgengine/netstack/netstack.go index 20eac06e6b8fd..0b8c67b061098 100644 --- a/wgengine/netstack/netstack.go +++ b/wgengine/netstack/netstack.go @@ -50,6 +50,7 @@ import ( "tailscale.com/types/netmap" "tailscale.com/types/nettype" "tailscale.com/util/clientmetric" + "tailscale.com/util/set" "tailscale.com/version" "tailscale.com/wgengine" "tailscale.com/wgengine/filter" @@ -200,6 +201,8 @@ type Impl struct { // updates. atomicIsLocalIPFunc syncs.AtomicValue[func(netip.Addr) bool] + atomicIsVIPServiceIPFunc syncs.AtomicValue[func(netip.Addr) bool] + // forwardDialFunc, if non-nil, is the net.Dialer.DialContext-style // function that is used to make outgoing connections when forwarding a // TCP connection to another host (e.g. in subnet router mode). @@ -387,6 +390,7 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi } ns.ctx, ns.ctxCancel = context.WithCancel(context.Background()) ns.atomicIsLocalIPFunc.Store(ipset.FalseContainsIPFunc()) + ns.atomicIsVIPServiceIPFunc.Store(ipset.FalseContainsIPFunc()) ns.tundev.PostFilterPacketInboundFromWireGuard = ns.injectInbound ns.tundev.PreFilterPacketOutboundToWireGuardNetstackIntercept = ns.handleLocalPackets stacksForMetrics.Store(ns, struct{}{}) @@ -532,7 +536,7 @@ func (ns *Impl) wrapTCPProtocolHandler(h protocolHandlerFunc) protocolHandlerFun // Dynamically reconfigure ns's subnet addresses as needed for // outbound traffic. - if !ns.isLocalIP(localIP) { + if !ns.isLocalIP(localIP) && !ns.isVIPServiceIP(localIP) { ns.addSubnetAddress(localIP) } @@ -621,10 +625,17 @@ var v4broadcast = netaddr.IPv4(255, 255, 255, 255) func (ns *Impl) UpdateNetstackIPs(nm *netmap.NetworkMap) { var selfNode tailcfg.NodeView if nm != nil { + vipServiceIPMap := nm.GetVIPServiceIPMap() + serviceAddrSet := set.Set[netip.Addr]{} + for _, addrs := range vipServiceIPMap { + serviceAddrSet.AddSlice(addrs) + } ns.atomicIsLocalIPFunc.Store(ipset.NewContainsIPFunc(nm.GetAddresses())) + ns.atomicIsVIPServiceIPFunc.Store(serviceAddrSet.Contains) selfNode = nm.SelfNode } else { ns.atomicIsLocalIPFunc.Store(ipset.FalseContainsIPFunc()) + ns.atomicIsVIPServiceIPFunc.Store(ipset.FalseContainsIPFunc()) } oldPfx := make(map[netip.Prefix]bool) @@ -952,6 +963,12 @@ func (ns *Impl) isLocalIP(ip netip.Addr) bool { return ns.atomicIsLocalIPFunc.Load()(ip) } +// isVIPServiceIP reports whether ip is an IP address that's +// assigned to a VIP service. +func (ns *Impl) isVIPServiceIP(ip netip.Addr) bool { + return ns.atomicIsVIPServiceIPFunc.Load()(ip) +} + func (ns *Impl) peerAPIPortAtomic(ip netip.Addr) *atomic.Uint32 { if ip.Is4() { return &ns.peerapiPort4Atomic @@ -968,6 +985,7 @@ func (ns *Impl) shouldProcessInbound(p *packet.Parsed, t *tstun.Wrapper) bool { // Handle incoming peerapi connections in netstack. dstIP := p.Dst.Addr() isLocal := ns.isLocalIP(dstIP) + isService := ns.isVIPServiceIP(dstIP) // Handle TCP connection to the Tailscale IP(s) in some cases: if ns.lb != nil && p.IPProto == ipproto.TCP && isLocal { @@ -990,6 +1008,13 @@ func (ns *Impl) shouldProcessInbound(p *packet.Parsed, t *tstun.Wrapper) bool { return true } } + if ns.lb != nil && p.IPProto == ipproto.TCP && isService { + // An assumption holds for this to work: when tun mode is on for a service, + // its tcp and web are not set. This is enforced in b.setServeConfigLocked. + if ns.lb.ShouldInterceptVIPServiceTCPPort(p.Dst) { + return true + } + } if p.IPVersion == 6 && !isLocal && viaRange.Contains(dstIP) { return ns.lb != nil && ns.lb.ShouldHandleViaIP(dstIP) } From ccd16430439e4cfc943fc7bfc40eebbfef0a9195 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:47:22 -0500 Subject: [PATCH 160/223] add copyright header Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- types/netmap/IPServiceMappings.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/types/netmap/IPServiceMappings.go b/types/netmap/IPServiceMappings.go index 0cd207fb865e5..4f02924ab0b90 100644 --- a/types/netmap/IPServiceMappings.go +++ b/types/netmap/IPServiceMappings.go @@ -1,3 +1,5 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause package netmap import "net/netip" From 0a57051f2eb7b3c98f32c98bfabc27a229b440bf Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:54:27 -0500 Subject: [PATCH 161/223] add blank line Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- types/netmap/IPServiceMappings.go | 1 + 1 file changed, 1 insertion(+) diff --git a/types/netmap/IPServiceMappings.go b/types/netmap/IPServiceMappings.go index 4f02924ab0b90..04e71b0bfda4d 100644 --- a/types/netmap/IPServiceMappings.go +++ b/types/netmap/IPServiceMappings.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package netmap import "net/netip" From 550923d95330d7809f28cf28daa41bb91a70770f Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:24:49 -0500 Subject: [PATCH 162/223] fix handler related and some nit Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- ipn/ipnlocal/local.go | 20 +++++++++++--------- ipn/ipnlocal/serve.go | 16 ++++++++-------- ipn/serve.go | 6 +++--- types/netmap/IPServiceMappings.go | 22 ---------------------- types/netmap/netmap.go | 18 +++++++++++++++++- 5 files changed, 39 insertions(+), 43 deletions(-) delete mode 100644 types/netmap/IPServiceMappings.go diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 05f56fcbdc80c..470824fde51ed 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -3442,11 +3442,13 @@ func (b *LocalBackend) setVIPServicesTCPPortsInterceptedLocked(svcPorts map[stri svcAddrPorts := make(map[netip.Addr]func(uint16) bool) // Only set the intercept function if the service has been assigned a VIP. for svcName, ports := range svcPorts { - if addrs, ok := vipServiceIPMap[svcName]; ok { - interceptFn := generateInterceptTCPPortFunc(ports) - for _, addr := range addrs { - svcAddrPorts[addr] = interceptFn - } + addrs, ok := vipServiceIPMap[svcName] + if !ok { + continue + } + interceptFn := generateInterceptTCPPortFunc(ports) + for _, addr := range addrs { + svcAddrPorts[addr] = interceptFn } } @@ -4214,7 +4216,7 @@ func (b *LocalBackend) TCPHandlerForDst(src, dst netip.AddrPort) (handler func(c } } - // TODO(corp#26001): Get handler for VIP services and Local IPs using + // TODO(tailscale/corp#26001): Get handler for VIP services and Local IPs using // the same function. if handler := b.tcpHandlerForVIPService(dst, src); handler != nil { return handler, opts @@ -6023,7 +6025,7 @@ func (b *LocalBackend) reloadServeConfigLocked(prefs ipn.PrefsView) { // b.mu must be held. func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn.PrefsView) { handlePorts := make([]uint16, 0, 4) - vipServicesPorts := make(map[string][]uint16) + var vipServicesPorts map[string][]uint16 if prefs.Valid() && prefs.RunSSH() && envknob.CanSSHD() { handlePorts = append(handlePorts, 22) @@ -6055,9 +6057,9 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn. } } if _, ok := vipServicesPorts[svc]; !ok { - vipServicesPorts[svc] = servicePorts + mak.Set(&vipServicesPorts, svc, servicePorts) } else { - vipServicesPorts[svc] = append(vipServicesPorts[svc], servicePorts...) + mak.Set(&vipServicesPorts, svc, append(vipServicesPorts[svc], servicePorts...)) } } diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index c20172a425900..a5247dd8c6734 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -55,7 +55,7 @@ var serveHTTPContextKey ctxkey.Key[*serveHTTPContext] type serveHTTPContext struct { SrcAddr netip.AddrPort - ForVIPService bool + ForVIPService string // VIP service name, empty string means local DestPort uint16 // provides funnel-specific context, nil if not funneled @@ -471,7 +471,7 @@ func (b *LocalBackend) tcpHandlerForVIPService(dstAddr, srcAddr netip.AddrPort) BaseContext: func(_ net.Listener) context.Context { return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{ SrcAddr: srcAddr, - ForVIPService: true, + ForVIPService: dstSvc, DestPort: dport, }) }, @@ -481,7 +481,7 @@ func (b *LocalBackend) tcpHandlerForVIPService(dstAddr, srcAddr netip.AddrPort) // hostnames, but for services this getTLSServeCetForPort will need a version that also take // in the hostname. How to store the TLS cert is still being discussed. hs.TLSConfig = &tls.Config{ - GetCertificate: b.getTLSServeCertForPort(dport, true), + GetCertificate: b.getTLSServeCertForPort(dport, dstSvc), } return func(c net.Conn) error { return hs.ServeTLS(netutil.NewOneConnListener(c, nil), "", "") @@ -568,7 +568,7 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort, } if tcph.HTTPS() { hs.TLSConfig = &tls.Config{ - GetCertificate: b.getTLSServeCertForPort(dport, false), + GetCertificate: b.getTLSServeCertForPort(dport, ""), } return func(c net.Conn) error { return hs.ServeTLS(netutil.NewOneConnListener(c, nil), "", "") @@ -1006,7 +1006,7 @@ func allNumeric(s string) bool { return s != "" } -func (b *LocalBackend) webServerConfig(hostname string, forVIPService bool, port uint16) (c ipn.WebServerConfigView, ok bool) { +func (b *LocalBackend) webServerConfig(hostname string, forVIPService string, port uint16) (c ipn.WebServerConfigView, ok bool) { key := ipn.HostPort(fmt.Sprintf("%s:%v", hostname, port)) b.mu.Lock() @@ -1015,13 +1015,13 @@ func (b *LocalBackend) webServerConfig(hostname string, forVIPService bool, port if !b.serveConfig.Valid() { return c, false } - if forVIPService { - return b.serveConfig.FindServiceWeb(key) + if forVIPService != "" { + return b.serveConfig.FindServiceWeb(forVIPService, key) } return b.serveConfig.FindWeb(key) } -func (b *LocalBackend) getTLSServeCertForPort(port uint16, forVIPService bool) func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { +func (b *LocalBackend) getTLSServeCertForPort(port uint16, forVIPService string) func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { return func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { if hi == nil || hi.ServerName == "" { return nil, errors.New("no SNI ServerName") diff --git a/ipn/serve.go b/ipn/serve.go index 472b327a369ff..4c2d2f158144e 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -626,9 +626,9 @@ func (v ServeConfigView) FindServiceTCP(svcName string, port uint16) (res TCPPor return svcCfg.TCP().GetOk(port) } -func (v ServeConfigView) FindServiceWeb(hp HostPort) (res WebServerConfigView, ok bool) { - for _, service := range v.Services().All() { - if res, ok := service.Web().GetOk(hp); ok { +func (v ServeConfigView) FindServiceWeb(svcName string, hp HostPort) (res WebServerConfigView, ok bool) { + if svcCfg, ok := v.Services().GetOk(svcName); ok { + if res, ok := svcCfg.Web().GetOk(hp); ok { return res, ok } } diff --git a/types/netmap/IPServiceMappings.go b/types/netmap/IPServiceMappings.go deleted file mode 100644 index 04e71b0bfda4d..0000000000000 --- a/types/netmap/IPServiceMappings.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package netmap - -import "net/netip" - -// IPServiceMappings maps IP addresses to service names. This is the inverse of -// [ServiceIPMappings], and is used to inform clients which services is an VIP -// address associated with. This is set to b.ipVIPServiceMap every time the -// netmap is updated. This is used to reduce the cost for looking up the service -// name for the dst IP address in the netStack packet processing workflow. -// -// This is of the form: -// -// { -// "100.65.32.1": "svc:samba", -// "fd7a:115c:a1e0::1234": "svc:samba", -// "100.102.42.3": "svc:web", -// "fd7a:115c:a1e0::abcd": "svc:web", -// } -type IPServiceMappings map[netip.Addr]string diff --git a/types/netmap/netmap.go b/types/netmap/netmap.go index c9f909b1a810d..1482e534e9755 100644 --- a/types/netmap/netmap.go +++ b/types/netmap/netmap.go @@ -105,7 +105,7 @@ func (nm *NetworkMap) GetAddresses() views.Slice[netip.Prefix] { // VIP addresses that correspond to the service. The service names are // with the prefix "svc:". // -// TODO(corp##25997): cache the result of decoding the capmap so that +// TODO(tailscale/corp##25997): cache the result of decoding the capmap so that // we don't have to decode it multiple times after each netmap update. func (nm *NetworkMap) GetVIPServiceIPMap() tailcfg.ServiceIPMappings { if nm == nil { @@ -425,3 +425,19 @@ const ( _ WGConfigFlags = 1 << iota AllowSubnetRoutes ) + +// IPServiceMappings maps IP addresses to service names. This is the inverse of +// [ServiceIPMappings], and is used to inform clients which services is an VIP +// address associated with. This is set to b.ipVIPServiceMap every time the +// netmap is updated. This is used to reduce the cost for looking up the service +// name for the dst IP address in the netStack packet processing workflow. +// +// This is of the form: +// +// { +// "100.65.32.1": "svc:samba", +// "fd7a:115c:a1e0::1234": "svc:samba", +// "100.102.42.3": "svc:web", +// "fd7a:115c:a1e0::abcd": "svc:web", +// } +type IPServiceMappings map[netip.Addr]string From e4779146b50f05bbcf07b4e99018902669ddd6cc Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:45:30 -0500 Subject: [PATCH 163/223] delete extra struct in tailcfg Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- tailcfg/tailcfg.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index b69139d340005..937f619e67430 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2997,19 +2997,3 @@ const LBHeader = "Ts-Lb" // correspond to those IPs. Any services that don't correspond to a service // this client is hosting can be ignored. type ServiceIPMappings map[string][]netip.Addr - -// IPServiceMappings maps IP addresses to service names. This is the inverse of -// [ServiceIPMappings], and is used to inform clients which services is an VIP -// address associated with. This is set to b.ipVIPServiceMap every time the -// netmap is updated. This is used to reduce the cost for looking up the service -// name for the dst IP address in the netStack packet processing workflow. -// -// This is of the form: -// -// { -// "100.65.32.1": "svc:samba", -// "fd7a:115c:a1e0::1234": "svc:samba", -// "100.102.42.3": "svc:web", -// "fd7a:115c:a1e0::abcd": "svc:web", -// } -type IPServiceMappings map[netip.Addr]string From 17022ad0e907d033418dbfc604999bedae8a3978 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 22 Jan 2025 06:33:05 -0800 Subject: [PATCH 164/223] tailcfg: remove now-unused TailscaleFunnelEnabled method As of tailscale/corp#26003 Updates tailscale/tailscale#11572 Change-Id: I5de2a0951b7b8972744178abc1b0e7948087d412 Signed-off-by: Brad Fitzpatrick --- tailcfg/tailcfg.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 937f619e67430..e1259b3f50cf0 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -927,14 +927,6 @@ func (hi *Hostinfo) TailscaleSSHEnabled() bool { func (v HostinfoView) TailscaleSSHEnabled() bool { return v.Đļ.TailscaleSSHEnabled() } -// TailscaleFunnelEnabled reports whether or not this node has explicitly -// enabled Funnel. -func (hi *Hostinfo) TailscaleFunnelEnabled() bool { - return hi != nil && hi.WireIngress -} - -func (v HostinfoView) TailscaleFunnelEnabled() bool { return v.Đļ.TailscaleFunnelEnabled() } - // NetInfo contains information about the host's network state. type NetInfo struct { // MappingVariesByDestIP says whether the host's NAT mappings From 8b65598614569b060a1af44258a706aa04d7aa5c Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 22 Jan 2025 10:05:43 -0800 Subject: [PATCH 165/223] util/slicesx: add AppendNonzero By request of @agottardo. Updates #cleanup Change-Id: I2f02314eb9533b1581e47b66b45b6fb8ac257bb7 Signed-off-by: Brad Fitzpatrick --- cmd/tailscale/cli/cli.go | 2 +- util/slicesx/slicesx.go | 11 +++++++++++ util/slicesx/slicesx_test.go | 13 +++++++++++++ util/syspolicy/internal/metrics/metrics.go | 4 +--- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index 645859038a1f3..b419417f9df06 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -242,7 +242,7 @@ change in the future. } func nonNilCmds(cmds ...*ffcli.Command) []*ffcli.Command { - return slicesx.Filter(cmds[:0], cmds, func(c *ffcli.Command) bool { return c != nil }) + return slicesx.AppendNonzero(cmds[:0], cmds) } func fatalf(format string, a ...any) { diff --git a/util/slicesx/slicesx.go b/util/slicesx/slicesx.go index 1a7e18d914114..ff9d473759fb0 100644 --- a/util/slicesx/slicesx.go +++ b/util/slicesx/slicesx.go @@ -95,6 +95,17 @@ func Filter[S ~[]T, T any](dst, src S, fn func(T) bool) S { return dst } +// AppendNonzero appends all non-zero elements of src to dst. +func AppendNonzero[S ~[]T, T comparable](dst, src S) S { + var zero T + for _, v := range src { + if v != zero { + dst = append(dst, v) + } + } + return dst +} + // AppendMatching appends elements in ps to dst if f(x) is true. func AppendMatching[T any](dst, ps []T, f func(T) bool) []T { for _, p := range ps { diff --git a/util/slicesx/slicesx_test.go b/util/slicesx/slicesx_test.go index 597b22b8335fe..34644928465d8 100644 --- a/util/slicesx/slicesx_test.go +++ b/util/slicesx/slicesx_test.go @@ -137,6 +137,19 @@ func TestFilterNoAllocations(t *testing.T) { } } +func TestAppendNonzero(t *testing.T) { + v := []string{"one", "two", "", "four"} + got := AppendNonzero(nil, v) + want := []string{"one", "two", "four"} + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v; want %v", got, want) + } + got = AppendNonzero(v[:0], v) + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v; want %v", got, want) + } +} + func TestAppendMatching(t *testing.T) { v := []string{"one", "two", "three", "four"} got := AppendMatching(v[:0], v, func(s string) bool { return len(s) > 3 }) diff --git a/util/syspolicy/internal/metrics/metrics.go b/util/syspolicy/internal/metrics/metrics.go index 0a2aa1192fc53..d8ba271a8e61c 100644 --- a/util/syspolicy/internal/metrics/metrics.go +++ b/util/syspolicy/internal/metrics/metrics.go @@ -289,7 +289,7 @@ func newSettingMetric(key setting.Key, scope setting.Scope, suffix string, typ c } func newMetric(nameParts []string, typ clientmetric.Type) metric { - name := strings.Join(slicesx.Filter([]string{internal.OS(), "syspolicy"}, nameParts, isNonEmpty), "_") + name := strings.Join(slicesx.AppendNonzero([]string{internal.OS(), "syspolicy"}, nameParts), "_") switch { case !ShouldReport(): return &funcMetric{name: name, typ: typ} @@ -304,8 +304,6 @@ func newMetric(nameParts []string, typ clientmetric.Type) metric { } } -func isNonEmpty(s string) bool { return s != "" } - func metricScopeName(scope setting.Scope) string { switch scope { case setting.DeviceSetting: From d1b378504c11cc2a6db896a02a4b963818b07170 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:46:13 -0700 Subject: [PATCH 166/223] .github: Bump slackapi/slack-github-action from 1.27.0 to 2.0.0 (#14141) Bumps [slackapi/slack-github-action](https://github.com/slackapi/slack-github-action) from 1.27.0 to 2.0.0. - [Release notes](https://github.com/slackapi/slack-github-action/releases) - [Commits](https://github.com/slackapi/slack-github-action/compare/37ebaef184d7626c5f204ab8d3baff4262dd30f0...485a9d42d3a73031f12ec201c457e2162c45d02d) --- updated-dependencies: - dependency-name: slackapi/slack-github-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/govulncheck.yml | 8 ++++---- .github/workflows/test.yml | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 4a5ad54f391e8..989e55fb112ea 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -24,13 +24,13 @@ jobs: - name: Post to slack if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }} + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 with: - channel-id: 'C05PXRM304B' + method: chat.postMessage + token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }} payload: | { + "channel": "C05PXRM304B", "blocks": [ { "type": "section", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20f215cd0ed42..d049323a3513d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -569,8 +569,10 @@ jobs: # By having the job always run, but skipping its only step as needed, we # let the CI output collapse nicely in PRs. if: failure() && github.event_name == 'push' - uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0 + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: incoming-webhook payload: | { "attachments": [{ @@ -582,9 +584,6 @@ jobs: "color": "danger" }] } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK check_mergeability: if: always() From 0fa7b4a236bc492d2c83e4ec319f4d0614d37774 Mon Sep 17 00:00:00 2001 From: Adrian Dewhurst Date: Tue, 21 Jan 2025 17:07:34 -0500 Subject: [PATCH 167/223] tailcfg: add ServiceName Rather than using a string everywhere and needing to clarify that the string should have the svc: prefix, create a separate type for Service names. Updates tailscale/corp#24607 Change-Id: I720e022f61a7221644bb60955b72cacf42f59960 Signed-off-by: Adrian Dewhurst --- cmd/k8s-operator/ingress-for-pg.go | 16 +++--- cmd/k8s-operator/ingress-for-pg_test.go | 2 +- cmd/tailscale/cli/advertise.go | 2 +- ipn/ipn_clone.go | 4 +- ipn/ipn_view.go | 4 +- ipn/ipnlocal/local.go | 17 +++--- ipn/ipnlocal/local_test.go | 16 +++--- ipn/ipnlocal/serve.go | 6 +-- ipn/ipnlocal/serve_test.go | 12 ++--- ipn/serve.go | 6 +-- tailcfg/tailcfg.go | 72 ++++++++++++++++--------- types/netmap/netmap.go | 10 ++-- 12 files changed, 95 insertions(+), 72 deletions(-) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 4dcaf7c6d1a86..e90187d586d12 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -222,13 +222,14 @@ func (a *IngressPGReconciler) maybeProvision(ctx context.Context, hostname strin }, }, } + serviceName := tailcfg.ServiceName("svc:" + hostname) var gotCfg *ipn.ServiceConfig if cfg != nil && cfg.Services != nil { - gotCfg = cfg.Services[hostname] + gotCfg = cfg.Services[serviceName] } if !reflect.DeepEqual(gotCfg, ingCfg) { logger.Infof("Updating serve config") - mak.Set(&cfg.Services, hostname, ingCfg) + mak.Set(&cfg.Services, serviceName, ingCfg) cfgBytes, err := json.Marshal(cfg) if err != nil { return fmt.Errorf("error marshaling serve config: %w", err) @@ -309,7 +310,7 @@ func (a *IngressPGReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG found := false for _, i := range ingList.Items { ingressHostname := hostnameForIngress(&i) - if ingressHostname == vipHostname { + if ingressHostname == vipHostname.WithoutPrefix() { found = true break } @@ -317,7 +318,7 @@ func (a *IngressPGReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG if !found { logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipHostname) - svc, err := a.getVIPService(ctx, vipHostname, logger) + svc, err := a.getVIPService(ctx, vipHostname.WithoutPrefix(), logger) if err != nil { errResp := &tailscale.ErrResponse{} if errors.As(err, &errResp) && errResp.Status == http.StatusNotFound { @@ -329,7 +330,7 @@ func (a *IngressPGReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG } if isVIPServiceForAnyIngress(svc) { logger.Infof("cleaning up orphaned VIPService %q", vipHostname) - if err := a.tsClient.deleteVIPServiceByName(ctx, vipHostname); err != nil { + if err := a.tsClient.deleteVIPServiceByName(ctx, vipHostname.WithoutPrefix()); err != nil { errResp := &tailscale.ErrResponse{} if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound { return fmt.Errorf("deleting VIPService %q: %w", vipHostname, err) @@ -374,11 +375,12 @@ func (a *IngressPGReconciler) maybeCleanup(ctx context.Context, hostname string, if err != nil { return fmt.Errorf("error getting ProxyGroup serve config: %w", err) } + serviceName := tailcfg.ServiceName("svc:" + hostname) // VIPService is always first added to serve config and only then created in the Tailscale API, so if it is not // found in the serve config, we can assume that there is no VIPService. TODO(irbekrm): once we have ingress // ProxyGroup, we will probably add currently exposed VIPServices to its status. At that point, we can use the // status rather than checking the serve config each time. - if cfg == nil || cfg.Services == nil || cfg.Services[hostname] == nil { + if cfg == nil || cfg.Services == nil || cfg.Services[serviceName] == nil { return nil } logger.Infof("Ensuring that VIPService %q configuration is cleaned up", hostname) @@ -390,7 +392,7 @@ func (a *IngressPGReconciler) maybeCleanup(ctx context.Context, hostname string, // 3. Remove the VIPService from the serve config for the ProxyGroup. logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg) - delete(cfg.Services, hostname) + delete(cfg.Services, serviceName) cfgBytes, err := json.Marshal(cfg) if err != nil { return fmt.Errorf("error marshaling serve config: %w", err) diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 2cd340962f993..9ef36f6968fd7 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -137,7 +137,7 @@ func TestIngressPGReconciler(t *testing.T) { t.Fatalf("unmarshaling serve config: %v", err) } - if cfg.Services["my-svc"] == nil { + if cfg.Services["svc:my-svc"] == nil { t.Error("expected serve config to contain VIPService configuration") } diff --git a/cmd/tailscale/cli/advertise.go b/cmd/tailscale/cli/advertise.go index 00b5024f0d3b8..83d1a35aa8a14 100644 --- a/cmd/tailscale/cli/advertise.go +++ b/cmd/tailscale/cli/advertise.go @@ -66,7 +66,7 @@ func parseServiceNames(servicesArg string) ([]string, error) { if servicesArg != "" { services = strings.Split(servicesArg, ",") for _, svc := range services { - err := tailcfg.CheckServiceName(svc) + err := tailcfg.ServiceName(svc).Validate() if err != nil { return nil, fmt.Errorf("service %q: %s", svc, err) } diff --git a/ipn/ipn_clone.go b/ipn/ipn_clone.go index 34d7ba9a66364..47cca71d024ab 100644 --- a/ipn/ipn_clone.go +++ b/ipn/ipn_clone.go @@ -106,7 +106,7 @@ func (src *ServeConfig) Clone() *ServeConfig { } } if dst.Services != nil { - dst.Services = map[string]*ServiceConfig{} + dst.Services = map[tailcfg.ServiceName]*ServiceConfig{} for k, v := range src.Services { if v == nil { dst.Services[k] = nil @@ -133,7 +133,7 @@ func (src *ServeConfig) Clone() *ServeConfig { var _ServeConfigCloneNeedsRegeneration = ServeConfig(struct { TCP map[uint16]*TCPPortHandler Web map[HostPort]*WebServerConfig - Services map[string]*ServiceConfig + Services map[tailcfg.ServiceName]*ServiceConfig AllowFunnel map[HostPort]bool Foreground map[string]*ServeConfig ETag string diff --git a/ipn/ipn_view.go b/ipn/ipn_view.go index 9cd5a466a6840..41b4ddbc89e09 100644 --- a/ipn/ipn_view.go +++ b/ipn/ipn_view.go @@ -195,7 +195,7 @@ func (v ServeConfigView) Web() views.MapFn[HostPort, *WebServerConfig, WebServer }) } -func (v ServeConfigView) Services() views.MapFn[string, *ServiceConfig, ServiceConfigView] { +func (v ServeConfigView) Services() views.MapFn[tailcfg.ServiceName, *ServiceConfig, ServiceConfigView] { return views.MapFnOf(v.Đļ.Services, func(t *ServiceConfig) ServiceConfigView { return t.View() }) @@ -216,7 +216,7 @@ func (v ServeConfigView) ETag() string { return v.Đļ.ETag } var _ServeConfigViewNeedsRegeneration = ServeConfig(struct { TCP map[uint16]*TCPPortHandler Web map[HostPort]*WebServerConfig - Services map[string]*ServiceConfig + Services map[tailcfg.ServiceName]*ServiceConfig AllowFunnel map[HostPort]bool Foreground map[string]*ServeConfig ETag string diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 470824fde51ed..2bd46b852ed6e 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -3417,13 +3417,13 @@ func generateInterceptVIPServicesTCPPortFunc(svcAddrPorts map[netip.Addr]func(ui // setVIPServicesTCPPortsIntercepted populates b.shouldInterceptVIPServicesTCPPortAtomic with an // efficient func for ShouldInterceptTCPPort to use, which is called on every incoming packet. -func (b *LocalBackend) setVIPServicesTCPPortsIntercepted(svcPorts map[string][]uint16) { +func (b *LocalBackend) setVIPServicesTCPPortsIntercepted(svcPorts map[tailcfg.ServiceName][]uint16) { b.mu.Lock() defer b.mu.Unlock() b.setVIPServicesTCPPortsInterceptedLocked(svcPorts) } -func (b *LocalBackend) setVIPServicesTCPPortsInterceptedLocked(svcPorts map[string][]uint16) { +func (b *LocalBackend) setVIPServicesTCPPortsInterceptedLocked(svcPorts map[tailcfg.ServiceName][]uint16) { if len(svcPorts) == 0 { b.shouldInterceptVIPServicesTCPPortAtomic.Store(func(netip.AddrPort) bool { return false }) return @@ -6025,7 +6025,7 @@ func (b *LocalBackend) reloadServeConfigLocked(prefs ipn.PrefsView) { // b.mu must be held. func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn.PrefsView) { handlePorts := make([]uint16, 0, 4) - var vipServicesPorts map[string][]uint16 + var vipServicesPorts map[tailcfg.ServiceName][]uint16 if prefs.Valid() && prefs.RunSSH() && envknob.CanSSHD() { handlePorts = append(handlePorts, 22) @@ -7815,7 +7815,7 @@ func (b *LocalBackend) vipServiceHash(services []*tailcfg.VIPService) string { func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcfg.VIPService { // keyed by service name - var services map[string]*tailcfg.VIPService + var services map[tailcfg.ServiceName]*tailcfg.VIPService if !b.serveConfig.Valid() { return nil } @@ -7828,12 +7828,13 @@ func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcf } for _, s := range prefs.AdvertiseServices().All() { - if services == nil || services[s] == nil { - mak.Set(&services, s, &tailcfg.VIPService{ - Name: s, + sn := tailcfg.ServiceName(s) + if services == nil || services[sn] == nil { + mak.Set(&services, sn, &tailcfg.VIPService{ + Name: sn, }) } - services[s].Active = true + services[sn].Active = true } return slicesx.MapValues(services) diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index f851bb0f8a018..b1a79860d1de0 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -2715,7 +2715,7 @@ func TestTCPHandlerForDstWithVIPService(t *testing.T) { err = b.setServeConfigLocked( &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:foo": { TCP: map[uint16]*ipn.TCPPortHandler{ 882: {HTTP: true}, @@ -4747,7 +4747,7 @@ func TestGetVIPServices(t *testing.T) { "served-only", []string{}, &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:abc": {Tun: true}, }, }, @@ -4762,7 +4762,7 @@ func TestGetVIPServices(t *testing.T) { "served-and-advertised", []string{"svc:abc"}, &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:abc": {Tun: true}, }, }, @@ -4778,7 +4778,7 @@ func TestGetVIPServices(t *testing.T) { "served-and-advertised-different-service", []string{"svc:def"}, &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:abc": {Tun: true}, }, }, @@ -4797,7 +4797,7 @@ func TestGetVIPServices(t *testing.T) { "served-with-port-ranges-one-range-single", []string{}, &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTPS: true}, }}, @@ -4814,7 +4814,7 @@ func TestGetVIPServices(t *testing.T) { "served-with-port-ranges-one-range-multiple", []string{}, &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTPS: true}, 81: {HTTPS: true}, @@ -4833,7 +4833,7 @@ func TestGetVIPServices(t *testing.T) { "served-with-port-ranges-multiple-ranges", []string{}, &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTPS: true}, 81: {HTTPS: true}, @@ -4866,7 +4866,7 @@ func TestGetVIPServices(t *testing.T) { } got := lb.vipServicesFromPrefsLocked(prefs.View()) slices.SortFunc(got, func(a, b *tailcfg.VIPService) int { - return strings.Compare(a.Name, b.Name) + return strings.Compare(a.Name.String(), b.Name.String()) }) if !reflect.DeepEqual(tt.want, got) { t.Logf("want:") diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index a5247dd8c6734..63cb2ef5553ae 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -55,7 +55,7 @@ var serveHTTPContextKey ctxkey.Key[*serveHTTPContext] type serveHTTPContext struct { SrcAddr netip.AddrPort - ForVIPService string // VIP service name, empty string means local + ForVIPService tailcfg.ServiceName // "" means local DestPort uint16 // provides funnel-specific context, nil if not funneled @@ -1006,7 +1006,7 @@ func allNumeric(s string) bool { return s != "" } -func (b *LocalBackend) webServerConfig(hostname string, forVIPService string, port uint16) (c ipn.WebServerConfigView, ok bool) { +func (b *LocalBackend) webServerConfig(hostname string, forVIPService tailcfg.ServiceName, port uint16) (c ipn.WebServerConfigView, ok bool) { key := ipn.HostPort(fmt.Sprintf("%s:%v", hostname, port)) b.mu.Lock() @@ -1021,7 +1021,7 @@ func (b *LocalBackend) webServerConfig(hostname string, forVIPService string, po return b.serveConfig.FindWeb(key) } -func (b *LocalBackend) getTLSServeCertForPort(port uint16, forVIPService string) func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { +func (b *LocalBackend) getTLSServeCertForPort(port uint16, forVIPService tailcfg.ServiceName) func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { return func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { if hi == nil || hi.ServerName == "" { return nil, errors.New("no SNI ServerName") diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index f2ea8e5cd6dbf..eb8169390785a 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -354,7 +354,7 @@ func TestServeConfigServices(t *testing.T) { { name: "one-incorrectly-configured-service", conf: &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:foo": { TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTP: true}, @@ -369,7 +369,7 @@ func TestServeConfigServices(t *testing.T) { // one correctly configured service with packet should be intercepted name: "one-service-intercept-packet", conf: &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:foo": { TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTP: true}, @@ -388,7 +388,7 @@ func TestServeConfigServices(t *testing.T) { // one correctly configured service with packet should not be intercepted name: "one-service-not-intercept-packet", conf: &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:foo": { TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTP: true}, @@ -406,10 +406,10 @@ func TestServeConfigServices(t *testing.T) { intercepted: false, }, { - //multiple correctly configured service with packet should be intercepted + // multiple correctly configured service with packet should be intercepted name: "multiple-service-intercept-packet", conf: &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:foo": { TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTP: true}, @@ -437,7 +437,7 @@ func TestServeConfigServices(t *testing.T) { // multiple correctly configured service with packet should not be intercepted name: "multiple-service-not-intercept-packet", conf: &ipn.ServeConfig{ - Services: map[string]*ipn.ServiceConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:foo": { TCP: map[uint16]*ipn.TCPPortHandler{ 80: {HTTP: true}, diff --git a/ipn/serve.go b/ipn/serve.go index 4c2d2f158144e..ac92287bdc08f 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -57,7 +57,7 @@ type ServeConfig struct { // Services maps from service name (in the form "svc:dns-label") to a ServiceConfig. // Which describes the L3, L4, and L7 forwarding information for the service. - Services map[string]*ServiceConfig `json:",omitempty"` + Services map[tailcfg.ServiceName]*ServiceConfig `json:",omitempty"` // AllowFunnel is the set of SNI:port values for which funnel // traffic is allowed, from trusted ingress peers. @@ -618,7 +618,7 @@ func (v ServeConfigView) Webs() iter.Seq2[HostPort, WebServerConfigView] { } // FindServiceTCP return the TCPPortHandlerView for the given service name and port. -func (v ServeConfigView) FindServiceTCP(svcName string, port uint16) (res TCPPortHandlerView, ok bool) { +func (v ServeConfigView) FindServiceTCP(svcName tailcfg.ServiceName, port uint16) (res TCPPortHandlerView, ok bool) { svcCfg, ok := v.Services().GetOk(svcName) if !ok { return res, ok @@ -626,7 +626,7 @@ func (v ServeConfigView) FindServiceTCP(svcName string, port uint16) (res TCPPor return svcCfg.TCP().GetOk(port) } -func (v ServeConfigView) FindServiceWeb(svcName string, hp HostPort) (res WebServerConfigView, ok bool) { +func (v ServeConfigView) FindServiceWeb(svcName tailcfg.ServiceName, hp HostPort) (res WebServerConfigView, ok bool) { if svcCfg, ok := v.Services().GetOk(svcName); ok { if res, ok := svcCfg.Web().GetOk(hp); ok { return res, ok diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index e1259b3f50cf0..c921a0c7d69a9 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -717,21 +717,6 @@ func CheckTag(tag string) error { return nil } -// CheckServiceName validates svc for use as a service name. -// We only allow valid DNS labels, since the expectation is that these will be -// used as parts of domain names. -func CheckServiceName(svc string) error { - var ok bool - svc, ok = strings.CutPrefix(svc, "svc:") - if !ok { - return errors.New("services must start with 'svc:'") - } - if svc == "" { - return errors.New("service names must not be empty") - } - return dnsname.ValidLabel(svc) -} - // CheckRequestTags checks that all of h.RequestTags are valid. func (h *Hostinfo) CheckRequestTags() error { if h == nil { @@ -897,16 +882,51 @@ type Hostinfo struct { // require changes to Hostinfo.Equal. } +// ServiceName is the name of a service, of the form `svc:dns-label`. Services +// represent some kind of application provided for users of the tailnet with a +// MagicDNS name and possibly dedicated IP addresses. Currently (2024-01-21), +// the only type of service is [VIPService]. +// This is not related to the older [Service] used in [Hostinfo.Services]. +type ServiceName string + +// Validate validates if the service name is formatted correctly. +// We only allow valid DNS labels, since the expectation is that these will be +// used as parts of domain names. +func (sn ServiceName) Validate() error { + bareName, ok := strings.CutPrefix(string(sn), "svc:") + if !ok { + return errors.New("services must start with 'svc:'") + } + if bareName == "" { + return errors.New("service names must not be empty") + } + return dnsname.ValidLabel(bareName) +} + +// String implements [fmt.Stringer]. +func (sn ServiceName) String() string { + return string(sn) +} + +// WithoutPrefix is the name of the service without the `svc:` prefix, used for +// DNS names. If the name does not include the prefix (which means +// [ServiceName.Validate] would return an error) then it returns "". +func (sn ServiceName) WithoutPrefix() string { + bareName, ok := strings.CutPrefix(string(sn), "svc:") + if !ok { + return "" + } + return bareName +} + // VIPService represents a service created on a tailnet from the // perspective of a node providing that service. These services // have an virtual IP (VIP) address pair distinct from the node's IPs. type VIPService struct { - // Name is the name of the service, of the form `svc:dns-label`. - // See CheckServiceName for a validation func. - // Name uniquely identifies a service on a particular tailnet, - // and so also corresponds uniquely to the pair of IP addresses - // belonging to the VIP service. - Name string + // Name is the name of the service. The Name uniquely identifies a service + // on a particular tailnet, and so also corresponds uniquely to the pair of + // IP addresses belonging to the VIP service. + Name ServiceName // Ports specify which ProtoPorts are made available by this node // on the service's IPs. @@ -2972,10 +2992,10 @@ type EarlyNoise struct { // vs NodeKey) const LBHeader = "Ts-Lb" -// ServiceIPMappings maps service names (strings that conform to -// [CheckServiceName]) to lists of IP addresses. This is used as the value of -// the [NodeAttrServiceHost] capability, to inform service hosts what IP -// addresses they need to listen on for each service that they are advertising. +// ServiceIPMappings maps ServiceName to lists of IP addresses. This is used +// as the value of the [NodeAttrServiceHost] capability, to inform service hosts +// what IP addresses they need to listen on for each service that they are +// advertising. // // This is of the form: // @@ -2988,4 +3008,4 @@ const LBHeader = "Ts-Lb" // provided in AllowedIPs, but this lets the client know which services // correspond to those IPs. Any services that don't correspond to a service // this client is hosting can be ignored. -type ServiceIPMappings map[string][]netip.Addr +type ServiceIPMappings map[ServiceName][]netip.Addr diff --git a/types/netmap/netmap.go b/types/netmap/netmap.go index 1482e534e9755..ab22eec3e3486 100644 --- a/types/netmap/netmap.go +++ b/types/netmap/netmap.go @@ -427,10 +427,10 @@ const ( ) // IPServiceMappings maps IP addresses to service names. This is the inverse of -// [ServiceIPMappings], and is used to inform clients which services is an VIP -// address associated with. This is set to b.ipVIPServiceMap every time the -// netmap is updated. This is used to reduce the cost for looking up the service -// name for the dst IP address in the netStack packet processing workflow. +// [tailcfg.ServiceIPMappings], and is used to inform track which service a VIP +// is associated with. This is set to b.ipVIPServiceMap every time the netmap is +// updated. This is used to reduce the cost for looking up the service name for +// the dst IP address in the netStack packet processing workflow. // // This is of the form: // @@ -440,4 +440,4 @@ const ( // "100.102.42.3": "svc:web", // "fd7a:115c:a1e0::abcd": "svc:web", // } -type IPServiceMappings map[netip.Addr]string +type IPServiceMappings map[netip.Addr]tailcfg.ServiceName From 3dabea0fc2c224249b2a503431fc610f8883d3e1 Mon Sep 17 00:00:00 2001 From: Andrea Gottardo Date: Wed, 22 Jan 2025 16:01:07 -0800 Subject: [PATCH 168/223] cmd/tailscale: define CLI tools to manipulate macOS network and system extensions (#14727) Updates tailscale/corp#25278 Adds definitions for new CLI commands getting added in v1.80. Refactors some pre-existing CLI commands within the `configure` tree to clean up code. Signed-off-by: Andrea Gottardo --- cmd/tailscale/cli/cli.go | 7 +- cmd/tailscale/cli/configure-kube.go | 28 +++--- cmd/tailscale/cli/configure-kube_omit.go | 13 +++ cmd/tailscale/cli/configure-synology-cert.go | 27 +++--- cmd/tailscale/cli/configure-synology.go | 55 +++++++---- cmd/tailscale/cli/configure.go | 43 +++++---- cmd/tailscale/cli/configure_apple-all.go | 11 +++ cmd/tailscale/cli/configure_apple.go | 97 ++++++++++++++++++++ 8 files changed, 213 insertions(+), 68 deletions(-) create mode 100644 cmd/tailscale/cli/configure-kube_omit.go create mode 100644 cmd/tailscale/cli/configure_apple-all.go create mode 100644 cmd/tailscale/cli/configure_apple.go diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index b419417f9df06..fd39b3b67d3fd 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -190,7 +190,7 @@ change in the future. loginCmd, logoutCmd, switchCmd, - configureCmd, + configureCmd(), syspolicyCmd, netcheckCmd, ipCmd, @@ -216,6 +216,7 @@ change in the future. driveCmd, idTokenCmd, advertiseCmd(), + configureHostCmd(), ), FlagSet: rootfs, Exec: func(ctx context.Context, args []string) error { @@ -226,10 +227,6 @@ change in the future. }, } - if runtime.GOOS == "linux" && distro.Get() == distro.Synology { - rootCmd.Subcommands = append(rootCmd.Subcommands, configureHostCmd) - } - walkCommands(rootCmd, func(w cmdWalk) bool { if w.UsageFunc == nil { w.UsageFunc = usageFunc diff --git a/cmd/tailscale/cli/configure-kube.go b/cmd/tailscale/cli/configure-kube.go index 6af15e3d9ae7b..6bc4e202efd4e 100644 --- a/cmd/tailscale/cli/configure-kube.go +++ b/cmd/tailscale/cli/configure-kube.go @@ -20,33 +20,31 @@ import ( "tailscale.com/version" ) -func init() { - configureCmd.Subcommands = append(configureCmd.Subcommands, configureKubeconfigCmd) -} - -var configureKubeconfigCmd = &ffcli.Command{ - Name: "kubeconfig", - ShortHelp: "[ALPHA] Connect to a Kubernetes cluster using a Tailscale Auth Proxy", - ShortUsage: "tailscale configure kubeconfig ", - LongHelp: strings.TrimSpace(` +func configureKubeconfigCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "kubeconfig", + ShortHelp: "[ALPHA] Connect to a Kubernetes cluster using a Tailscale Auth Proxy", + ShortUsage: "tailscale configure kubeconfig ", + LongHelp: strings.TrimSpace(` Run this command to configure kubectl to connect to a Kubernetes cluster over Tailscale. The hostname argument should be set to the Tailscale hostname of the peer running as an auth proxy in the cluster. See: https://tailscale.com/s/k8s-auth-proxy `), - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("kubeconfig") - return fs - })(), - Exec: runConfigureKubeconfig, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("kubeconfig") + return fs + })(), + Exec: runConfigureKubeconfig, + } } // kubeconfigPath returns the path to the kubeconfig file for the current user. func kubeconfigPath() (string, error) { if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { if version.IsSandboxedMacOS() { - return "", errors.New("$KUBECONFIG is incompatible with the App Store version") + return "", errors.New("cannot read $KUBECONFIG on GUI builds of the macOS client: this requires the open-source tailscaled distribution") } var out string for _, out = range filepath.SplitList(kubeconfig) { diff --git a/cmd/tailscale/cli/configure-kube_omit.go b/cmd/tailscale/cli/configure-kube_omit.go new file mode 100644 index 0000000000000..130f2870fab44 --- /dev/null +++ b/cmd/tailscale/cli/configure-kube_omit.go @@ -0,0 +1,13 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_omit_kube + +package cli + +import "github.com/peterbourgon/ff/v3/ffcli" + +func configureKubeconfigCmd() *ffcli.Command { + // omitted from the build when the ts_omit_kube build tag is set + return nil +} diff --git a/cmd/tailscale/cli/configure-synology-cert.go b/cmd/tailscale/cli/configure-synology-cert.go index aabcb8dfad866..663d0c8790456 100644 --- a/cmd/tailscale/cli/configure-synology-cert.go +++ b/cmd/tailscale/cli/configure-synology-cert.go @@ -22,22 +22,27 @@ import ( "tailscale.com/version/distro" ) -var synologyConfigureCertCmd = &ffcli.Command{ - Name: "synology-cert", - Exec: runConfigureSynologyCert, - ShortHelp: "Configure Synology with a TLS certificate for your tailnet", - ShortUsage: "synology-cert [--domain ]", - LongHelp: strings.TrimSpace(` +func synologyConfigureCertCmd() *ffcli.Command { + if runtime.GOOS != "linux" || distro.Get() != distro.Synology { + return nil + } + return &ffcli.Command{ + Name: "synology-cert", + Exec: runConfigureSynologyCert, + ShortHelp: "Configure Synology with a TLS certificate for your tailnet", + ShortUsage: "synology-cert [--domain ]", + LongHelp: strings.TrimSpace(` This command is intended to run periodically as root on a Synology device to create or refresh the TLS certificate for the tailnet domain. See: https://tailscale.com/kb/1153/enabling-https `), - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("synology-cert") - fs.StringVar(&synologyConfigureCertArgs.domain, "domain", "", "Tailnet domain to create or refresh certificates for. Ignored if only one domain exists.") - return fs - })(), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("synology-cert") + fs.StringVar(&synologyConfigureCertArgs.domain, "domain", "", "Tailnet domain to create or refresh certificates for. Ignored if only one domain exists.") + return fs + })(), + } } var synologyConfigureCertArgs struct { diff --git a/cmd/tailscale/cli/configure-synology.go b/cmd/tailscale/cli/configure-synology.go index 9d674e56dd79a..f0f05f75765b9 100644 --- a/cmd/tailscale/cli/configure-synology.go +++ b/cmd/tailscale/cli/configure-synology.go @@ -21,34 +21,49 @@ import ( // configureHostCmd is the "tailscale configure-host" command which was once // used to configure Synology devices, but is now a compatibility alias to // "tailscale configure synology". -var configureHostCmd = &ffcli.Command{ - Name: "configure-host", - Exec: runConfigureSynology, - ShortUsage: "tailscale configure-host\n" + synologyConfigureCmd.ShortUsage, - ShortHelp: synologyConfigureCmd.ShortHelp, - LongHelp: hidden + synologyConfigureCmd.LongHelp, - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("configure-host") - return fs - })(), +// +// It returns nil if the actual "tailscale configure synology" command is not +// available. +func configureHostCmd() *ffcli.Command { + synologyConfigureCmd := synologyConfigureCmd() + if synologyConfigureCmd == nil { + // No need to offer this compatibility alias if the actual command is not available. + return nil + } + return &ffcli.Command{ + Name: "configure-host", + Exec: runConfigureSynology, + ShortUsage: "tailscale configure-host\n" + synologyConfigureCmd.ShortUsage, + ShortHelp: synologyConfigureCmd.ShortHelp, + LongHelp: hidden + synologyConfigureCmd.LongHelp, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("configure-host") + return fs + })(), + } } -var synologyConfigureCmd = &ffcli.Command{ - Name: "synology", - Exec: runConfigureSynology, - ShortUsage: "tailscale configure synology", - ShortHelp: "Configure Synology to enable outbound connections", - LongHelp: strings.TrimSpace(` +func synologyConfigureCmd() *ffcli.Command { + if runtime.GOOS != "linux" || distro.Get() != distro.Synology { + return nil + } + return &ffcli.Command{ + Name: "synology", + Exec: runConfigureSynology, + ShortUsage: "tailscale configure synology", + ShortHelp: "Configure Synology to enable outbound connections", + LongHelp: strings.TrimSpace(` This command is intended to run at boot as root on a Synology device to create the /dev/net/tun device and give the tailscaled binary permission to use it. See: https://tailscale.com/s/synology-outbound `), - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("synology") - return fs - })(), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("synology") + return fs + })(), + } } func runConfigureSynology(ctx context.Context, args []string) error { diff --git a/cmd/tailscale/cli/configure.go b/cmd/tailscale/cli/configure.go index fd136d766360d..acb416755a586 100644 --- a/cmd/tailscale/cli/configure.go +++ b/cmd/tailscale/cli/configure.go @@ -5,32 +5,41 @@ package cli import ( "flag" - "runtime" "strings" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/version/distro" ) -var configureCmd = &ffcli.Command{ - Name: "configure", - ShortUsage: "tailscale configure ", - ShortHelp: "[ALPHA] Configure the host to enable more Tailscale features", - LongHelp: strings.TrimSpace(` +func configureCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "configure", + ShortUsage: "tailscale configure ", + ShortHelp: "Configure the host to enable more Tailscale features", + LongHelp: strings.TrimSpace(` The 'configure' set of commands are intended to provide a way to enable different services on the host to use Tailscale in more ways. `), - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("configure") - return fs - })(), - Subcommands: configureSubcommands(), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("configure") + return fs + })(), + Subcommands: nonNilCmds( + configureKubeconfigCmd(), + synologyConfigureCmd(), + synologyConfigureCertCmd(), + ccall(maybeSysExtCmd), + ccall(maybeVPNConfigCmd), + ), + } } -func configureSubcommands() (out []*ffcli.Command) { - if runtime.GOOS == "linux" && distro.Get() == distro.Synology { - out = append(out, synologyConfigureCmd) - out = append(out, synologyConfigureCertCmd) +// ccall calls the function f if it is non-nil, and returns its result. +// +// It returns the zero value of the type T if f is nil. +func ccall[T any](f func() T) T { + var zero T + if f == nil { + return zero } - return out + return f() } diff --git a/cmd/tailscale/cli/configure_apple-all.go b/cmd/tailscale/cli/configure_apple-all.go new file mode 100644 index 0000000000000..5f0da9b95420e --- /dev/null +++ b/cmd/tailscale/cli/configure_apple-all.go @@ -0,0 +1,11 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import "github.com/peterbourgon/ff/v3/ffcli" + +var ( + maybeSysExtCmd func() *ffcli.Command // non-nil only on macOS, see configure_apple.go + maybeVPNConfigCmd func() *ffcli.Command // non-nil only on macOS, see configure_apple.go +) diff --git a/cmd/tailscale/cli/configure_apple.go b/cmd/tailscale/cli/configure_apple.go new file mode 100644 index 0000000000000..edd9ec1abe5bc --- /dev/null +++ b/cmd/tailscale/cli/configure_apple.go @@ -0,0 +1,97 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin + +package cli + +import ( + "context" + "errors" + + "github.com/peterbourgon/ff/v3/ffcli" +) + +func init() { + maybeSysExtCmd = sysExtCmd + maybeVPNConfigCmd = vpnConfigCmd +} + +// Functions in this file provide a dummy Exec function that only prints an error message for users of the open-source +// tailscaled distribution. On GUI builds, the Swift code in the macOS client handles these commands by not passing the +// flow of execution to the CLI. + +// sysExtCmd returns a command for managing the Tailscale system extension on macOS +// (for the Standalone variant of the client only). +func sysExtCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "sysext", + ShortUsage: "tailscale configure sysext [activate|deactivate|status]", + ShortHelp: "Manages the system extension for macOS (Standalone variant)", + LongHelp: "The sysext set of commands provides a way to activate, deactivate, or manage the state of the Tailscale system extension on macOS. " + + "This is only relevant if you are running the Standalone variant of the Tailscale client for macOS. " + + "To access more detailed information about system extensions installed on this Mac, run 'systemextensionsctl list'.", + Subcommands: []*ffcli.Command{ + { + Name: "activate", + ShortUsage: "tailscale sysext activate", + ShortHelp: "Register the Tailscale system extension with macOS.", + LongHelp: "This command registers the Tailscale system extension with macOS. To run Tailscale, you'll also need to install the VPN configuration separately (run `tailscale configure vpn-config install`). After running this command, you need to approve the extension in System Settings > Login Items and Extensions > Network Extensions.", + Exec: requiresStandalone, + }, + { + Name: "deactivate", + ShortUsage: "tailscale sysext deactivate", + ShortHelp: "Deactivate the Tailscale system extension on macOS", + LongHelp: "This command deactivates the Tailscale system extension on macOS. To completely remove Tailscale, you'll also need to delete the VPN configuration separately (use `tailscale configure vpn-config uninstall`).", + Exec: requiresStandalone, + }, + { + Name: "status", + ShortUsage: "tailscale sysext status", + ShortHelp: "Print the enablement status of the Tailscale system extension", + LongHelp: "This command prints the enablement status of the Tailscale system extension. If the extension is not enabled, run `tailscale sysext activate` to enable it.", + Exec: requiresStandalone, + }, + }, + Exec: requiresStandalone, + } +} + +// vpnConfigCmd returns a command for managing the Tailscale VPN configuration on macOS +// (the entry that appears in System Settings > VPN). +func vpnConfigCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "mac-vpn", + ShortUsage: "tailscale configure mac-vpn [install|uninstall]", + ShortHelp: "Manage the VPN configuration on macOS (App Store and Standalone variants)", + LongHelp: "The vpn-config set of commands provides a way to add or remove the Tailscale VPN configuration from the macOS settings. This is the entry that appears in System Settings > VPN.", + Subcommands: []*ffcli.Command{ + { + Name: "install", + ShortUsage: "tailscale mac-vpn install", + ShortHelp: "Write the Tailscale VPN configuration to the macOS settings", + LongHelp: "This command writes the Tailscale VPN configuration to the macOS settings. This is the entry that appears in System Settings > VPN. If you are running the Standalone variant of the client, you'll also need to install the system extension separately (run `tailscale configure sysext activate`).", + Exec: requiresGUI, + }, + { + Name: "uninstall", + ShortUsage: "tailscale mac-vpn uninstall", + ShortHelp: "Delete the Tailscale VPN configuration from the macOS settings", + LongHelp: "This command removes the Tailscale VPN configuration from the macOS settings. This is the entry that appears in System Settings > VPN. If you are running the Standalone variant of the client, you'll also need to deactivate the system extension separately (run `tailscale configure sysext deactivate`).", + Exec: requiresGUI, + }, + }, + Exec: func(ctx context.Context, args []string) error { + return errors.New("unsupported command: requires a GUI build of the macOS client") + }, + } +} + +func requiresStandalone(ctx context.Context, args []string) error { + return errors.New("unsupported command: requires the Standalone (.pkg installer) GUI build of the client") +} + +func requiresGUI(ctx context.Context, args []string) error { + return errors.New("unsupported command: requires a GUI build of the macOS client") +} From 3fb8a1f6bf4bdc8f438430014721486f3a6f20f6 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Wed, 22 Jan 2025 16:50:25 -0800 Subject: [PATCH 169/223] ipn/ipnlocal: re-advertise appc routes on startup, take 2 (#14740) * Reapply "ipn/ipnlocal: re-advertise appc routes on startup (#14609)" This reverts commit 51adaec35a3e4d25df88d81e6264584e151bd33d. Signed-off-by: Andrew Lytvynov * ipn/ipnlocal: fix a deadlock in readvertiseAppConnectorRoutes Don't hold LocalBackend.mu while calling the methods of appc.AppConnector. Those methods could call back into LocalBackend and try to acquire it's mutex. Fixes https://github.com/tailscale/corp/issues/25965 Fixes #14606 Signed-off-by: Andrew Lytvynov --------- Signed-off-by: Andrew Lytvynov --- ipn/ipnlocal/local.go | 43 +++++++++++++++++++++++++++++++--- ipn/ipnlocal/local_test.go | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 2bd46b852ed6e..4ff3f3db47edf 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4418,6 +4418,41 @@ func (b *LocalBackend) reconfigAppConnectorLocked(nm *netmap.NetworkMap, prefs i b.appConnector.UpdateDomainsAndRoutes(domains, routes) } +func (b *LocalBackend) readvertiseAppConnectorRoutes() { + // Note: we should never call b.appConnector methods while holding b.mu. + // This can lead to a deadlock, like + // https://github.com/tailscale/corp/issues/25965. + // + // Grab a copy of the field, since b.mu only guards access to the + // b.appConnector field itself. + b.mu.Lock() + appConnector := b.appConnector + b.mu.Unlock() + + if appConnector == nil { + return + } + domainRoutes := appConnector.DomainRoutes() + if domainRoutes == nil { + return + } + + // Re-advertise the stored routes, in case stored state got out of + // sync with previously advertised routes in prefs. + var prefixes []netip.Prefix + for _, ips := range domainRoutes { + for _, ip := range ips { + prefixes = append(prefixes, netip.PrefixFrom(ip, ip.BitLen())) + } + } + // Note: AdvertiseRoute will trim routes that are already + // advertised, so if everything is already being advertised this is + // a noop. + if err := b.AdvertiseRoute(prefixes...); err != nil { + b.logf("error advertising stored app connector routes: %v", err) + } +} + // authReconfig pushes a new configuration into wgengine, if engine // updates are not currently blocked, based on the cached netmap and // user prefs. @@ -4496,6 +4531,7 @@ func (b *LocalBackend) authReconfig() { } b.initPeerAPIListener() + b.readvertiseAppConnectorRoutes() } // shouldUseOneCGNATRoute reports whether we should prefer to make one big @@ -7261,7 +7297,7 @@ var ErrDisallowedAutoRoute = errors.New("route is not allowed") // If the route is disallowed, ErrDisallowedAutoRoute is returned. func (b *LocalBackend) AdvertiseRoute(ipps ...netip.Prefix) error { finalRoutes := b.Prefs().AdvertiseRoutes().AsSlice() - newRoutes := false + var newRoutes []netip.Prefix for _, ipp := range ipps { if !allowedAutoRoute(ipp) { @@ -7277,13 +7313,14 @@ func (b *LocalBackend) AdvertiseRoute(ipps ...netip.Prefix) error { } finalRoutes = append(finalRoutes, ipp) - newRoutes = true + newRoutes = append(newRoutes, ipp) } - if !newRoutes { + if len(newRoutes) == 0 { return nil } + b.logf("advertising new app connector routes: %v", newRoutes) _, err := b.EditPrefs(&ipn.MaskedPrefs{ Prefs: ipn.Prefs{ AdvertiseRoutes: finalRoutes, diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index b1a79860d1de0..b7b81ada8b07b 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1501,6 +1501,53 @@ func TestReconfigureAppConnector(t *testing.T) { } } +func TestBackfillAppConnectorRoutes(t *testing.T) { + // Create backend with an empty app connector. + b := newTestBackend(t) + if err := b.Start(ipn.Options{}); err != nil { + t.Fatal(err) + } + if _, err := b.EditPrefs(&ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AppConnector: ipn.AppConnectorPrefs{Advertise: true}, + }, + AppConnectorSet: true, + }); err != nil { + t.Fatal(err) + } + b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs) + + // Smoke check that AdvertiseRoutes doesn't have the test IP. + ip := netip.MustParseAddr("1.2.3.4") + routes := b.Prefs().AdvertiseRoutes().AsSlice() + if slices.Contains(routes, netip.PrefixFrom(ip, ip.BitLen())) { + t.Fatalf("AdvertiseRoutes %v on a fresh backend already contains advertised route for %v", routes, ip) + } + + // Store the test IP in profile data, but not in Prefs.AdvertiseRoutes. + b.ControlKnobs().AppCStoreRoutes.Store(true) + if err := b.storeRouteInfo(&appc.RouteInfo{ + Domains: map[string][]netip.Addr{ + "example.com": {ip}, + }, + }); err != nil { + t.Fatal(err) + } + + // Mimic b.authReconfigure for the app connector bits. + b.mu.Lock() + b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs) + b.mu.Unlock() + b.readvertiseAppConnectorRoutes() + + // Check that Prefs.AdvertiseRoutes got backfilled with routes stored in + // profile data. + routes = b.Prefs().AdvertiseRoutes().AsSlice() + if !slices.Contains(routes, netip.PrefixFrom(ip, ip.BitLen())) { + t.Fatalf("AdvertiseRoutes %v was not backfilled from stored app connector routes with %v", routes, ip) + } +} + func resolversEqual(t *testing.T, a, b []*dnstype.Resolver) bool { if a == nil && b == nil { return true From 1562a6f2f2b8017a65ae147e48f23e1ec113ac2f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 22 Jan 2025 11:56:36 -0800 Subject: [PATCH 170/223] feature/*: make Wake-on-LAN conditional, start supporting modular features This pulls out the Wake-on-LAN (WoL) code out into its own package (feature/wakeonlan) that registers itself with various new hooks around tailscaled. Then a new build tag (ts_omit_wakeonlan) causes the package to not even be linked in the binary. Ohter new packages include: * feature: to just record which features are loaded. Future: dependencies between features. * feature/condregister: the package with all the build tags that tailscaled, tsnet, and the Tailscale Xcode project extension can empty (underscore) import to load features as a function of the defined build tags. Future commits will move of our "ts_omit_foo" build tags into this style. Updates #12614 Change-Id: I9c5378dafb1113b62b816aabef02714db3fc9c4a Signed-off-by: Brad Fitzpatrick --- build_dist.sh | 2 +- cmd/k8s-operator/depaware.txt | 5 +- cmd/tailscaled/depaware.txt | 5 +- cmd/tailscaled/tailscaled.go | 1 + feature/condregister/condregister.go | 7 + feature/condregister/maybe_wakeonlan.go | 8 + feature/feature.go | 15 ++ feature/wakeonlan/wakeonlan.go | 243 ++++++++++++++++++ hostinfo/hostinfo.go | 15 +- hostinfo/wol.go | 106 -------- ipn/ipnlocal/c2n.go | 67 +---- ipn/ipnlocal/peerapi.go | 117 +++------ tsnet/tsnet.go | 1 + .../tailscaled_deps_test_darwin.go | 1 + .../tailscaled_deps_test_freebsd.go | 1 + .../integration/tailscaled_deps_test_linux.go | 1 + .../tailscaled_deps_test_openbsd.go | 1 + .../tailscaled_deps_test_windows.go | 1 + 18 files changed, 355 insertions(+), 242 deletions(-) create mode 100644 feature/condregister/condregister.go create mode 100644 feature/condregister/maybe_wakeonlan.go create mode 100644 feature/feature.go create mode 100644 feature/wakeonlan/wakeonlan.go delete mode 100644 hostinfo/wol.go diff --git a/build_dist.sh b/build_dist.sh index 66afa8f745c4c..9a29e5201123a 100755 --- a/build_dist.sh +++ b/build_dist.sh @@ -37,7 +37,7 @@ while [ "$#" -gt 1 ]; do --extra-small) shift ldflags="$ldflags -w -s" - tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion" + tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion,ts_omit_ssh,ts_omit_wakeonlan" ;; --box) shift diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index a27e1761d83c5..bdcf3417a623b 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -156,7 +156,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd - github.com/kortschak/wol from tailscale.com/ipn/ipnlocal + github.com/kortschak/wol from tailscale.com/feature/wakeonlan github.com/mailru/easyjson/buffer from github.com/mailru/easyjson/jwriter đŸ’Ŗ github.com/mailru/easyjson/jlexer from github.com/go-openapi/swag github.com/mailru/easyjson/jwriter from github.com/go-openapi/swag @@ -801,6 +801,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/drive from tailscale.com/client/tailscale+ tailscale.com/envknob from tailscale.com/client/tailscale+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ + tailscale.com/feature from tailscale.com/feature/wakeonlan + tailscale.com/feature/condregister from tailscale.com/tsnet + tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal tailscale.com/hostinfo from tailscale.com/client/web+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 1fc1b8d70060a..5246b82b92485 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -127,7 +127,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd - github.com/kortschak/wol from tailscale.com/ipn/ipnlocal + github.com/kortschak/wol from tailscale.com/feature/wakeonlan LD github.com/kr/fs from github.com/pkg/sftp L github.com/mdlayher/genetlink from tailscale.com/net/tstun L đŸ’Ŗ github.com/mdlayher/netlink from github.com/google/nftables+ @@ -259,6 +259,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/drive/driveimpl/shared from tailscale.com/drive/driveimpl+ tailscale.com/envknob from tailscale.com/client/tailscale+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ + tailscale.com/feature from tailscale.com/feature/wakeonlan + tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled + tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal tailscale.com/hostinfo from tailscale.com/client/web+ diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 9dd00ddd95624..bab3bc75a3586 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -35,6 +35,7 @@ import ( "tailscale.com/control/controlclient" "tailscale.com/drive/driveimpl" "tailscale.com/envknob" + _ "tailscale.com/feature/condregister" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/conffile" diff --git a/feature/condregister/condregister.go b/feature/condregister/condregister.go new file mode 100644 index 0000000000000..f9025095147f1 --- /dev/null +++ b/feature/condregister/condregister.go @@ -0,0 +1,7 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// The condregister package registers all conditional features guarded +// by build tags. It is one central package that callers can empty import +// to ensure all conditional features are registered. +package condregister diff --git a/feature/condregister/maybe_wakeonlan.go b/feature/condregister/maybe_wakeonlan.go new file mode 100644 index 0000000000000..14cae605d1468 --- /dev/null +++ b/feature/condregister/maybe_wakeonlan.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_wakeonlan + +package condregister + +import _ "tailscale.com/feature/wakeonlan" diff --git a/feature/feature.go b/feature/feature.go new file mode 100644 index 0000000000000..ea290c43ac0d4 --- /dev/null +++ b/feature/feature.go @@ -0,0 +1,15 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package feature tracks which features are linked into the binary. +package feature + +var in = map[string]bool{} + +// Register notes that the named feature is linked into the binary. +func Register(name string) { + if _, ok := in[name]; ok { + panic("duplicate feature registration for " + name) + } + in[name] = true +} diff --git a/feature/wakeonlan/wakeonlan.go b/feature/wakeonlan/wakeonlan.go new file mode 100644 index 0000000000000..96c424084dcc6 --- /dev/null +++ b/feature/wakeonlan/wakeonlan.go @@ -0,0 +1,243 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package wakeonlan registers the Wake-on-LAN feature. +package wakeonlan + +import ( + "encoding/json" + "log" + "net" + "net/http" + "runtime" + "sort" + "strings" + "unicode" + + "github.com/kortschak/wol" + "tailscale.com/envknob" + "tailscale.com/feature" + "tailscale.com/hostinfo" + "tailscale.com/ipn/ipnlocal" + "tailscale.com/tailcfg" + "tailscale.com/util/clientmetric" +) + +func init() { + feature.Register("wakeonlan") + ipnlocal.RegisterC2N("POST /wol", handleC2NWoL) + ipnlocal.RegisterPeerAPIHandler("/v0/wol", handlePeerAPIWakeOnLAN) + hostinfo.RegisterHostinfoNewHook(func(h *tailcfg.Hostinfo) { + h.WoLMACs = getWoLMACs() + }) +} + +func handleC2NWoL(b *ipnlocal.LocalBackend, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + var macs []net.HardwareAddr + for _, macStr := range r.Form["mac"] { + mac, err := net.ParseMAC(macStr) + if err != nil { + http.Error(w, "bad 'mac' param", http.StatusBadRequest) + return + } + macs = append(macs, mac) + } + var res struct { + SentTo []string + Errors []string + } + st := b.NetMon().InterfaceState() + if st == nil { + res.Errors = append(res.Errors, "no interface state") + writeJSON(w, &res) + return + } + var password []byte // TODO(bradfitz): support? does anything use WoL passwords? + for _, mac := range macs { + for ifName, ips := range st.InterfaceIPs { + for _, ip := range ips { + if ip.Addr().IsLoopback() || ip.Addr().Is6() { + continue + } + local := &net.UDPAddr{ + IP: ip.Addr().AsSlice(), + Port: 0, + } + remote := &net.UDPAddr{ + IP: net.IPv4bcast, + Port: 0, + } + if err := wol.Wake(mac, password, local, remote); err != nil { + res.Errors = append(res.Errors, err.Error()) + } else { + res.SentTo = append(res.SentTo, ifName) + } + break // one per interface is enough + } + } + } + sort.Strings(res.SentTo) + writeJSON(w, &res) +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +func canWakeOnLAN(h ipnlocal.PeerAPIHandler) bool { + if h.Peer().UnsignedPeerAPIOnly() { + return false + } + return h.IsSelfUntagged() || h.PeerCaps().HasCapability(tailcfg.PeerCapabilityWakeOnLAN) +} + +var metricWakeOnLANCalls = clientmetric.NewCounter("peerapi_wol") + +func handlePeerAPIWakeOnLAN(h ipnlocal.PeerAPIHandler, w http.ResponseWriter, r *http.Request) { + metricWakeOnLANCalls.Add(1) + if !canWakeOnLAN(h) { + http.Error(w, "no WoL access", http.StatusForbidden) + return + } + if r.Method != "POST" { + http.Error(w, "bad method", http.StatusMethodNotAllowed) + return + } + macStr := r.FormValue("mac") + if macStr == "" { + http.Error(w, "missing 'mac' param", http.StatusBadRequest) + return + } + mac, err := net.ParseMAC(macStr) + if err != nil { + http.Error(w, "bad 'mac' param", http.StatusBadRequest) + return + } + var password []byte // TODO(bradfitz): support? does anything use WoL passwords? + st := h.LocalBackend().NetMon().InterfaceState() + if st == nil { + http.Error(w, "failed to get interfaces state", http.StatusInternalServerError) + return + } + var res struct { + SentTo []string + Errors []string + } + for ifName, ips := range st.InterfaceIPs { + for _, ip := range ips { + if ip.Addr().IsLoopback() || ip.Addr().Is6() { + continue + } + local := &net.UDPAddr{ + IP: ip.Addr().AsSlice(), + Port: 0, + } + remote := &net.UDPAddr{ + IP: net.IPv4bcast, + Port: 0, + } + if err := wol.Wake(mac, password, local, remote); err != nil { + res.Errors = append(res.Errors, err.Error()) + } else { + res.SentTo = append(res.SentTo, ifName) + } + break // one per interface is enough + } + } + sort.Strings(res.SentTo) + writeJSON(w, res) +} + +// TODO(bradfitz): this is all too simplistic and static. It needs to run +// continuously in response to netmon events (USB ethernet adapters might get +// plugged in) and look for the media type/status/etc. Right now on macOS it +// still detects a half dozen "up" en0, en1, en2, en3 etc interfaces that don't +// have any media. We should only report the one that's actually connected. +// But it works for now (2023-10-05) for fleshing out the rest. + +var wakeMAC = envknob.RegisterString("TS_WAKE_MAC") // mac address, "false" or "auto". for https://github.com/tailscale/tailscale/issues/306 + +// getWoLMACs returns up to 10 MAC address of the local machine to send +// wake-on-LAN packets to in order to wake it up. The returned MACs are in +// lowercase hex colon-separated form ("xx:xx:xx:xx:xx:xx"). +// +// If TS_WAKE_MAC=auto, it tries to automatically find the MACs based on the OS +// type and interface properties. (TODO(bradfitz): incomplete) If TS_WAKE_MAC is +// set to a MAC address, that sole MAC address is returned. +func getWoLMACs() (macs []string) { + switch runtime.GOOS { + case "ios", "android": + return nil + } + if s := wakeMAC(); s != "" { + switch s { + case "auto": + ifs, _ := net.Interfaces() + for _, iface := range ifs { + if iface.Flags&net.FlagLoopback != 0 { + continue + } + if iface.Flags&net.FlagBroadcast == 0 || + iface.Flags&net.FlagRunning == 0 || + iface.Flags&net.FlagUp == 0 { + continue + } + if keepMAC(iface.Name, iface.HardwareAddr) { + macs = append(macs, iface.HardwareAddr.String()) + } + if len(macs) == 10 { + break + } + } + return macs + case "false", "off": // fast path before ParseMAC error + return nil + } + mac, err := net.ParseMAC(s) + if err != nil { + log.Printf("invalid MAC %q", s) + return nil + } + return []string{mac.String()} + } + return nil +} + +var ignoreWakeOUI = map[[3]byte]bool{ + {0x00, 0x15, 0x5d}: true, // Hyper-V + {0x00, 0x50, 0x56}: true, // VMware + {0x00, 0x1c, 0x14}: true, // VMware + {0x00, 0x05, 0x69}: true, // VMware + {0x00, 0x0c, 0x29}: true, // VMware + {0x00, 0x1c, 0x42}: true, // Parallels + {0x08, 0x00, 0x27}: true, // VirtualBox + {0x00, 0x21, 0xf6}: true, // VirtualBox + {0x00, 0x14, 0x4f}: true, // VirtualBox + {0x00, 0x0f, 0x4b}: true, // VirtualBox + {0x52, 0x54, 0x00}: true, // VirtualBox/Vagrant +} + +func keepMAC(ifName string, mac []byte) bool { + if len(mac) != 6 { + return false + } + base := strings.TrimRightFunc(ifName, unicode.IsNumber) + switch runtime.GOOS { + case "darwin": + switch base { + case "llw", "awdl", "utun", "bridge", "lo", "gif", "stf", "anpi", "ap": + return false + } + } + if mac[0] == 0x02 && mac[1] == 0x42 { + // Docker container. + return false + } + oui := [3]byte{mac[0], mac[1], mac[2]} + if ignoreWakeOUI[oui] { + return false + } + return true +} diff --git a/hostinfo/hostinfo.go b/hostinfo/hostinfo.go index 89968e1e6db87..d952ce60399e7 100644 --- a/hostinfo/hostinfo.go +++ b/hostinfo/hostinfo.go @@ -32,11 +32,19 @@ import ( var started = time.Now() +var newHooks []func(*tailcfg.Hostinfo) + +// RegisterHostinfoNewHook registers a callback to be called on a non-nil +// [tailcfg.Hostinfo] before it is returned by [New]. +func RegisterHostinfoNewHook(f func(*tailcfg.Hostinfo)) { + newHooks = append(newHooks, f) +} + // New returns a partially populated Hostinfo for the current host. func New() *tailcfg.Hostinfo { hostname, _ := os.Hostname() hostname = dnsname.FirstLabel(hostname) - return &tailcfg.Hostinfo{ + hi := &tailcfg.Hostinfo{ IPNVersion: version.Long(), Hostname: hostname, App: appTypeCached(), @@ -57,8 +65,11 @@ func New() *tailcfg.Hostinfo { Cloud: string(cloudenv.Get()), NoLogsNoSupport: envknob.NoLogsNoSupport(), AllowsUpdate: envknob.AllowsRemoteUpdate(), - WoLMACs: getWoLMACs(), } + for _, f := range newHooks { + f(hi) + } + return hi } // non-nil on some platforms diff --git a/hostinfo/wol.go b/hostinfo/wol.go deleted file mode 100644 index 3a30af2fe3a37..0000000000000 --- a/hostinfo/wol.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package hostinfo - -import ( - "log" - "net" - "runtime" - "strings" - "unicode" - - "tailscale.com/envknob" -) - -// TODO(bradfitz): this is all too simplistic and static. It needs to run -// continuously in response to netmon events (USB ethernet adapaters might get -// plugged in) and look for the media type/status/etc. Right now on macOS it -// still detects a half dozen "up" en0, en1, en2, en3 etc interfaces that don't -// have any media. We should only report the one that's actually connected. -// But it works for now (2023-10-05) for fleshing out the rest. - -var wakeMAC = envknob.RegisterString("TS_WAKE_MAC") // mac address, "false" or "auto". for https://github.com/tailscale/tailscale/issues/306 - -// getWoLMACs returns up to 10 MAC address of the local machine to send -// wake-on-LAN packets to in order to wake it up. The returned MACs are in -// lowercase hex colon-separated form ("xx:xx:xx:xx:xx:xx"). -// -// If TS_WAKE_MAC=auto, it tries to automatically find the MACs based on the OS -// type and interface properties. (TODO(bradfitz): incomplete) If TS_WAKE_MAC is -// set to a MAC address, that sole MAC address is returned. -func getWoLMACs() (macs []string) { - switch runtime.GOOS { - case "ios", "android": - return nil - } - if s := wakeMAC(); s != "" { - switch s { - case "auto": - ifs, _ := net.Interfaces() - for _, iface := range ifs { - if iface.Flags&net.FlagLoopback != 0 { - continue - } - if iface.Flags&net.FlagBroadcast == 0 || - iface.Flags&net.FlagRunning == 0 || - iface.Flags&net.FlagUp == 0 { - continue - } - if keepMAC(iface.Name, iface.HardwareAddr) { - macs = append(macs, iface.HardwareAddr.String()) - } - if len(macs) == 10 { - break - } - } - return macs - case "false", "off": // fast path before ParseMAC error - return nil - } - mac, err := net.ParseMAC(s) - if err != nil { - log.Printf("invalid MAC %q", s) - return nil - } - return []string{mac.String()} - } - return nil -} - -var ignoreWakeOUI = map[[3]byte]bool{ - {0x00, 0x15, 0x5d}: true, // Hyper-V - {0x00, 0x50, 0x56}: true, // VMware - {0x00, 0x1c, 0x14}: true, // VMware - {0x00, 0x05, 0x69}: true, // VMware - {0x00, 0x0c, 0x29}: true, // VMware - {0x00, 0x1c, 0x42}: true, // Parallels - {0x08, 0x00, 0x27}: true, // VirtualBox - {0x00, 0x21, 0xf6}: true, // VirtualBox - {0x00, 0x14, 0x4f}: true, // VirtualBox - {0x00, 0x0f, 0x4b}: true, // VirtualBox - {0x52, 0x54, 0x00}: true, // VirtualBox/Vagrant -} - -func keepMAC(ifName string, mac []byte) bool { - if len(mac) != 6 { - return false - } - base := strings.TrimRightFunc(ifName, unicode.IsNumber) - switch runtime.GOOS { - case "darwin": - switch base { - case "llw", "awdl", "utun", "bridge", "lo", "gif", "stf", "anpi", "ap": - return false - } - } - if mac[0] == 0x02 && mac[1] == 0x42 { - // Docker container. - return false - } - oui := [3]byte{mac[0], mac[1], mac[2]} - if ignoreWakeOUI[oui] { - return false - } - return true -} diff --git a/ipn/ipnlocal/c2n.go b/ipn/ipnlocal/c2n.go index 04f91954ff441..e919215339f9a 100644 --- a/ipn/ipnlocal/c2n.go +++ b/ipn/ipnlocal/c2n.go @@ -10,19 +10,16 @@ import ( "errors" "fmt" "io" - "net" "net/http" "os" "os/exec" "path" "path/filepath" "runtime" - "sort" "strconv" "strings" "time" - "github.com/kortschak/wol" "tailscale.com/clientupdate" "tailscale.com/envknob" "tailscale.com/ipn" @@ -66,9 +63,6 @@ var c2nHandlers = map[methodAndPath]c2nHandler{ req("GET /update"): handleC2NUpdateGet, req("POST /update"): handleC2NUpdatePost, - // Wake-on-LAN. - req("POST /wol"): handleC2NWoL, - // Device posture. req("GET /posture/identity"): handleC2NPostureIdentityGet, @@ -82,6 +76,18 @@ var c2nHandlers = map[methodAndPath]c2nHandler{ req("GET /vip-services"): handleC2NVIPServicesGet, } +// RegisterC2N registers a new c2n handler for the given pattern. +// +// A pattern is like "GET /foo" (specific to an HTTP method) or "/foo" (all +// methods). It panics if the pattern is already registered. +func RegisterC2N(pattern string, h func(*LocalBackend, http.ResponseWriter, *http.Request)) { + k := req(pattern) + if _, ok := c2nHandlers[k]; ok { + panic(fmt.Sprintf("c2n: duplicate handler for %q", pattern)) + } + c2nHandlers[k] = h +} + type c2nHandler func(*LocalBackend, http.ResponseWriter, *http.Request) type methodAndPath struct { @@ -503,55 +509,6 @@ func regularFileExists(path string) bool { return err == nil && fi.Mode().IsRegular() } -func handleC2NWoL(b *LocalBackend, w http.ResponseWriter, r *http.Request) { - r.ParseForm() - var macs []net.HardwareAddr - for _, macStr := range r.Form["mac"] { - mac, err := net.ParseMAC(macStr) - if err != nil { - http.Error(w, "bad 'mac' param", http.StatusBadRequest) - return - } - macs = append(macs, mac) - } - var res struct { - SentTo []string - Errors []string - } - st := b.sys.NetMon.Get().InterfaceState() - if st == nil { - res.Errors = append(res.Errors, "no interface state") - writeJSON(w, &res) - return - } - var password []byte // TODO(bradfitz): support? does anything use WoL passwords? - for _, mac := range macs { - for ifName, ips := range st.InterfaceIPs { - for _, ip := range ips { - if ip.Addr().IsLoopback() || ip.Addr().Is6() { - continue - } - local := &net.UDPAddr{ - IP: ip.Addr().AsSlice(), - Port: 0, - } - remote := &net.UDPAddr{ - IP: net.IPv4bcast, - Port: 0, - } - if err := wol.Wake(mac, password, local, remote); err != nil { - res.Errors = append(res.Errors, err.Error()) - } else { - res.SentTo = append(res.SentTo, ifName) - } - break // one per interface is enough - } - } - } - sort.Strings(res.SentTo) - writeJSON(w, &res) -} - // handleC2NTLSCertStatus returns info about the last TLS certificate issued for the // provided domain. This can be called by the controlplane to clean up DNS TXT // records when they're no longer needed by LetsEncrypt. diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index 4d05489171e3e..f79fb200b7e71 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -20,13 +20,11 @@ import ( "path/filepath" "runtime" "slices" - "sort" "strconv" "strings" "sync" "time" - "github.com/kortschak/wol" "golang.org/x/net/dns/dnsmessage" "golang.org/x/net/http/httpguts" "tailscale.com/drive" @@ -226,6 +224,23 @@ type peerAPIHandler struct { peerUser tailcfg.UserProfile // profile of peerNode } +// PeerAPIHandler is the interface implemented by [peerAPIHandler] and needed by +// module features registered via tailscale.com/feature/*. +type PeerAPIHandler interface { + Peer() tailcfg.NodeView + PeerCaps() tailcfg.PeerCapMap + Self() tailcfg.NodeView + LocalBackend() *LocalBackend + IsSelfUntagged() bool // whether the peer is untagged and the same as this user +} + +func (h *peerAPIHandler) IsSelfUntagged() bool { + return !h.selfNode.IsTagged() && !h.peerNode.IsTagged() && h.isSelf +} +func (h *peerAPIHandler) Peer() tailcfg.NodeView { return h.peerNode } +func (h *peerAPIHandler) Self() tailcfg.NodeView { return h.selfNode } +func (h *peerAPIHandler) LocalBackend() *LocalBackend { return h.ps.b } + func (h *peerAPIHandler) logf(format string, a ...any) { h.ps.b.logf("peerapi: "+format, a...) } @@ -302,6 +317,20 @@ func peerAPIRequestShouldGetSecurityHeaders(r *http.Request) bool { return false } +// RegisterPeerAPIHandler registers a PeerAPI handler. +// +// The path should be of the form "/v0/foo". +// +// It panics if the path is already registered. +func RegisterPeerAPIHandler(path string, f func(PeerAPIHandler, http.ResponseWriter, *http.Request)) { + if _, ok := peerAPIHandlers[path]; ok { + panic(fmt.Sprintf("duplicate PeerAPI handler %q", path)) + } + peerAPIHandlers[path] = f +} + +var peerAPIHandlers = map[string]func(PeerAPIHandler, http.ResponseWriter, *http.Request){} // by URL.Path + func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err := h.validatePeerAPIRequest(r); err != nil { metricInvalidRequests.Add(1) @@ -346,10 +375,6 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "/v0/dnsfwd": h.handleServeDNSFwd(w, r) return - case "/v0/wol": - metricWakeOnLANCalls.Add(1) - h.handleWakeOnLAN(w, r) - return case "/v0/interfaces": h.handleServeInterfaces(w, r) return @@ -364,6 +389,10 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.handleServeIngress(w, r) return } + if ph, ok := peerAPIHandlers[r.URL.Path]; ok { + ph(h, w, r) + return + } who := h.peerUser.DisplayName fmt.Fprintf(w, ` @@ -624,14 +653,6 @@ func (h *peerAPIHandler) canDebug() bool { return h.isSelf || h.peerHasCap(tailcfg.PeerCapabilityDebugPeer) } -// canWakeOnLAN reports whether h can send a Wake-on-LAN packet from this node. -func (h *peerAPIHandler) canWakeOnLAN() bool { - if h.peerNode.UnsignedPeerAPIOnly() { - return false - } - return h.isSelf || h.peerHasCap(tailcfg.PeerCapabilityWakeOnLAN) -} - var allowSelfIngress = envknob.RegisterBool("TS_ALLOW_SELF_INGRESS") // canIngress reports whether h can send ingress requests to this node. @@ -640,10 +661,10 @@ func (h *peerAPIHandler) canIngress() bool { } func (h *peerAPIHandler) peerHasCap(wantCap tailcfg.PeerCapability) bool { - return h.peerCaps().HasCapability(wantCap) + return h.PeerCaps().HasCapability(wantCap) } -func (h *peerAPIHandler) peerCaps() tailcfg.PeerCapMap { +func (h *peerAPIHandler) PeerCaps() tailcfg.PeerCapMap { return h.ps.b.PeerCaps(h.remoteAddr.Addr()) } @@ -817,61 +838,6 @@ func (h *peerAPIHandler) handleServeDNSFwd(w http.ResponseWriter, r *http.Reques dh.ServeHTTP(w, r) } -func (h *peerAPIHandler) handleWakeOnLAN(w http.ResponseWriter, r *http.Request) { - if !h.canWakeOnLAN() { - http.Error(w, "no WoL access", http.StatusForbidden) - return - } - if r.Method != "POST" { - http.Error(w, "bad method", http.StatusMethodNotAllowed) - return - } - macStr := r.FormValue("mac") - if macStr == "" { - http.Error(w, "missing 'mac' param", http.StatusBadRequest) - return - } - mac, err := net.ParseMAC(macStr) - if err != nil { - http.Error(w, "bad 'mac' param", http.StatusBadRequest) - return - } - var password []byte // TODO(bradfitz): support? does anything use WoL passwords? - st := h.ps.b.sys.NetMon.Get().InterfaceState() - if st == nil { - http.Error(w, "failed to get interfaces state", http.StatusInternalServerError) - return - } - var res struct { - SentTo []string - Errors []string - } - for ifName, ips := range st.InterfaceIPs { - for _, ip := range ips { - if ip.Addr().IsLoopback() || ip.Addr().Is6() { - continue - } - local := &net.UDPAddr{ - IP: ip.Addr().AsSlice(), - Port: 0, - } - remote := &net.UDPAddr{ - IP: net.IPv4bcast, - Port: 0, - } - if err := wol.Wake(mac, password, local, remote); err != nil { - res.Errors = append(res.Errors, err.Error()) - } else { - res.SentTo = append(res.SentTo, ifName) - } - break // one per interface is enough - } - } - sort.Strings(res.SentTo) - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(res) -} - func (h *peerAPIHandler) replyToDNSQueries() bool { if h.isSelf { // If the peer is owned by the same user, just allow it @@ -1150,7 +1116,7 @@ func (h *peerAPIHandler) handleServeDrive(w http.ResponseWriter, r *http.Request return } - capsMap := h.peerCaps() + capsMap := h.PeerCaps() driveCaps, ok := capsMap[tailcfg.PeerCapabilityTaildrive] if !ok { h.logf("taildrive: not permitted") @@ -1274,8 +1240,7 @@ var ( metricInvalidRequests = clientmetric.NewCounter("peerapi_invalid_requests") // Non-debug PeerAPI endpoints. - metricPutCalls = clientmetric.NewCounter("peerapi_put") - metricDNSCalls = clientmetric.NewCounter("peerapi_dns") - metricWakeOnLANCalls = clientmetric.NewCounter("peerapi_wol") - metricIngressCalls = clientmetric.NewCounter("peerapi_ingress") + metricPutCalls = clientmetric.NewCounter("peerapi_put") + metricDNSCalls = clientmetric.NewCounter("peerapi_dns") + metricIngressCalls = clientmetric.NewCounter("peerapi_ingress") ) diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index b769e719cbabd..3505c94539d48 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -29,6 +29,7 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/control/controlclient" "tailscale.com/envknob" + _ "tailscale.com/feature/condregister" "tailscale.com/health" "tailscale.com/hostinfo" "tailscale.com/ipn" diff --git a/tstest/integration/tailscaled_deps_test_darwin.go b/tstest/integration/tailscaled_deps_test_darwin.go index 6676ee22cbd1c..d04dc6aa18ab5 100644 --- a/tstest/integration/tailscaled_deps_test_darwin.go +++ b/tstest/integration/tailscaled_deps_test_darwin.go @@ -17,6 +17,7 @@ import ( _ "tailscale.com/derp/derphttp" _ "tailscale.com/drive/driveimpl" _ "tailscale.com/envknob" + _ "tailscale.com/feature/condregister" _ "tailscale.com/health" _ "tailscale.com/hostinfo" _ "tailscale.com/ipn" diff --git a/tstest/integration/tailscaled_deps_test_freebsd.go b/tstest/integration/tailscaled_deps_test_freebsd.go index 6676ee22cbd1c..d04dc6aa18ab5 100644 --- a/tstest/integration/tailscaled_deps_test_freebsd.go +++ b/tstest/integration/tailscaled_deps_test_freebsd.go @@ -17,6 +17,7 @@ import ( _ "tailscale.com/derp/derphttp" _ "tailscale.com/drive/driveimpl" _ "tailscale.com/envknob" + _ "tailscale.com/feature/condregister" _ "tailscale.com/health" _ "tailscale.com/hostinfo" _ "tailscale.com/ipn" diff --git a/tstest/integration/tailscaled_deps_test_linux.go b/tstest/integration/tailscaled_deps_test_linux.go index 6676ee22cbd1c..d04dc6aa18ab5 100644 --- a/tstest/integration/tailscaled_deps_test_linux.go +++ b/tstest/integration/tailscaled_deps_test_linux.go @@ -17,6 +17,7 @@ import ( _ "tailscale.com/derp/derphttp" _ "tailscale.com/drive/driveimpl" _ "tailscale.com/envknob" + _ "tailscale.com/feature/condregister" _ "tailscale.com/health" _ "tailscale.com/hostinfo" _ "tailscale.com/ipn" diff --git a/tstest/integration/tailscaled_deps_test_openbsd.go b/tstest/integration/tailscaled_deps_test_openbsd.go index 6676ee22cbd1c..d04dc6aa18ab5 100644 --- a/tstest/integration/tailscaled_deps_test_openbsd.go +++ b/tstest/integration/tailscaled_deps_test_openbsd.go @@ -17,6 +17,7 @@ import ( _ "tailscale.com/derp/derphttp" _ "tailscale.com/drive/driveimpl" _ "tailscale.com/envknob" + _ "tailscale.com/feature/condregister" _ "tailscale.com/health" _ "tailscale.com/hostinfo" _ "tailscale.com/ipn" diff --git a/tstest/integration/tailscaled_deps_test_windows.go b/tstest/integration/tailscaled_deps_test_windows.go index bbf46d8c21938..5eda22327cd8a 100644 --- a/tstest/integration/tailscaled_deps_test_windows.go +++ b/tstest/integration/tailscaled_deps_test_windows.go @@ -24,6 +24,7 @@ import ( _ "tailscale.com/derp/derphttp" _ "tailscale.com/drive/driveimpl" _ "tailscale.com/envknob" + _ "tailscale.com/feature/condregister" _ "tailscale.com/health" _ "tailscale.com/hostinfo" _ "tailscale.com/ipn" From 3033a96b02c7bad11a1260c92e06668f0ac970bc Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Thu, 23 Jan 2025 10:47:21 +0000 Subject: [PATCH 171/223] cmd/k8s-operator: fix reconciler name clash (#14712) The new ProxyGroup-based Ingress reconciler is causing a fatal log at startup because it has the same name as the existing Ingress reconciler. Explicitly name both to ensure they have unique names that are consistent with other explicitly named reconcilers. Updates #14583 Change-Id: Ie76e3eaf3a96b1cec3d3615ea254a847447372ea Signed-off-by: Tom Proctor --- cmd/k8s-operator/operator.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 6368698d87638..f349e7848ee27 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -315,6 +315,7 @@ func runReconcilers(opts reconcilerOpts) { err = builder. ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). + Named("ingress-reconciler"). Watches(&appsv1.StatefulSet{}, ingressChildFilter). Watches(&corev1.Secret{}, ingressChildFilter). Watches(&corev1.Service{}, svcHandlerForIngress). @@ -336,6 +337,7 @@ func runReconcilers(opts reconcilerOpts) { err = builder. ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). + Named("ingress-pg-reconciler"). Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))). Complete(&IngressPGReconciler{ recorder: eventRecorder, @@ -357,6 +359,7 @@ func runReconcilers(opts reconcilerOpts) { proxyClassFilterForConnector := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForConnector(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.Connector{}). + Named("connector-reconciler"). Watches(&appsv1.StatefulSet{}, connectorFilter). Watches(&corev1.Secret{}, connectorFilter). Watches(&tsapi.ProxyClass{}, proxyClassFilterForConnector). @@ -376,6 +379,7 @@ func runReconcilers(opts reconcilerOpts) { nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver")) err = builder.ControllerManagedBy(mgr). For(&tsapi.DNSConfig{}). + Named("nameserver-reconciler"). Watches(&appsv1.Deployment{}, nameserverFilter). Watches(&corev1.ConfigMap{}, nameserverFilter). Watches(&corev1.Service{}, nameserverFilter). @@ -455,6 +459,7 @@ func runReconcilers(opts reconcilerOpts) { serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyClass{}). + Named("proxyclass-reconciler"). Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter). Complete(&ProxyClassReconciler{ Client: mgr.GetClient(), @@ -498,6 +503,7 @@ func runReconcilers(opts reconcilerOpts) { recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{}) err = builder.ControllerManagedBy(mgr). For(&tsapi.Recorder{}). + Named("recorder-reconciler"). Watches(&appsv1.StatefulSet{}, recorderFilter). Watches(&corev1.ServiceAccount{}, recorderFilter). Watches(&corev1.Secret{}, recorderFilter). @@ -520,6 +526,7 @@ func runReconcilers(opts reconcilerOpts) { proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). + Named("proxygroup-reconciler"). Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter). Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter). From a00623e8c4f4fa94ae705ac992f361a594b7caa1 Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Thu, 23 Jan 2025 09:04:03 -0500 Subject: [PATCH 172/223] derp,wgengine/magicsock: remove unexpected label (#14711) Remove "unexpected" labelling of PeerGoneReasonNotHere. A peer being no longer connected to a DERP server is not an unexpected case and causes confusion in looking at logs. Fixes tailscale/corp#25609 Signed-off-by: Mike O'Driscoll --- derp/derp.go | 7 +++---- wgengine/magicsock/derp.go | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/derp/derp.go b/derp/derp.go index 878188cd20625..6a7b3b73552d5 100644 --- a/derp/derp.go +++ b/derp/derp.go @@ -79,8 +79,7 @@ const ( // framePeerGone to B so B can forget that a reverse path // exists on that connection to get back to A. It is also sent // if A tries to send a CallMeMaybe to B and the server has no - // record of B (which currently would only happen if there was - // a bug). + // record of B framePeerGone = frameType(0x08) // 32B pub key of peer that's gone + 1 byte reason // framePeerPresent is like framePeerGone, but for other members of the DERP @@ -131,8 +130,8 @@ const ( type PeerGoneReasonType byte const ( - PeerGoneReasonDisconnected = PeerGoneReasonType(0x00) // peer disconnected from this server - PeerGoneReasonNotHere = PeerGoneReasonType(0x01) // server doesn't know about this peer, unexpected + PeerGoneReasonDisconnected = PeerGoneReasonType(0x00) // is only sent when a peer disconnects from this server + PeerGoneReasonNotHere = PeerGoneReasonType(0x01) // server doesn't know about this peer PeerGoneReasonMeshConnBroke = PeerGoneReasonType(0xf0) // invented by Client.RunWatchConnectionLoop on disconnect; not sent on the wire ) diff --git a/wgengine/magicsock/derp.go b/wgengine/magicsock/derp.go index e9f07086271d5..7c8ffc01af213 100644 --- a/wgengine/magicsock/derp.go +++ b/wgengine/magicsock/derp.go @@ -632,7 +632,7 @@ func (c *Conn) runDerpReader(ctx context.Context, regionID int, dc *derphttp.Cli // Do nothing. case derp.PeerGoneReasonNotHere: metricRecvDiscoDERPPeerNotHere.Add(1) - c.logf("[unexpected] magicsock: derp-%d does not know about peer %s, removing route", + c.logf("magicsock: derp-%d does not know about peer %s, removing route", regionID, key.NodePublic(m.Peer).ShortString()) default: metricRecvDiscoDERPPeerGoneUnknown.Add(1) From f1710f4a429911b461fb5e25a1e33642317bdedf Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Thu, 23 Jan 2025 09:03:56 -0800 Subject: [PATCH 173/223] appc,ipn/ipnlocal: log DNS parsing errors in app connectors (#14607) If we fail to parse the upstream DNS response in an app connector, we might miss new IPs for the target domain. Log parsing errors to be able to diagnose that. Updates #14606 Signed-off-by: Andrew Lytvynov --- appc/appconnector.go | 21 +++++------ appc/appconnector_test.go | 72 ++++++++++++++++++++++++++++---------- ipn/ipnlocal/local.go | 6 ++-- ipn/ipnlocal/local_test.go | 8 +++-- ipn/ipnlocal/peerapi.go | 6 +++- 5 files changed, 78 insertions(+), 35 deletions(-) diff --git a/appc/appconnector.go b/appc/appconnector.go index 063381cd7e9a0..f4857fcc612aa 100644 --- a/appc/appconnector.go +++ b/appc/appconnector.go @@ -374,13 +374,13 @@ func (e *AppConnector) DomainRoutes() map[string][]netip.Addr { // response is being returned over the PeerAPI. The response is parsed and // matched against the configured domains, if matched the routeAdvertiser is // advised to advertise the discovered route. -func (e *AppConnector) ObserveDNSResponse(res []byte) { +func (e *AppConnector) ObserveDNSResponse(res []byte) error { var p dnsmessage.Parser if _, err := p.Start(res); err != nil { - return + return err } if err := p.SkipAllQuestions(); err != nil { - return + return err } // cnameChain tracks a chain of CNAMEs for a given query in order to reverse @@ -399,12 +399,12 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) { break } if err != nil { - return + return err } if h.Class != dnsmessage.ClassINET { if err := p.SkipAnswer(); err != nil { - return + return err } continue } @@ -413,7 +413,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) { case dnsmessage.TypeCNAME, dnsmessage.TypeA, dnsmessage.TypeAAAA: default: if err := p.SkipAnswer(); err != nil { - return + return err } continue @@ -427,7 +427,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) { if h.Type == dnsmessage.TypeCNAME { res, err := p.CNAMEResource() if err != nil { - return + return err } cname := strings.TrimSuffix(strings.ToLower(res.CNAME.String()), ".") if len(cname) == 0 { @@ -441,20 +441,20 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) { case dnsmessage.TypeA: r, err := p.AResource() if err != nil { - return + return err } addr := netip.AddrFrom4(r.A) mak.Set(&addressRecords, domain, append(addressRecords[domain], addr)) case dnsmessage.TypeAAAA: r, err := p.AAAAResource() if err != nil { - return + return err } addr := netip.AddrFrom16(r.AAAA) mak.Set(&addressRecords, domain, append(addressRecords[domain], addr)) default: if err := p.SkipAnswer(); err != nil { - return + return err } continue } @@ -485,6 +485,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) { e.scheduleAdvertisement(domain, toAdvertise...) } } + return nil } // starting from the given domain that resolved to an address, find it, or any diff --git a/appc/appconnector_test.go b/appc/appconnector_test.go index 36ec7a119dfbd..fd0001224984a 100644 --- a/appc/appconnector_test.go +++ b/appc/appconnector_test.go @@ -69,7 +69,9 @@ func TestUpdateRoutes(t *testing.T) { a.updateDomains([]string{"*.example.com"}) // This route should be collapsed into the range - a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1")) + if err := a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) { @@ -77,7 +79,9 @@ func TestUpdateRoutes(t *testing.T) { } // This route should not be collapsed or removed - a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1")) + if err := a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")} @@ -130,7 +134,9 @@ func TestDomainRoutes(t *testing.T) { a = NewAppConnector(t.Logf, rc, nil, nil) } a.updateDomains([]string{"example.com"}) - a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")) + if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(context.Background()) want := map[string][]netip.Addr{ @@ -155,7 +161,9 @@ func TestObserveDNSResponse(t *testing.T) { } // a has no domains configured, so it should not advertise any routes - a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")) + if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) { t.Errorf("got %v; want %v", got, want) } @@ -163,7 +171,9 @@ func TestObserveDNSResponse(t *testing.T) { wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")} a.updateDomains([]string{"example.com"}) - a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")) + if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) { t.Errorf("got %v; want %v", got, want) @@ -172,7 +182,9 @@ func TestObserveDNSResponse(t *testing.T) { // a CNAME record chain should result in a route being added if the chain // matches a routed domain. a.updateDomains([]string{"www.example.com", "example.com"}) - a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com.")) + if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com.")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32")) if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) { @@ -181,7 +193,9 @@ func TestObserveDNSResponse(t *testing.T) { // a CNAME record chain should result in a route being added if the chain // even if only found in the middle of the chain - a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org.")) + if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org.")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32")) if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) { @@ -190,14 +204,18 @@ func TestObserveDNSResponse(t *testing.T) { wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128")) - a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")) + if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) { t.Errorf("got %v; want %v", got, want) } // don't re-advertise routes that have already been advertised - a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")) + if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) if !slices.Equal(rc.Routes(), wantRoutes) { t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes) @@ -207,7 +225,9 @@ func TestObserveDNSResponse(t *testing.T) { pfx := netip.MustParsePrefix("192.0.2.0/24") a.updateRoutes([]netip.Prefix{pfx}) wantRoutes = append(wantRoutes, pfx) - a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1")) + if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) if !slices.Equal(rc.Routes(), wantRoutes) { t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes) @@ -230,7 +250,9 @@ func TestWildcardDomains(t *testing.T) { } a.updateDomains([]string{"*.example.com"}) - a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8")) + if err := a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } a.Wait(ctx) if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) { t.Errorf("routes: got %v; want %v", got, want) @@ -438,10 +460,16 @@ func TestUpdateDomainRouteRemoval(t *testing.T) { // adding domains doesn't immediately cause any routes to be advertised assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{}) - a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1")) - a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2")) - a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3")) - a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4")) + for _, res := range [][]byte{ + dnsResponse("a.example.com.", "1.2.3.1"), + dnsResponse("a.example.com.", "1.2.3.2"), + dnsResponse("b.example.com.", "1.2.3.3"), + dnsResponse("b.example.com.", "1.2.3.4"), + } { + if err := a.ObserveDNSResponse(res); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } + } a.Wait(ctx) // observing dns responses causes routes to be advertised assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{}) @@ -487,10 +515,16 @@ func TestUpdateWildcardRouteRemoval(t *testing.T) { // adding domains doesn't immediately cause any routes to be advertised assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{}) - a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1")) - a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2")) - a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3")) - a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4")) + for _, res := range [][]byte{ + dnsResponse("a.example.com.", "1.2.3.1"), + dnsResponse("a.example.com.", "1.2.3.2"), + dnsResponse("1.b.example.com.", "1.2.3.3"), + dnsResponse("2.b.example.com.", "1.2.3.4"), + } { + if err := a.ObserveDNSResponse(res); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } + } a.Wait(ctx) // observing dns responses causes routes to be advertised assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{}) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 4ff3f3db47edf..33ce9f331d94b 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -7276,17 +7276,17 @@ func (b *LocalBackend) DoSelfUpdate() { // ObserveDNSResponse passes a DNS response from the PeerAPI DNS server to the // App Connector to enable route discovery. -func (b *LocalBackend) ObserveDNSResponse(res []byte) { +func (b *LocalBackend) ObserveDNSResponse(res []byte) error { var appConnector *appc.AppConnector b.mu.Lock() if b.appConnector == nil { b.mu.Unlock() - return + return nil } appConnector = b.appConnector b.mu.Unlock() - appConnector.ObserveDNSResponse(res) + return appConnector.ObserveDNSResponse(res) } // ErrDisallowedAutoRoute is returned by AdvertiseRoute when a route that is not allowed is requested. diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index b7b81ada8b07b..de9ebf9fb3b10 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1372,7 +1372,9 @@ func TestObserveDNSResponse(t *testing.T) { b := newTestBackend(t) // ensure no error when no app connector is configured - b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")) + if err := b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } rc := &appctest.RouteCollector{} if shouldStore { @@ -1383,7 +1385,9 @@ func TestObserveDNSResponse(t *testing.T) { b.appConnector.UpdateDomains([]string{"example.com"}) b.appConnector.Wait(context.Background()) - b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")) + if err := b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil { + t.Errorf("ObserveDNSResponse: %v", err) + } b.appConnector.Wait(context.Background()) wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")} if !slices.Equal(rc.Routes(), wantRoutes) { diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index f79fb200b7e71..ab2093c139e81 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -932,7 +932,11 @@ func (h *peerAPIHandler) handleDNSQuery(w http.ResponseWriter, r *http.Request) // instead to avoid re-parsing the DNS response for improved performance in // the future. if h.ps.b.OfferingAppConnector() { - h.ps.b.ObserveDNSResponse(res) + if err := h.ps.b.ObserveDNSResponse(res); err != nil { + h.logf("ObserveDNSResponse error: %v", err) + // This is not fatal, we probably just failed to parse the upstream + // response. Return it to the caller anyway. + } } if pretty { From d6abbc2e610f605cdb114f97182e14d54794dc87 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 22 Jan 2025 18:07:57 -0800 Subject: [PATCH 174/223] net/tstun: move TAP support out to separate package feature/tap Still behind the same ts_omit_tap build tag. See #14738 for background on the pattern. Updates #12614 Change-Id: I03fb3d2bf137111e727415bd8e713d8568156ecc Signed-off-by: Brad Fitzpatrick --- cmd/k8s-operator/depaware.txt | 7 +++-- cmd/tailscaled/depaware.txt | 7 +++-- feature/condregister/maybe_tap.go | 8 +++++ feature/feature.go | 39 +++++++++++++++++++++++++ {net/tstun => feature/tap}/tap_linux.go | 24 +++++++++++---- net/tstun/tun.go | 9 +++--- net/tstun/wrap.go | 11 ++++--- 7 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 feature/condregister/maybe_tap.go rename {net/tstun => feature/tap}/tap_linux.go (95%) diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index bdcf3417a623b..11a9201d4028b 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -140,7 +140,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L đŸ’Ŗ github.com/illarion/gonotify/v2 from tailscale.com/net/dns - L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/net/tstun + L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/feature/tap L github.com/insomniacslk/dhcp/iana from github.com/insomniacslk/dhcp/dhcpv4 L github.com/insomniacslk/dhcp/interfaces from github.com/insomniacslk/dhcp/dhcpv4 L github.com/insomniacslk/dhcp/rfc1035label from github.com/insomniacslk/dhcp/dhcpv4 @@ -302,7 +302,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ gvisor.dev/gvisor/pkg/tcpip/network/internal/ip from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ - gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/net/tstun+ + gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/feature/tap+ gvisor.dev/gvisor/pkg/tcpip/network/ipv6 from tailscale.com/wgengine/netstack+ gvisor.dev/gvisor/pkg/tcpip/ports from gvisor.dev/gvisor/pkg/tcpip/stack+ gvisor.dev/gvisor/pkg/tcpip/seqnum from gvisor.dev/gvisor/pkg/tcpip/header+ @@ -801,8 +801,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/drive from tailscale.com/client/tailscale+ tailscale.com/envknob from tailscale.com/client/tailscale+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ - tailscale.com/feature from tailscale.com/feature/wakeonlan + tailscale.com/feature from tailscale.com/feature/wakeonlan+ tailscale.com/feature/condregister from tailscale.com/tsnet + L tailscale.com/feature/tap from tailscale.com/feature/condregister tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 5246b82b92485..4f81d93dda9fe 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -112,7 +112,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L đŸ’Ŗ github.com/illarion/gonotify/v2 from tailscale.com/net/dns - L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/net/tstun + L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/feature/tap L github.com/insomniacslk/dhcp/iana from github.com/insomniacslk/dhcp/dhcpv4 L github.com/insomniacslk/dhcp/interfaces from github.com/insomniacslk/dhcp/dhcpv4 L github.com/insomniacslk/dhcp/rfc1035label from github.com/insomniacslk/dhcp/dhcpv4 @@ -214,7 +214,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ gvisor.dev/gvisor/pkg/tcpip/network/internal/ip from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ - gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/net/tstun+ + gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/feature/tap+ gvisor.dev/gvisor/pkg/tcpip/network/ipv6 from tailscale.com/wgengine/netstack+ gvisor.dev/gvisor/pkg/tcpip/ports from gvisor.dev/gvisor/pkg/tcpip/stack+ gvisor.dev/gvisor/pkg/tcpip/seqnum from gvisor.dev/gvisor/pkg/tcpip/header+ @@ -259,8 +259,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/drive/driveimpl/shared from tailscale.com/drive/driveimpl+ tailscale.com/envknob from tailscale.com/client/tailscale+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ - tailscale.com/feature from tailscale.com/feature/wakeonlan + tailscale.com/feature from tailscale.com/feature/wakeonlan+ tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled + L tailscale.com/feature/tap from tailscale.com/feature/condregister tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal diff --git a/feature/condregister/maybe_tap.go b/feature/condregister/maybe_tap.go new file mode 100644 index 0000000000000..eca4fc3ac84af --- /dev/null +++ b/feature/condregister/maybe_tap.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux && !ts_omit_tap + +package condregister + +import _ "tailscale.com/feature/tap" diff --git a/feature/feature.go b/feature/feature.go index ea290c43ac0d4..6415cfc4a00d8 100644 --- a/feature/feature.go +++ b/feature/feature.go @@ -4,6 +4,8 @@ // Package feature tracks which features are linked into the binary. package feature +import "reflect" + var in = map[string]bool{} // Register notes that the named feature is linked into the binary. @@ -13,3 +15,40 @@ func Register(name string) { } in[name] = true } + +// Hook is a func that can only be set once. +// +// It is not safe for concurrent use. +type Hook[Func any] struct { + f Func + ok bool +} + +// IsSet reports whether the hook has been set. +func (h *Hook[Func]) IsSet() bool { + return h.ok +} + +// Set sets the hook function, panicking if it's already been set +// or f is the zero value. +// +// It's meant to be called in init. +func (h *Hook[Func]) Set(f Func) { + if h.ok { + panic("Set on already-set feature hook") + } + if reflect.ValueOf(f).IsZero() { + panic("Set with zero value") + } + h.f = f + h.ok = true +} + +// Get returns the hook function, or panics if it hasn't been set. +// Use IsSet to check if it's been set. +func (h *Hook[Func]) Get() Func { + if !h.ok { + panic("Get on unset feature hook, without IsSet") + } + return h.f +} diff --git a/net/tstun/tap_linux.go b/feature/tap/tap_linux.go similarity index 95% rename from net/tstun/tap_linux.go rename to feature/tap/tap_linux.go index 8a00a96927c4d..58ac00593d3a8 100644 --- a/net/tstun/tap_linux.go +++ b/feature/tap/tap_linux.go @@ -1,9 +1,8 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build !ts_omit_tap - -package tstun +// Package tap registers Tailscale's experimental (demo) Linux TAP (Layer 2) support. +package tap import ( "bytes" @@ -26,6 +25,7 @@ import ( "tailscale.com/net/netaddr" "tailscale.com/net/packet" "tailscale.com/net/tsaddr" + "tailscale.com/net/tstun" "tailscale.com/syncs" "tailscale.com/types/ipproto" "tailscale.com/types/logger" @@ -38,7 +38,11 @@ import ( // For now just hard code it. var ourMAC = net.HardwareAddr{0x30, 0x2D, 0x66, 0xEC, 0x7A, 0x93} -func init() { createTAP = createTAPLinux } +const tapDebug = tstun.TAPDebug + +func init() { + tstun.CreateTAP.Set(createTAPLinux) +} func createTAPLinux(logf logger.Logf, tapName, bridgeName string) (tun.Device, error) { fd, err := unix.Open("/dev/net/tun", unix.O_RDWR, 0) @@ -87,7 +91,10 @@ var ( etherTypeIPv6 = etherType{0x86, 0xDD} ) -const ipv4HeaderLen = 20 +const ( + ipv4HeaderLen = 20 + ethernetFrameSize = 14 // 2 six byte MACs, 2 bytes ethertype +) const ( consumePacket = true @@ -186,6 +193,11 @@ var ( cgnatNetMask = net.IPMask(net.ParseIP("255.192.0.0").To4()) ) +// parsedPacketPool holds a pool of Parsed structs for use in filtering. +// This is needed because escape analysis cannot see that parsed packets +// do not escape through {Pre,Post}Filter{In,Out}. +var parsedPacketPool = sync.Pool{New: func() any { return new(packet.Parsed) }} + // handleDHCPRequest handles receiving a raw TAP ethernet frame and reports whether // it's been handled as a DHCP request. That is, it reports whether the frame should // be ignored by the caller and not passed on. @@ -392,7 +404,7 @@ type tapDevice struct { destMACAtomic syncs.AtomicValue[[6]byte] } -var _ setIPer = (*tapDevice)(nil) +var _ tstun.SetIPer = (*tapDevice)(nil) func (t *tapDevice) SetIP(ipV4, ipV6TODO netip.Addr) error { t.clientIPv4.Store(ipV4.String()) diff --git a/net/tstun/tun.go b/net/tstun/tun.go index 56c66c83a27e5..44ccdfc991de8 100644 --- a/net/tstun/tun.go +++ b/net/tstun/tun.go @@ -14,11 +14,12 @@ import ( "time" "github.com/tailscale/wireguard-go/tun" + "tailscale.com/feature" "tailscale.com/types/logger" ) -// createTAP is non-nil on Linux. -var createTAP func(logf logger.Logf, tapName, bridgeName string) (tun.Device, error) +// CrateTAP is the hook set by feature/tap. +var CreateTAP feature.Hook[func(logf logger.Logf, tapName, bridgeName string) (tun.Device, error)] // New returns a tun.Device for the requested device name, along with // the OS-dependent name that was allocated to the device. @@ -29,7 +30,7 @@ func New(logf logger.Logf, tunName string) (tun.Device, string, error) { if runtime.GOOS != "linux" { return nil, "", errors.New("tap only works on Linux") } - if createTAP == nil { // if the ts_omit_tap tag is used + if !CreateTAP.IsSet() { // if the ts_omit_tap tag is used return nil, "", errors.New("tap is not supported in this build") } f := strings.Split(tunName, ":") @@ -42,7 +43,7 @@ func New(logf logger.Logf, tunName string) (tun.Device, string, error) { default: return nil, "", errors.New("bogus tap argument") } - dev, err = createTAP(logf, tapName, bridgeName) + dev, err = CreateTAP.Get()(logf, tapName, bridgeName) } else { dev, err = tun.CreateTUN(tunName, int(DefaultTUNMTU())) } diff --git a/net/tstun/wrap.go b/net/tstun/wrap.go index e4ff36b49322b..2d56f376832c9 100644 --- a/net/tstun/wrap.go +++ b/net/tstun/wrap.go @@ -53,7 +53,8 @@ const PacketStartOffset = device.MessageTransportHeaderSize // of a packet that can be injected into a tstun.Wrapper. const MaxPacketSize = device.MaxContentSize -const tapDebug = false // for super verbose TAP debugging +// TAPDebug is whether super verbose TAP debugging is enabled. +const TAPDebug = false var ( // ErrClosed is returned when attempting an operation on a closed Wrapper. @@ -459,7 +460,7 @@ func (t *Wrapper) pollVector() { return } n, err = reader(t.vectorBuffer[:], sizes, readOffset) - if t.isTAP && tapDebug { + if t.isTAP && TAPDebug { s := fmt.Sprintf("% x", t.vectorBuffer[0][:]) for strings.HasSuffix(s, " 00") { s = strings.TrimSuffix(s, " 00") @@ -792,7 +793,9 @@ func (pc *peerConfigTable) outboundPacketIsJailed(p *packet.Parsed) bool { return c.jailed } -type setIPer interface { +// SetIPer is the interface expected to be implemented by the TAP implementation +// of tun.Device. +type SetIPer interface { // SetIP sets the IP addresses of the TAP device. SetIP(ipV4, ipV6 netip.Addr) error } @@ -800,7 +803,7 @@ type setIPer interface { // SetWGConfig is called when a new NetworkMap is received. func (t *Wrapper) SetWGConfig(wcfg *wgcfg.Config) { if t.isTAP { - if sip, ok := t.tdev.(setIPer); ok { + if sip, ok := t.tdev.(SetIPer); ok { sip.SetIP(findV4(wcfg.Addresses), findV6(wcfg.Addresses)) } } From 413fb5b93311972e3a8d724bb696607ef3afe6f2 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 23 Jan 2025 12:45:06 -0800 Subject: [PATCH 175/223] control/controlclient: delete unreferenced mapSession UserProfiles This was a slow memory leak on busy tailnets with lots of tagged ephemeral nodes. Updates tailscale/corp#26058 Change-Id: I298e7d438e3ffbb3cde795640e344671d244c632 Signed-off-by: Brad Fitzpatrick --- control/controlclient/map.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 13b11d6df431b..1a54fc5430fcc 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -195,6 +195,10 @@ func (ms *mapSession) HandleNonKeepAliveMapResponse(ctx context.Context, resp *t ms.updateStateFromResponse(resp) + // Occasionally clean up old userprofile if it grows too much + // from e.g. ephemeral tagged nodes. + ms.cleanLastUserProfile() + if ms.tryHandleIncrementally(resp) { ms.occasionallyPrintSummary(ms.lastNetmapSummary) return nil @@ -292,7 +296,6 @@ func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) { for _, up := range resp.UserProfiles { ms.lastUserProfile[up.ID] = up } - // TODO(bradfitz): clean up old user profiles? maybe not worth it. if dm := resp.DERPMap; dm != nil { ms.vlogf("netmap: new map contains DERP map") @@ -532,6 +535,32 @@ func (ms *mapSession) addUserProfile(nm *netmap.NetworkMap, userID tailcfg.UserI } } +// cleanLastUserProfile deletes any entries from lastUserProfile +// that are not referenced by any peer or the self node. +// +// This is expensive enough that we don't do this on every message +// from the server, but only when it's grown enough to matter. +func (ms *mapSession) cleanLastUserProfile() { + if len(ms.lastUserProfile) < len(ms.peers)*2 { + // Hasn't grown enough to be worth cleaning. + return + } + + keep := set.Set[tailcfg.UserID]{} + if node := ms.lastNode; node.Valid() { + keep.Add(node.User()) + } + for _, n := range ms.peers { + keep.Add(n.User()) + keep.Add(n.Sharer()) + } + for userID := range ms.lastUserProfile { + if !keep.Contains(userID) { + delete(ms.lastUserProfile, userID) + } + } +} + var debugPatchifyPeer = envknob.RegisterBool("TS_DEBUG_PATCHIFY_PEER") // patchifyPeersChanged mutates resp to promote PeersChanged entries to PeersChangedPatch From f0db47338e61dbf803cbfe3beba936282fa04c2a Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 16 Jan 2025 15:48:07 -0600 Subject: [PATCH 176/223] cmd/tailscaled,util/syspolicy/source,util/winutil/gp: disallow acquiring the GP lock during service startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In v1.78, we started acquiring the GP lock when reading policy settings. This led to a deadlock during Tailscale installation via Group Policy Software Installation because the GP engine holds the write lock for the duration of policy processing, which in turn waits for the installation to complete, which in turn waits for the service to enter the running state. In this PR, we prevent the acquisition of GP locks (aka EnterCriticalPolicySection) during service startup and update the Windows Registry-based util/syspolicy/source.PlatformPolicyStore to handle this failure gracefully. The GP lock is somewhat optional; it’s safe to read policy settings without it, but acquiring the lock is recommended when reading multiple values to prevent the Group Policy engine from modifying settings mid-read and to avoid inconsistent results. Fixes #14416 Signed-off-by: Nick Khyl --- cmd/tailscaled/tailscaled_windows.go | 25 +++++- .../tailscaled_deps_test_windows.go | 1 + util/syspolicy/source/policy_store_windows.go | 82 ++++++++++++++++++- util/winutil/gp/policylock_windows.go | 38 ++++++++- 4 files changed, 138 insertions(+), 8 deletions(-) diff --git a/cmd/tailscaled/tailscaled_windows.go b/cmd/tailscaled/tailscaled_windows.go index 786c5d8330939..7208e03dac9d6 100644 --- a/cmd/tailscaled/tailscaled_windows.go +++ b/cmd/tailscaled/tailscaled_windows.go @@ -55,6 +55,7 @@ import ( "tailscale.com/util/osdiag" "tailscale.com/util/syspolicy" "tailscale.com/util/winutil" + "tailscale.com/util/winutil/gp" "tailscale.com/version" "tailscale.com/wf" ) @@ -70,6 +71,22 @@ func init() { } } +// permitPolicyLocks is a function to be called to lift the restriction on acquiring +// [gp.PolicyLock]s once the service is running. +// It is safe to be called multiple times. +var permitPolicyLocks = func() {} + +func init() { + if isWindowsService() { + // We prevent [gp.PolicyLock]s from being acquired until the service enters the running state. + // Otherwise, if tailscaled starts due to a GPSI policy installing Tailscale, it may deadlock + // while waiting for the write counterpart of the GP lock to be released by Group Policy, + // which is itself waiting for the installation to complete and tailscaled to start. + // See tailscale/tailscale#14416 for more information. + permitPolicyLocks = gp.RestrictPolicyLocks() + } +} + const serviceName = "Tailscale" // Application-defined command codes between 128 and 255 @@ -109,13 +126,13 @@ func tstunNewWithWindowsRetries(logf logger.Logf, tunName string) (_ tun.Device, } } -func isWindowsService() bool { +var isWindowsService = sync.OnceValue(func() bool { v, err := svc.IsWindowsService() if err != nil { log.Fatalf("svc.IsWindowsService failed: %v", err) } return v -} +}) // syslogf is a logger function that writes to the Windows event log (ie, the // one that you see in the Windows Event Viewer). tailscaled may optionally @@ -180,6 +197,10 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch changes <- svc.Status{State: svc.Running, Accepts: svcAccepts} syslogf("Service running") + // It is safe to allow GP locks to be acquired now that the service + // is running. + permitPolicyLocks() + for { select { case <-doneCh: diff --git a/tstest/integration/tailscaled_deps_test_windows.go b/tstest/integration/tailscaled_deps_test_windows.go index 5eda22327cd8a..b0d1c896847aa 100644 --- a/tstest/integration/tailscaled_deps_test_windows.go +++ b/tstest/integration/tailscaled_deps_test_windows.go @@ -60,6 +60,7 @@ import ( _ "tailscale.com/util/osshare" _ "tailscale.com/util/syspolicy" _ "tailscale.com/util/winutil" + _ "tailscale.com/util/winutil/gp" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wf" diff --git a/util/syspolicy/source/policy_store_windows.go b/util/syspolicy/source/policy_store_windows.go index 86e2254e0a381..621701e84f23c 100644 --- a/util/syspolicy/source/policy_store_windows.go +++ b/util/syspolicy/source/policy_store_windows.go @@ -12,6 +12,7 @@ import ( "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" "tailscale.com/util/set" + "tailscale.com/util/syspolicy/internal/loggerx" "tailscale.com/util/syspolicy/setting" "tailscale.com/util/winutil/gp" ) @@ -29,6 +30,18 @@ var ( _ Expirable = (*PlatformPolicyStore)(nil) ) +// lockableCloser is a [Lockable] that can also be closed. +// It is implemented by [gp.PolicyLock] and [optionalPolicyLock]. +type lockableCloser interface { + Lockable + Close() error +} + +var ( + _ lockableCloser = (*gp.PolicyLock)(nil) + _ lockableCloser = (*optionalPolicyLock)(nil) +) + // PlatformPolicyStore implements [Store] by providing read access to // Registry-based Tailscale policies, such as those configured via Group Policy or MDM. // For better performance and consistency, it is recommended to lock it when @@ -55,7 +68,7 @@ type PlatformPolicyStore struct { // they are being read. // // When both policyLock and mu need to be taken, mu must be taken before policyLock. - policyLock *gp.PolicyLock + policyLock lockableCloser mu sync.Mutex tsKeys []registry.Key // or nil if the [PlatformPolicyStore] hasn't been locked. @@ -108,7 +121,7 @@ func newPlatformPolicyStore(scope gp.Scope, softwareKey registry.Key, policyLock scope: scope, softwareKey: softwareKey, done: make(chan struct{}), - policyLock: policyLock, + policyLock: &optionalPolicyLock{PolicyLock: policyLock}, } } @@ -448,3 +461,68 @@ func tailscaleKeyNamesFor(scope gp.Scope) []string { panic("unreachable") } } + +type gpLockState int + +const ( + gpUnlocked = gpLockState(iota) + gpLocked + gpLockRestricted // the lock could not be acquired due to a restriction in place +) + +// optionalPolicyLock is a wrapper around [gp.PolicyLock] that locks +// and unlocks the underlying [gp.PolicyLock]. +// +// If the [gp.PolicyLock.Lock] returns [gp.ErrLockRestricted], the error is ignored, +// and calling [optionalPolicyLock.Unlock] is a no-op. +// +// The underlying GP lock is kinda optional: it is safe to read policy settings +// from the Registry without acquiring it, but it is recommended to lock it anyway +// when reading multiple policy settings to avoid potentially inconsistent results. +// +// It is not safe for concurrent use. +type optionalPolicyLock struct { + *gp.PolicyLock + state gpLockState +} + +// Lock acquires the underlying [gp.PolicyLock], returning an error on failure. +// If the lock cannot be acquired due to a restriction in place +// (e.g., attempting to acquire a lock while the service is starting), +// the lock is considered to be held, the method returns nil, and a subsequent +// call to [Unlock] is a no-op. +// It is a runtime error to call Lock when the lock is already held. +func (o *optionalPolicyLock) Lock() error { + if o.state != gpUnlocked { + panic("already locked") + } + switch err := o.PolicyLock.Lock(); err { + case nil: + o.state = gpLocked + return nil + case gp.ErrLockRestricted: + loggerx.Errorf("GP lock not acquired: %v", err) + o.state = gpLockRestricted + return nil + default: + return err + } +} + +// Unlock releases the underlying [gp.PolicyLock], if it was previously acquired. +// It is a runtime error to call Unlock when the lock is not held. +func (o *optionalPolicyLock) Unlock() { + switch o.state { + case gpLocked: + o.PolicyLock.Unlock() + case gpLockRestricted: + // The GP lock wasn't acquired due to a restriction in place + // when [optionalPolicyLock.Lock] was called. Unlock is a no-op. + case gpUnlocked: + panic("not locked") + default: + panic("unreachable") + } + + o.state = gpUnlocked +} diff --git a/util/winutil/gp/policylock_windows.go b/util/winutil/gp/policylock_windows.go index 95453aa16b110..69c5ff01697f4 100644 --- a/util/winutil/gp/policylock_windows.go +++ b/util/winutil/gp/policylock_windows.go @@ -48,10 +48,35 @@ type policyLockResult struct { } var ( - // ErrInvalidLockState is returned by (*PolicyLock).Lock if the lock has a zero value or has already been closed. + // ErrInvalidLockState is returned by [PolicyLock.Lock] if the lock has a zero value or has already been closed. ErrInvalidLockState = errors.New("the lock has not been created or has already been closed") + // ErrLockRestricted is returned by [PolicyLock.Lock] if the lock cannot be acquired due to a restriction in place, + // such as when [RestrictPolicyLocks] has been called. + ErrLockRestricted = errors.New("the lock cannot be acquired due to a restriction in place") ) +var policyLockRestricted atomic.Int32 + +// RestrictPolicyLocks forces all [PolicyLock.Lock] calls to return [ErrLockRestricted] +// until the returned function is called to remove the restriction. +// +// It is safe to call the returned function multiple times, but the restriction will only +// be removed once. If [RestrictPolicyLocks] is called multiple times, each call must be +// matched by a corresponding call to the returned function to fully remove the restrictions. +// +// It is primarily used to prevent certain deadlocks, such as when tailscaled attempts to acquire +// a policy lock during startup. If the service starts due to Tailscale being installed by GPSI, +// the write lock will be held by the Group Policy service throughout the installation, +// preventing tailscaled from acquiring the read lock. Since Group Policy waits for the installation +// to complete, and therefore for tailscaled to start, before releasing the write lock, this scenario +// would result in a deadlock. See tailscale/tailscale#14416 for more information. +func RestrictPolicyLocks() (removeRestriction func()) { + policyLockRestricted.Add(1) + return sync.OnceFunc(func() { + policyLockRestricted.Add(-1) + }) +} + // NewMachinePolicyLock creates a PolicyLock that facilitates pausing the // application of computer policy. To avoid deadlocks when acquiring both // machine and user locks, acquire the user lock before the machine lock. @@ -103,13 +128,18 @@ func NewUserPolicyLock(token windows.Token) (*PolicyLock, error) { } // Lock locks l. -// It returns ErrNotInitialized if l has a zero value or has already been closed, -// or an Errno if the underlying Group Policy lock cannot be acquired. +// It returns [ErrInvalidLockState] if l has a zero value or has already been closed, +// [ErrLockRestricted] if the lock cannot be acquired due to a restriction in place, +// or a [syscall.Errno] if the underlying Group Policy lock cannot be acquired. // -// As a special case, it fails with windows.ERROR_ACCESS_DENIED +// As a special case, it fails with [windows.ERROR_ACCESS_DENIED] // if l is a user policy lock, and the corresponding user is not logged in // interactively at the time of the call. func (l *PolicyLock) Lock() error { + if policyLockRestricted.Load() > 0 { + return ErrLockRestricted + } + l.mu.Lock() defer l.mu.Unlock() if l.lockCnt.Add(2)&1 == 0 { From 61bea750928b291d5e7bca4ea87d64d503d8a7ac Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 23 Jan 2025 18:40:17 -0800 Subject: [PATCH 177/223] cmd/tailscale: fix, test some recent doc inconsistencies 3dabea0fc2c added some docs with inconsistent usage docs. This fixes them, and adds a test. It also adds some other tests and fixes other verb tense inconsistencies. Updates tailscale/corp#25278 Change-Id: I94c2a8940791bddd7c35c1c3d5fb791a317370c2 Signed-off-by: Brad Fitzpatrick --- cmd/tailscale/cli/cli_test.go | 43 ++++++++++++++++++++++++++++ cmd/tailscale/cli/configure_apple.go | 12 ++++---- cmd/tailscale/cli/debug.go | 8 +++--- cmd/tailscale/cli/dns.go | 2 +- cmd/tailscale/cli/exitnode.go | 2 +- cmd/tailscale/cli/metrics.go | 4 +-- cmd/tailscale/cli/network-lock.go | 19 ++++++------ cmd/tailscale/cli/switch.go | 2 +- cmd/tailscale/cli/syspolicy.go | 4 +-- 9 files changed, 68 insertions(+), 28 deletions(-) diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index dccb69876d3b4..6f43814e8bb2c 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -17,6 +17,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/google/go-cmp/cmp" + "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/envknob" "tailscale.com/health/healthmsg" "tailscale.com/ipn" @@ -1525,3 +1526,45 @@ func TestHelpAlias(t *testing.T) { t.Fatalf("Run: %v", err) } } + +func TestDocs(t *testing.T) { + root := newRootCmd() + check := func(t *testing.T, c *ffcli.Command) { + shortVerb, _, ok := strings.Cut(c.ShortHelp, " ") + if !ok || shortVerb == "" { + t.Errorf("couldn't find verb+space in ShortHelp") + } else { + if strings.HasSuffix(shortVerb, ".") { + t.Errorf("ShortHelp shouldn't end in period; got %q", c.ShortHelp) + } + if b := shortVerb[0]; b >= 'a' && b <= 'z' { + t.Errorf("ShortHelp should start with upper-case letter; got %q", c.ShortHelp) + } + if strings.HasSuffix(shortVerb, "s") && shortVerb != "Does" { + t.Errorf("verb %q ending in 's' is unexpected, from %q", shortVerb, c.ShortHelp) + } + } + + name := t.Name() + wantPfx := strings.ReplaceAll(strings.TrimPrefix(name, "TestDocs/"), "/", " ") + switch name { + case "TestDocs/tailscale/completion/bash", + "TestDocs/tailscale/completion/zsh": + wantPfx = "" // special-case exceptions + } + if !strings.HasPrefix(c.ShortUsage, wantPfx) { + t.Errorf("ShortUsage should start with %q; got %q", wantPfx, c.ShortUsage) + } + } + + var walk func(t *testing.T, c *ffcli.Command) + walk = func(t *testing.T, c *ffcli.Command) { + t.Run(c.Name, func(t *testing.T) { + check(t, c) + for _, sub := range c.Subcommands { + walk(t, sub) + } + }) + } + walk(t, root) +} diff --git a/cmd/tailscale/cli/configure_apple.go b/cmd/tailscale/cli/configure_apple.go index edd9ec1abe5bc..c0d99b90aa2c4 100644 --- a/cmd/tailscale/cli/configure_apple.go +++ b/cmd/tailscale/cli/configure_apple.go @@ -27,28 +27,28 @@ func sysExtCmd() *ffcli.Command { return &ffcli.Command{ Name: "sysext", ShortUsage: "tailscale configure sysext [activate|deactivate|status]", - ShortHelp: "Manages the system extension for macOS (Standalone variant)", + ShortHelp: "Manage the system extension for macOS (Standalone variant)", LongHelp: "The sysext set of commands provides a way to activate, deactivate, or manage the state of the Tailscale system extension on macOS. " + "This is only relevant if you are running the Standalone variant of the Tailscale client for macOS. " + "To access more detailed information about system extensions installed on this Mac, run 'systemextensionsctl list'.", Subcommands: []*ffcli.Command{ { Name: "activate", - ShortUsage: "tailscale sysext activate", + ShortUsage: "tailscale configure sysext activate", ShortHelp: "Register the Tailscale system extension with macOS.", LongHelp: "This command registers the Tailscale system extension with macOS. To run Tailscale, you'll also need to install the VPN configuration separately (run `tailscale configure vpn-config install`). After running this command, you need to approve the extension in System Settings > Login Items and Extensions > Network Extensions.", Exec: requiresStandalone, }, { Name: "deactivate", - ShortUsage: "tailscale sysext deactivate", + ShortUsage: "tailscale configure sysext deactivate", ShortHelp: "Deactivate the Tailscale system extension on macOS", LongHelp: "This command deactivates the Tailscale system extension on macOS. To completely remove Tailscale, you'll also need to delete the VPN configuration separately (use `tailscale configure vpn-config uninstall`).", Exec: requiresStandalone, }, { Name: "status", - ShortUsage: "tailscale sysext status", + ShortUsage: "tailscale configure sysext status", ShortHelp: "Print the enablement status of the Tailscale system extension", LongHelp: "This command prints the enablement status of the Tailscale system extension. If the extension is not enabled, run `tailscale sysext activate` to enable it.", Exec: requiresStandalone, @@ -69,14 +69,14 @@ func vpnConfigCmd() *ffcli.Command { Subcommands: []*ffcli.Command{ { Name: "install", - ShortUsage: "tailscale mac-vpn install", + ShortUsage: "tailscale configure mac-vpn install", ShortHelp: "Write the Tailscale VPN configuration to the macOS settings", LongHelp: "This command writes the Tailscale VPN configuration to the macOS settings. This is the entry that appears in System Settings > VPN. If you are running the Standalone variant of the client, you'll also need to install the system extension separately (run `tailscale configure sysext activate`).", Exec: requiresGUI, }, { Name: "uninstall", - ShortUsage: "tailscale mac-vpn uninstall", + ShortUsage: "tailscale configure mac-vpn uninstall", ShortHelp: "Delete the Tailscale VPN configuration from the macOS settings", LongHelp: "This command removes the Tailscale VPN configuration from the macOS settings. This is the entry that appears in System Settings > VPN. If you are running the Standalone variant of the client, you'll also need to deactivate the system extension separately (run `tailscale configure sysext deactivate`).", Exec: requiresGUI, diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index 04b343e760e3c..f84dd25f0049f 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -289,7 +289,7 @@ var debugCmd = &ffcli.Command{ Name: "capture", ShortUsage: "tailscale debug capture", Exec: runCapture, - ShortHelp: "Streams pcaps for debugging", + ShortHelp: "Stream pcaps for debugging", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("capture") fs.StringVar(&captureArgs.outFile, "o", "", "path to stream the pcap (or - for stdout), leave empty to start wireshark") @@ -315,13 +315,13 @@ var debugCmd = &ffcli.Command{ Name: "peer-endpoint-changes", ShortUsage: "tailscale debug peer-endpoint-changes ", Exec: runPeerEndpointChanges, - ShortHelp: "Prints debug information about a peer's endpoint changes", + ShortHelp: "Print debug information about a peer's endpoint changes", }, { Name: "dial-types", ShortUsage: "tailscale debug dial-types ", Exec: runDebugDialTypes, - ShortHelp: "Prints debug information about connecting to a given host or IP", + ShortHelp: "Print debug information about connecting to a given host or IP", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("dial-types") fs.StringVar(&debugDialTypesArgs.network, "network", "tcp", `network type to dial ("tcp", "udp", etc.)`) @@ -342,7 +342,7 @@ var debugCmd = &ffcli.Command{ { Name: "go-buildinfo", ShortUsage: "tailscale debug go-buildinfo", - ShortHelp: "Prints Go's runtime/debug.BuildInfo", + ShortHelp: "Print Go's runtime/debug.BuildInfo", Exec: runGoBuildInfo, }, }, diff --git a/cmd/tailscale/cli/dns.go b/cmd/tailscale/cli/dns.go index 042ce1a94161a..402f0cedf0a1e 100644 --- a/cmd/tailscale/cli/dns.go +++ b/cmd/tailscale/cli/dns.go @@ -20,7 +20,7 @@ var dnsCmd = &ffcli.Command{ Name: "status", ShortUsage: "tailscale dns status [--all]", Exec: runDNSStatus, - ShortHelp: "Prints the current DNS status and configuration", + ShortHelp: "Print the current DNS status and configuration", LongHelp: dnsStatusLongHelp(), FlagSet: (func() *flag.FlagSet { fs := newFlagSet("status") diff --git a/cmd/tailscale/cli/exitnode.go b/cmd/tailscale/cli/exitnode.go index 941c6be8d1add..ad7a8ccee5b42 100644 --- a/cmd/tailscale/cli/exitnode.go +++ b/cmd/tailscale/cli/exitnode.go @@ -41,7 +41,7 @@ func exitNodeCmd() *ffcli.Command { { Name: "suggest", ShortUsage: "tailscale exit-node suggest", - ShortHelp: "Suggests the best available exit node", + ShortHelp: "Suggest the best available exit node", Exec: runExitNodeSuggest, }}, (func() []*ffcli.Command { diff --git a/cmd/tailscale/cli/metrics.go b/cmd/tailscale/cli/metrics.go index d5fe9ad81cb70..dbdedd5a61037 100644 --- a/cmd/tailscale/cli/metrics.go +++ b/cmd/tailscale/cli/metrics.go @@ -33,13 +33,13 @@ https://tailscale.com/s/client-metrics Name: "print", ShortUsage: "tailscale metrics print", Exec: runMetricsPrint, - ShortHelp: "Prints current metric values in the Prometheus text exposition format", + ShortHelp: "Print current metric values in Prometheus text format", }, { Name: "write", ShortUsage: "tailscale metrics write ", Exec: runMetricsWrite, - ShortHelp: "Writes metric values to a file", + ShortHelp: "Write metric values to a file", LongHelp: strings.TrimSpace(` The 'tailscale metrics write' command writes metric values to a text file provided as its diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/network-lock.go index 45f989f1057a7..c7776707422ec 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/network-lock.go @@ -191,8 +191,7 @@ var nlStatusArgs struct { var nlStatusCmd = &ffcli.Command{ Name: "status", ShortUsage: "tailscale lock status", - ShortHelp: "Outputs the state of tailnet lock", - LongHelp: "Outputs the state of tailnet lock", + ShortHelp: "Output the state of tailnet lock", Exec: runNetworkLockStatus, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock status") @@ -293,8 +292,7 @@ func runNetworkLockStatus(ctx context.Context, args []string) error { var nlAddCmd = &ffcli.Command{ Name: "add", ShortUsage: "tailscale lock add ...", - ShortHelp: "Adds one or more trusted signing keys to tailnet lock", - LongHelp: "Adds one or more trusted signing keys to tailnet lock", + ShortHelp: "Add one or more trusted signing keys to tailnet lock", Exec: func(ctx context.Context, args []string) error { return runNetworkLockModify(ctx, args, nil) }, @@ -307,8 +305,7 @@ var nlRemoveArgs struct { var nlRemoveCmd = &ffcli.Command{ Name: "remove", ShortUsage: "tailscale lock remove [--re-sign=false] ...", - ShortHelp: "Removes one or more trusted signing keys from tailnet lock", - LongHelp: "Removes one or more trusted signing keys from tailnet lock", + ShortHelp: "Remove one or more trusted signing keys from tailnet lock", Exec: runNetworkLockRemove, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock remove") @@ -448,7 +445,7 @@ func runNetworkLockModify(ctx context.Context, addArgs, removeArgs []string) err var nlSignCmd = &ffcli.Command{ Name: "sign", ShortUsage: "tailscale lock sign []\ntailscale lock sign ", - ShortHelp: "Signs a node or pre-approved auth key", + ShortHelp: "Sign a node or pre-approved auth key", LongHelp: `Either: - signs a node key and transmits the signature to the coordination server, or @@ -510,7 +507,7 @@ func runNetworkLockSign(ctx context.Context, args []string) error { var nlDisableCmd = &ffcli.Command{ Name: "disable", ShortUsage: "tailscale lock disable ", - ShortHelp: "Consumes a disablement secret to shut down tailnet lock for the tailnet", + ShortHelp: "Consume a disablement secret to shut down tailnet lock for the tailnet", LongHelp: strings.TrimSpace(` The 'tailscale lock disable' command uses the specified disablement @@ -539,7 +536,7 @@ func runNetworkLockDisable(ctx context.Context, args []string) error { var nlLocalDisableCmd = &ffcli.Command{ Name: "local-disable", ShortUsage: "tailscale lock local-disable", - ShortHelp: "Disables tailnet lock for this node only", + ShortHelp: "Disable tailnet lock for this node only", LongHelp: strings.TrimSpace(` The 'tailscale lock local-disable' command disables tailnet lock for only @@ -561,8 +558,8 @@ func runNetworkLockLocalDisable(ctx context.Context, args []string) error { var nlDisablementKDFCmd = &ffcli.Command{ Name: "disablement-kdf", ShortUsage: "tailscale lock disablement-kdf ", - ShortHelp: "Computes a disablement value from a disablement secret (advanced users only)", - LongHelp: "Computes a disablement value from a disablement secret (advanced users only)", + ShortHelp: "Compute a disablement value from a disablement secret (advanced users only)", + LongHelp: "Compute a disablement value from a disablement secret (advanced users only)", Exec: runNetworkLockDisablementKDF, } diff --git a/cmd/tailscale/cli/switch.go b/cmd/tailscale/cli/switch.go index 731492daaa976..af8b513263d37 100644 --- a/cmd/tailscale/cli/switch.go +++ b/cmd/tailscale/cli/switch.go @@ -20,7 +20,7 @@ import ( var switchCmd = &ffcli.Command{ Name: "switch", ShortUsage: "tailscale switch ", - ShortHelp: "Switches to a different Tailscale account", + ShortHelp: "Switch to a different Tailscale account", LongHelp: `"tailscale switch" switches between logged in accounts. You can use the ID that's returned from 'tailnet switch -list' to pick which profile you want to switch to. Alternatively, you diff --git a/cmd/tailscale/cli/syspolicy.go b/cmd/tailscale/cli/syspolicy.go index 0e903db397c7d..a71952a9f7f62 100644 --- a/cmd/tailscale/cli/syspolicy.go +++ b/cmd/tailscale/cli/syspolicy.go @@ -31,7 +31,7 @@ var syspolicyCmd = &ffcli.Command{ Name: "list", ShortUsage: "tailscale syspolicy list", Exec: runSysPolicyList, - ShortHelp: "Prints effective policy settings", + ShortHelp: "Print effective policy settings", LongHelp: "The 'tailscale syspolicy list' subcommand displays the effective policy settings and their sources (e.g., MDM or environment variables).", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("syspolicy list") @@ -43,7 +43,7 @@ var syspolicyCmd = &ffcli.Command{ Name: "reload", ShortUsage: "tailscale syspolicy reload", Exec: runSysPolicyReload, - ShortHelp: "Forces a reload of policy settings, even if no changes are detected, and prints the result", + ShortHelp: "Force a reload of policy settings, even if no changes are detected, and prints the result", LongHelp: "The 'tailscale syspolicy reload' subcommand forces a reload of policy settings, even if no changes are detected, and prints the result.", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("syspolicy reload") From 3a39f08735abd5d9e757f8ac2dd7b0e8eb359c03 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 20 Dec 2024 09:09:53 +0100 Subject: [PATCH 178/223] util/usermetric: add more drop labels Updates #14280 Signed-off-by: Kristoffer Dalby --- util/usermetric/metrics.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/util/usermetric/metrics.go b/util/usermetric/metrics.go index 7f85989ff062a..0c5511759da5f 100644 --- a/util/usermetric/metrics.go +++ b/util/usermetric/metrics.go @@ -28,6 +28,19 @@ const ( // ReasonACL means that the packet was not permitted by ACL. ReasonACL DropReason = "acl" + // ReasonMulticast means that the packet was dropped because it was a multicast packet. + ReasonMulticast DropReason = "multicast" + + // ReasonLinkLocalUnicast means that the packet was dropped because it was a link-local unicast packet. + ReasonLinkLocalUnicast DropReason = "link_local_unicast" + + // ReasonTooShort means that the packet was dropped because it was a bad packet, + // this could be due to a short packet. + ReasonTooShort DropReason = "too_short" + + // ReasonFragment means that the packet was dropped because it was an IP fragment. + ReasonFragment DropReason = "fragment" + // ReasonError means that the packet was dropped because of an error. ReasonError DropReason = "error" ) From 5756bc17049fbfa65c531eff08a8db5aea66b14a Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 20 Dec 2024 09:09:06 +0100 Subject: [PATCH 179/223] wgengine/filter: return drop reason for metrics Updates #14280 Signed-off-by: Kristoffer Dalby --- wgengine/filter/filter.go | 25 +++++++++++++------------ wgengine/filter/filter_test.go | 32 +++++++++++++++++--------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/wgengine/filter/filter.go b/wgengine/filter/filter.go index 9e5d8a37f2b24..6269b08ebe8bf 100644 --- a/wgengine/filter/filter.go +++ b/wgengine/filter/filter.go @@ -24,6 +24,7 @@ import ( "tailscale.com/types/views" "tailscale.com/util/mak" "tailscale.com/util/slicesx" + "tailscale.com/util/usermetric" "tailscale.com/wgengine/filter/filtertype" ) @@ -410,7 +411,7 @@ func (f *Filter) ShieldsUp() bool { return f.shieldsUp } // Tailscale peer. func (f *Filter) RunIn(q *packet.Parsed, rf RunFlags) Response { dir := in - r := f.pre(q, rf, dir) + r, _ := f.pre(q, rf, dir) if r == Accept || r == Drop { // already logged return r @@ -431,16 +432,16 @@ func (f *Filter) RunIn(q *packet.Parsed, rf RunFlags) Response { // RunOut determines whether this node is allowed to send q to a // Tailscale peer. -func (f *Filter) RunOut(q *packet.Parsed, rf RunFlags) Response { +func (f *Filter) RunOut(q *packet.Parsed, rf RunFlags) (Response, usermetric.DropReason) { dir := out - r := f.pre(q, rf, dir) + r, reason := f.pre(q, rf, dir) if r == Accept || r == Drop { // already logged - return r + return r, reason } r, why := f.runOut(q) f.logRateLimit(rf, q, dir, r, why) - return r + return r, "" } var unknownProtoStringCache sync.Map // ipproto.Proto -> string @@ -610,33 +611,33 @@ var gcpDNSAddr = netaddr.IPv4(169, 254, 169, 254) // pre runs the direction-agnostic filter logic. dir is only used for // logging. -func (f *Filter) pre(q *packet.Parsed, rf RunFlags, dir direction) Response { +func (f *Filter) pre(q *packet.Parsed, rf RunFlags, dir direction) (Response, usermetric.DropReason) { if len(q.Buffer()) == 0 { // wireguard keepalive packet, always permit. - return Accept + return Accept, "" } if len(q.Buffer()) < 20 { f.logRateLimit(rf, q, dir, Drop, "too short") - return Drop + return Drop, usermetric.ReasonTooShort } if q.Dst.Addr().IsMulticast() { f.logRateLimit(rf, q, dir, Drop, "multicast") - return Drop + return Drop, usermetric.ReasonMulticast } if q.Dst.Addr().IsLinkLocalUnicast() && q.Dst.Addr() != gcpDNSAddr { f.logRateLimit(rf, q, dir, Drop, "link-local-unicast") - return Drop + return Drop, usermetric.ReasonLinkLocalUnicast } if q.IPProto == ipproto.Fragment { // Fragments after the first always need to be passed through. // Very small fragments are considered Junk by Parsed. f.logRateLimit(rf, q, dir, Accept, "fragment") - return Accept + return Accept, "" } - return noVerdict + return noVerdict, "" } // loggingAllowed reports whether p can appear in logs at all. diff --git a/wgengine/filter/filter_test.go b/wgengine/filter/filter_test.go index e7f71e6a45310..68f206778a98b 100644 --- a/wgengine/filter/filter_test.go +++ b/wgengine/filter/filter_test.go @@ -30,6 +30,7 @@ import ( "tailscale.com/types/views" "tailscale.com/util/must" "tailscale.com/util/slicesx" + "tailscale.com/util/usermetric" "tailscale.com/wgengine/filter/filtertype" ) @@ -211,7 +212,7 @@ func TestUDPState(t *testing.T) { t.Fatalf("incoming initial packet not dropped, got=%v: %v", got, a4) } // We talk to that peer - if got := acl.RunOut(&b4, flags); got != Accept { + if got, _ := acl.RunOut(&b4, flags); got != Accept { t.Fatalf("outbound packet didn't egress, got=%v: %v", got, b4) } // Now, the same packet as before is allowed back. @@ -227,7 +228,7 @@ func TestUDPState(t *testing.T) { t.Fatalf("incoming initial packet not dropped: %v", a4) } // We talk to that peer - if got := acl.RunOut(&b6, flags); got != Accept { + if got, _ := acl.RunOut(&b6, flags); got != Accept { t.Fatalf("outbound packet didn't egress: %v", b4) } // Now, the same packet as before is allowed back. @@ -382,25 +383,26 @@ func BenchmarkFilter(b *testing.B) { func TestPreFilter(t *testing.T) { packets := []struct { - desc string - want Response - b []byte + desc string + want Response + wantReason usermetric.DropReason + b []byte }{ - {"empty", Accept, []byte{}}, - {"short", Drop, []byte("short")}, - {"junk", Drop, raw4default(ipproto.Unknown, 10)}, - {"fragment", Accept, raw4default(ipproto.Fragment, 40)}, - {"tcp", noVerdict, raw4default(ipproto.TCP, 0)}, - {"udp", noVerdict, raw4default(ipproto.UDP, 0)}, - {"icmp", noVerdict, raw4default(ipproto.ICMPv4, 0)}, + {"empty", Accept, "", []byte{}}, + {"short", Drop, usermetric.ReasonTooShort, []byte("short")}, + {"junk", Drop, "", raw4default(ipproto.Unknown, 10)}, + {"fragment", Accept, "", raw4default(ipproto.Fragment, 40)}, + {"tcp", noVerdict, "", raw4default(ipproto.TCP, 0)}, + {"udp", noVerdict, "", raw4default(ipproto.UDP, 0)}, + {"icmp", noVerdict, "", raw4default(ipproto.ICMPv4, 0)}, } f := NewAllowNone(t.Logf, &netipx.IPSet{}) for _, testPacket := range packets { p := &packet.Parsed{} p.Decode(testPacket.b) - got := f.pre(p, LogDrops|LogAccepts, in) - if got != testPacket.want { - t.Errorf("%q got=%v want=%v packet:\n%s", testPacket.desc, got, testPacket.want, packet.Hexdump(testPacket.b)) + got, gotReason := f.pre(p, LogDrops|LogAccepts, in) + if got != testPacket.want || gotReason != testPacket.wantReason { + t.Errorf("%q got=%v want=%v gotReason=%s wantReason=%s packet:\n%s", testPacket.desc, got, testPacket.want, gotReason, testPacket.wantReason, packet.Hexdump(testPacket.b)) } } } From f39ee8e5200a3c65491f024b968e5dddee97d872 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 20 Dec 2024 09:12:37 +0100 Subject: [PATCH 180/223] net/tstun: add back outgoing drop metric Using new labels returned from the filter Updates #14280 Signed-off-by: Kristoffer Dalby --- net/tstun/wrap.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/net/tstun/wrap.go b/net/tstun/wrap.go index 2d56f376832c9..b26239632bf06 100644 --- a/net/tstun/wrap.go +++ b/net/tstun/wrap.go @@ -877,12 +877,13 @@ func (t *Wrapper) filterPacketOutboundToWireGuard(p *packet.Parsed, pc *peerConf return filter.Drop, gro } - if filt.RunOut(p, t.filterFlags) != filter.Accept { + if resp, reason := filt.RunOut(p, t.filterFlags); resp != filter.Accept { metricPacketOutDropFilter.Add(1) - // TODO(#14280): increment a t.metrics.outboundDroppedPacketsTotal here - // once we figure out & document what labels to use for multicast, - // link-local-unicast, IP fragments, etc. But they're not - // usermetric.ReasonACL. + if reason != "" { + t.metrics.outboundDroppedPacketsTotal.Add(usermetric.DropLabels{ + Reason: reason, + }, 1) + } return filter.Drop, gro } From f0b63d0eecd01dc2562d14b7eb25710c3d351218 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 8 Jan 2025 10:53:07 +0100 Subject: [PATCH 181/223] wgengine/filter: add check for unknown proto Updates #14280 Signed-off-by: Kristoffer Dalby --- util/usermetric/metrics.go | 3 +++ wgengine/filter/filter.go | 5 +++++ wgengine/filter/filter_test.go | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/util/usermetric/metrics.go b/util/usermetric/metrics.go index 0c5511759da5f..044b4d65f7120 100644 --- a/util/usermetric/metrics.go +++ b/util/usermetric/metrics.go @@ -41,6 +41,9 @@ const ( // ReasonFragment means that the packet was dropped because it was an IP fragment. ReasonFragment DropReason = "fragment" + // ReasonUnknownProtocol means that the packet was dropped because it was an unknown protocol. + ReasonUnknownProtocol DropReason = "unknown_protocol" + // ReasonError means that the packet was dropped because of an error. ReasonError DropReason = "error" ) diff --git a/wgengine/filter/filter.go b/wgengine/filter/filter.go index 6269b08ebe8bf..987fcee0153a6 100644 --- a/wgengine/filter/filter.go +++ b/wgengine/filter/filter.go @@ -621,6 +621,11 @@ func (f *Filter) pre(q *packet.Parsed, rf RunFlags, dir direction) (Response, us return Drop, usermetric.ReasonTooShort } + if q.IPProto == ipproto.Unknown { + f.logRateLimit(rf, q, dir, Drop, "unknown proto") + return Drop, usermetric.ReasonUnknownProtocol + } + if q.Dst.Addr().IsMulticast() { f.logRateLimit(rf, q, dir, Drop, "multicast") return Drop, usermetric.ReasonMulticast diff --git a/wgengine/filter/filter_test.go b/wgengine/filter/filter_test.go index 68f206778a98b..ae39eeb08692f 100644 --- a/wgengine/filter/filter_test.go +++ b/wgengine/filter/filter_test.go @@ -390,7 +390,8 @@ func TestPreFilter(t *testing.T) { }{ {"empty", Accept, "", []byte{}}, {"short", Drop, usermetric.ReasonTooShort, []byte("short")}, - {"junk", Drop, "", raw4default(ipproto.Unknown, 10)}, + {"short-junk", Drop, usermetric.ReasonTooShort, raw4default(ipproto.Unknown, 10)}, + {"long-junk", Drop, usermetric.ReasonUnknownProtocol, raw4default(ipproto.Unknown, 21)}, {"fragment", Accept, "", raw4default(ipproto.Fragment, 40)}, {"tcp", noVerdict, "", raw4default(ipproto.TCP, 0)}, {"udp", noVerdict, "", raw4default(ipproto.UDP, 0)}, From 5e9056a35641d06bacc0e73864cba6d659869f9c Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Thu, 23 Jan 2025 09:08:54 -0600 Subject: [PATCH 182/223] derp: move Conn interface to derp.go This interface is used both by the DERP client as well as the server. Defining the interface in derp.go makes it clear that it is shared. Updates tailscale/corp#26045 Signed-off-by: Percy Wegmann --- derp/derp.go | 12 ++++++++++++ derp/derp_server.go | 12 ------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/derp/derp.go b/derp/derp.go index 6a7b3b73552d5..65acd43210234 100644 --- a/derp/derp.go +++ b/derp/derp.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "io" + "net" "time" ) @@ -254,3 +255,14 @@ func writeFrame(bw *bufio.Writer, t frameType, b []byte) error { } return bw.Flush() } + +// Conn is the subset of the underlying net.Conn the DERP Server needs. +// It is a defined type so that non-net connections can be used. +type Conn interface { + io.WriteCloser + LocalAddr() net.Addr + // The *Deadline methods follow the semantics of net.Conn. + SetDeadline(time.Time) error + SetReadDeadline(time.Time) error + SetWriteDeadline(time.Time) error +} diff --git a/derp/derp_server.go b/derp/derp_server.go index 983b5dc002c83..4b5cc2f78cc01 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -23,7 +23,6 @@ import ( "math" "math/big" "math/rand/v2" - "net" "net/http" "net/netip" "os" @@ -341,17 +340,6 @@ type PacketForwarder interface { String() string } -// Conn is the subset of the underlying net.Conn the DERP Server needs. -// It is a defined type so that non-net connections can be used. -type Conn interface { - io.WriteCloser - LocalAddr() net.Addr - // The *Deadline methods follow the semantics of net.Conn. - SetDeadline(time.Time) error - SetReadDeadline(time.Time) error - SetWriteDeadline(time.Time) error -} - var packetsDropped = metrics.NewMultiLabelMap[dropReasonKindLabels]( "derp_packets_dropped", "counter", From 450bc9a6b8a70904aa58614999c5400eed6d273c Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Thu, 23 Jan 2025 14:32:22 -0600 Subject: [PATCH 183/223] cmd/derper,derp: make TCP write timeout configurable The timeout still defaults to 2 seconds, but can now be changed via command-line flag. Updates tailscale/corp#26045 Signed-off-by: Percy Wegmann --- cmd/derper/derper.go | 3 +++ derp/derp_server.go | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 46ff644b26ddd..2c6ecd1753a95 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -77,6 +77,8 @@ var ( tcpKeepAlive = flag.Duration("tcp-keepalive-time", 10*time.Minute, "TCP keepalive time") // tcpUserTimeout is intentionally short, so that hung connections are cleaned up promptly. DERPs should be nearby users. tcpUserTimeout = flag.Duration("tcp-user-timeout", 15*time.Second, "TCP user timeout") + // tcpWriteTimeout is the timeout for writing to client TCP connections. It does not apply to mesh connections. + tcpWriteTimeout = flag.Duration("tcp-write-timeout", derp.DefaultTCPWiteTimeout, "TCP write timeout; 0 results in no timeout being set on writes") ) var ( @@ -173,6 +175,7 @@ func main() { s.SetVerifyClient(*verifyClients) s.SetVerifyClientURL(*verifyClientURL) s.SetVerifyClientURLFailOpen(*verifyFailOpen) + s.SetTCPWriteTimeout(*tcpWriteTimeout) if *meshPSKFile != "" { b, err := os.ReadFile(*meshPSKFile) diff --git a/derp/derp_server.go b/derp/derp_server.go index 4b5cc2f78cc01..0389eed6427c0 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -84,7 +84,7 @@ func init() { const ( defaultPerClientSendQueueDepth = 32 // default packets buffered for sending - writeTimeout = 2 * time.Second + DefaultTCPWiteTimeout = 2 * time.Second privilegedWriteTimeout = 30 * time.Second // for clients with the mesh key ) @@ -201,6 +201,8 @@ type Server struct { // Sets the client send queue depth for the server. perClientSendQueueDepth int + tcpWriteTimeout time.Duration + clock tstime.Clock } @@ -377,6 +379,7 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server { bufferedWriteFrames: metrics.NewHistogram([]float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 50, 100}), keyOfAddr: map[netip.AddrPort]key.NodePublic{}, clock: tstime.StdClock{}, + tcpWriteTimeout: DefaultTCPWiteTimeout, } s.initMetacert() s.packetsRecvDisco = s.packetsRecvByKind.Get(string(packetKindDisco)) @@ -481,6 +484,13 @@ func (s *Server) SetVerifyClientURLFailOpen(v bool) { s.verifyClientsURLFailOpen = v } +// SetTCPWriteTimeout sets the timeout for writing to connected clients. +// This timeout does not apply to mesh connections. +// Defaults to 2 seconds. +func (s *Server) SetTCPWriteTimeout(d time.Duration) { + s.tcpWriteTimeout = d +} + // HasMeshKey reports whether the server is configured with a mesh key. func (s *Server) HasMeshKey() bool { return s.meshKey != "" } @@ -1805,7 +1815,7 @@ func (c *sclient) sendLoop(ctx context.Context) error { } func (c *sclient) setWriteDeadline() { - d := writeTimeout + d := c.s.tcpWriteTimeout if c.canMesh { // Trusted peers get more tolerance. // @@ -1817,7 +1827,10 @@ func (c *sclient) setWriteDeadline() { // of connected peers. d = privilegedWriteTimeout } - c.nc.SetWriteDeadline(time.Now().Add(d)) + // Ignore the error from setting the write deadline. In practice, + // setting the deadline will only fail if the connection is closed + // or closing, so the subsequent Write() will fail anyway. + _ = c.nc.SetWriteDeadline(time.Now().Add(d)) } // sendKeepAlive sends a keep-alive frame, without flushing. From 05afa31df3e3840549d946ff90392cf6d73ccf12 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 24 Jan 2025 12:51:01 +0100 Subject: [PATCH 184/223] util/clientmetric: use counter in aggcounter Fixes #14743 Signed-off-by: Kristoffer Dalby --- util/clientmetric/clientmetric.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/clientmetric/clientmetric.go b/util/clientmetric/clientmetric.go index 584a24f73dca8..5c11160194fdc 100644 --- a/util/clientmetric/clientmetric.go +++ b/util/clientmetric/clientmetric.go @@ -270,7 +270,7 @@ func (c *AggregateCounter) UnregisterAll() { // a sum of expvar variables registered with it. func NewAggregateCounter(name string) *AggregateCounter { c := &AggregateCounter{counters: set.Set[*expvar.Int]{}} - NewGaugeFunc(name, c.Value) + NewCounterFunc(name, c.Value) return c } From d69c70ee5b446f74721ce20fd3677b8c6e642cb5 Mon Sep 17 00:00:00 2001 From: Adrian Dewhurst Date: Thu, 23 Jan 2025 14:26:16 -0500 Subject: [PATCH 185/223] tailcfg: adjust ServiceName.Validate to use vizerror Updates #cleanup Change-Id: I163b3f762b9d45c2155afe1c0a36860606833a22 Signed-off-by: Adrian Dewhurst --- tailcfg/tailcfg.go | 7 ++++--- util/dnsname/dnsname.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index c921a0c7d69a9..738c8a5dca96d 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -27,6 +27,7 @@ import ( "tailscale.com/types/tkatype" "tailscale.com/util/dnsname" "tailscale.com/util/slicesx" + "tailscale.com/util/vizerror" ) // CapabilityVersion represents the client's capability level. That @@ -891,14 +892,14 @@ type ServiceName string // Validate validates if the service name is formatted correctly. // We only allow valid DNS labels, since the expectation is that these will be -// used as parts of domain names. +// used as parts of domain names. All errors are [vizerror.Error]. func (sn ServiceName) Validate() error { bareName, ok := strings.CutPrefix(string(sn), "svc:") if !ok { - return errors.New("services must start with 'svc:'") + return vizerror.Errorf("%q is not a valid service name: must start with 'svc:'", sn) } if bareName == "" { - return errors.New("service names must not be empty") + return vizerror.Errorf("%q is not a valid service name: must not be empty after the 'svc:' prefix", sn) } return dnsname.ValidLabel(bareName) } diff --git a/util/dnsname/dnsname.go b/util/dnsname/dnsname.go index 131bdd14b632e..6404a9af1cc2f 100644 --- a/util/dnsname/dnsname.go +++ b/util/dnsname/dnsname.go @@ -94,7 +94,8 @@ func (f FQDN) Contains(other FQDN) bool { return strings.HasSuffix(other.WithTrailingDot(), cmp) } -// ValidLabel reports whether label is a valid DNS label. +// ValidLabel reports whether label is a valid DNS label. All errors are +// [vizerror.Error]. func ValidLabel(label string) error { if len(label) == 0 { return vizerror.New("empty DNS label") From 69bc164c621b8dc920b4208b389bd4a8f87c3d9f Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Fri, 24 Jan 2025 17:04:26 +0000 Subject: [PATCH 186/223] ipn/ipnlocal: include DNS SAN in cert CSR (#14764) The CN field is technically deprecated; set the requested name in a DNS SAN extension in addition to maximise compatibility with RFC 8555. Fixes #14762 Change-Id: If5d27f1e7abc519ec86489bf034ac98b2e613043 Signed-off-by: Tom Proctor --- ipn/ipnlocal/cert.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ipn/ipnlocal/cert.go b/ipn/ipnlocal/cert.go index 0d92c7cf809b4..71ae8ac86373f 100644 --- a/ipn/ipnlocal/cert.go +++ b/ipn/ipnlocal/cert.go @@ -556,6 +556,7 @@ func (b *LocalBackend) getCertPEM(ctx context.Context, cs certStore, logf logger } logf("requesting cert...") + traceACME(csr) der, _, err := ac.CreateOrderCert(ctx, order.FinalizeURL, csr, true) if err != nil { return nil, fmt.Errorf("CreateOrder: %v", err) @@ -578,10 +579,10 @@ func (b *LocalBackend) getCertPEM(ctx context.Context, cs certStore, logf logger } // certRequest generates a CSR for the given common name cn and optional SANs. -func certRequest(key crypto.Signer, cn string, ext []pkix.Extension, san ...string) ([]byte, error) { +func certRequest(key crypto.Signer, name string, ext []pkix.Extension) ([]byte, error) { req := &x509.CertificateRequest{ - Subject: pkix.Name{CommonName: cn}, - DNSNames: san, + Subject: pkix.Name{CommonName: name}, + DNSNames: []string{name}, ExtraExtensions: ext, } return x509.CreateCertificateRequest(rand.Reader, req, key) From 716e4fcc97759308f79875ff1809da945df70574 Mon Sep 17 00:00:00 2001 From: Mario Minardi Date: Fri, 24 Jan 2025 16:29:58 -0700 Subject: [PATCH 187/223] client/web: remove advanced options from web client login (#14770) Removing the advanced options collapsible from the web client login for now ahead of our next client release. Updates https://github.com/tailscale/tailscale/issues/14568 Signed-off-by: Mario Minardi --- .../web/src/components/views/login-view.tsx | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/client/web/src/components/views/login-view.tsx b/client/web/src/components/views/login-view.tsx index b2868bb46c991..f8c15b16dbcaa 100644 --- a/client/web/src/components/views/login-view.tsx +++ b/client/web/src/components/views/login-view.tsx @@ -1,13 +1,11 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -import React, { useState } from "react" +import React from "react" import { useAPI } from "src/api" import TailscaleIcon from "src/assets/icons/tailscale-icon.svg?react" import { NodeData } from "src/types" import Button from "src/ui/button" -import Collapsible from "src/ui/collapsible" -import Input from "src/ui/input" /** * LoginView is rendered when the client is not authenticated @@ -15,8 +13,6 @@ import Input from "src/ui/input" */ export default function LoginView({ data }: { data: NodeData }) { const api = useAPI() - const [controlURL, setControlURL] = useState("") - const [authKey, setAuthKey] = useState("") return (
@@ -88,8 +84,6 @@ export default function LoginView({ data }: { data: NodeData }) { action: "up", data: { Reauthenticate: true, - ControlURL: controlURL, - AuthKey: authKey, }, }) } @@ -98,34 +92,6 @@ export default function LoginView({ data }: { data: NodeData }) { > Log In - -

Auth Key

-

- Connect with a pre-authenticated key.{" "} - - Learn more → - -

- setAuthKey(e.target.value)} - placeholder="tskey-auth-XXX" - /> -

Server URL

-

Base URL of control server.

- setControlURL(e.target.value)} - placeholder="https://login.tailscale.com/" - /> - )}
From cbf1a9abe188ae3c121b98aad00b3cc4439fe677 Mon Sep 17 00:00:00 2001 From: Mario Minardi Date: Fri, 24 Jan 2025 17:04:12 -0700 Subject: [PATCH 188/223] go.{mod,sum}: update web-client-prebuilt (#14772) Manually update the `web-client-prebuilt` package as the GitHub action is failing for some reason. Updates https://github.com/tailscale/tailscale/issues/14568 Signed-off-by: Mario Minardi --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4265953a4023d..22193ee6efe8d 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/tailscale/mkctr v0.0.0-20250110151924-54977352e4a6 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc - github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 + github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e diff --git a/go.sum b/go.sum index 2623cb6e93667..20dbe73063589 100644 --- a/go.sum +++ b/go.sum @@ -933,8 +933,8 @@ github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 h1:uFsXVBE9Qr4 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0= github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc h1:24heQPtnFR+yfntqhI3oAu9i27nEojcQ4NuBQOo5ZFA= github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc/go.mod h1:f93CXfllFsO9ZQVq+Zocb1Gp4G5Fz0b0rXHLOzt/Djc= -github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:tdUdyPqJ0C97SJfjB9tW6EylTtreyee9C44de+UBG0g= -github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= +github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:UBPHPtv8+nEAy2PD8RyAhOYvau1ek0HDJqLS/Pysi14= +github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19 h1:BcEJP2ewTIK2ZCsqgl6YGpuO6+oKqqag5HHb7ehljKw= From 1a7274fccb0617f6d0bc31a45d835b61a9d5c5b7 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 24 Jan 2025 13:09:21 -0800 Subject: [PATCH 189/223] control/controlclient: skip SetControlClientStatus when queue has newer results later Updates #1909 Updates #12542 Updates tailscale/corp#26058 Change-Id: I3033d235ca49f9739fdf3deaf603eea4ec3e407e Signed-off-by: Brad Fitzpatrick --- control/controlclient/auto.go | 71 ++++++++++++++++- control/controlclient/controlclient_test.go | 85 +++++++++++++++++++++ control/controlknobs/controlknobs.go | 8 ++ tailcfg/tailcfg.go | 5 ++ 4 files changed, 167 insertions(+), 2 deletions(-) diff --git a/control/controlclient/auto.go b/control/controlclient/auto.go index edd0ae29c645d..a5397594e864f 100644 --- a/control/controlclient/auto.go +++ b/control/controlclient/auto.go @@ -21,6 +21,7 @@ import ( "tailscale.com/types/netmap" "tailscale.com/types/persist" "tailscale.com/types/structs" + "tailscale.com/util/clientmetric" "tailscale.com/util/execqueue" ) @@ -131,6 +132,8 @@ type Auto struct { // the server. lastUpdateGen updateGen + lastStatus atomic.Pointer[Status] + paused bool // whether we should stop making HTTP requests unpauseWaiters []chan bool // chans that gets sent true (once) on wake, or false on Shutdown loggedIn bool // true if currently logged in @@ -596,21 +599,85 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM // not logged in. nm = nil } - new := Status{ + newSt := &Status{ URL: url, Persist: p, NetMap: nm, Err: err, state: state, } + c.lastStatus.Store(newSt) // Launch a new goroutine to avoid blocking the caller while the observer // does its thing, which may result in a call back into the client. + metricQueued.Add(1) c.observerQueue.Add(func() { - c.observer.SetControlClientStatus(c, new) + if canSkipStatus(newSt, c.lastStatus.Load()) { + metricSkippable.Add(1) + if !c.direct.controlKnobs.DisableSkipStatusQueue.Load() { + metricSkipped.Add(1) + return + } + } + c.observer.SetControlClientStatus(c, *newSt) + // Best effort stop retaining the memory now that + // we've sent it to the observer (LocalBackend). + // We CAS here because the caller goroutine is + // doing a Store which we want to want to win + // a race. This is only a memory optimization + // and is for correctness: + c.lastStatus.CompareAndSwap(newSt, nil) }) } +var ( + metricQueued = clientmetric.NewCounter("controlclient_auto_status_queued") + metricSkippable = clientmetric.NewCounter("controlclient_auto_status_queue_skippable") + metricSkipped = clientmetric.NewCounter("controlclient_auto_status_queue_skipped") +) + +// canSkipStatus reports whether we can skip sending s1, knowing +// that s2 is enqueued sometime in the future after s1. +// +// s1 must be non-nil. s2 may be nil. +func canSkipStatus(s1, s2 *Status) bool { + if s2 == nil { + // Nothing in the future. + return false + } + if s1 == s2 { + // If the last item in the queue is the same as s1, + // we can't skip it. + return false + } + if s1.Err != nil || s1.URL != "" { + // If s1 has an error or a URL, we shouldn't skip it, lest the error go + // away in s2 or in-between. We want to make sure all the subsystems see + // it. Plus there aren't many of these, so not worth skipping. + return false + } + if !s1.Persist.Equals(s2.Persist) || s1.state != s2.state { + // If s1 has a different Persist or state than s2, + // don't skip it. We only care about skipping the typical + // entries where the only difference is the NetMap. + return false + } + // If nothing above precludes it, and both s1 and s2 have NetMaps, then + // we can skip it, because s2's NetMap is a newer version and we can + // jump straight from whatever state we had before to s2's state, + // without passing through s1's state first. A NetMap is regrettably a + // full snapshot of the state, not an incremental delta. We're slowly + // moving towards passing around only deltas around internally at all + // layers, but this is explicitly the case where we didn't have a delta + // path for the message we received over the wire and had to resort + // to the legacy full NetMap path. And then we can get behind processing + // these full NetMap snapshots in LocalBackend/wgengine/magicsock/netstack + // and this path (when it returns true) lets us skip over useless work + // and not get behind in the queue. This matters in particular for tailnets + // that are both very large + very churny. + return s1.NetMap != nil && s2.NetMap != nil +} + func (c *Auto) Login(flags LoginFlags) { c.logf("client.Login(%v)", flags) diff --git a/control/controlclient/controlclient_test.go b/control/controlclient/controlclient_test.go index b376234511c09..6885b5851d70d 100644 --- a/control/controlclient/controlclient_test.go +++ b/control/controlclient/controlclient_test.go @@ -4,8 +4,13 @@ package controlclient import ( + "io" "reflect" + "slices" "testing" + + "tailscale.com/types/netmap" + "tailscale.com/types/persist" ) func fieldsOf(t reflect.Type) (fields []string) { @@ -62,3 +67,83 @@ func TestStatusEqual(t *testing.T) { } } } + +// tests [canSkipStatus]. +func TestCanSkipStatus(t *testing.T) { + st := new(Status) + nm1 := &netmap.NetworkMap{} + nm2 := &netmap.NetworkMap{} + + tests := []struct { + name string + s1, s2 *Status + want bool + }{ + { + name: "nil-s2", + s1: st, + s2: nil, + want: false, + }, + { + name: "equal", + s1: st, + s2: st, + want: false, + }, + { + name: "s1-error", + s1: &Status{Err: io.EOF, NetMap: nm1}, + s2: &Status{NetMap: nm2}, + want: false, + }, + { + name: "s1-url", + s1: &Status{URL: "foo", NetMap: nm1}, + s2: &Status{NetMap: nm2}, + want: false, + }, + { + name: "s1-persist-diff", + s1: &Status{Persist: new(persist.Persist).View(), NetMap: nm1}, + s2: &Status{NetMap: nm2}, + want: false, + }, + { + name: "s1-state-diff", + s1: &Status{state: 123, NetMap: nm1}, + s2: &Status{NetMap: nm2}, + want: false, + }, + { + name: "s1-no-netmap1", + s1: &Status{NetMap: nil}, + s2: &Status{NetMap: nm2}, + want: false, + }, + { + name: "s1-no-netmap2", + s1: &Status{NetMap: nm1}, + s2: &Status{NetMap: nil}, + want: false, + }, + { + name: "skip", + s1: &Status{NetMap: nm1}, + s2: &Status{NetMap: nm2}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := canSkipStatus(tt.s1, tt.s2); got != tt.want { + t.Errorf("canSkipStatus = %v, want %v", got, tt.want) + } + }) + } + + want := []string{"Err", "URL", "NetMap", "Persist", "state"} + if f := fieldsOf(reflect.TypeFor[Status]()); !slices.Equal(f, want) { + t.Errorf("Status fields = %q; this code was only written to handle fields %q", f, want) + } +} diff --git a/control/controlknobs/controlknobs.go b/control/controlknobs/controlknobs.go index dd76a3abdba5b..c7933be5a148d 100644 --- a/control/controlknobs/controlknobs.go +++ b/control/controlknobs/controlknobs.go @@ -103,6 +103,11 @@ type Knobs struct { // DisableCaptivePortalDetection is whether the node should not perform captive portal detection // automatically when the network state changes. DisableCaptivePortalDetection atomic.Bool + + // DisableSkipStatusQueue is whether the node should disable skipping + // of queued netmap.NetworkMap between the controlclient and LocalBackend. + // See tailscale/tailscale#14768. + DisableSkipStatusQueue atomic.Bool } // UpdateFromNodeAttributes updates k (if non-nil) based on the provided self @@ -132,6 +137,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) { disableLocalDNSOverrideViaNRPT = has(tailcfg.NodeAttrDisableLocalDNSOverrideViaNRPT) disableCryptorouting = has(tailcfg.NodeAttrDisableMagicSockCryptoRouting) disableCaptivePortalDetection = has(tailcfg.NodeAttrDisableCaptivePortalDetection) + disableSkipStatusQueue = has(tailcfg.NodeAttrDisableSkipStatusQueue) ) if has(tailcfg.NodeAttrOneCGNATEnable) { @@ -159,6 +165,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) { k.DisableLocalDNSOverrideViaNRPT.Store(disableLocalDNSOverrideViaNRPT) k.DisableCryptorouting.Store(disableCryptorouting) k.DisableCaptivePortalDetection.Store(disableCaptivePortalDetection) + k.DisableSkipStatusQueue.Store(disableSkipStatusQueue) } // AsDebugJSON returns k as something that can be marshalled with json.Marshal @@ -187,5 +194,6 @@ func (k *Knobs) AsDebugJSON() map[string]any { "DisableLocalDNSOverrideViaNRPT": k.DisableLocalDNSOverrideViaNRPT.Load(), "DisableCryptorouting": k.DisableCryptorouting.Load(), "DisableCaptivePortalDetection": k.DisableCaptivePortalDetection.Load(), + "DisableSkipStatusQueue": k.DisableSkipStatusQueue.Load(), } } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 738c8a5dca96d..c17cd5f45df6e 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2470,6 +2470,11 @@ const ( // automatically when the network state changes. NodeAttrDisableCaptivePortalDetection NodeCapability = "disable-captive-portal-detection" + // NodeAttrDisableSkipStatusQueue is set when the node should disable skipping + // of queued netmap.NetworkMap between the controlclient and LocalBackend. + // See tailscale/tailscale#14768. + NodeAttrDisableSkipStatusQueue NodeCapability = "disable-skip-status-queue" + // NodeAttrSSHEnvironmentVariables enables logic for handling environment variables sent // via SendEnv in the SSH server and applying them to the SSH session. NodeAttrSSHEnvironmentVariables NodeCapability = "ssh-env-vars" From ca39c4e150366b0cdcb766a62c9c8bc3fb116083 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Thu, 23 Jan 2025 16:23:41 -0800 Subject: [PATCH 190/223] cmd/natc,wgengine/netstack: tune buffer size and segment lifetime in natc Some natc instances have been observed with excessive memory growth, dominant in gvisor buffers. It is likely that the connection buffers are sticking around for too long due to the default long segment time, and uptuned buffer size applied by default in wgengine/netstack. Apply configurations in natc specifically which are a better match for the natc use case, most notably a 5s maximum segment lifetime. Updates tailscale/corp#25169 Signed-off-by: James Tucker --- cmd/natc/natc.go | 31 +++++++++++++++++++++++++++++++ wgengine/netstack/netstack.go | 8 ++++++++ 2 files changed, 39 insertions(+) diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index d94523c6e4161..b28f4a1d5a664 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -26,6 +26,8 @@ import ( "github.com/inetaf/tcpproxy" "github.com/peterbourgon/ff/v3" "golang.org/x/net/dns/dnsmessage" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" "tailscale.com/client/tailscale" "tailscale.com/envknob" "tailscale.com/hostinfo" @@ -37,6 +39,7 @@ import ( "tailscale.com/tsweb" "tailscale.com/util/dnsname" "tailscale.com/util/mak" + "tailscale.com/wgengine/netstack" ) func main() { @@ -112,6 +115,7 @@ func main() { ts.Port = uint16(*wgPort) } defer ts.Close() + if *verboseTSNet { ts.Logf = log.Printf } @@ -129,6 +133,33 @@ func main() { log.Fatalf("debug serve: %v", http.Serve(dln, mux)) }() } + + if err := ts.Start(); err != nil { + log.Fatalf("ts.Start: %v", err) + } + // TODO(raggi): this is not a public interface or guarantee. + ns := ts.Sys().Netstack.Get().(*netstack.Impl) + tcpRXBufOpt := tcpip.TCPReceiveBufferSizeRangeOption{ + Min: tcp.MinBufferSize, + Default: tcp.DefaultReceiveBufferSize, + Max: tcp.MaxBufferSize, + } + if err := ns.SetTransportProtocolOption(tcp.ProtocolNumber, &tcpRXBufOpt); err != nil { + log.Fatalf("could not set TCP RX buf size: %v", err) + } + tcpTXBufOpt := tcpip.TCPSendBufferSizeRangeOption{ + Min: tcp.MinBufferSize, + Default: tcp.DefaultSendBufferSize, + Max: tcp.MaxBufferSize, + } + if err := ns.SetTransportProtocolOption(tcp.ProtocolNumber, &tcpTXBufOpt); err != nil { + log.Fatalf("could not set TCP TX buf size: %v", err) + } + mslOpt := tcpip.TCPTimeWaitTimeoutOption(5 * time.Second) + if err := ns.SetTransportProtocolOption(tcp.ProtocolNumber, &mslOpt); err != nil { + log.Fatalf("could not set TCP MSL: %v", err) + } + lc, err := ts.LocalClient() if err != nil { log.Fatalf("LocalClient() failed: %v", err) diff --git a/wgengine/netstack/netstack.go b/wgengine/netstack/netstack.go index 0b8c67b061098..f0c4c527163af 100644 --- a/wgengine/netstack/netstack.go +++ b/wgengine/netstack/netstack.go @@ -405,6 +405,14 @@ func (ns *Impl) Close() error { return nil } +// SetTransportProtocolOption forwards to the underlying +// [stack.Stack.SetTransportProtocolOption]. Callers are responsible for +// ensuring that the options are valid, compatible and appropriate for their use +// case. Compatibility may change at any version. +func (ns *Impl) SetTransportProtocolOption(transport tcpip.TransportProtocolNumber, option tcpip.SettableTransportProtocolOption) tcpip.Error { + return ns.ipstack.SetTransportProtocolOption(transport, option) +} + // A single process might have several netstacks running at the same time. // Exported clientmetric counters will have a sum of counters of all of them. var stacksForMetrics syncs.Map[*Impl, struct{}] From 2089f4b603e36501dd1a7497ab4de691b1560dd7 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Sat, 25 Jan 2025 00:29:00 +0000 Subject: [PATCH 191/223] ipn/ipnlocal: add debug envknob for ACME directory URL (#14771) Adds an envknob setting for changing the client's ACME directory URL. This allows testing cert issuing against LE's staging environment, as well as enabling local-only test environments, which is useful for avoiding the production rate limits in test and development scenarios. Fixes #14761 Change-Id: I191c840c0ca143a20e4fa54ea3b2f9b7cbfc889f Signed-off-by: Tom Proctor --- ipn/ipnlocal/cert.go | 5 +++-- ipn/ipnlocal/cert_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ipn/ipnlocal/cert.go b/ipn/ipnlocal/cert.go index 71ae8ac86373f..3361fc70ba54e 100644 --- a/ipn/ipnlocal/cert.go +++ b/ipn/ipnlocal/cert.go @@ -659,8 +659,9 @@ func acmeClient(cs certStore) (*acme.Client, error) { // LetsEncrypt), we should make sure that they support ARI extension (see // shouldStartDomainRenewalARI). return &acme.Client{ - Key: key, - UserAgent: "tailscaled/" + version.Long(), + Key: key, + UserAgent: "tailscaled/" + version.Long(), + DirectoryURL: envknob.String("TS_DEBUG_ACME_DIRECTORY_URL"), }, nil } diff --git a/ipn/ipnlocal/cert_test.go b/ipn/ipnlocal/cert_test.go index 3ae7870e3174f..21741ca9522e6 100644 --- a/ipn/ipnlocal/cert_test.go +++ b/ipn/ipnlocal/cert_test.go @@ -199,3 +199,19 @@ func TestShouldStartDomainRenewal(t *testing.T) { }) } } + +func TestDebugACMEDirectoryURL(t *testing.T) { + for _, tc := range []string{"", "https://acme-staging-v02.api.letsencrypt.org/directory"} { + const setting = "TS_DEBUG_ACME_DIRECTORY_URL" + t.Run(tc, func(t *testing.T) { + t.Setenv(setting, tc) + ac, err := acmeClient(certStateStore{StateStore: new(mem.Store)}) + if err != nil { + t.Fatalf("acmeClient creation err: %v", err) + } + if ac.DirectoryURL != tc { + t.Fatalf("acmeClient.DirectoryURL = %q, want %q", ac.DirectoryURL, tc) + } + }) + } +} From 82e41ddc427aa6d41b875642788cb12f765ed40c Mon Sep 17 00:00:00 2001 From: James Tucker Date: Thu, 23 Jan 2025 16:31:40 -0800 Subject: [PATCH 192/223] cmd/natc: expose netstack metrics in client metrics in natc Updates tailscale/corp#25169 Signed-off-by: James Tucker --- cmd/natc/natc.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index b28f4a1d5a664..069eabefd7089 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -10,6 +10,7 @@ import ( "context" "encoding/binary" "errors" + "expvar" "flag" "fmt" "log" @@ -159,6 +160,9 @@ func main() { if err := ns.SetTransportProtocolOption(tcp.ProtocolNumber, &mslOpt); err != nil { log.Fatalf("could not set TCP MSL: %v", err) } + if *debugPort != 0 { + expvar.Publish("netstack", ns.ExpVar()) + } lc, err := ts.LocalClient() if err != nil { From 2c98c44d9a7c0b67aef7e72e7fed0766a7e7b1e6 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 23 Jan 2025 13:58:40 -0800 Subject: [PATCH 193/223] control/controlclient: sanitize invalid DERPMap nil Region from control Fixes #14752 Change-Id: If364603eefb9ac6dc5ec6df84a0d5e16c94dda8d Signed-off-by: Brad Fitzpatrick --- control/controlclient/map.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 1a54fc5430fcc..f0a11bdf1a0c2 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -300,6 +300,15 @@ func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) { if dm := resp.DERPMap; dm != nil { ms.vlogf("netmap: new map contains DERP map") + // Guard against the control server accidentally sending + // a nil region definition, which at least Headscale was + // observed to send. + for rid, r := range dm.Regions { + if r == nil { + delete(dm.Regions, rid) + } + } + // Zero-valued fields in a DERPMap mean that we're not changing // anything and are using the previous value(s). if ldm := ms.lastDERPMap; ldm != nil { From 68a66ee81b8e59de355a4b1a0688f28adf2c59b6 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 23 Jan 2025 20:39:28 -0800 Subject: [PATCH 194/223] feature/capture: move packet capture to feature/*, out of iOS + CLI We had the debug packet capture code + Lua dissector in the CLI + the iOS app. Now we don't, with tests to lock it in. As a bonus, tailscale.com/net/packet and tailscale.com/net/flowtrack no longer appear in the CLI's binary either. A new build tag ts_omit_capture disables the packet capture code and was added to build_dist.sh's --extra-small mode. Updates #12614 Change-Id: I79b0628c0d59911bd4d510c732284d97b0160f10 Signed-off-by: Brad Fitzpatrick --- build_dist.sh | 2 +- cmd/k8s-operator/depaware.txt | 4 +- cmd/tailscale/cli/cli.go | 2 +- cmd/tailscale/cli/cli_test.go | 30 + cmd/tailscale/cli/debug-capture.go | 80 +++ cmd/tailscale/cli/debug.go | 638 ++++++++---------- cmd/tailscale/depaware.txt | 6 +- cmd/tailscaled/depaware.txt | 4 +- {wgengine => feature}/capture/capture.go | 74 +- feature/capture/dissector/dissector.go | 12 + .../capture/dissector}/ts-dissector.lua | 0 feature/condregister/maybe_capture.go | 8 + ipn/ipnlocal/local.go | 80 +-- ipn/localapi/localapi.go | 34 +- net/packet/capture.go | 75 ++ net/packet/packet.go | 8 - net/tstun/wrap.go | 15 +- net/tstun/wrap_test.go | 13 +- tstest/iosdeps/iosdeps_test.go | 1 + wgengine/magicsock/magicsock.go | 7 +- wgengine/userspace.go | 3 +- wgengine/watchdog.go | 4 +- wgengine/wgengine.go | 4 +- 23 files changed, 620 insertions(+), 484 deletions(-) create mode 100644 cmd/tailscale/cli/debug-capture.go rename {wgengine => feature}/capture/capture.go (79%) create mode 100644 feature/capture/dissector/dissector.go rename {wgengine/capture => feature/capture/dissector}/ts-dissector.lua (100%) create mode 100644 feature/condregister/maybe_capture.go create mode 100644 net/packet/capture.go diff --git a/build_dist.sh b/build_dist.sh index 9a29e5201123a..ccd4ac8b164dd 100755 --- a/build_dist.sh +++ b/build_dist.sh @@ -37,7 +37,7 @@ while [ "$#" -gt 1 ]; do --extra-small) shift ldflags="$ldflags -w -s" - tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion,ts_omit_ssh,ts_omit_wakeonlan" + tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion,ts_omit_ssh,ts_omit_wakeonlan,ts_omit_capture" ;; --box) shift diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 11a9201d4028b..fc2f8854ae3d2 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -802,6 +802,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/envknob from tailscale.com/client/tailscale+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/feature/wakeonlan+ + tailscale.com/feature/capture from tailscale.com/feature/condregister tailscale.com/feature/condregister from tailscale.com/tsnet L tailscale.com/feature/tap from tailscale.com/feature/condregister tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister @@ -814,7 +815,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ đŸ’Ŗ tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnlocal+ tailscale.com/ipn/ipnlocal from tailscale.com/ipn/localapi+ tailscale.com/ipn/ipnstate from tailscale.com/client/tailscale+ - tailscale.com/ipn/localapi from tailscale.com/tsnet + tailscale.com/ipn/localapi from tailscale.com/tsnet+ tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+ L tailscale.com/ipn/store/awsstore from tailscale.com/ipn/store @@ -969,7 +970,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/version from tailscale.com/client/web+ tailscale.com/version/distro from tailscale.com/client/web+ tailscale.com/wgengine from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/capture from tailscale.com/ipn/ipnlocal+ tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ đŸ’Ŗ tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index fd39b3b67d3fd..d80d0c02f3cfa 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -212,7 +212,7 @@ change in the future. exitNodeCmd(), updateCmd, whoisCmd, - debugCmd, + debugCmd(), driveCmd, idTokenCmd, advertiseCmd(), diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index 6f43814e8bb2c..2d02b6b7a191c 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -25,10 +25,12 @@ import ( "tailscale.com/tailcfg" "tailscale.com/tka" "tailscale.com/tstest" + "tailscale.com/tstest/deptest" "tailscale.com/types/logger" "tailscale.com/types/opt" "tailscale.com/types/persist" "tailscale.com/types/preftype" + "tailscale.com/util/set" "tailscale.com/version/distro" ) @@ -1568,3 +1570,31 @@ func TestDocs(t *testing.T) { } walk(t, root) } + +func TestDeps(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "arm64", + WantDeps: set.Of( + "tailscale.com/feature/capture/dissector", // want the Lua by default + ), + BadDeps: map[string]string{ + "tailscale.com/feature/capture": "don't link capture code", + "tailscale.com/net/packet": "why we passing packets in the CLI?", + "tailscale.com/net/flowtrack": "why we tracking flows in the CLI?", + }, + }.Check(t) +} + +func TestDepsNoCapture(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "arm64", + Tags: "ts_omit_capture", + BadDeps: map[string]string{ + "tailscale.com/feature/capture": "don't link capture code", + "tailscale.com/feature/capture/dissector": "don't like the Lua", + }, + }.Check(t) + +} diff --git a/cmd/tailscale/cli/debug-capture.go b/cmd/tailscale/cli/debug-capture.go new file mode 100644 index 0000000000000..a54066fa614cb --- /dev/null +++ b/cmd/tailscale/cli/debug-capture.go @@ -0,0 +1,80 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ios && !ts_omit_capture + +package cli + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "os/exec" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/feature/capture/dissector" +) + +func init() { + debugCaptureCmd = mkDebugCaptureCmd +} + +func mkDebugCaptureCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "capture", + ShortUsage: "tailscale debug capture", + Exec: runCapture, + ShortHelp: "Stream pcaps for debugging", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("capture") + fs.StringVar(&captureArgs.outFile, "o", "", "path to stream the pcap (or - for stdout), leave empty to start wireshark") + return fs + })(), + } +} + +var captureArgs struct { + outFile string +} + +func runCapture(ctx context.Context, args []string) error { + stream, err := localClient.StreamDebugCapture(ctx) + if err != nil { + return err + } + defer stream.Close() + + switch captureArgs.outFile { + case "-": + fmt.Fprintln(Stderr, "Press Ctrl-C to stop the capture.") + _, err = io.Copy(os.Stdout, stream) + return err + case "": + lua, err := os.CreateTemp("", "ts-dissector") + if err != nil { + return err + } + defer os.Remove(lua.Name()) + io.WriteString(lua, dissector.Lua) + if err := lua.Close(); err != nil { + return err + } + + wireshark := exec.CommandContext(ctx, "wireshark", "-X", "lua_script:"+lua.Name(), "-k", "-i", "-") + wireshark.Stdin = stream + wireshark.Stdout = os.Stdout + wireshark.Stderr = os.Stderr + return wireshark.Run() + } + + f, err := os.OpenFile(captureArgs.outFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer f.Close() + fmt.Fprintln(Stderr, "Press Ctrl-C to stop the capture.") + _, err = io.Copy(f, stream) + return err +} diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index f84dd25f0049f..ce5edd8d344f4 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -20,7 +20,6 @@ import ( "net/netip" "net/url" "os" - "os/exec" "runtime" "runtime/debug" "strconv" @@ -45,307 +44,302 @@ import ( "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/util/must" - "tailscale.com/wgengine/capture" ) -var debugCmd = &ffcli.Command{ - Name: "debug", - Exec: runDebug, - ShortUsage: "tailscale debug ", - ShortHelp: "Debug commands", - LongHelp: hidden + `"tailscale debug" contains misc debug facilities; it is not a stable interface.`, - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("debug") - fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME") - fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-seconds seconds and write it to this file; - for stdout") - fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout") - fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty") - return fs - })(), - Subcommands: []*ffcli.Command{ - { - Name: "derp-map", - ShortUsage: "tailscale debug derp-map", - Exec: runDERPMap, - ShortHelp: "Print DERP map", - }, - { - Name: "component-logs", - ShortUsage: "tailscale debug component-logs [" + strings.Join(ipn.DebuggableComponents, "|") + "]", - Exec: runDebugComponentLogs, - ShortHelp: "Enable/disable debug logs for a component", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("component-logs") - fs.DurationVar(&debugComponentLogsArgs.forDur, "for", time.Hour, "how long to enable debug logs for; zero or negative means to disable") - return fs - })(), - }, - { - Name: "daemon-goroutines", - ShortUsage: "tailscale debug daemon-goroutines", - Exec: runDaemonGoroutines, - ShortHelp: "Print tailscaled's goroutines", - }, - { - Name: "daemon-logs", - ShortUsage: "tailscale debug daemon-logs", - Exec: runDaemonLogs, - ShortHelp: "Watch tailscaled's server logs", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("daemon-logs") - fs.IntVar(&daemonLogsArgs.verbose, "verbose", 0, "verbosity level") - fs.BoolVar(&daemonLogsArgs.time, "time", false, "include client time") - return fs - })(), - }, - { - Name: "metrics", - ShortUsage: "tailscale debug metrics", - Exec: runDaemonMetrics, - ShortHelp: "Print tailscaled's metrics", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("metrics") - fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values") - return fs - })(), - }, - { - Name: "env", - ShortUsage: "tailscale debug env", - Exec: runEnv, - ShortHelp: "Print cmd/tailscale environment", - }, - { - Name: "stat", - ShortUsage: "tailscale debug stat ", - Exec: runStat, - ShortHelp: "Stat a file", - }, - { - Name: "hostinfo", - ShortUsage: "tailscale debug hostinfo", - Exec: runHostinfo, - ShortHelp: "Print hostinfo", - }, - { - Name: "local-creds", - ShortUsage: "tailscale debug local-creds", - Exec: runLocalCreds, - ShortHelp: "Print how to access Tailscale LocalAPI", - }, - { - Name: "restun", - ShortUsage: "tailscale debug restun", - Exec: localAPIAction("restun"), - ShortHelp: "Force a magicsock restun", - }, - { - Name: "rebind", - ShortUsage: "tailscale debug rebind", - Exec: localAPIAction("rebind"), - ShortHelp: "Force a magicsock rebind", - }, - { - Name: "derp-set-on-demand", - ShortUsage: "tailscale debug derp-set-on-demand", - Exec: localAPIAction("derp-set-homeless"), - ShortHelp: "Enable DERP on-demand mode (breaks reachability)", - }, - { - Name: "derp-unset-on-demand", - ShortUsage: "tailscale debug derp-unset-on-demand", - Exec: localAPIAction("derp-unset-homeless"), - ShortHelp: "Disable DERP on-demand mode", - }, - { - Name: "break-tcp-conns", - ShortUsage: "tailscale debug break-tcp-conns", - Exec: localAPIAction("break-tcp-conns"), - ShortHelp: "Break any open TCP connections from the daemon", - }, - { - Name: "break-derp-conns", - ShortUsage: "tailscale debug break-derp-conns", - Exec: localAPIAction("break-derp-conns"), - ShortHelp: "Break any open DERP connections from the daemon", - }, - { - Name: "pick-new-derp", - ShortUsage: "tailscale debug pick-new-derp", - Exec: localAPIAction("pick-new-derp"), - ShortHelp: "Switch to some other random DERP home region for a short time", - }, - { - Name: "force-prefer-derp", - ShortUsage: "tailscale debug force-prefer-derp", - Exec: forcePreferDERP, - ShortHelp: "Prefer the given region ID if reachable (until restart, or 0 to clear)", - }, - { - Name: "force-netmap-update", - ShortUsage: "tailscale debug force-netmap-update", - Exec: localAPIAction("force-netmap-update"), - ShortHelp: "Force a full no-op netmap update (for load testing)", - }, - { - // TODO(bradfitz,maisem): eventually promote this out of debug - Name: "reload-config", - ShortUsage: "tailscale debug reload-config", - Exec: reloadConfig, - ShortHelp: "Reload config", - }, - { - Name: "control-knobs", - ShortUsage: "tailscale debug control-knobs", - Exec: debugControlKnobs, - ShortHelp: "See current control knobs", - }, - { - Name: "prefs", - ShortUsage: "tailscale debug prefs", - Exec: runPrefs, - ShortHelp: "Print prefs", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("prefs") - fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output") - return fs - })(), - }, - { - Name: "watch-ipn", - ShortUsage: "tailscale debug watch-ipn", - Exec: runWatchIPN, - ShortHelp: "Subscribe to IPN message bus", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("watch-ipn") - fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages") - fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status") - fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messags") - fs.BoolVar(&watchIPNArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap") - fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever") - return fs - })(), - }, - { - Name: "netmap", - ShortUsage: "tailscale debug netmap", - Exec: runNetmap, - ShortHelp: "Print the current network map", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("netmap") - fs.BoolVar(&netmapArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap") - return fs - })(), - }, - { - Name: "via", - ShortUsage: "tailscale debug via \n" + - "tailscale debug via ", - Exec: runVia, - ShortHelp: "Convert between site-specific IPv4 CIDRs and IPv6 'via' routes", - }, - { - Name: "ts2021", - ShortUsage: "tailscale debug ts2021", - Exec: runTS2021, - ShortHelp: "Debug ts2021 protocol connectivity", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("ts2021") - fs.StringVar(&ts2021Args.host, "host", "controlplane.tailscale.com", "hostname of control plane") - fs.IntVar(&ts2021Args.version, "version", int(tailcfg.CurrentCapabilityVersion), "protocol version") - fs.BoolVar(&ts2021Args.verbose, "verbose", false, "be extra verbose") - return fs - })(), - }, - { - Name: "set-expire", - ShortUsage: "tailscale debug set-expire --in=1m", - Exec: runSetExpire, - ShortHelp: "Manipulate node key expiry for testing", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("set-expire") - fs.DurationVar(&setExpireArgs.in, "in", 0, "if non-zero, set node key to expire this duration from now") - return fs - })(), - }, - { - Name: "dev-store-set", - ShortUsage: "tailscale debug dev-store-set", - Exec: runDevStoreSet, - ShortHelp: "Set a key/value pair during development", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("store-set") - fs.BoolVar(&devStoreSetArgs.danger, "danger", false, "accept danger") - return fs - })(), - }, - { - Name: "derp", - ShortUsage: "tailscale debug derp", - Exec: runDebugDERP, - ShortHelp: "Test a DERP configuration", - }, - { - Name: "capture", - ShortUsage: "tailscale debug capture", - Exec: runCapture, - ShortHelp: "Stream pcaps for debugging", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("capture") - fs.StringVar(&captureArgs.outFile, "o", "", "path to stream the pcap (or - for stdout), leave empty to start wireshark") - return fs - })(), - }, - { - Name: "portmap", - ShortUsage: "tailscale debug portmap", - Exec: debugPortmap, - ShortHelp: "Run portmap debugging", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("portmap") - fs.DurationVar(&debugPortmapArgs.duration, "duration", 5*time.Second, "timeout for port mapping") - fs.StringVar(&debugPortmapArgs.ty, "type", "", `portmap debug type (one of "", "pmp", "pcp", or "upnp")`) - fs.StringVar(&debugPortmapArgs.gatewayAddr, "gateway-addr", "", `override gateway IP (must also pass --self-addr)`) - fs.StringVar(&debugPortmapArgs.selfAddr, "self-addr", "", `override self IP (must also pass --gateway-addr)`) - fs.BoolVar(&debugPortmapArgs.logHTTP, "log-http", false, `print all HTTP requests and responses to the log`) - return fs - })(), - }, - { - Name: "peer-endpoint-changes", - ShortUsage: "tailscale debug peer-endpoint-changes ", - Exec: runPeerEndpointChanges, - ShortHelp: "Print debug information about a peer's endpoint changes", - }, - { - Name: "dial-types", - ShortUsage: "tailscale debug dial-types ", - Exec: runDebugDialTypes, - ShortHelp: "Print debug information about connecting to a given host or IP", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("dial-types") - fs.StringVar(&debugDialTypesArgs.network, "network", "tcp", `network type to dial ("tcp", "udp", etc.)`) - return fs - })(), - }, - { - Name: "resolve", - ShortUsage: "tailscale debug resolve ", - Exec: runDebugResolve, - ShortHelp: "Does a DNS lookup", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("resolve") - fs.StringVar(&resolveArgs.net, "net", "ip", "network type to resolve (ip, ip4, ip6)") - return fs - })(), - }, - { - Name: "go-buildinfo", - ShortUsage: "tailscale debug go-buildinfo", - ShortHelp: "Print Go's runtime/debug.BuildInfo", - Exec: runGoBuildInfo, - }, - }, +var ( + debugCaptureCmd func() *ffcli.Command // or nil +) + +func debugCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "debug", + Exec: runDebug, + ShortUsage: "tailscale debug ", + ShortHelp: "Debug commands", + LongHelp: hidden + `"tailscale debug" contains misc debug facilities; it is not a stable interface.`, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("debug") + fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME") + fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-seconds seconds and write it to this file; - for stdout") + fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout") + fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty") + return fs + })(), + Subcommands: nonNilCmds([]*ffcli.Command{ + { + Name: "derp-map", + ShortUsage: "tailscale debug derp-map", + Exec: runDERPMap, + ShortHelp: "Print DERP map", + }, + { + Name: "component-logs", + ShortUsage: "tailscale debug component-logs [" + strings.Join(ipn.DebuggableComponents, "|") + "]", + Exec: runDebugComponentLogs, + ShortHelp: "Enable/disable debug logs for a component", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("component-logs") + fs.DurationVar(&debugComponentLogsArgs.forDur, "for", time.Hour, "how long to enable debug logs for; zero or negative means to disable") + return fs + })(), + }, + { + Name: "daemon-goroutines", + ShortUsage: "tailscale debug daemon-goroutines", + Exec: runDaemonGoroutines, + ShortHelp: "Print tailscaled's goroutines", + }, + { + Name: "daemon-logs", + ShortUsage: "tailscale debug daemon-logs", + Exec: runDaemonLogs, + ShortHelp: "Watch tailscaled's server logs", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("daemon-logs") + fs.IntVar(&daemonLogsArgs.verbose, "verbose", 0, "verbosity level") + fs.BoolVar(&daemonLogsArgs.time, "time", false, "include client time") + return fs + })(), + }, + { + Name: "metrics", + ShortUsage: "tailscale debug metrics", + Exec: runDaemonMetrics, + ShortHelp: "Print tailscaled's metrics", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("metrics") + fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values") + return fs + })(), + }, + { + Name: "env", + ShortUsage: "tailscale debug env", + Exec: runEnv, + ShortHelp: "Print cmd/tailscale environment", + }, + { + Name: "stat", + ShortUsage: "tailscale debug stat ", + Exec: runStat, + ShortHelp: "Stat a file", + }, + { + Name: "hostinfo", + ShortUsage: "tailscale debug hostinfo", + Exec: runHostinfo, + ShortHelp: "Print hostinfo", + }, + { + Name: "local-creds", + ShortUsage: "tailscale debug local-creds", + Exec: runLocalCreds, + ShortHelp: "Print how to access Tailscale LocalAPI", + }, + { + Name: "restun", + ShortUsage: "tailscale debug restun", + Exec: localAPIAction("restun"), + ShortHelp: "Force a magicsock restun", + }, + { + Name: "rebind", + ShortUsage: "tailscale debug rebind", + Exec: localAPIAction("rebind"), + ShortHelp: "Force a magicsock rebind", + }, + { + Name: "derp-set-on-demand", + ShortUsage: "tailscale debug derp-set-on-demand", + Exec: localAPIAction("derp-set-homeless"), + ShortHelp: "Enable DERP on-demand mode (breaks reachability)", + }, + { + Name: "derp-unset-on-demand", + ShortUsage: "tailscale debug derp-unset-on-demand", + Exec: localAPIAction("derp-unset-homeless"), + ShortHelp: "Disable DERP on-demand mode", + }, + { + Name: "break-tcp-conns", + ShortUsage: "tailscale debug break-tcp-conns", + Exec: localAPIAction("break-tcp-conns"), + ShortHelp: "Break any open TCP connections from the daemon", + }, + { + Name: "break-derp-conns", + ShortUsage: "tailscale debug break-derp-conns", + Exec: localAPIAction("break-derp-conns"), + ShortHelp: "Break any open DERP connections from the daemon", + }, + { + Name: "pick-new-derp", + ShortUsage: "tailscale debug pick-new-derp", + Exec: localAPIAction("pick-new-derp"), + ShortHelp: "Switch to some other random DERP home region for a short time", + }, + { + Name: "force-prefer-derp", + ShortUsage: "tailscale debug force-prefer-derp", + Exec: forcePreferDERP, + ShortHelp: "Prefer the given region ID if reachable (until restart, or 0 to clear)", + }, + { + Name: "force-netmap-update", + ShortUsage: "tailscale debug force-netmap-update", + Exec: localAPIAction("force-netmap-update"), + ShortHelp: "Force a full no-op netmap update (for load testing)", + }, + { + // TODO(bradfitz,maisem): eventually promote this out of debug + Name: "reload-config", + ShortUsage: "tailscale debug reload-config", + Exec: reloadConfig, + ShortHelp: "Reload config", + }, + { + Name: "control-knobs", + ShortUsage: "tailscale debug control-knobs", + Exec: debugControlKnobs, + ShortHelp: "See current control knobs", + }, + { + Name: "prefs", + ShortUsage: "tailscale debug prefs", + Exec: runPrefs, + ShortHelp: "Print prefs", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("prefs") + fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output") + return fs + })(), + }, + { + Name: "watch-ipn", + ShortUsage: "tailscale debug watch-ipn", + Exec: runWatchIPN, + ShortHelp: "Subscribe to IPN message bus", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("watch-ipn") + fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages") + fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status") + fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messags") + fs.BoolVar(&watchIPNArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap") + fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever") + return fs + })(), + }, + { + Name: "netmap", + ShortUsage: "tailscale debug netmap", + Exec: runNetmap, + ShortHelp: "Print the current network map", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("netmap") + fs.BoolVar(&netmapArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap") + return fs + })(), + }, + { + Name: "via", + ShortUsage: "tailscale debug via \n" + + "tailscale debug via ", + Exec: runVia, + ShortHelp: "Convert between site-specific IPv4 CIDRs and IPv6 'via' routes", + }, + { + Name: "ts2021", + ShortUsage: "tailscale debug ts2021", + Exec: runTS2021, + ShortHelp: "Debug ts2021 protocol connectivity", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("ts2021") + fs.StringVar(&ts2021Args.host, "host", "controlplane.tailscale.com", "hostname of control plane") + fs.IntVar(&ts2021Args.version, "version", int(tailcfg.CurrentCapabilityVersion), "protocol version") + fs.BoolVar(&ts2021Args.verbose, "verbose", false, "be extra verbose") + return fs + })(), + }, + { + Name: "set-expire", + ShortUsage: "tailscale debug set-expire --in=1m", + Exec: runSetExpire, + ShortHelp: "Manipulate node key expiry for testing", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("set-expire") + fs.DurationVar(&setExpireArgs.in, "in", 0, "if non-zero, set node key to expire this duration from now") + return fs + })(), + }, + { + Name: "dev-store-set", + ShortUsage: "tailscale debug dev-store-set", + Exec: runDevStoreSet, + ShortHelp: "Set a key/value pair during development", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("store-set") + fs.BoolVar(&devStoreSetArgs.danger, "danger", false, "accept danger") + return fs + })(), + }, + { + Name: "derp", + ShortUsage: "tailscale debug derp", + Exec: runDebugDERP, + ShortHelp: "Test a DERP configuration", + }, + ccall(debugCaptureCmd), + { + Name: "portmap", + ShortUsage: "tailscale debug portmap", + Exec: debugPortmap, + ShortHelp: "Run portmap debugging", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("portmap") + fs.DurationVar(&debugPortmapArgs.duration, "duration", 5*time.Second, "timeout for port mapping") + fs.StringVar(&debugPortmapArgs.ty, "type", "", `portmap debug type (one of "", "pmp", "pcp", or "upnp")`) + fs.StringVar(&debugPortmapArgs.gatewayAddr, "gateway-addr", "", `override gateway IP (must also pass --self-addr)`) + fs.StringVar(&debugPortmapArgs.selfAddr, "self-addr", "", `override self IP (must also pass --gateway-addr)`) + fs.BoolVar(&debugPortmapArgs.logHTTP, "log-http", false, `print all HTTP requests and responses to the log`) + return fs + })(), + }, + { + Name: "peer-endpoint-changes", + ShortUsage: "tailscale debug peer-endpoint-changes ", + Exec: runPeerEndpointChanges, + ShortHelp: "Print debug information about a peer's endpoint changes", + }, + { + Name: "dial-types", + ShortUsage: "tailscale debug dial-types ", + Exec: runDebugDialTypes, + ShortHelp: "Print debug information about connecting to a given host or IP", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("dial-types") + fs.StringVar(&debugDialTypesArgs.network, "network", "tcp", `network type to dial ("tcp", "udp", etc.)`) + return fs + })(), + }, + { + Name: "resolve", + ShortUsage: "tailscale debug resolve ", + Exec: runDebugResolve, + ShortHelp: "Does a DNS lookup", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("resolve") + fs.StringVar(&resolveArgs.net, "net", "ip", "network type to resolve (ip, ip4, ip6)") + return fs + })(), + }, + { + Name: "go-buildinfo", + ShortUsage: "tailscale debug go-buildinfo", + ShortHelp: "Print Go's runtime/debug.BuildInfo", + Exec: runGoBuildInfo, + }, + }...), + } } func runGoBuildInfo(ctx context.Context, args []string) error { @@ -1036,50 +1030,6 @@ func runSetExpire(ctx context.Context, args []string) error { return localClient.DebugSetExpireIn(ctx, setExpireArgs.in) } -var captureArgs struct { - outFile string -} - -func runCapture(ctx context.Context, args []string) error { - stream, err := localClient.StreamDebugCapture(ctx) - if err != nil { - return err - } - defer stream.Close() - - switch captureArgs.outFile { - case "-": - fmt.Fprintln(Stderr, "Press Ctrl-C to stop the capture.") - _, err = io.Copy(os.Stdout, stream) - return err - case "": - lua, err := os.CreateTemp("", "ts-dissector") - if err != nil { - return err - } - defer os.Remove(lua.Name()) - lua.Write([]byte(capture.DissectorLua)) - if err := lua.Close(); err != nil { - return err - } - - wireshark := exec.CommandContext(ctx, "wireshark", "-X", "lua_script:"+lua.Name(), "-k", "-i", "-") - wireshark.Stdin = stream - wireshark.Stdout = os.Stdout - wireshark.Stderr = os.Stderr - return wireshark.Run() - } - - f, err := os.OpenFile(captureArgs.outFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - defer f.Close() - fmt.Fprintln(Stderr, "Press Ctrl-C to stop the capture.") - _, err = io.Copy(f, stream) - return err -} - var debugPortmapArgs struct { duration time.Duration gatewayAddr string diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 774d97d8ef373..47ba03cb97b74 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -88,6 +88,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/drive from tailscale.com/client/tailscale+ tailscale.com/envknob from tailscale.com/client/tailscale+ tailscale.com/envknob/featureknob from tailscale.com/client/web + tailscale.com/feature/capture/dissector from tailscale.com/cmd/tailscale/cli tailscale.com/health from tailscale.com/net/tlsdial+ tailscale.com/health/healthmsg from tailscale.com/cmd/tailscale/cli tailscale.com/hostinfo from tailscale.com/client/web+ @@ -102,7 +103,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback tailscale.com/net/dnscache from tailscale.com/control/controlhttp+ tailscale.com/net/dnsfallback from tailscale.com/control/controlhttp+ - tailscale.com/net/flowtrack from tailscale.com/net/packet tailscale.com/net/netaddr from tailscale.com/ipn+ tailscale.com/net/netcheck from tailscale.com/cmd/tailscale/cli tailscale.com/net/neterror from tailscale.com/net/netcheck+ @@ -110,7 +110,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep đŸ’Ŗ tailscale.com/net/netmon from tailscale.com/cmd/tailscale/cli+ đŸ’Ŗ tailscale.com/net/netns from tailscale.com/derp/derphttp+ tailscale.com/net/netutil from tailscale.com/client/tailscale+ - tailscale.com/net/packet from tailscale.com/wgengine/capture tailscale.com/net/ping from tailscale.com/net/netcheck tailscale.com/net/portmapper from tailscale.com/cmd/tailscale/cli+ tailscale.com/net/sockstats from tailscale.com/control/controlhttp+ @@ -133,7 +132,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/tsweb/varz from tailscale.com/util/usermetric tailscale.com/types/dnstype from tailscale.com/tailcfg+ tailscale.com/types/empty from tailscale.com/ipn - tailscale.com/types/ipproto from tailscale.com/net/flowtrack+ + tailscale.com/types/ipproto from tailscale.com/ipn+ tailscale.com/types/key from tailscale.com/client/tailscale+ tailscale.com/types/lazy from tailscale.com/util/testenv+ tailscale.com/types/logger from tailscale.com/client/web+ @@ -185,7 +184,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep W đŸ’Ŗ tailscale.com/util/winutil/winenv from tailscale.com/hostinfo+ tailscale.com/version from tailscale.com/client/web+ tailscale.com/version/distro from tailscale.com/client/web+ - tailscale.com/wgengine/capture from tailscale.com/cmd/tailscale/cli tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 4f81d93dda9fe..1e0b2061a3a21 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -260,6 +260,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/envknob from tailscale.com/client/tailscale+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/feature/wakeonlan+ + tailscale.com/feature/capture from tailscale.com/feature/condregister tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled L tailscale.com/feature/tap from tailscale.com/feature/condregister tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister @@ -273,7 +274,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/ipn/ipnlocal from tailscale.com/cmd/tailscaled+ tailscale.com/ipn/ipnserver from tailscale.com/cmd/tailscaled tailscale.com/ipn/ipnstate from tailscale.com/client/tailscale+ - tailscale.com/ipn/localapi from tailscale.com/ipn/ipnserver + tailscale.com/ipn/localapi from tailscale.com/ipn/ipnserver+ tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal tailscale.com/ipn/store from tailscale.com/cmd/tailscaled+ L tailscale.com/ipn/store/awsstore from tailscale.com/ipn/store @@ -422,7 +423,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/version/distro from tailscale.com/client/web+ W tailscale.com/wf from tailscale.com/cmd/tailscaled tailscale.com/wgengine from tailscale.com/cmd/tailscaled+ - tailscale.com/wgengine/capture from tailscale.com/ipn/ipnlocal+ tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ đŸ’Ŗ tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ diff --git a/wgengine/capture/capture.go b/feature/capture/capture.go similarity index 79% rename from wgengine/capture/capture.go rename to feature/capture/capture.go index 6ea5a9549b4f1..e5e150de8e761 100644 --- a/wgengine/capture/capture.go +++ b/feature/capture/capture.go @@ -13,21 +13,44 @@ import ( "sync" "time" - _ "embed" - + "tailscale.com/feature" + "tailscale.com/ipn/localapi" "tailscale.com/net/packet" "tailscale.com/util/set" ) -//go:embed ts-dissector.lua -var DissectorLua string +func init() { + feature.Register("capture") + localapi.Register("debug-capture", serveLocalAPIDebugCapture) +} + +func serveLocalAPIDebugCapture(h *localapi.Handler, w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + if !h.PermitWrite { + http.Error(w, "debug access denied", http.StatusForbidden) + return + } + if r.Method != "POST" { + http.Error(w, "POST required", http.StatusMethodNotAllowed) + return + } + + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + + b := h.LocalBackend() + s := b.GetOrSetCaptureSink(newSink) -// Callback describes a function which is called to -// record packets when debugging packet-capture. -// Such callbacks must not take ownership of the -// provided data slice: it may only copy out of it -// within the lifetime of the function. -type Callback func(Path, time.Time, []byte, packet.CaptureMeta) + unregister := s.RegisterOutput(w) + + select { + case <-ctx.Done(): + case <-s.WaitCh(): + } + unregister() + + b.ClearCaptureSink() +} var bufferPool = sync.Pool{ New: func() any { @@ -57,29 +80,8 @@ func writePktHeader(w *bytes.Buffer, when time.Time, length int) { binary.Write(w, binary.LittleEndian, uint32(length)) // total length } -// Path describes where in the data path the packet was captured. -type Path uint8 - -// Valid Path values. -const ( - // FromLocal indicates the packet was logged as it traversed the FromLocal path: - // i.e.: A packet from the local system into the TUN. - FromLocal Path = 0 - // FromPeer indicates the packet was logged upon reception from a remote peer. - FromPeer Path = 1 - // SynthesizedToLocal indicates the packet was generated from within tailscaled, - // and is being routed to the local machine's network stack. - SynthesizedToLocal Path = 2 - // SynthesizedToPeer indicates the packet was generated from within tailscaled, - // and is being routed to a remote Wireguard peer. - SynthesizedToPeer Path = 3 - - // PathDisco indicates the packet is information about a disco frame. - PathDisco Path = 254 -) - -// New creates a new capture sink. -func New() *Sink { +// newSink creates a new capture sink. +func newSink() packet.CaptureSink { ctx, c := context.WithCancel(context.Background()) return &Sink{ ctx: ctx, @@ -126,6 +128,10 @@ func (s *Sink) RegisterOutput(w io.Writer) (unregister func()) { } } +func (s *Sink) CaptureCallback() packet.CaptureCallback { + return s.LogPacket +} + // NumOutputs returns the number of outputs registered with the sink. func (s *Sink) NumOutputs() int { s.mu.Lock() @@ -174,7 +180,7 @@ func customDataLen(meta packet.CaptureMeta) int { // LogPacket is called to insert a packet into the capture. // // This function does not take ownership of the provided data slice. -func (s *Sink) LogPacket(path Path, when time.Time, data []byte, meta packet.CaptureMeta) { +func (s *Sink) LogPacket(path packet.CapturePath, when time.Time, data []byte, meta packet.CaptureMeta) { select { case <-s.ctx.Done(): return diff --git a/feature/capture/dissector/dissector.go b/feature/capture/dissector/dissector.go new file mode 100644 index 0000000000000..ab2f6c2ec1607 --- /dev/null +++ b/feature/capture/dissector/dissector.go @@ -0,0 +1,12 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package dissector contains the Lua dissector for Tailscale packets. +package dissector + +import ( + _ "embed" +) + +//go:embed ts-dissector.lua +var Lua string diff --git a/wgengine/capture/ts-dissector.lua b/feature/capture/dissector/ts-dissector.lua similarity index 100% rename from wgengine/capture/ts-dissector.lua rename to feature/capture/dissector/ts-dissector.lua diff --git a/feature/condregister/maybe_capture.go b/feature/condregister/maybe_capture.go new file mode 100644 index 0000000000000..0c68331f101cd --- /dev/null +++ b/feature/condregister/maybe_capture.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ios && !ts_omit_capture + +package condregister + +import _ "tailscale.com/feature/capture" diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 33ce9f331d94b..58cd4025f44a9 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -73,6 +73,7 @@ import ( "tailscale.com/net/netmon" "tailscale.com/net/netns" "tailscale.com/net/netutil" + "tailscale.com/net/packet" "tailscale.com/net/tsaddr" "tailscale.com/net/tsdial" "tailscale.com/paths" @@ -115,7 +116,6 @@ import ( "tailscale.com/version" "tailscale.com/version/distro" "tailscale.com/wgengine" - "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/magicsock" "tailscale.com/wgengine/router" @@ -209,7 +209,7 @@ type LocalBackend struct { // Tailscale on port 5252. exposeRemoteWebClientAtomicBool atomic.Bool shutdownCalled bool // if Shutdown has been called - debugSink *capture.Sink + debugSink packet.CaptureSink sockstatLogger *sockstatlog.Logger // getTCPHandlerForFunnelFlow returns a handler for an incoming TCP flow for @@ -948,6 +948,40 @@ func (b *LocalBackend) onHealthChange(w *health.Warnable, us *health.UnhealthySt } } +// GetOrSetCaptureSink returns the current packet capture sink, creating it +// with the provided newSink function if it does not already exist. +func (b *LocalBackend) GetOrSetCaptureSink(newSink func() packet.CaptureSink) packet.CaptureSink { + b.mu.Lock() + defer b.mu.Unlock() + + if b.debugSink != nil { + return b.debugSink + } + s := newSink() + b.debugSink = s + b.e.InstallCaptureHook(s.CaptureCallback()) + return s +} + +func (b *LocalBackend) ClearCaptureSink() { + // Shut down & uninstall the sink if there are no longer + // any outputs on it. + b.mu.Lock() + defer b.mu.Unlock() + + select { + case <-b.ctx.Done(): + return + default: + } + if b.debugSink != nil && b.debugSink.NumOutputs() == 0 { + s := b.debugSink + b.e.InstallCaptureHook(nil) + b.debugSink = nil + s.Close() + } +} + // Shutdown halts the backend and all its sub-components. The backend // can no longer be used after Shutdown returns. func (b *LocalBackend) Shutdown() { @@ -7154,48 +7188,6 @@ func (b *LocalBackend) ResetAuth() error { return b.resetForProfileChangeLockedOnEntry(unlock) } -// StreamDebugCapture writes a pcap stream of packets traversing -// tailscaled to the provided response writer. -func (b *LocalBackend) StreamDebugCapture(ctx context.Context, w io.Writer) error { - var s *capture.Sink - - b.mu.Lock() - if b.debugSink == nil { - s = capture.New() - b.debugSink = s - b.e.InstallCaptureHook(s.LogPacket) - } else { - s = b.debugSink - } - b.mu.Unlock() - - unregister := s.RegisterOutput(w) - - select { - case <-ctx.Done(): - case <-s.WaitCh(): - } - unregister() - - // Shut down & uninstall the sink if there are no longer - // any outputs on it. - b.mu.Lock() - defer b.mu.Unlock() - - select { - case <-b.ctx.Done(): - return nil - default: - } - if b.debugSink != nil && b.debugSink.NumOutputs() == 0 { - s := b.debugSink - b.e.InstallCaptureHook(nil) - b.debugSink = nil - return s.Close() - } - return nil -} - func (b *LocalBackend) GetPeerEndpointChanges(ctx context.Context, ip netip.Addr) ([]magicsock.EndpointChange, error) { pip, ok := b.e.PeerForIP(ip) if !ok { diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 157f72a65be03..e6b537d8ff122 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -68,12 +68,12 @@ import ( "tailscale.com/wgengine/magicsock" ) -type localAPIHandler func(*Handler, http.ResponseWriter, *http.Request) +type LocalAPIHandler func(*Handler, http.ResponseWriter, *http.Request) // handler is the set of LocalAPI handlers, keyed by the part of the // Request.URL.Path after "/localapi/v0/". If the key ends with a trailing slash // then it's a prefix match. -var handler = map[string]localAPIHandler{ +var handler = map[string]LocalAPIHandler{ // The prefix match handlers end with a slash: "cert/": (*Handler).serveCert, "file-put/": (*Handler).serveFilePut, @@ -90,7 +90,6 @@ var handler = map[string]localAPIHandler{ "check-udp-gro-forwarding": (*Handler).serveCheckUDPGROForwarding, "component-debug-logging": (*Handler).serveComponentDebugLogging, "debug": (*Handler).serveDebug, - "debug-capture": (*Handler).serveDebugCapture, "debug-derp-region": (*Handler).serveDebugDERPRegion, "debug-dial-types": (*Handler).serveDebugDialTypes, "debug-log": (*Handler).serveDebugLog, @@ -152,6 +151,14 @@ var handler = map[string]localAPIHandler{ "whois": (*Handler).serveWhoIs, } +// Register registers a new LocalAPI handler for the given name. +func Register(name string, fn LocalAPIHandler) { + if _, ok := handler[name]; ok { + panic("duplicate LocalAPI handler registration: " + name) + } + handler[name] = fn +} + var ( // The clientmetrics package is stateful, but we want to expose a simple // imperative API to local clients, so we need to keep track of @@ -196,6 +203,10 @@ type Handler struct { clock tstime.Clock } +func (h *Handler) LocalBackend() *ipnlocal.LocalBackend { + return h.b +} + func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if h.b == nil { http.Error(w, "server has no local backend", http.StatusInternalServerError) @@ -260,7 +271,7 @@ func (h *Handler) validHost(hostname string) bool { // handlerForPath returns the LocalAPI handler for the provided Request.URI.Path. // (the path doesn't include any query parameters) -func handlerForPath(urlPath string) (h localAPIHandler, ok bool) { +func handlerForPath(urlPath string) (h LocalAPIHandler, ok bool) { if urlPath == "/" { return (*Handler).serveLocalAPIRoot, true } @@ -2689,21 +2700,6 @@ func defBool(a string, def bool) bool { return v } -func (h *Handler) serveDebugCapture(w http.ResponseWriter, r *http.Request) { - if !h.PermitWrite { - http.Error(w, "debug access denied", http.StatusForbidden) - return - } - if r.Method != "POST" { - http.Error(w, "POST required", http.StatusMethodNotAllowed) - return - } - - w.WriteHeader(http.StatusOK) - w.(http.Flusher).Flush() - h.b.StreamDebugCapture(r.Context(), w) -} - func (h *Handler) serveDebugLog(w http.ResponseWriter, r *http.Request) { if !h.PermitRead { http.Error(w, "debug-log access denied", http.StatusForbidden) diff --git a/net/packet/capture.go b/net/packet/capture.go new file mode 100644 index 0000000000000..dd0ca411f2051 --- /dev/null +++ b/net/packet/capture.go @@ -0,0 +1,75 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package packet + +import ( + "io" + "net/netip" + "time" +) + +// Callback describes a function which is called to +// record packets when debugging packet-capture. +// Such callbacks must not take ownership of the +// provided data slice: it may only copy out of it +// within the lifetime of the function. +type CaptureCallback func(CapturePath, time.Time, []byte, CaptureMeta) + +// CaptureSink is the minimal interface from [tailscale.com/feature/capture]'s +// Sink type that is needed by the core (magicsock/LocalBackend/wgengine/etc). +// This lets the relativel heavy feature/capture package be optionally linked. +type CaptureSink interface { + // Close closes + Close() error + + // NumOutputs returns the number of outputs registered with the sink. + NumOutputs() int + + // CaptureCallback returns a callback which can be used to + // write packets to the sink. + CaptureCallback() CaptureCallback + + // WaitCh returns a channel which blocks until + // the sink is closed. + WaitCh() <-chan struct{} + + // RegisterOutput connects an output to this sink, which + // will be written to with a pcap stream as packets are logged. + // A function is returned which unregisters the output when + // called. + // + // If w implements io.Closer, it will be closed upon error + // or when the sink is closed. If w implements http.Flusher, + // it will be flushed periodically. + RegisterOutput(w io.Writer) (unregister func()) +} + +// CaptureMeta contains metadata that is used when debugging. +type CaptureMeta struct { + DidSNAT bool // SNAT was performed & the address was updated. + OriginalSrc netip.AddrPort // The source address before SNAT was performed. + DidDNAT bool // DNAT was performed & the address was updated. + OriginalDst netip.AddrPort // The destination address before DNAT was performed. +} + +// CapturePath describes where in the data path the packet was captured. +type CapturePath uint8 + +// CapturePath values +const ( + // FromLocal indicates the packet was logged as it traversed the FromLocal path: + // i.e.: A packet from the local system into the TUN. + FromLocal CapturePath = 0 + // FromPeer indicates the packet was logged upon reception from a remote peer. + FromPeer CapturePath = 1 + // SynthesizedToLocal indicates the packet was generated from within tailscaled, + // and is being routed to the local machine's network stack. + SynthesizedToLocal CapturePath = 2 + // SynthesizedToPeer indicates the packet was generated from within tailscaled, + // and is being routed to a remote Wireguard peer. + SynthesizedToPeer CapturePath = 3 + + // PathDisco indicates the packet is information about a disco frame. + PathDisco CapturePath = 254 +) diff --git a/net/packet/packet.go b/net/packet/packet.go index c9521ad4667c2..b683b22126948 100644 --- a/net/packet/packet.go +++ b/net/packet/packet.go @@ -34,14 +34,6 @@ const ( TCPECNBits TCPFlag = TCPECNEcho | TCPCWR ) -// CaptureMeta contains metadata that is used when debugging. -type CaptureMeta struct { - DidSNAT bool // SNAT was performed & the address was updated. - OriginalSrc netip.AddrPort // The source address before SNAT was performed. - DidDNAT bool // DNAT was performed & the address was updated. - OriginalDst netip.AddrPort // The destination address before DNAT was performed. -} - // Parsed is a minimal decoding of a packet suitable for use in filters. type Parsed struct { // b is the byte buffer that this decodes. diff --git a/net/tstun/wrap.go b/net/tstun/wrap.go index b26239632bf06..442184065aa92 100644 --- a/net/tstun/wrap.go +++ b/net/tstun/wrap.go @@ -36,7 +36,6 @@ import ( "tailscale.com/types/logger" "tailscale.com/util/clientmetric" "tailscale.com/util/usermetric" - "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/netstack/gro" "tailscale.com/wgengine/wgcfg" @@ -208,7 +207,7 @@ type Wrapper struct { // stats maintains per-connection counters. stats atomic.Pointer[connstats.Statistics] - captureHook syncs.AtomicValue[capture.Callback] + captureHook syncs.AtomicValue[packet.CaptureCallback] metrics *metrics } @@ -955,7 +954,7 @@ func (t *Wrapper) Read(buffs [][]byte, sizes []int, offset int) (int, error) { } } if captHook != nil { - captHook(capture.FromLocal, t.now(), p.Buffer(), p.CaptureMeta) + captHook(packet.FromLocal, t.now(), p.Buffer(), p.CaptureMeta) } if !t.disableFilter { var response filter.Response @@ -1101,9 +1100,9 @@ func (t *Wrapper) injectedRead(res tunInjectedRead, outBuffs [][]byte, sizes []i return n, err } -func (t *Wrapper) filterPacketInboundFromWireGuard(p *packet.Parsed, captHook capture.Callback, pc *peerConfigTable, gro *gro.GRO) (filter.Response, *gro.GRO) { +func (t *Wrapper) filterPacketInboundFromWireGuard(p *packet.Parsed, captHook packet.CaptureCallback, pc *peerConfigTable, gro *gro.GRO) (filter.Response, *gro.GRO) { if captHook != nil { - captHook(capture.FromPeer, t.now(), p.Buffer(), p.CaptureMeta) + captHook(packet.FromPeer, t.now(), p.Buffer(), p.CaptureMeta) } if p.IPProto == ipproto.TSMP { @@ -1317,7 +1316,7 @@ func (t *Wrapper) InjectInboundPacketBuffer(pkt *stack.PacketBuffer, buffs [][]b p.Decode(buf) captHook := t.captureHook.Load() if captHook != nil { - captHook(capture.SynthesizedToLocal, t.now(), p.Buffer(), p.CaptureMeta) + captHook(packet.SynthesizedToLocal, t.now(), p.Buffer(), p.CaptureMeta) } invertGSOChecksum(buf, pkt.GSOOptions) @@ -1449,7 +1448,7 @@ func (t *Wrapper) InjectOutboundPacketBuffer(pkt *stack.PacketBuffer) error { } if capt := t.captureHook.Load(); capt != nil { b := pkt.ToBuffer() - capt(capture.SynthesizedToPeer, t.now(), b.Flatten(), packet.CaptureMeta{}) + capt(packet.SynthesizedToPeer, t.now(), b.Flatten(), packet.CaptureMeta{}) } t.injectOutbound(tunInjectedRead{packet: pkt}) @@ -1491,6 +1490,6 @@ var ( metricPacketOutDropSelfDisco = clientmetric.NewCounter("tstun_out_to_wg_drop_self_disco") ) -func (t *Wrapper) InstallCaptureHook(cb capture.Callback) { +func (t *Wrapper) InstallCaptureHook(cb packet.CaptureCallback) { t.captureHook.Store(cb) } diff --git a/net/tstun/wrap_test.go b/net/tstun/wrap_test.go index a3dfe7d86c914..223ee34f4336a 100644 --- a/net/tstun/wrap_test.go +++ b/net/tstun/wrap_test.go @@ -40,7 +40,6 @@ import ( "tailscale.com/types/views" "tailscale.com/util/must" "tailscale.com/util/usermetric" - "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/wgcfg" ) @@ -871,14 +870,14 @@ func TestPeerCfg_NAT(t *testing.T) { // with the correct parameters when various packet operations are performed. func TestCaptureHook(t *testing.T) { type captureRecord struct { - path capture.Path + path packet.CapturePath now time.Time pkt []byte meta packet.CaptureMeta } var captured []captureRecord - hook := func(path capture.Path, now time.Time, pkt []byte, meta packet.CaptureMeta) { + hook := func(path packet.CapturePath, now time.Time, pkt []byte, meta packet.CaptureMeta) { captured = append(captured, captureRecord{ path: path, now: now, @@ -935,19 +934,19 @@ func TestCaptureHook(t *testing.T) { // Assert that the right packets are captured. want := []captureRecord{ { - path: capture.FromPeer, + path: packet.FromPeer, pkt: []byte("Write1"), }, { - path: capture.FromPeer, + path: packet.FromPeer, pkt: []byte("Write2"), }, { - path: capture.SynthesizedToLocal, + path: packet.SynthesizedToLocal, pkt: []byte("InjectInboundPacketBuffer"), }, { - path: capture.SynthesizedToPeer, + path: packet.SynthesizedToPeer, pkt: []byte("InjectOutboundPacketBuffer"), }, } diff --git a/tstest/iosdeps/iosdeps_test.go b/tstest/iosdeps/iosdeps_test.go index ab69f1c2b0649..b533724eb4b3d 100644 --- a/tstest/iosdeps/iosdeps_test.go +++ b/tstest/iosdeps/iosdeps_test.go @@ -24,6 +24,7 @@ func TestDeps(t *testing.T) { "github.com/google/uuid": "see tailscale/tailscale#13760", "tailscale.com/clientupdate/distsign": "downloads via AppStore, not distsign", "github.com/tailscale/hujson": "no config file support on iOS", + "tailscale.com/feature/capture": "no debug packet capture on iOS", }, }.Check(t) } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 98cb63b888b59..acf7114e14f74 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -61,7 +61,6 @@ import ( "tailscale.com/util/set" "tailscale.com/util/testenv" "tailscale.com/util/usermetric" - "tailscale.com/wgengine/capture" "tailscale.com/wgengine/wgint" ) @@ -238,7 +237,7 @@ type Conn struct { stats atomic.Pointer[connstats.Statistics] // captureHook, if non-nil, is the pcap logging callback when capturing. - captureHook syncs.AtomicValue[capture.Callback] + captureHook syncs.AtomicValue[packet.CaptureCallback] // discoPrivate is the private naclbox key used for active // discovery traffic. It is always present, and immutable. @@ -655,7 +654,7 @@ func deregisterMetrics(m *metrics) { // log debug information into the pcap stream. This function // can be called with a nil argument to uninstall the capture // hook. -func (c *Conn) InstallCaptureHook(cb capture.Callback) { +func (c *Conn) InstallCaptureHook(cb packet.CaptureCallback) { c.captureHook.Store(cb) } @@ -1709,7 +1708,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke // Emit information about the disco frame into the pcap stream // if a capture hook is installed. if cb := c.captureHook.Load(); cb != nil { - cb(capture.PathDisco, time.Now(), disco.ToPCAPFrame(src, derpNodeSrc, payload), packet.CaptureMeta{}) + cb(packet.PathDisco, time.Now(), disco.ToPCAPFrame(src, derpNodeSrc, payload), packet.CaptureMeta{}) } dm, err := disco.Parse(payload) diff --git a/wgengine/userspace.go b/wgengine/userspace.go index 81f8000e0d557..b51b2c8ead7ab 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -51,7 +51,6 @@ import ( "tailscale.com/util/testenv" "tailscale.com/util/usermetric" "tailscale.com/version" - "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/magicsock" "tailscale.com/wgengine/netlog" @@ -1594,7 +1593,7 @@ var ( metricNumMinorChanges = clientmetric.NewCounter("wgengine_minor_changes") ) -func (e *userspaceEngine) InstallCaptureHook(cb capture.Callback) { +func (e *userspaceEngine) InstallCaptureHook(cb packet.CaptureCallback) { e.tundev.InstallCaptureHook(cb) e.magicConn.InstallCaptureHook(cb) } diff --git a/wgengine/watchdog.go b/wgengine/watchdog.go index 232591f5eca60..74a1917488dd8 100644 --- a/wgengine/watchdog.go +++ b/wgengine/watchdog.go @@ -17,10 +17,10 @@ import ( "tailscale.com/envknob" "tailscale.com/ipn/ipnstate" "tailscale.com/net/dns" + "tailscale.com/net/packet" "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/types/netmap" - "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/router" "tailscale.com/wgengine/wgcfg" @@ -162,7 +162,7 @@ func (e *watchdogEngine) Done() <-chan struct{} { return e.wrap.Done() } -func (e *watchdogEngine) InstallCaptureHook(cb capture.Callback) { +func (e *watchdogEngine) InstallCaptureHook(cb packet.CaptureCallback) { e.wrap.InstallCaptureHook(cb) } diff --git a/wgengine/wgengine.go b/wgengine/wgengine.go index c165ccdf3c3aa..6aaf567ad01ee 100644 --- a/wgengine/wgengine.go +++ b/wgengine/wgengine.go @@ -11,10 +11,10 @@ import ( "tailscale.com/ipn/ipnstate" "tailscale.com/net/dns" + "tailscale.com/net/packet" "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/types/netmap" - "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/router" "tailscale.com/wgengine/wgcfg" @@ -129,5 +129,5 @@ type Engine interface { // InstallCaptureHook registers a function to be called to capture // packets traversing the data path. The hook can be uninstalled by // calling this function with a nil value. - InstallCaptureHook(capture.Callback) + InstallCaptureHook(packet.CaptureCallback) } From 66b2e9fd07f2c635b809aa82d657fd82de3f9323 Mon Sep 17 00:00:00 2001 From: Derek Kaser <11674153+dkaser@users.noreply.github.com> Date: Sun, 26 Jan 2025 10:35:58 -0500 Subject: [PATCH 195/223] envknob/featureknob: allow use of exit node on unraid (#14754) Fixes #14372 Signed-off-by: Derek Kaser <11674153+dkaser@users.noreply.github.com> --- envknob/featureknob/featureknob.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/envknob/featureknob/featureknob.go b/envknob/featureknob/featureknob.go index d7af80d239782..210414bfe6624 100644 --- a/envknob/featureknob/featureknob.go +++ b/envknob/featureknob/featureknob.go @@ -55,8 +55,7 @@ func CanRunTailscaleSSH() error { func CanUseExitNode() error { switch dist := distro.Get(); dist { case distro.Synology, // see https://github.com/tailscale/tailscale/issues/1995 - distro.QNAP, - distro.Unraid: + distro.QNAP: return errors.New("Tailscale exit nodes cannot be used on " + string(dist)) } From e701fde6b389a4a69b4d33aace8969530b25de8d Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 26 Jan 2025 18:23:38 +0000 Subject: [PATCH 196/223] control/controlknobs: make Knobs.AsDebugJSON automatic, not require maintenance The AsDebugJSON method (used only for a LocalAPI debug call) always needed to be updated whenever a new controlknob was added. We had a test for it, which was nice, but it was a tedious step we don't need to do. Use reflect instead. Updates #14788 Change-Id: If59cd776920f3ce7c748f86ed2eddd9323039a0b Signed-off-by: Brad Fitzpatrick --- control/controlknobs/controlknobs.go | 37 ++++++++++------------- control/controlknobs/controlknobs_test.go | 3 ++ 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/control/controlknobs/controlknobs.go b/control/controlknobs/controlknobs.go index c7933be5a148d..a86f0af53a829 100644 --- a/control/controlknobs/controlknobs.go +++ b/control/controlknobs/controlknobs.go @@ -6,6 +6,8 @@ package controlknobs import ( + "fmt" + "reflect" "sync/atomic" "tailscale.com/syncs" @@ -174,26 +176,19 @@ func (k *Knobs) AsDebugJSON() map[string]any { if k == nil { return nil } - return map[string]any{ - "DisableUPnP": k.DisableUPnP.Load(), - "KeepFullWGConfig": k.KeepFullWGConfig.Load(), - "RandomizeClientPort": k.RandomizeClientPort.Load(), - "OneCGNAT": k.OneCGNAT.Load(), - "ForceBackgroundSTUN": k.ForceBackgroundSTUN.Load(), - "DisableDeltaUpdates": k.DisableDeltaUpdates.Load(), - "PeerMTUEnable": k.PeerMTUEnable.Load(), - "DisableDNSForwarderTCPRetries": k.DisableDNSForwarderTCPRetries.Load(), - "SilentDisco": k.SilentDisco.Load(), - "LinuxForceIPTables": k.LinuxForceIPTables.Load(), - "LinuxForceNfTables": k.LinuxForceNfTables.Load(), - "SeamlessKeyRenewal": k.SeamlessKeyRenewal.Load(), - "ProbeUDPLifetime": k.ProbeUDPLifetime.Load(), - "AppCStoreRoutes": k.AppCStoreRoutes.Load(), - "UserDialUseRoutes": k.UserDialUseRoutes.Load(), - "DisableSplitDNSWhenNoCustomResolvers": k.DisableSplitDNSWhenNoCustomResolvers.Load(), - "DisableLocalDNSOverrideViaNRPT": k.DisableLocalDNSOverrideViaNRPT.Load(), - "DisableCryptorouting": k.DisableCryptorouting.Load(), - "DisableCaptivePortalDetection": k.DisableCaptivePortalDetection.Load(), - "DisableSkipStatusQueue": k.DisableSkipStatusQueue.Load(), + ret := map[string]any{} + rt := reflect.TypeFor[Knobs]() + rv := reflect.ValueOf(k).Elem() // of *k + for i := 0; i < rt.NumField(); i++ { + name := rt.Field(i).Name + switch v := rv.Field(i).Addr().Interface().(type) { + case *atomic.Bool: + ret[name] = v.Load() + case *syncs.AtomicValue[opt.Bool]: + ret[name] = v.Load() + default: + panic(fmt.Sprintf("unknown field type %T for %v", v, name)) + } } + return ret } diff --git a/control/controlknobs/controlknobs_test.go b/control/controlknobs/controlknobs_test.go index a78a486f3aaae..7618b7121c500 100644 --- a/control/controlknobs/controlknobs_test.go +++ b/control/controlknobs/controlknobs_test.go @@ -6,6 +6,8 @@ package controlknobs import ( "reflect" "testing" + + "tailscale.com/types/logger" ) func TestAsDebugJSON(t *testing.T) { @@ -18,4 +20,5 @@ func TestAsDebugJSON(t *testing.T) { if want := reflect.TypeFor[Knobs]().NumField(); len(got) != want { t.Errorf("AsDebugJSON map has %d fields; want %v", len(got), want) } + t.Logf("Got: %v", logger.AsJSON(got)) } From 04029b857f5ef8699e5cd2c80f57048b34b32825 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 26 Jan 2025 18:12:46 +0000 Subject: [PATCH 197/223] tstest/deptest: verify that tailscale.com BadDeps actually exist This protects against rearranging packages and not catching that a BadDeps package got moved. That would then effectively remove a test. Updates #12614 Change-Id: I257f1eeda9e3569c867b7628d5bfb252d3354ba6 Signed-off-by: Brad Fitzpatrick --- tstest/deptest/deptest.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tstest/deptest/deptest.go b/tstest/deptest/deptest.go index 00faa8a386db8..2393733e60dfd 100644 --- a/tstest/deptest/deptest.go +++ b/tstest/deptest/deptest.go @@ -15,6 +15,7 @@ import ( "runtime" "slices" "strings" + "sync" "testing" "tailscale.com/util/set" @@ -54,11 +55,35 @@ func (c DepChecker) Check(t *testing.T) { t.Fatal(err) } + tsRoot := sync.OnceValue(func() string { + out, err := exec.Command("go", "list", "-f", "{{.Dir}}", "tailscale.com").Output() + if err != nil { + t.Fatalf("failed to find tailscale.com root: %v", err) + } + return strings.TrimSpace(string(out)) + }) + for _, dep := range res.Deps { if why, ok := c.BadDeps[dep]; ok { t.Errorf("package %q is not allowed as a dependency (env: %q); reason: %s", dep, extraEnv, why) } } + // Make sure the BadDeps packages actually exists. If they got renamed or + // moved around, we should update the test referencing the old name. + // Doing this in the general case requires network access at runtime + // (resolving a package path to its module, possibly doing the ?go-get=1 + // meta tag dance), so we just check the common case of + // "tailscale.com/*" packages for now, with the assumption that all + // "tailscale.com/*" packages are in the same module, which isn't + // necessarily true in the general case. + for dep := range c.BadDeps { + if suf, ok := strings.CutPrefix(dep, "tailscale.com/"); ok { + pkgDir := filepath.Join(tsRoot(), suf) + if _, err := os.Stat(pkgDir); err != nil { + t.Errorf("listed BadDep %q doesn't seem to exist anymore: %v", dep, err) + } + } + } for dep := range c.WantDeps { if !slices.Contains(res.Deps, dep) { t.Errorf("expected package %q to be a dependency (env: %q)", dep, extraEnv) From 8c925899e115ce18e47ba4ec4c630696140e63df Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 26 Jan 2025 17:09:44 +0000 Subject: [PATCH 198/223] go.mod: bump depaware, add --internal flag to stop hiding internal packages The hiding of internal packages has hidden things I wanted to see a few times now. Stop hiding them. This makes depaware.txt output a bit longer, but not too much. Plus we only really look at it with diffs & greps anyway; it's not like anybody reads the whole thing. Updates #12614 Change-Id: I868c89eeeddcaaab63e82371651003629bc9bda8 Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 2 +- Makefile | 4 +-- cmd/derper/depaware.txt | 59 +++++++++++++++++++++++++++++++ cmd/k8s-operator/depaware.txt | 66 +++++++++++++++++++++++++++++++++++ cmd/stund/depaware.txt | 58 ++++++++++++++++++++++++++++++ cmd/tailscale/depaware.txt | 61 ++++++++++++++++++++++++++++++++ cmd/tailscaled/depaware.txt | 64 +++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +-- 9 files changed, 314 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d049323a3513d..cc773e4a94d3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -481,7 +481,7 @@ jobs: - name: check depaware run: | export PATH=$(./tool/go env GOROOT)/bin:$PATH - find . -name 'depaware.txt' | xargs -n1 dirname | xargs ./tool/go run github.com/tailscale/depaware --check + find . -name 'depaware.txt' | xargs -n1 dirname | xargs ./tool/go run github.com/tailscale/depaware --check --internal go_generate: runs-on: ubuntu-22.04 diff --git a/Makefile b/Makefile index d3e50af0571b7..30ac5327a023e 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ lint: ## Run golangci-lint updatedeps: ## Update depaware deps # depaware (via x/tools/go/packages) shells back to "go", so make sure the "go" # it finds in its $$PATH is the right one. - PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update \ + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --internal \ tailscale.com/cmd/tailscaled \ tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ @@ -27,7 +27,7 @@ updatedeps: ## Update depaware deps depaware: ## Run depaware checks # depaware (via x/tools/go/packages) shells back to "go", so make sure the "go" # it finds in its $$PATH is the right one. - PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check \ + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --internal \ tailscale.com/cmd/tailscaled \ tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 3a730dd997437..5a39c110e105e 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -189,6 +189,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ golang.org/x/crypto/curve25519 from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/hkdf from crypto/tls+ + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/nacl/box from tailscale.com/types/key golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ @@ -201,6 +203,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa golang.org/x/net/http/httpproxy from net/http+ golang.org/x/net/http2/hpack from net/http golang.org/x/net/idna from golang.org/x/crypto/acme/autocert+ + golang.org/x/net/internal/socks from golang.org/x/net/proxy golang.org/x/net/proxy from tailscale.com/net/netns D golang.org/x/net/route from net+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ @@ -232,6 +235,18 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ crypto/hmac from crypto/tls+ + crypto/internal/alias from crypto/aes+ + crypto/internal/bigmod from crypto/ecdsa+ + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/edwards25519 from crypto/ed25519 + crypto/internal/edwards25519/field from crypto/ecdh+ + crypto/internal/hpke from crypto/tls + crypto/internal/mlkem768 from crypto/tls + crypto/internal/nistec from crypto/ecdh+ + crypto/internal/nistec/fiat from crypto/internal/nistec + crypto/internal/randutil from crypto/dsa+ crypto/md5 from crypto/tls+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls @@ -242,6 +257,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa crypto/subtle from crypto/aes+ crypto/tls from golang.org/x/crypto/acme+ crypto/x509 from crypto/tls+ + D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509+ embed from crypto/internal/nistec+ encoding from encoding/json+ @@ -263,6 +279,44 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa hash/maphash from go4.org/mem html from net/http/pprof+ html/template from tailscale.com/cmd/derper + internal/abi from crypto/x509/internal/macos+ + internal/asan from syscall + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/aes+ + internal/chacha8rand from math/rand/v2+ + internal/concurrent from unique + internal/coverage/rtcov from runtime + internal/cpu from crypto/aes+ + internal/filepathlite from os+ + internal/fmtsort from fmt+ + internal/goarch from crypto/aes+ + internal/godebug from crypto/tls+ + internal/godebugs from internal/godebug+ + internal/goexperiment from runtime + internal/goos from crypto/x509+ + internal/itoa from internal/poll+ + internal/msan from syscall + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profile from net/http/pprof + internal/profilerecord from runtime+ + internal/race from internal/poll+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/exithook from runtime + L internal/runtime/syscall from runtime+ + internal/singleflight from net + internal/stringslite from embed+ + internal/syscall/execenv from os+ + LD internal/syscall/unix from crypto/rand+ + W internal/syscall/windows from crypto/rand+ + W internal/syscall/windows/registry from mime+ + W internal/syscall/windows/sysdll from internal/syscall/windows+ + internal/testlog from os + internal/unsafeheader from internal/reflectlite+ + internal/weak from unique io from bufio+ io/fs from crypto/x509+ L io/ioutil from github.com/mitchellh/go-ps+ @@ -282,6 +336,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa net/http from expvar+ net/http/httptrace from net/http+ net/http/internal from net/http + net/http/internal/ascii from net/http net/http/pprof from tailscale.com/tsweb net/netip from go4.org/netipx+ net/textproto from golang.org/x/net/http/httpguts+ @@ -295,7 +350,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa reflect from crypto/x509+ regexp from github.com/coreos/go-iptables/iptables+ regexp/syntax from regexp + runtime from crypto/internal/nistec+ runtime/debug from github.com/prometheus/client_golang/prometheus+ + runtime/internal/math from runtime + runtime/internal/sys from runtime runtime/metrics from github.com/prometheus/client_golang/prometheus+ runtime/pprof from net/http/pprof runtime/trace from net/http/pprof @@ -314,3 +372,4 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa unicode/utf16 from crypto/x509+ unicode/utf8 from bufio+ unique from net/netip + unsafe from bytes+ diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index fc2f8854ae3d2..32af3b25e3ad0 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -992,6 +992,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ golang.org/x/crypto/curve25519 from github.com/tailscale/golang-x-crypto/ssh+ golang.org/x/crypto/hkdf from crypto/tls+ + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/nacl/box from tailscale.com/types/key golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device @@ -1009,6 +1011,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/net/http2/hpack from golang.org/x/net/http2+ golang.org/x/net/icmp from github.com/prometheus-community/pro-bing+ golang.org/x/net/idna from golang.org/x/net/http/httpguts+ + golang.org/x/net/internal/iana from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socks from golang.org/x/net/proxy golang.org/x/net/ipv4 from github.com/miekg/dns+ golang.org/x/net/ipv6 from github.com/miekg/dns+ golang.org/x/net/proxy from tailscale.com/net/netns @@ -1050,6 +1055,18 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ crypto/hmac from crypto/tls+ + crypto/internal/alias from crypto/aes+ + crypto/internal/bigmod from crypto/ecdsa+ + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/edwards25519 from crypto/ed25519 + crypto/internal/edwards25519/field from crypto/ecdh+ + crypto/internal/hpke from crypto/tls + crypto/internal/mlkem768 from crypto/tls + crypto/internal/nistec from crypto/ecdh+ + crypto/internal/nistec/fiat from crypto/internal/nistec + crypto/internal/randutil from crypto/dsa+ crypto/md5 from crypto/tls+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ @@ -1060,6 +1077,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/subtle from crypto/aes+ crypto/tls from github.com/aws/aws-sdk-go-v2/aws/transport/http+ crypto/x509 from crypto/tls+ + D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509+ database/sql from github.com/prometheus/client_golang/prometheus/collectors database/sql/driver from database/sql+ @@ -1085,6 +1103,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ go/build/constraint from go/parser go/doc from k8s.io/apimachinery/pkg/runtime go/doc/comment from go/doc + go/internal/typeparams from go/parser go/parser from k8s.io/apimachinery/pkg/runtime go/scanner from go/ast+ go/token from go/ast+ @@ -1095,6 +1114,46 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ hash/maphash from go4.org/mem html from html/template+ html/template from github.com/gorilla/csrf + internal/abi from crypto/x509/internal/macos+ + internal/asan from syscall + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/aes+ + internal/chacha8rand from math/rand/v2+ + internal/concurrent from unique + internal/coverage/rtcov from runtime + internal/cpu from crypto/aes+ + internal/filepathlite from os+ + internal/fmtsort from fmt+ + internal/goarch from crypto/aes+ + internal/godebug from archive/tar+ + internal/godebugs from internal/godebug+ + internal/goexperiment from runtime + internal/goos from crypto/x509+ + internal/itoa from internal/poll+ + internal/lazyregexp from go/doc + internal/msan from syscall + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profile from net/http/pprof + internal/profilerecord from runtime+ + internal/race from internal/poll+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/exithook from runtime + L internal/runtime/syscall from runtime+ + internal/saferio from debug/pe+ + internal/singleflight from net + internal/stringslite from embed+ + internal/syscall/execenv from os+ + LD internal/syscall/unix from crypto/rand+ + W internal/syscall/windows from crypto/rand+ + W internal/syscall/windows/registry from mime+ + W internal/syscall/windows/sysdll from internal/syscall/windows+ + internal/testlog from os + internal/unsafeheader from internal/reflectlite+ + internal/weak from unique io from archive/tar+ io/fs from archive/tar+ io/ioutil from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ @@ -1103,6 +1162,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ log/internal from log+ log/slog from github.com/go-logr/logr+ log/slog/internal from log/slog + log/slog/internal/buffer from log/slog maps from sigs.k8s.io/controller-runtime/pkg/predicate+ math from archive/tar+ math/big from crypto/dsa+ @@ -1118,6 +1178,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ net/http/httptrace from github.com/prometheus-community/pro-bing+ net/http/httputil from github.com/aws/smithy-go/transport/http+ net/http/internal from net/http+ + net/http/internal/ascii from net/http+ + net/http/internal/testcert from net/http/httptest net/http/pprof from sigs.k8s.io/controller-runtime/pkg/manager+ net/netip from github.com/gaissmai/bart+ net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ @@ -1131,7 +1193,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ reflect from archive/tar+ regexp from github.com/aws/aws-sdk-go-v2/internal/endpoints+ regexp/syntax from regexp + runtime from archive/tar+ runtime/debug from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+ + runtime/internal/math from runtime + runtime/internal/sys from runtime runtime/metrics from github.com/prometheus/client_golang/prometheus+ runtime/pprof from net/http/pprof+ runtime/trace from net/http/pprof @@ -1150,3 +1215,4 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ unicode/utf16 from crypto/x509+ unicode/utf8 from bufio+ unique from net/netip + unsafe from bytes+ diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index 52d649a1d6c39..c553b9be58811 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -89,6 +89,8 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ golang.org/x/crypto/curve25519 from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/hkdf from crypto/tls+ + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/nacl/box from tailscale.com/types/key golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ @@ -123,6 +125,18 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ crypto/hmac from crypto/tls+ + crypto/internal/alias from crypto/aes+ + crypto/internal/bigmod from crypto/ecdsa+ + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/edwards25519 from crypto/ed25519 + crypto/internal/edwards25519/field from crypto/ecdh+ + crypto/internal/hpke from crypto/tls + crypto/internal/mlkem768 from crypto/tls + crypto/internal/nistec from crypto/ecdh+ + crypto/internal/nistec/fiat from crypto/internal/nistec + crypto/internal/randutil from crypto/dsa+ crypto/md5 from crypto/tls+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls @@ -133,6 +147,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar crypto/subtle from crypto/aes+ crypto/tls from net/http+ crypto/x509 from crypto/tls + D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509 embed from crypto/internal/nistec+ encoding from encoding/json+ @@ -153,6 +168,44 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar hash/fnv from google.golang.org/protobuf/internal/detrand hash/maphash from go4.org/mem html from net/http/pprof+ + internal/abi from crypto/x509/internal/macos+ + internal/asan from syscall + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/aes+ + internal/chacha8rand from math/rand/v2+ + internal/concurrent from unique + internal/coverage/rtcov from runtime + internal/cpu from crypto/aes+ + internal/filepathlite from os+ + internal/fmtsort from fmt + internal/goarch from crypto/aes+ + internal/godebug from crypto/tls+ + internal/godebugs from internal/godebug+ + internal/goexperiment from runtime + internal/goos from crypto/x509+ + internal/itoa from internal/poll+ + internal/msan from syscall + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profile from net/http/pprof + internal/profilerecord from runtime+ + internal/race from internal/poll+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/exithook from runtime + L internal/runtime/syscall from runtime+ + internal/singleflight from net + internal/stringslite from embed+ + internal/syscall/execenv from os + LD internal/syscall/unix from crypto/rand+ + W internal/syscall/windows from crypto/rand+ + W internal/syscall/windows/registry from mime+ + W internal/syscall/windows/sysdll from internal/syscall/windows+ + internal/testlog from os + internal/unsafeheader from internal/reflectlite+ + internal/weak from unique io from bufio+ io/fs from crypto/x509+ iter from maps+ @@ -171,6 +224,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar net/http from expvar+ net/http/httptrace from net/http net/http/internal from net/http + net/http/internal/ascii from net/http net/http/pprof from tailscale.com/tsweb net/netip from go4.org/netipx+ net/textproto from golang.org/x/net/http/httpguts+ @@ -182,7 +236,10 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar reflect from crypto/x509+ regexp from github.com/prometheus/client_golang/prometheus/internal+ regexp/syntax from regexp + runtime from crypto/internal/nistec+ runtime/debug from github.com/prometheus/client_golang/prometheus+ + runtime/internal/math from runtime + runtime/internal/sys from runtime runtime/metrics from github.com/prometheus/client_golang/prometheus+ runtime/pprof from net/http/pprof runtime/trace from net/http/pprof @@ -199,3 +256,4 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar unicode/utf16 from crypto/x509+ unicode/utf8 from bufio+ unique from net/netip + unsafe from bytes+ diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 47ba03cb97b74..6d1fcfd03a7ac 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -194,6 +194,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ golang.org/x/crypto/curve25519 from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/hkdf from crypto/tls+ + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/nacl/box from tailscale.com/types/key golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/pbkdf2 from software.sslmate.com/src/go-pkcs12 @@ -209,6 +211,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep golang.org/x/net/http2/hpack from net/http+ golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpguts+ + golang.org/x/net/internal/iana from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socks from golang.org/x/net/proxy golang.org/x/net/ipv4 from github.com/miekg/dns+ golang.org/x/net/ipv6 from github.com/miekg/dns+ golang.org/x/net/proxy from tailscale.com/net/netns @@ -247,6 +252,18 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ crypto/hmac from crypto/tls+ + crypto/internal/alias from crypto/aes+ + crypto/internal/bigmod from crypto/ecdsa+ + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/edwards25519 from crypto/ed25519 + crypto/internal/edwards25519/field from crypto/ecdh+ + crypto/internal/hpke from crypto/tls + crypto/internal/mlkem768 from crypto/tls + crypto/internal/nistec from crypto/ecdh+ + crypto/internal/nistec/fiat from crypto/internal/nistec + crypto/internal/randutil from crypto/dsa+ crypto/md5 from crypto/tls+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls @@ -257,6 +274,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/subtle from crypto/aes+ crypto/tls from github.com/miekg/dns+ crypto/x509 from crypto/tls+ + D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509+ DW database/sql/driver from github.com/google/uuid W debug/dwarf from debug/pe @@ -285,6 +303,44 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep image from github.com/skip2/go-qrcode+ image/color from github.com/skip2/go-qrcode+ image/png from github.com/skip2/go-qrcode + internal/abi from crypto/x509/internal/macos+ + internal/asan from syscall + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/aes+ + internal/chacha8rand from math/rand/v2+ + internal/concurrent from unique + internal/coverage/rtcov from runtime + internal/cpu from crypto/aes+ + internal/filepathlite from os+ + internal/fmtsort from fmt+ + internal/goarch from crypto/aes+ + internal/godebug from archive/tar+ + internal/godebugs from internal/godebug+ + internal/goexperiment from runtime + internal/goos from crypto/x509+ + internal/itoa from internal/poll+ + internal/msan from syscall + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profilerecord from runtime + internal/race from internal/poll+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/exithook from runtime + L internal/runtime/syscall from runtime+ + internal/saferio from debug/pe+ + internal/singleflight from net + internal/stringslite from embed+ + internal/syscall/execenv from os+ + LD internal/syscall/unix from crypto/rand+ + W internal/syscall/windows from crypto/rand+ + W internal/syscall/windows/registry from mime+ + W internal/syscall/windows/sysdll from internal/syscall/windows+ + internal/testlog from os + internal/unsafeheader from internal/reflectlite+ + internal/weak from unique io from archive/tar+ io/fs from archive/tar+ io/ioutil from github.com/mitchellh/go-ps+ @@ -306,6 +362,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep net/http/httptrace from golang.org/x/net/http2+ net/http/httputil from tailscale.com/client/web+ net/http/internal from net/http+ + net/http/internal/ascii from net/http+ net/netip from go4.org/netipx+ net/textproto from golang.org/x/net/http/httpguts+ net/url from crypto/x509+ @@ -318,7 +375,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep reflect from archive/tar+ regexp from github.com/coreos/go-iptables/iptables+ regexp/syntax from regexp + runtime from archive/tar+ runtime/debug from tailscale.com+ + runtime/internal/math from runtime + runtime/internal/sys from runtime slices from tailscale.com/client/web+ sort from compress/flate+ strconv from archive/tar+ @@ -334,3 +394,4 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep unicode/utf16 from crypto/x509+ unicode/utf8 from bufio+ unique from net/netip + unsafe from bytes+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 1e0b2061a3a21..e0ed51ebb1309 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -445,12 +445,15 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ golang.org/x/crypto/curve25519 from github.com/tailscale/golang-x-crypto/ssh+ golang.org/x/crypto/hkdf from crypto/tls+ + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/nacl/box from tailscale.com/types/key golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/sha3 from crypto/internal/mlkem768+ LD golang.org/x/crypto/ssh from github.com/pkg/sftp+ + LD golang.org/x/crypto/ssh/internal/bcrypt_pbkdf from golang.org/x/crypto/ssh golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ golang.org/x/exp/maps from tailscale.com/ipn/store/mem+ golang.org/x/net/bpf from github.com/mdlayher/genetlink+ @@ -462,6 +465,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/net/http2/hpack from golang.org/x/net/http2+ golang.org/x/net/icmp from tailscale.com/net/ping+ golang.org/x/net/idna from golang.org/x/net/http/httpguts+ + golang.org/x/net/internal/iana from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socks from golang.org/x/net/proxy golang.org/x/net/ipv4 from github.com/miekg/dns+ golang.org/x/net/ipv6 from github.com/miekg/dns+ golang.org/x/net/proxy from tailscale.com/net/netns @@ -501,6 +507,18 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ crypto/hmac from crypto/tls+ + crypto/internal/alias from crypto/aes+ + crypto/internal/bigmod from crypto/ecdsa+ + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/edwards25519 from crypto/ed25519 + crypto/internal/edwards25519/field from crypto/ecdh+ + crypto/internal/hpke from crypto/tls + crypto/internal/mlkem768 from crypto/tls + crypto/internal/nistec from crypto/ecdh+ + crypto/internal/nistec/fiat from crypto/internal/nistec + crypto/internal/randutil from crypto/dsa+ crypto/md5 from crypto/tls+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ @@ -511,6 +529,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/subtle from crypto/aes+ crypto/tls from github.com/aws/aws-sdk-go-v2/aws/transport/http+ crypto/x509 from crypto/tls+ + D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509+ DW database/sql/driver from github.com/google/uuid W debug/dwarf from debug/pe @@ -536,6 +555,45 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de hash/maphash from go4.org/mem html from html/template+ html/template from github.com/gorilla/csrf + internal/abi from crypto/x509/internal/macos+ + internal/asan from syscall + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/aes+ + internal/chacha8rand from math/rand/v2+ + internal/concurrent from unique + internal/coverage/rtcov from runtime + internal/cpu from crypto/aes+ + internal/filepathlite from os+ + internal/fmtsort from fmt+ + internal/goarch from crypto/aes+ + internal/godebug from archive/tar+ + internal/godebugs from internal/godebug+ + internal/goexperiment from runtime + internal/goos from crypto/x509+ + internal/itoa from internal/poll+ + internal/msan from syscall + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profile from net/http/pprof + internal/profilerecord from runtime+ + internal/race from internal/poll+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/exithook from runtime + L internal/runtime/syscall from runtime+ + internal/saferio from debug/pe+ + internal/singleflight from net + internal/stringslite from embed+ + internal/syscall/execenv from os+ + LD internal/syscall/unix from crypto/rand+ + W internal/syscall/windows from crypto/rand+ + W internal/syscall/windows/registry from mime+ + W internal/syscall/windows/sysdll from internal/syscall/windows+ + internal/testlog from os + internal/unsafeheader from internal/reflectlite+ + internal/weak from unique io from archive/tar+ io/fs from archive/tar+ io/ioutil from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ @@ -558,6 +616,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de net/http/httptrace from github.com/prometheus-community/pro-bing+ net/http/httputil from github.com/aws/smithy-go/transport/http+ net/http/internal from net/http+ + net/http/internal/ascii from net/http+ + net/http/internal/testcert from net/http/httptest net/http/pprof from tailscale.com/cmd/tailscaled+ net/netip from github.com/tailscale/wireguard-go/conn+ net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ @@ -571,7 +631,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de reflect from archive/tar+ regexp from github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn+ regexp/syntax from regexp + runtime from archive/tar+ runtime/debug from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+ + runtime/internal/math from runtime + runtime/internal/sys from runtime runtime/pprof from net/http/pprof+ runtime/trace from net/http/pprof slices from tailscale.com/appc+ @@ -589,3 +652,4 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de unicode/utf16 from crypto/x509+ unicode/utf8 from bufio+ unique from net/netip + unsafe from bytes+ diff --git a/go.mod b/go.mod index 22193ee6efe8d..8e52a9ab337b0 100644 --- a/go.mod +++ b/go.mod @@ -74,7 +74,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/studio-b12/gowebdav v0.9.0 github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e - github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 + github.com/tailscale/depaware v0.0.0-20250112153213-b748de04d81b github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41 github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 diff --git a/go.sum b/go.sum index 20dbe73063589..c1c82ad7794c7 100644 --- a/go.sum +++ b/go.sum @@ -915,8 +915,8 @@ github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c h1:+aPplB github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c/go.mod h1:SbErYREK7xXdsRiigaQiQkI9McGRzYMvlKYaP3Nimdk= github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e h1:PtWT87weP5LWHEY//SWsYkSO3RWRZo4OSWagh3YD2vQ= github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e/go.mod h1:XrBNfAFN+pwoWuksbFS9Ccxnopa15zJGgXRFN90l3K4= -github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE= -github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= +github.com/tailscale/depaware v0.0.0-20250112153213-b748de04d81b h1:ewWb4cA+YO9/3X+v5UhdV+eKFsNBOPcGRh39Glshx/4= +github.com/tailscale/depaware v0.0.0-20250112153213-b748de04d81b/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 h1:Gzfnfk2TWrk8Jj4P4c1a3CtQyMaTVCznlkLZI++hok4= github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55/go.mod h1:4k4QO+dQ3R5FofL+SanAUZe+/QfeK0+OIuwDIRu2vSg= github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41 h1:/V2rCMMWcsjYaYO2MeovLw+ClP63OtXgCF2Y1eb8+Ns= From bce05ec6c3f3cb2dad1086472e99e2e69b2cfadc Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 26 Jan 2025 17:06:06 +0000 Subject: [PATCH 199/223] control/controlclient,tempfork/httprec: don't link httptest, test certs for c2n The c2n handling code was using the Go httptest package's ResponseRecorder code but that's in a test package which brings in Go's test certs, etc. This forks the httptest recorder type into its own package that only has the recorder and adds a test that we don't re-introduce a dependency on httptest. Updates #12614 Change-Id: I3546f49972981e21813ece9064cc2be0b74f4b16 Signed-off-by: Brad Fitzpatrick --- cmd/k8s-operator/depaware.txt | 3 +- cmd/tailscaled/depaware.txt | 5 +- cmd/tailscaled/tailscaled_test.go | 2 + control/controlclient/direct.go | 4 +- tempfork/httprec/httprec.go | 258 ++++++++++++++++++++++++++++++ 5 files changed, 265 insertions(+), 7 deletions(-) create mode 100644 tempfork/httprec/httprec.go diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 32af3b25e3ad0..fab29ba0317b8 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -889,6 +889,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/tailcfg from tailscale.com/client/tailscale+ tailscale.com/taildrop from tailscale.com/ipn/ipnlocal+ tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock + tailscale.com/tempfork/httprec from tailscale.com/control/controlclient tailscale.com/tka from tailscale.com/client/tailscale+ tailscale.com/tsconst from tailscale.com/net/netmon+ tailscale.com/tsd from tailscale.com/ipn/ipnlocal+ @@ -1174,12 +1175,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ - net/http/httptest from tailscale.com/control/controlclient net/http/httptrace from github.com/prometheus-community/pro-bing+ net/http/httputil from github.com/aws/smithy-go/transport/http+ net/http/internal from net/http+ net/http/internal/ascii from net/http+ - net/http/internal/testcert from net/http/httptest net/http/pprof from sigs.k8s.io/controller-runtime/pkg/manager+ net/netip from github.com/gaissmai/bart+ net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index e0ed51ebb1309..36b6063d5cb9b 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -341,6 +341,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/taildrop from tailscale.com/ipn/ipnlocal+ LD tailscale.com/tempfork/gliderlabs/ssh from tailscale.com/ssh/tailssh tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock + tailscale.com/tempfork/httprec from tailscale.com/control/controlclient tailscale.com/tka from tailscale.com/client/tailscale+ tailscale.com/tsconst from tailscale.com/net/netmon+ tailscale.com/tsd from tailscale.com/cmd/tailscaled+ @@ -547,7 +548,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de encoding/xml from github.com/aws/aws-sdk-go-v2/aws/protocol/xml+ errors from archive/tar+ expvar from tailscale.com/derp+ - flag from net/http/httptest+ + flag from tailscale.com/cmd/tailscaled+ fmt from archive/tar+ hash from compress/zlib+ hash/adler32 from compress/zlib+ @@ -612,12 +613,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ - net/http/httptest from tailscale.com/control/controlclient net/http/httptrace from github.com/prometheus-community/pro-bing+ net/http/httputil from github.com/aws/smithy-go/transport/http+ net/http/internal from net/http+ net/http/internal/ascii from net/http+ - net/http/internal/testcert from net/http/httptest net/http/pprof from tailscale.com/cmd/tailscaled+ net/netip from github.com/tailscale/wireguard-go/conn+ net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ diff --git a/cmd/tailscaled/tailscaled_test.go b/cmd/tailscaled/tailscaled_test.go index f36120f1300b5..c50c237591170 100644 --- a/cmd/tailscaled/tailscaled_test.go +++ b/cmd/tailscaled/tailscaled_test.go @@ -22,6 +22,8 @@ func TestDeps(t *testing.T) { BadDeps: map[string]string{ "testing": "do not use testing package in production code", "gvisor.dev/gvisor/pkg/hostarch": "will crash on non-4K page sizes; see https://github.com/tailscale/tailscale/issues/8658", + "net/http/httptest": "do not use httptest in production code", + "net/http/internal/testcert": "do not use httptest in production code", }, }.Check(t) diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index c436bc8b19926..f327ecc2afb5c 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -15,7 +15,6 @@ import ( "log" "net" "net/http" - "net/http/httptest" "net/netip" "net/url" "os" @@ -42,6 +41,7 @@ import ( "tailscale.com/net/tsdial" "tailscale.com/net/tshttpproxy" "tailscale.com/tailcfg" + "tailscale.com/tempfork/httprec" "tailscale.com/tka" "tailscale.com/tstime" "tailscale.com/types/key" @@ -1384,7 +1384,7 @@ func answerC2NPing(logf logger.Logf, c2nHandler http.Handler, c *http.Client, pr handlerCtx, cancel := context.WithTimeout(context.Background(), handlerTimeout) defer cancel() hreq = hreq.WithContext(handlerCtx) - rec := httptest.NewRecorder() + rec := httprec.NewRecorder() c2nHandler.ServeHTTP(rec, hreq) cancel() diff --git a/tempfork/httprec/httprec.go b/tempfork/httprec/httprec.go new file mode 100644 index 0000000000000..13786aaf60e05 --- /dev/null +++ b/tempfork/httprec/httprec.go @@ -0,0 +1,258 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httprec is a copy of the Go standard library's httptest.ResponseRecorder +// type, which we want to use in non-test code without pulling in the rest of +// the httptest package and its test certs, etc. +package httprec + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/textproto" + "strconv" + "strings" + + "golang.org/x/net/http/httpguts" +) + +// ResponseRecorder is an implementation of [http.ResponseWriter] that +// records its mutations for later inspection in tests. +type ResponseRecorder struct { + // Code is the HTTP response code set by WriteHeader. + // + // Note that if a Handler never calls WriteHeader or Write, + // this might end up being 0, rather than the implicit + // http.StatusOK. To get the implicit value, use the Result + // method. + Code int + + // HeaderMap contains the headers explicitly set by the Handler. + // It is an internal detail. + // + // Deprecated: HeaderMap exists for historical compatibility + // and should not be used. To access the headers returned by a handler, + // use the Response.Header map as returned by the Result method. + HeaderMap http.Header + + // Body is the buffer to which the Handler's Write calls are sent. + // If nil, the Writes are silently discarded. + Body *bytes.Buffer + + // Flushed is whether the Handler called Flush. + Flushed bool + + result *http.Response // cache of Result's return value + snapHeader http.Header // snapshot of HeaderMap at first Write + wroteHeader bool +} + +// NewRecorder returns an initialized [ResponseRecorder]. +func NewRecorder() *ResponseRecorder { + return &ResponseRecorder{ + HeaderMap: make(http.Header), + Body: new(bytes.Buffer), + Code: 200, + } +} + +// DefaultRemoteAddr is the default remote address to return in RemoteAddr if +// an explicit DefaultRemoteAddr isn't set on [ResponseRecorder]. +const DefaultRemoteAddr = "1.2.3.4" + +// Header implements [http.ResponseWriter]. It returns the response +// headers to mutate within a handler. To test the headers that were +// written after a handler completes, use the [ResponseRecorder.Result] method and see +// the returned Response value's Header. +func (rw *ResponseRecorder) Header() http.Header { + m := rw.HeaderMap + if m == nil { + m = make(http.Header) + rw.HeaderMap = m + } + return m +} + +// writeHeader writes a header if it was not written yet and +// detects Content-Type if needed. +// +// bytes or str are the beginning of the response body. +// We pass both to avoid unnecessarily generate garbage +// in rw.WriteString which was created for performance reasons. +// Non-nil bytes win. +func (rw *ResponseRecorder) writeHeader(b []byte, str string) { + if rw.wroteHeader { + return + } + if len(str) > 512 { + str = str[:512] + } + + m := rw.Header() + + _, hasType := m["Content-Type"] + hasTE := m.Get("Transfer-Encoding") != "" + if !hasType && !hasTE { + if b == nil { + b = []byte(str) + } + m.Set("Content-Type", http.DetectContentType(b)) + } + + rw.WriteHeader(200) +} + +// Write implements http.ResponseWriter. The data in buf is written to +// rw.Body, if not nil. +func (rw *ResponseRecorder) Write(buf []byte) (int, error) { + rw.writeHeader(buf, "") + if rw.Body != nil { + rw.Body.Write(buf) + } + return len(buf), nil +} + +// WriteString implements [io.StringWriter]. The data in str is written +// to rw.Body, if not nil. +func (rw *ResponseRecorder) WriteString(str string) (int, error) { + rw.writeHeader(nil, str) + if rw.Body != nil { + rw.Body.WriteString(str) + } + return len(str), nil +} + +func checkWriteHeaderCode(code int) { + // Issue 22880: require valid WriteHeader status codes. + // For now we only enforce that it's three digits. + // In the future we might block things over 599 (600 and above aren't defined + // at https://httpwg.org/specs/rfc7231.html#status.codes) + // and we might block under 200 (once we have more mature 1xx support). + // But for now any three digits. + // + // We used to send "HTTP/1.1 000 0" on the wire in responses but there's + // no equivalent bogus thing we can realistically send in HTTP/2, + // so we'll consistently panic instead and help people find their bugs + // early. (We can't return an error from WriteHeader even if we wanted to.) + if code < 100 || code > 999 { + panic(fmt.Sprintf("invalid WriteHeader code %v", code)) + } +} + +// WriteHeader implements [http.ResponseWriter]. +func (rw *ResponseRecorder) WriteHeader(code int) { + if rw.wroteHeader { + return + } + + checkWriteHeaderCode(code) + rw.Code = code + rw.wroteHeader = true + if rw.HeaderMap == nil { + rw.HeaderMap = make(http.Header) + } + rw.snapHeader = rw.HeaderMap.Clone() +} + +// Flush implements [http.Flusher]. To test whether Flush was +// called, see rw.Flushed. +func (rw *ResponseRecorder) Flush() { + if !rw.wroteHeader { + rw.WriteHeader(200) + } + rw.Flushed = true +} + +// Result returns the response generated by the handler. +// +// The returned Response will have at least its StatusCode, +// Header, Body, and optionally Trailer populated. +// More fields may be populated in the future, so callers should +// not DeepEqual the result in tests. +// +// The Response.Header is a snapshot of the headers at the time of the +// first write call, or at the time of this call, if the handler never +// did a write. +// +// The Response.Body is guaranteed to be non-nil and Body.Read call is +// guaranteed to not return any error other than [io.EOF]. +// +// Result must only be called after the handler has finished running. +func (rw *ResponseRecorder) Result() *http.Response { + if rw.result != nil { + return rw.result + } + if rw.snapHeader == nil { + rw.snapHeader = rw.HeaderMap.Clone() + } + res := &http.Response{ + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + StatusCode: rw.Code, + Header: rw.snapHeader, + } + rw.result = res + if res.StatusCode == 0 { + res.StatusCode = 200 + } + res.Status = fmt.Sprintf("%03d %s", res.StatusCode, http.StatusText(res.StatusCode)) + if rw.Body != nil { + res.Body = io.NopCloser(bytes.NewReader(rw.Body.Bytes())) + } else { + res.Body = http.NoBody + } + res.ContentLength = parseContentLength(res.Header.Get("Content-Length")) + + if trailers, ok := rw.snapHeader["Trailer"]; ok { + res.Trailer = make(http.Header, len(trailers)) + for _, k := range trailers { + for _, k := range strings.Split(k, ",") { + k = http.CanonicalHeaderKey(textproto.TrimString(k)) + if !httpguts.ValidTrailerHeader(k) { + // Ignore since forbidden by RFC 7230, section 4.1.2. + continue + } + vv, ok := rw.HeaderMap[k] + if !ok { + continue + } + vv2 := make([]string, len(vv)) + copy(vv2, vv) + res.Trailer[k] = vv2 + } + } + } + for k, vv := range rw.HeaderMap { + if !strings.HasPrefix(k, http.TrailerPrefix) { + continue + } + if res.Trailer == nil { + res.Trailer = make(http.Header) + } + for _, v := range vv { + res.Trailer.Add(strings.TrimPrefix(k, http.TrailerPrefix), v) + } + } + return res +} + +// parseContentLength trims whitespace from s and returns -1 if no value +// is set, or the value if it's >= 0. +// +// This a modified version of same function found in net/http/transfer.go. This +// one just ignores an invalid header. +func parseContentLength(cl string) int64 { + cl = textproto.TrimString(cl) + if cl == "" { + return -1 + } + n, err := strconv.ParseUint(cl, 10, 63) + if err != nil { + return -1 + } + return int64(n) +} From 3fec806523dbc650afa30f4a500b02f0fe40d641 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 12:36:04 -0700 Subject: [PATCH 200/223] .github: Bump actions/setup-go from 5.2.0 to 5.3.0 (#14793) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.2.0 to 5.3.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/3041bf56c941b39c61721a86cd11f3bb1338122a...f111f3307d8850f501ac008e886eec1fd1932a34) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 605f0939b2c84..ecac2851c5b3b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: # Install a more recent Go that understands modern go.mod content. - name: Install Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version-file: go.mod diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index ad135f784adec..58e6115918050 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version-file: go.mod cache: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc773e4a94d3e..a6ef6c36ec133 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,7 +153,7 @@ jobs: uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version-file: go.mod cache: false From 76dc028b389b72f111e976af2cff8ed1080996c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 12:36:42 -0700 Subject: [PATCH 201/223] .github: Bump github/codeql-action from 3.28.1 to 3.28.5 (#14794) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.1 to 3.28.5. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/b6a472f63d85b9c78a3ac5e89422239fc15e9b3c...f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ecac2851c5b3b..928240c5375d5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 + uses: github/codeql-action/init@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 + uses: github/codeql-action/autobuild@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 + uses: github/codeql-action/analyze@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5 From bfde8079a0919269fd9f435ba0c6e1bfc35988c2 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 27 Jan 2025 13:37:49 +0000 Subject: [PATCH 202/223] health: do Warnable dependency filtering in tailscaled Previously we were depending on the GUI(s) to do it. By doing it in tailscaled, GUIs can be simplified and be guaranteed to render consistent results. If warnable A depends on warnable B, if both A & B are unhealhy, only B will be shown to the GUI as unhealthy. Once B clears up, only then will A be presented as unhealthy. Updates #14687 Change-Id: Id8566f2672d8d2d699740fa053d4e2a2c8009e83 Signed-off-by: Brad Fitzpatrick --- health/health.go | 9 +++++++-- health/health_test.go | 8 +++++++- health/state.go | 25 +++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/health/health.go b/health/health.go index 079b3195c8e86..fa608ea7307d8 100644 --- a/health/health.go +++ b/health/health.go @@ -214,9 +214,11 @@ type Warnable struct { // TODO(angott): turn this into a SeverityFunc, which allows the Warnable to change its severity based on // the Args of the unhappy state, just like we do in the Text function. Severity Severity - // DependsOn is a set of Warnables that this Warnable depends, on and need to be healthy - // before this Warnable can also be healthy again. The GUI can use this information to ignore + // DependsOn is a set of Warnables that this Warnable depends on and need to be healthy + // before this Warnable is relevant. The GUI can use this information to ignore // this Warnable if one of its dependencies is unhealthy. + // That is, if any of these Warnables are unhealthy, then this Warnable is not relevant + // and should be considered healthy to bother the user about. DependsOn []*Warnable // MapDebugFlag is a MapRequest.DebugFlag that is sent to control when this Warnable is unhealthy @@ -940,6 +942,9 @@ func (t *Tracker) stringsLocked() []string { // Do not append invisible warnings. continue } + if t.isEffectivelyHealthyLocked(w) { + continue + } if ws.Args == nil { result = append(result, w.Text(Args{})) } else { diff --git a/health/health_test.go b/health/health_test.go index ebdddc988edc7..cc7b9d5aa287a 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -257,9 +257,15 @@ func TestCheckDependsOnAppearsInUnhealthyState(t *testing.T) { } ht.SetUnhealthy(w2, Args{ArgError: "w2 is also unhealthy now"}) us2, ok := ht.CurrentState().Warnings[w2.Code] + if ok { + t.Fatalf("Saw w2 being unhealthy but it shouldn't be, as it depends on unhealthy w1") + } + ht.SetHealthy(w1) + us2, ok = ht.CurrentState().Warnings[w2.Code] if !ok { - t.Fatalf("Expected an UnhealthyState for w2, got nothing") + t.Fatalf("w2 wasn't unhealthy; want it to be unhealthy now that w1 is back healthy") } + wantDependsOn = slices.Concat([]WarnableCode{w1.Code}, wantDependsOn) if !reflect.DeepEqual(us2.DependsOn, wantDependsOn) { t.Fatalf("Expected DependsOn = %v in the unhealthy state, got: %v", wantDependsOn, us2.DependsOn) diff --git a/health/state.go b/health/state.go index 17a646794b252..3bfa6f99bf6dd 100644 --- a/health/state.go +++ b/health/state.go @@ -90,6 +90,11 @@ func (t *Tracker) CurrentState() *State { // Skip invisible Warnables. continue } + if t.isEffectivelyHealthyLocked(w) { + // Skip Warnables that are unhealthy if they have dependencies + // that are unhealthy. + continue + } wm[w.Code] = *w.unhealthyState(ws) } @@ -97,3 +102,23 @@ func (t *Tracker) CurrentState() *State { Warnings: wm, } } + +// isEffectivelyHealthyLocked reports whether w is effectively healthy. +// That means it's either actually healthy or it has a dependency that +// that's unhealthy, so we should treat w as healthy to not spam users +// with multiple warnings when only the root cause is relevant. +func (t *Tracker) isEffectivelyHealthyLocked(w *Warnable) bool { + if _, ok := t.warnableVal[w]; !ok { + // Warnable not found in the tracker. So healthy. + return true + } + for _, d := range w.DependsOn { + if !t.isEffectivelyHealthyLocked(d) { + // If one of our deps is unhealthy, we're healthy. + return true + } + } + // If we have no unhealthy deps and had warnableVal set, + // we're unhealthy. + return false +} From bd9725c5f80b2e47da8ddb09a788036822531488 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 27 Jan 2025 14:21:25 +0000 Subject: [PATCH 203/223] health: relax no-derp-home warnable to not fire if not in map poll Fixes #14687 Change-Id: I05035df7e075e94dd39b2192bee34d878c15310d Signed-off-by: Brad Fitzpatrick --- health/health.go | 64 ++++++++++++++++++++++++++++--------------- health/health_test.go | 45 ++++++++++++++++++++++++++++++ health/state.go | 2 +- 3 files changed, 88 insertions(+), 23 deletions(-) diff --git a/health/health.go b/health/health.go index fa608ea7307d8..b0733f353317d 100644 --- a/health/health.go +++ b/health/health.go @@ -22,6 +22,7 @@ import ( "tailscale.com/envknob" "tailscale.com/metrics" "tailscale.com/tailcfg" + "tailscale.com/tstime" "tailscale.com/types/opt" "tailscale.com/util/cibuild" "tailscale.com/util/mak" @@ -73,6 +74,8 @@ type Tracker struct { // mu should not be held during init. initOnce sync.Once + testClock tstime.Clock // nil means use time.Now / tstime.StdClock{} + // mu guards everything that follows. mu sync.Mutex @@ -80,13 +83,13 @@ type Tracker struct { warnableVal map[*Warnable]*warningState // pendingVisibleTimers contains timers for Warnables that are unhealthy, but are // not visible to the user yet, because they haven't been unhealthy for TimeToVisible - pendingVisibleTimers map[*Warnable]*time.Timer + pendingVisibleTimers map[*Warnable]tstime.TimerController // sysErr maps subsystems to their current error (or nil if the subsystem is healthy) // Deprecated: using Warnables should be preferred sysErr map[Subsystem]error watchers set.HandleSet[func(*Warnable, *UnhealthyState)] // opt func to run if error state changes - timer *time.Timer + timer tstime.TimerController latestVersion *tailcfg.ClientVersion // or nil checkForUpdates bool @@ -115,6 +118,20 @@ type Tracker struct { metricHealthMessage *metrics.MultiLabelMap[metricHealthMessageLabel] } +func (t *Tracker) now() time.Time { + if t.testClock != nil { + return t.testClock.Now() + } + return time.Now() +} + +func (t *Tracker) clock() tstime.Clock { + if t.testClock != nil { + return t.testClock + } + return tstime.StdClock{} +} + // Subsystem is the name of a subsystem whose health can be monitored. // // Deprecated: Registering a Warnable using Register() and updating its health state @@ -311,11 +328,11 @@ func (ws *warningState) Equal(other *warningState) bool { // IsVisible returns whether the Warnable should be visible to the user, based on the TimeToVisible // field of the Warnable and the BrokenSince time when the Warnable became unhealthy. -func (w *Warnable) IsVisible(ws *warningState) bool { +func (w *Warnable) IsVisible(ws *warningState, clockNow func() time.Time) bool { if ws == nil || w.TimeToVisible == 0 { return true } - return time.Since(ws.BrokenSince) >= w.TimeToVisible + return clockNow().Sub(ws.BrokenSince) >= w.TimeToVisible } // SetMetricsRegistry sets up the metrics for the Tracker. It takes @@ -365,7 +382,7 @@ func (t *Tracker) setUnhealthyLocked(w *Warnable, args Args) { // If we already have a warningState for this Warnable with an earlier BrokenSince time, keep that // BrokenSince time. - brokenSince := time.Now() + brokenSince := t.now() if existingWS := t.warnableVal[w]; existingWS != nil { brokenSince = existingWS.BrokenSince } @@ -384,15 +401,15 @@ func (t *Tracker) setUnhealthyLocked(w *Warnable, args Args) { // If the Warnable has been unhealthy for more than its TimeToVisible, the callback should be // executed immediately. Otherwise, the callback should be enqueued to run once the Warnable // becomes visible. - if w.IsVisible(ws) { + if w.IsVisible(ws, t.now) { go cb(w, w.unhealthyState(ws)) continue } // The time remaining until the Warnable will be visible to the user is the TimeToVisible // minus the time that has already passed since the Warnable became unhealthy. - visibleIn := w.TimeToVisible - time.Since(brokenSince) - mak.Set(&t.pendingVisibleTimers, w, time.AfterFunc(visibleIn, func() { + visibleIn := w.TimeToVisible - t.now().Sub(brokenSince) + var tc tstime.TimerController = t.clock().AfterFunc(visibleIn, func() { t.mu.Lock() defer t.mu.Unlock() // Check if the Warnable is still unhealthy, as it could have become healthy between the time @@ -401,7 +418,8 @@ func (t *Tracker) setUnhealthyLocked(w *Warnable, args Args) { go cb(w, w.unhealthyState(ws)) delete(t.pendingVisibleTimers, w) } - })) + }) + mak.Set(&t.pendingVisibleTimers, w, tc) } } } @@ -476,7 +494,7 @@ func (t *Tracker) RegisterWatcher(cb func(w *Warnable, r *UnhealthyState)) (unre } handle := t.watchers.Add(cb) if t.timer == nil { - t.timer = time.AfterFunc(time.Minute, t.timerSelfCheck) + t.timer = t.clock().AfterFunc(time.Minute, t.timerSelfCheck) } return func() { t.mu.Lock() @@ -640,10 +658,10 @@ func (t *Tracker) GotStreamedMapResponse() { } t.mu.Lock() defer t.mu.Unlock() - t.lastStreamedMapResponse = time.Now() + t.lastStreamedMapResponse = t.now() if !t.inMapPoll { t.inMapPoll = true - t.inMapPollSince = time.Now() + t.inMapPollSince = t.now() } t.selfCheckLocked() } @@ -660,7 +678,7 @@ func (t *Tracker) SetOutOfPollNetMap() { return } t.inMapPoll = false - t.lastMapPollEndedAt = time.Now() + t.lastMapPollEndedAt = t.now() t.selfCheckLocked() } @@ -702,7 +720,7 @@ func (t *Tracker) NoteMapRequestHeard(mr *tailcfg.MapRequest) { // against SetMagicSockDERPHome and // SetDERPRegionConnectedState - t.lastMapRequestHeard = time.Now() + t.lastMapRequestHeard = t.now() t.selfCheckLocked() } @@ -740,7 +758,7 @@ func (t *Tracker) NoteDERPRegionReceivedFrame(region int) { } t.mu.Lock() defer t.mu.Unlock() - mak.Set(&t.derpRegionLastFrame, region, time.Now()) + mak.Set(&t.derpRegionLastFrame, region, t.now()) t.selfCheckLocked() } @@ -799,9 +817,9 @@ func (t *Tracker) SetIPNState(state string, wantRunning bool) { // The first time we see wantRunning=true and it used to be false, it means the user requested // the backend to start. We store this timestamp and use it to silence some warnings that are // expected during startup. - t.ipnWantRunningLastTrue = time.Now() + t.ipnWantRunningLastTrue = t.now() t.setUnhealthyLocked(warmingUpWarnable, nil) - time.AfterFunc(warmingUpWarnableDuration, func() { + t.clock().AfterFunc(warmingUpWarnableDuration, func() { t.mu.Lock() t.updateWarmingUpWarnableLocked() t.mu.Unlock() @@ -938,7 +956,7 @@ func (t *Tracker) Strings() []string { func (t *Tracker) stringsLocked() []string { result := []string{} for w, ws := range t.warnableVal { - if !w.IsVisible(ws) { + if !w.IsVisible(ws, t.now) { // Do not append invisible warnings. continue } @@ -1010,7 +1028,7 @@ func (t *Tracker) updateBuiltinWarnablesLocked() { t.setHealthyLocked(localLogWarnable) } - now := time.Now() + now := t.now() // How long we assume we'll have heard a DERP frame or a MapResponse // KeepAlive by. @@ -1020,8 +1038,10 @@ func (t *Tracker) updateBuiltinWarnablesLocked() { recentlyOn := now.Sub(t.ipnWantRunningLastTrue) < 5*time.Second homeDERP := t.derpHomeRegion - if recentlyOn { + if recentlyOn || !t.inMapPoll { // If user just turned Tailscale on, don't warn for a bit. + // Also, if we're not in a map poll, that means we don't yet + // have a DERPMap or aren't in a state where we even want t.setHealthyLocked(noDERPHomeWarnable) t.setHealthyLocked(noDERPConnectionWarnable) t.setHealthyLocked(derpTimeoutWarnable) @@ -1170,7 +1190,7 @@ func (t *Tracker) updateBuiltinWarnablesLocked() { // updateWarmingUpWarnableLocked ensures the warmingUpWarnable is healthy if wantRunning has been set to true // for more than warmingUpWarnableDuration. func (t *Tracker) updateWarmingUpWarnableLocked() { - if !t.ipnWantRunningLastTrue.IsZero() && time.Now().After(t.ipnWantRunningLastTrue.Add(warmingUpWarnableDuration)) { + if !t.ipnWantRunningLastTrue.IsZero() && t.now().After(t.ipnWantRunningLastTrue.Add(warmingUpWarnableDuration)) { t.setHealthyLocked(warmingUpWarnable) } } @@ -1282,7 +1302,7 @@ func (t *Tracker) LastNoiseDialWasRecent() bool { t.mu.Lock() defer t.mu.Unlock() - now := time.Now() + now := t.now() dur := now.Sub(t.lastNoiseDial) t.lastNoiseDial = now return dur < 2*time.Minute diff --git a/health/health_test.go b/health/health_test.go index cc7b9d5aa287a..abc0ec07e17fa 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -12,6 +12,7 @@ import ( "time" "tailscale.com/tailcfg" + "tailscale.com/tstest" "tailscale.com/types/opt" "tailscale.com/util/usermetric" "tailscale.com/version" @@ -406,3 +407,47 @@ func TestHealthMetric(t *testing.T) { }) } } + +// TestNoDERPHomeWarnable checks that we don't +// complain about no DERP home if we're not in a +// map poll. +func TestNoDERPHomeWarnable(t *testing.T) { + t.Skip("TODO: fix https://github.com/tailscale/tailscale/issues/14798 to make this test not deadlock") + clock := tstest.NewClock(tstest.ClockOpts{ + Start: time.Unix(123, 0), + FollowRealTime: false, + }) + ht := &Tracker{ + testClock: clock, + } + ht.SetIPNState("NeedsLogin", true) + + // Advance 30 seconds to get past the "recentlyLoggedIn" check. + clock.Advance(30 * time.Second) + ht.updateBuiltinWarnablesLocked() + + // Advance to get past the the TimeToVisible delay. + clock.Advance(noDERPHomeWarnable.TimeToVisible * 2) + + ht.updateBuiltinWarnablesLocked() + if ws, ok := ht.CurrentState().Warnings[noDERPHomeWarnable.Code]; ok { + t.Fatalf("got unexpected noDERPHomeWarnable warnable: %v", ws) + } +} + +// TestNoDERPHomeWarnableManual is like TestNoDERPHomeWarnable +// but doesn't use tstest.Clock so avoids the deadlock +// I hit: https://github.com/tailscale/tailscale/issues/14798 +func TestNoDERPHomeWarnableManual(t *testing.T) { + ht := &Tracker{} + ht.SetIPNState("NeedsLogin", true) + + // Avoid wantRunning: + ht.ipnWantRunningLastTrue = ht.ipnWantRunningLastTrue.Add(-10 * time.Second) + ht.updateBuiltinWarnablesLocked() + + ws, ok := ht.warnableVal[noDERPHomeWarnable] + if ok { + t.Fatalf("got unexpected noDERPHomeWarnable warnable: %v", ws) + } +} diff --git a/health/state.go b/health/state.go index 3bfa6f99bf6dd..c06f6ef59c8ed 100644 --- a/health/state.go +++ b/health/state.go @@ -86,7 +86,7 @@ func (t *Tracker) CurrentState() *State { wm := map[WarnableCode]UnhealthyState{} for w, ws := range t.warnableVal { - if !w.IsVisible(ws) { + if !w.IsVisible(ws, t.now) { // Skip invisible Warnables. continue } From 2691b9f6be2925188159d914411298e13dc409df Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 27 Jan 2025 03:07:21 +0000 Subject: [PATCH 204/223] tempfork/acme: add new package for x/crypto package acme fork, move We've been maintaining temporary dev forks of golang.org/x/crypto/{acme,ssh} in https://github.com/tailscale/golang-x-crypto instead of using this repo's tempfork directory as we do with other packages. The reason we were doing that was because x/crypto/ssh depended on x/crypto/ssh/internal/poly1305 and I hadn't noticed there are forwarding wrappers already available in x/crypto/poly1305. It also depended internal/bcrypt_pbkdf but we don't use that so it's easy to just delete that calling code in our tempfork/ssh. Now that our SSH changes have been upstreamed, we can soon unfork from SSH. That leaves ACME remaining. This change copies our tailscale/golang-x-crypto/acme code to tempfork/acme but adds a test that our vendored copied still matches our tailscale/golang-x-crypto repo, where we can continue to do development work and rebases with upstream. A comment on the new test describes the expected workflow. While we could continue to just import & use tailscale/golang-x-crypto/acme, it seems a bit nicer to not have that entire-fork-of-x-crypto visible at all in our transitive deps and the questions that invites. Showing just a fork of an ACME client is much less scary. It does add a step to the process of hacking on the ACME client code, but we do that approximately never anyway, and the extra step is very incremental compared to the existing tedious steps. Updates #8593 Updates #10238 Change-Id: I8af4378c04c1f82e63d31bf4d16dba9f510f9199 Signed-off-by: Brad Fitzpatrick --- cmd/k8s-operator/depaware.txt | 2 +- cmd/tailscaled/depaware.txt | 2 +- ipn/ipnlocal/cert.go | 2 +- tempfork/acme/README.md | 14 + tempfork/acme/acme.go | 861 ++++++++++++++++++++ tempfork/acme/acme_test.go | 973 +++++++++++++++++++++++ tempfork/acme/http.go | 325 ++++++++ tempfork/acme/http_test.go | 255 ++++++ tempfork/acme/jws.go | 257 ++++++ tempfork/acme/jws_test.go | 550 +++++++++++++ tempfork/acme/rfc8555.go | 476 +++++++++++ tempfork/acme/rfc8555_test.go | 1017 ++++++++++++++++++++++++ tempfork/acme/sync_to_upstream_test.go | 70 ++ tempfork/acme/types.go | 632 +++++++++++++++ tempfork/acme/types_test.go | 219 +++++ tempfork/acme/version_go112.go | 27 + 16 files changed, 5679 insertions(+), 3 deletions(-) create mode 100644 tempfork/acme/README.md create mode 100644 tempfork/acme/acme.go create mode 100644 tempfork/acme/acme_test.go create mode 100644 tempfork/acme/http.go create mode 100644 tempfork/acme/http_test.go create mode 100644 tempfork/acme/jws.go create mode 100644 tempfork/acme/jws_test.go create mode 100644 tempfork/acme/rfc8555.go create mode 100644 tempfork/acme/rfc8555_test.go create mode 100644 tempfork/acme/sync_to_upstream_test.go create mode 100644 tempfork/acme/types.go create mode 100644 tempfork/acme/types_test.go create mode 100644 tempfork/acme/version_go112.go diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index fab29ba0317b8..e32fd4a2b0d8f 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -197,7 +197,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - github.com/tailscale/golang-x-crypto/acme from tailscale.com/ipn/ipnlocal LD github.com/tailscale/golang-x-crypto/internal/poly1305 from github.com/tailscale/golang-x-crypto/ssh LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal LD github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh @@ -888,6 +887,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/syncs from tailscale.com/control/controlknobs+ tailscale.com/tailcfg from tailscale.com/client/tailscale+ tailscale.com/taildrop from tailscale.com/ipn/ipnlocal+ + tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock tailscale.com/tempfork/httprec from tailscale.com/control/controlclient tailscale.com/tka from tailscale.com/client/tailscale+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 36b6063d5cb9b..a7ad83818d317 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -152,7 +152,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - github.com/tailscale/golang-x-crypto/acme from tailscale.com/ipn/ipnlocal LD github.com/tailscale/golang-x-crypto/internal/poly1305 from github.com/tailscale/golang-x-crypto/ssh LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal+ LD github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh @@ -339,6 +338,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/syncs from tailscale.com/cmd/tailscaled+ tailscale.com/tailcfg from tailscale.com/client/tailscale+ tailscale.com/taildrop from tailscale.com/ipn/ipnlocal+ + tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal LD tailscale.com/tempfork/gliderlabs/ssh from tailscale.com/ssh/tailssh tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock tailscale.com/tempfork/httprec from tailscale.com/control/controlclient diff --git a/ipn/ipnlocal/cert.go b/ipn/ipnlocal/cert.go index 3361fc70ba54e..cfa4fe1bafdf8 100644 --- a/ipn/ipnlocal/cert.go +++ b/ipn/ipnlocal/cert.go @@ -32,7 +32,6 @@ import ( "sync" "time" - "github.com/tailscale/golang-x-crypto/acme" "tailscale.com/atomicfile" "tailscale.com/envknob" "tailscale.com/hostinfo" @@ -41,6 +40,7 @@ import ( "tailscale.com/ipn/store" "tailscale.com/ipn/store/mem" "tailscale.com/net/bakedroots" + "tailscale.com/tempfork/acme" "tailscale.com/types/logger" "tailscale.com/util/testenv" "tailscale.com/version" diff --git a/tempfork/acme/README.md b/tempfork/acme/README.md new file mode 100644 index 0000000000000..def357fc1e500 --- /dev/null +++ b/tempfork/acme/README.md @@ -0,0 +1,14 @@ +# tempfork/acme + +This is a vendored copy of Tailscale's https://github.com/tailscale/golang-x-crypto, +which is a fork of golang.org/x/crypto/acme. + +See https://github.com/tailscale/tailscale/issues/10238 for unforking +status. + +The https://github.com/tailscale/golang-x-crypto location exists to +let us do rebases from upstream easily, and then we update tempfork/acme +in the same commit we go get github.com/tailscale/golang-x-crypto@main. +See the comment on the TestSyncedToUpstream test for details. That +test should catch that forgotten step. + diff --git a/tempfork/acme/acme.go b/tempfork/acme/acme.go new file mode 100644 index 0000000000000..8bc2ac16ee1d8 --- /dev/null +++ b/tempfork/acme/acme.go @@ -0,0 +1,861 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package acme provides an implementation of the +// Automatic Certificate Management Environment (ACME) spec, +// most famously used by Let's Encrypt. +// +// The initial implementation of this package was based on an early version +// of the spec. The current implementation supports only the modern +// RFC 8555 but some of the old API surface remains for compatibility. +// While code using the old API will still compile, it will return an error. +// Note the deprecation comments to update your code. +// +// See https://tools.ietf.org/html/rfc8555 for the spec. +// +// Most common scenarios will want to use autocert subdirectory instead, +// which provides automatic access to certificates from Let's Encrypt +// and any other ACME-based CA. +package acme + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/http" + "strings" + "sync" + "time" +) + +const ( + // LetsEncryptURL is the Directory endpoint of Let's Encrypt CA. + LetsEncryptURL = "https://acme-v02.api.letsencrypt.org/directory" + + // ALPNProto is the ALPN protocol name used by a CA server when validating + // tls-alpn-01 challenges. + // + // Package users must ensure their servers can negotiate the ACME ALPN in + // order for tls-alpn-01 challenge verifications to succeed. + // See the crypto/tls package's Config.NextProtos field. + ALPNProto = "acme-tls/1" +) + +// idPeACMEIdentifier is the OID for the ACME extension for the TLS-ALPN challenge. +// https://tools.ietf.org/html/draft-ietf-acme-tls-alpn-05#section-5.1 +var idPeACMEIdentifier = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 31} + +const ( + maxChainLen = 5 // max depth and breadth of a certificate chain + maxCertSize = 1 << 20 // max size of a certificate, in DER bytes + // Used for decoding certs from application/pem-certificate-chain response, + // the default when in RFC mode. + maxCertChainSize = maxCertSize * maxChainLen + + // Max number of collected nonces kept in memory. + // Expect usual peak of 1 or 2. + maxNonces = 100 +) + +// Client is an ACME client. +// +// The only required field is Key. An example of creating a client with a new key +// is as follows: +// +// key, err := rsa.GenerateKey(rand.Reader, 2048) +// if err != nil { +// log.Fatal(err) +// } +// client := &Client{Key: key} +type Client struct { + // Key is the account key used to register with a CA and sign requests. + // Key.Public() must return a *rsa.PublicKey or *ecdsa.PublicKey. + // + // The following algorithms are supported: + // RS256, ES256, ES384 and ES512. + // See RFC 7518 for more details about the algorithms. + Key crypto.Signer + + // HTTPClient optionally specifies an HTTP client to use + // instead of http.DefaultClient. + HTTPClient *http.Client + + // DirectoryURL points to the CA directory endpoint. + // If empty, LetsEncryptURL is used. + // Mutating this value after a successful call of Client's Discover method + // will have no effect. + DirectoryURL string + + // RetryBackoff computes the duration after which the nth retry of a failed request + // should occur. The value of n for the first call on failure is 1. + // The values of r and resp are the request and response of the last failed attempt. + // If the returned value is negative or zero, no more retries are done and an error + // is returned to the caller of the original method. + // + // Requests which result in a 4xx client error are not retried, + // except for 400 Bad Request due to "bad nonce" errors and 429 Too Many Requests. + // + // If RetryBackoff is nil, a truncated exponential backoff algorithm + // with the ceiling of 10 seconds is used, where each subsequent retry n + // is done after either ("Retry-After" + jitter) or (2^n seconds + jitter), + // preferring the former if "Retry-After" header is found in the resp. + // The jitter is a random value up to 1 second. + RetryBackoff func(n int, r *http.Request, resp *http.Response) time.Duration + + // UserAgent is prepended to the User-Agent header sent to the ACME server, + // which by default is this package's name and version. + // + // Reusable libraries and tools in particular should set this value to be + // identifiable by the server, in case they are causing issues. + UserAgent string + + cacheMu sync.Mutex + dir *Directory // cached result of Client's Discover method + // KID is the key identifier provided by the CA. If not provided it will be + // retrieved from the CA by making a call to the registration endpoint. + KID KeyID + + noncesMu sync.Mutex + nonces map[string]struct{} // nonces collected from previous responses +} + +// accountKID returns a key ID associated with c.Key, the account identity +// provided by the CA during RFC based registration. +// It assumes c.Discover has already been called. +// +// accountKID requires at most one network roundtrip. +// It caches only successful result. +// +// When in pre-RFC mode or when c.getRegRFC responds with an error, accountKID +// returns noKeyID. +func (c *Client) accountKID(ctx context.Context) KeyID { + c.cacheMu.Lock() + defer c.cacheMu.Unlock() + if c.KID != noKeyID { + return c.KID + } + a, err := c.getRegRFC(ctx) + if err != nil { + return noKeyID + } + c.KID = KeyID(a.URI) + return c.KID +} + +var errPreRFC = errors.New("acme: server does not support the RFC 8555 version of ACME") + +// Discover performs ACME server discovery using c.DirectoryURL. +// +// It caches successful result. So, subsequent calls will not result in +// a network round-trip. This also means mutating c.DirectoryURL after successful call +// of this method will have no effect. +func (c *Client) Discover(ctx context.Context) (Directory, error) { + c.cacheMu.Lock() + defer c.cacheMu.Unlock() + if c.dir != nil { + return *c.dir, nil + } + + res, err := c.get(ctx, c.directoryURL(), wantStatus(http.StatusOK)) + if err != nil { + return Directory{}, err + } + defer res.Body.Close() + c.addNonce(res.Header) + + var v struct { + Reg string `json:"newAccount"` + Authz string `json:"newAuthz"` + Order string `json:"newOrder"` + Revoke string `json:"revokeCert"` + Nonce string `json:"newNonce"` + KeyChange string `json:"keyChange"` + RenewalInfo string `json:"renewalInfo"` + Meta struct { + Terms string `json:"termsOfService"` + Website string `json:"website"` + CAA []string `json:"caaIdentities"` + ExternalAcct bool `json:"externalAccountRequired"` + } + } + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return Directory{}, err + } + if v.Order == "" { + return Directory{}, errPreRFC + } + c.dir = &Directory{ + RegURL: v.Reg, + AuthzURL: v.Authz, + OrderURL: v.Order, + RevokeURL: v.Revoke, + NonceURL: v.Nonce, + KeyChangeURL: v.KeyChange, + RenewalInfoURL: v.RenewalInfo, + Terms: v.Meta.Terms, + Website: v.Meta.Website, + CAA: v.Meta.CAA, + ExternalAccountRequired: v.Meta.ExternalAcct, + } + return *c.dir, nil +} + +func (c *Client) directoryURL() string { + if c.DirectoryURL != "" { + return c.DirectoryURL + } + return LetsEncryptURL +} + +// CreateCert was part of the old version of ACME. It is incompatible with RFC 8555. +// +// Deprecated: this was for the pre-RFC 8555 version of ACME. Callers should use CreateOrderCert. +func (c *Client) CreateCert(ctx context.Context, csr []byte, exp time.Duration, bundle bool) (der [][]byte, certURL string, err error) { + return nil, "", errPreRFC +} + +// FetchCert retrieves already issued certificate from the given url, in DER format. +// It retries the request until the certificate is successfully retrieved, +// context is cancelled by the caller or an error response is received. +// +// If the bundle argument is true, the returned value also contains the CA (issuer) +// certificate chain. +// +// FetchCert returns an error if the CA's response or chain was unreasonably large. +// Callers are encouraged to parse the returned value to ensure the certificate is valid +// and has expected features. +func (c *Client) FetchCert(ctx context.Context, url string, bundle bool) ([][]byte, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + return c.fetchCertRFC(ctx, url, bundle) +} + +// RevokeCert revokes a previously issued certificate cert, provided in DER format. +// +// The key argument, used to sign the request, must be authorized +// to revoke the certificate. It's up to the CA to decide which keys are authorized. +// For instance, the key pair of the certificate may be authorized. +// If the key is nil, c.Key is used instead. +func (c *Client) RevokeCert(ctx context.Context, key crypto.Signer, cert []byte, reason CRLReasonCode) error { + if _, err := c.Discover(ctx); err != nil { + return err + } + return c.revokeCertRFC(ctx, key, cert, reason) +} + +// FetchRenewalInfo retrieves the RenewalInfo from Directory.RenewalInfoURL. +func (c *Client) FetchRenewalInfo(ctx context.Context, leaf []byte) (*RenewalInfo, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + + parsedLeaf, err := x509.ParseCertificate(leaf) + if err != nil { + return nil, fmt.Errorf("parsing leaf certificate: %w", err) + } + + renewalURL, err := c.getRenewalURL(parsedLeaf) + if err != nil { + return nil, fmt.Errorf("generating renewal info URL: %w", err) + } + + res, err := c.get(ctx, renewalURL, wantStatus(http.StatusOK)) + if err != nil { + return nil, fmt.Errorf("fetching renewal info: %w", err) + } + defer res.Body.Close() + + var info RenewalInfo + if err := json.NewDecoder(res.Body).Decode(&info); err != nil { + return nil, fmt.Errorf("parsing renewal info response: %w", err) + } + return &info, nil +} + +func (c *Client) getRenewalURL(cert *x509.Certificate) (string, error) { + // See https://www.ietf.org/archive/id/draft-ietf-acme-ari-04.html#name-the-renewalinfo-resource + // for how the request URL is built. + url := c.dir.RenewalInfoURL + if !strings.HasSuffix(url, "/") { + url += "/" + } + aki := base64.RawURLEncoding.EncodeToString(cert.AuthorityKeyId) + serial := base64.RawURLEncoding.EncodeToString(cert.SerialNumber.Bytes()) + return fmt.Sprintf("%s%s.%s", url, aki, serial), nil +} + +// AcceptTOS always returns true to indicate the acceptance of a CA's Terms of Service +// during account registration. See Register method of Client for more details. +func AcceptTOS(tosURL string) bool { return true } + +// Register creates a new account with the CA using c.Key. +// It returns the registered account. The account acct is not modified. +// +// The registration may require the caller to agree to the CA's Terms of Service (TOS). +// If so, and the account has not indicated the acceptance of the terms (see Account for details), +// Register calls prompt with a TOS URL provided by the CA. Prompt should report +// whether the caller agrees to the terms. To always accept the terms, the caller can use AcceptTOS. +// +// When interfacing with an RFC-compliant CA, non-RFC 8555 fields of acct are ignored +// and prompt is called if Directory's Terms field is non-zero. +// Also see Error's Instance field for when a CA requires already registered accounts to agree +// to an updated Terms of Service. +func (c *Client) Register(ctx context.Context, acct *Account, prompt func(tosURL string) bool) (*Account, error) { + if c.Key == nil { + return nil, errors.New("acme: client.Key must be set to Register") + } + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + return c.registerRFC(ctx, acct, prompt) +} + +// GetReg retrieves an existing account associated with c.Key. +// +// The url argument is a legacy artifact of the pre-RFC 8555 API +// and is ignored. +func (c *Client) GetReg(ctx context.Context, url string) (*Account, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + return c.getRegRFC(ctx) +} + +// UpdateReg updates an existing registration. +// It returns an updated account copy. The provided account is not modified. +// +// The account's URI is ignored and the account URL associated with +// c.Key is used instead. +func (c *Client) UpdateReg(ctx context.Context, acct *Account) (*Account, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + return c.updateRegRFC(ctx, acct) +} + +// AccountKeyRollover attempts to transition a client's account key to a new key. +// On success client's Key is updated which is not concurrency safe. +// On failure an error will be returned. +// The new key is already registered with the ACME provider if the following is true: +// - error is of type acme.Error +// - StatusCode should be 409 (Conflict) +// - Location header will have the KID of the associated account +// +// More about account key rollover can be found at +// https://tools.ietf.org/html/rfc8555#section-7.3.5. +func (c *Client) AccountKeyRollover(ctx context.Context, newKey crypto.Signer) error { + return c.accountKeyRollover(ctx, newKey) +} + +// Authorize performs the initial step in the pre-authorization flow, +// as opposed to order-based flow. +// The caller will then need to choose from and perform a set of returned +// challenges using c.Accept in order to successfully complete authorization. +// +// Once complete, the caller can use AuthorizeOrder which the CA +// should provision with the already satisfied authorization. +// For pre-RFC CAs, the caller can proceed directly to requesting a certificate +// using CreateCert method. +// +// If an authorization has been previously granted, the CA may return +// a valid authorization which has its Status field set to StatusValid. +// +// More about pre-authorization can be found at +// https://tools.ietf.org/html/rfc8555#section-7.4.1. +func (c *Client) Authorize(ctx context.Context, domain string) (*Authorization, error) { + return c.authorize(ctx, "dns", domain) +} + +// AuthorizeIP is the same as Authorize but requests IP address authorization. +// Clients which successfully obtain such authorization may request to issue +// a certificate for IP addresses. +// +// See the ACME spec extension for more details about IP address identifiers: +// https://tools.ietf.org/html/draft-ietf-acme-ip. +func (c *Client) AuthorizeIP(ctx context.Context, ipaddr string) (*Authorization, error) { + return c.authorize(ctx, "ip", ipaddr) +} + +func (c *Client) authorize(ctx context.Context, typ, val string) (*Authorization, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + + type authzID struct { + Type string `json:"type"` + Value string `json:"value"` + } + req := struct { + Resource string `json:"resource"` + Identifier authzID `json:"identifier"` + }{ + Resource: "new-authz", + Identifier: authzID{Type: typ, Value: val}, + } + res, err := c.post(ctx, nil, c.dir.AuthzURL, req, wantStatus(http.StatusCreated)) + if err != nil { + return nil, err + } + defer res.Body.Close() + + var v wireAuthz + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("acme: invalid response: %v", err) + } + if v.Status != StatusPending && v.Status != StatusValid { + return nil, fmt.Errorf("acme: unexpected status: %s", v.Status) + } + return v.authorization(res.Header.Get("Location")), nil +} + +// GetAuthorization retrieves an authorization identified by the given URL. +// +// If a caller needs to poll an authorization until its status is final, +// see the WaitAuthorization method. +func (c *Client) GetAuthorization(ctx context.Context, url string) (*Authorization, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + + res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Body.Close() + var v wireAuthz + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("acme: invalid response: %v", err) + } + return v.authorization(url), nil +} + +// RevokeAuthorization relinquishes an existing authorization identified +// by the given URL. +// The url argument is an Authorization.URI value. +// +// If successful, the caller will be required to obtain a new authorization +// using the Authorize or AuthorizeOrder methods before being able to request +// a new certificate for the domain associated with the authorization. +// +// It does not revoke existing certificates. +func (c *Client) RevokeAuthorization(ctx context.Context, url string) error { + if _, err := c.Discover(ctx); err != nil { + return err + } + + req := struct { + Resource string `json:"resource"` + Status string `json:"status"` + Delete bool `json:"delete"` + }{ + Resource: "authz", + Status: "deactivated", + Delete: true, + } + res, err := c.post(ctx, nil, url, req, wantStatus(http.StatusOK)) + if err != nil { + return err + } + defer res.Body.Close() + return nil +} + +// WaitAuthorization polls an authorization at the given URL +// until it is in one of the final states, StatusValid or StatusInvalid, +// the ACME CA responded with a 4xx error code, or the context is done. +// +// It returns a non-nil Authorization only if its Status is StatusValid. +// In all other cases WaitAuthorization returns an error. +// If the Status is StatusInvalid, the returned error is of type *AuthorizationError. +func (c *Client) WaitAuthorization(ctx context.Context, url string) (*Authorization, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + for { + res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted)) + if err != nil { + return nil, err + } + + var raw wireAuthz + err = json.NewDecoder(res.Body).Decode(&raw) + res.Body.Close() + switch { + case err != nil: + // Skip and retry. + case raw.Status == StatusValid: + return raw.authorization(url), nil + case raw.Status == StatusInvalid: + return nil, raw.error(url) + } + + // Exponential backoff is implemented in c.get above. + // This is just to prevent continuously hitting the CA + // while waiting for a final authorization status. + d := retryAfter(res.Header.Get("Retry-After")) + if d == 0 { + // Given that the fastest challenges TLS-SNI and HTTP-01 + // require a CA to make at least 1 network round trip + // and most likely persist a challenge state, + // this default delay seems reasonable. + d = time.Second + } + t := time.NewTimer(d) + select { + case <-ctx.Done(): + t.Stop() + return nil, ctx.Err() + case <-t.C: + // Retry. + } + } +} + +// GetChallenge retrieves the current status of an challenge. +// +// A client typically polls a challenge status using this method. +func (c *Client) GetChallenge(ctx context.Context, url string) (*Challenge, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + + res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted)) + if err != nil { + return nil, err + } + + defer res.Body.Close() + v := wireChallenge{URI: url} + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("acme: invalid response: %v", err) + } + return v.challenge(), nil +} + +// Accept informs the server that the client accepts one of its challenges +// previously obtained with c.Authorize. +// +// The server will then perform the validation asynchronously. +func (c *Client) Accept(ctx context.Context, chal *Challenge) (*Challenge, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + + res, err := c.post(ctx, nil, chal.URI, json.RawMessage("{}"), wantStatus( + http.StatusOK, // according to the spec + http.StatusAccepted, // Let's Encrypt: see https://goo.gl/WsJ7VT (acme-divergences.md) + )) + if err != nil { + return nil, err + } + defer res.Body.Close() + + var v wireChallenge + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("acme: invalid response: %v", err) + } + return v.challenge(), nil +} + +// DNS01ChallengeRecord returns a DNS record value for a dns-01 challenge response. +// A TXT record containing the returned value must be provisioned under +// "_acme-challenge" name of the domain being validated. +// +// The token argument is a Challenge.Token value. +func (c *Client) DNS01ChallengeRecord(token string) (string, error) { + ka, err := keyAuth(c.Key.Public(), token) + if err != nil { + return "", err + } + b := sha256.Sum256([]byte(ka)) + return base64.RawURLEncoding.EncodeToString(b[:]), nil +} + +// HTTP01ChallengeResponse returns the response for an http-01 challenge. +// Servers should respond with the value to HTTP requests at the URL path +// provided by HTTP01ChallengePath to validate the challenge and prove control +// over a domain name. +// +// The token argument is a Challenge.Token value. +func (c *Client) HTTP01ChallengeResponse(token string) (string, error) { + return keyAuth(c.Key.Public(), token) +} + +// HTTP01ChallengePath returns the URL path at which the response for an http-01 challenge +// should be provided by the servers. +// The response value can be obtained with HTTP01ChallengeResponse. +// +// The token argument is a Challenge.Token value. +func (c *Client) HTTP01ChallengePath(token string) string { + return "/.well-known/acme-challenge/" + token +} + +// TLSSNI01ChallengeCert creates a certificate for TLS-SNI-01 challenge response. +// +// Deprecated: This challenge type is unused in both draft-02 and RFC versions of the ACME spec. +func (c *Client) TLSSNI01ChallengeCert(token string, opt ...CertOption) (cert tls.Certificate, name string, err error) { + ka, err := keyAuth(c.Key.Public(), token) + if err != nil { + return tls.Certificate{}, "", err + } + b := sha256.Sum256([]byte(ka)) + h := hex.EncodeToString(b[:]) + name = fmt.Sprintf("%s.%s.acme.invalid", h[:32], h[32:]) + cert, err = tlsChallengeCert([]string{name}, opt) + if err != nil { + return tls.Certificate{}, "", err + } + return cert, name, nil +} + +// TLSSNI02ChallengeCert creates a certificate for TLS-SNI-02 challenge response. +// +// Deprecated: This challenge type is unused in both draft-02 and RFC versions of the ACME spec. +func (c *Client) TLSSNI02ChallengeCert(token string, opt ...CertOption) (cert tls.Certificate, name string, err error) { + b := sha256.Sum256([]byte(token)) + h := hex.EncodeToString(b[:]) + sanA := fmt.Sprintf("%s.%s.token.acme.invalid", h[:32], h[32:]) + + ka, err := keyAuth(c.Key.Public(), token) + if err != nil { + return tls.Certificate{}, "", err + } + b = sha256.Sum256([]byte(ka)) + h = hex.EncodeToString(b[:]) + sanB := fmt.Sprintf("%s.%s.ka.acme.invalid", h[:32], h[32:]) + + cert, err = tlsChallengeCert([]string{sanA, sanB}, opt) + if err != nil { + return tls.Certificate{}, "", err + } + return cert, sanA, nil +} + +// TLSALPN01ChallengeCert creates a certificate for TLS-ALPN-01 challenge response. +// Servers can present the certificate to validate the challenge and prove control +// over a domain name. For more details on TLS-ALPN-01 see +// https://tools.ietf.org/html/draft-shoemaker-acme-tls-alpn-00#section-3 +// +// The token argument is a Challenge.Token value. +// If a WithKey option is provided, its private part signs the returned cert, +// and the public part is used to specify the signee. +// If no WithKey option is provided, a new ECDSA key is generated using P-256 curve. +// +// The returned certificate is valid for the next 24 hours and must be presented only when +// the server name in the TLS ClientHello matches the domain, and the special acme-tls/1 ALPN protocol +// has been specified. +func (c *Client) TLSALPN01ChallengeCert(token, domain string, opt ...CertOption) (cert tls.Certificate, err error) { + ka, err := keyAuth(c.Key.Public(), token) + if err != nil { + return tls.Certificate{}, err + } + shasum := sha256.Sum256([]byte(ka)) + extValue, err := asn1.Marshal(shasum[:]) + if err != nil { + return tls.Certificate{}, err + } + acmeExtension := pkix.Extension{ + Id: idPeACMEIdentifier, + Critical: true, + Value: extValue, + } + + tmpl := defaultTLSChallengeCertTemplate() + + var newOpt []CertOption + for _, o := range opt { + switch o := o.(type) { + case *certOptTemplate: + t := *(*x509.Certificate)(o) // shallow copy is ok + tmpl = &t + default: + newOpt = append(newOpt, o) + } + } + tmpl.ExtraExtensions = append(tmpl.ExtraExtensions, acmeExtension) + newOpt = append(newOpt, WithTemplate(tmpl)) + return tlsChallengeCert([]string{domain}, newOpt) +} + +// popNonce returns a nonce value previously stored with c.addNonce +// or fetches a fresh one from c.dir.NonceURL. +// If NonceURL is empty, it first tries c.directoryURL() and, failing that, +// the provided url. +func (c *Client) popNonce(ctx context.Context, url string) (string, error) { + c.noncesMu.Lock() + defer c.noncesMu.Unlock() + if len(c.nonces) == 0 { + if c.dir != nil && c.dir.NonceURL != "" { + return c.fetchNonce(ctx, c.dir.NonceURL) + } + dirURL := c.directoryURL() + v, err := c.fetchNonce(ctx, dirURL) + if err != nil && url != dirURL { + v, err = c.fetchNonce(ctx, url) + } + return v, err + } + var nonce string + for nonce = range c.nonces { + delete(c.nonces, nonce) + break + } + return nonce, nil +} + +// clearNonces clears any stored nonces +func (c *Client) clearNonces() { + c.noncesMu.Lock() + defer c.noncesMu.Unlock() + c.nonces = make(map[string]struct{}) +} + +// addNonce stores a nonce value found in h (if any) for future use. +func (c *Client) addNonce(h http.Header) { + v := nonceFromHeader(h) + if v == "" { + return + } + c.noncesMu.Lock() + defer c.noncesMu.Unlock() + if len(c.nonces) >= maxNonces { + return + } + if c.nonces == nil { + c.nonces = make(map[string]struct{}) + } + c.nonces[v] = struct{}{} +} + +func (c *Client) fetchNonce(ctx context.Context, url string) (string, error) { + r, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return "", err + } + resp, err := c.doNoRetry(ctx, r) + if err != nil { + return "", err + } + defer resp.Body.Close() + nonce := nonceFromHeader(resp.Header) + if nonce == "" { + if resp.StatusCode > 299 { + return "", responseError(resp) + } + return "", errors.New("acme: nonce not found") + } + return nonce, nil +} + +func nonceFromHeader(h http.Header) string { + return h.Get("Replay-Nonce") +} + +// linkHeader returns URI-Reference values of all Link headers +// with relation-type rel. +// See https://tools.ietf.org/html/rfc5988#section-5 for details. +func linkHeader(h http.Header, rel string) []string { + var links []string + for _, v := range h["Link"] { + parts := strings.Split(v, ";") + for _, p := range parts { + p = strings.TrimSpace(p) + if !strings.HasPrefix(p, "rel=") { + continue + } + if v := strings.Trim(p[4:], `"`); v == rel { + links = append(links, strings.Trim(parts[0], "<>")) + } + } + } + return links +} + +// keyAuth generates a key authorization string for a given token. +func keyAuth(pub crypto.PublicKey, token string) (string, error) { + th, err := JWKThumbprint(pub) + if err != nil { + return "", err + } + return fmt.Sprintf("%s.%s", token, th), nil +} + +// defaultTLSChallengeCertTemplate is a template used to create challenge certs for TLS challenges. +func defaultTLSChallengeCertTemplate() *x509.Certificate { + return &x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } +} + +// tlsChallengeCert creates a temporary certificate for TLS-SNI challenges +// with the given SANs and auto-generated public/private key pair. +// The Subject Common Name is set to the first SAN to aid debugging. +// To create a cert with a custom key pair, specify WithKey option. +func tlsChallengeCert(san []string, opt []CertOption) (tls.Certificate, error) { + var key crypto.Signer + tmpl := defaultTLSChallengeCertTemplate() + for _, o := range opt { + switch o := o.(type) { + case *certOptKey: + if key != nil { + return tls.Certificate{}, errors.New("acme: duplicate key option") + } + key = o.key + case *certOptTemplate: + t := *(*x509.Certificate)(o) // shallow copy is ok + tmpl = &t + default: + // package's fault, if we let this happen: + panic(fmt.Sprintf("unsupported option type %T", o)) + } + } + if key == nil { + var err error + if key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader); err != nil { + return tls.Certificate{}, err + } + } + tmpl.DNSNames = san + if len(san) > 0 { + tmpl.Subject.CommonName = san[0] + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, key.Public(), key) + if err != nil { + return tls.Certificate{}, err + } + return tls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: key, + }, nil +} + +// encodePEM returns b encoded as PEM with block of type typ. +func encodePEM(typ string, b []byte) []byte { + pb := &pem.Block{Type: typ, Bytes: b} + return pem.EncodeToMemory(pb) +} + +// timeNow is time.Now, except in tests which can mess with it. +var timeNow = time.Now diff --git a/tempfork/acme/acme_test.go b/tempfork/acme/acme_test.go new file mode 100644 index 0000000000000..dcd2148968a92 --- /dev/null +++ b/tempfork/acme/acme_test.go @@ -0,0 +1,973 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "sort" + "strings" + "testing" + "time" +) + +// newTestClient creates a client with a non-nil Directory so that it skips +// the discovery which is otherwise done on the first call of almost every +// exported method. +func newTestClient() *Client { + return &Client{ + Key: testKeyEC, + dir: &Directory{}, // skip discovery + } +} + +// newTestClientWithMockDirectory creates a client with a non-nil Directory +// that contains mock field values. +func newTestClientWithMockDirectory() *Client { + return &Client{ + Key: testKeyEC, + dir: &Directory{ + RenewalInfoURL: "https://example.com/acme/renewal-info/", + }, + } +} + +// Decodes a JWS-encoded request and unmarshals the decoded JSON into a provided +// interface. +func decodeJWSRequest(t *testing.T, v interface{}, r io.Reader) { + // Decode request + var req struct{ Payload string } + if err := json.NewDecoder(r).Decode(&req); err != nil { + t.Fatal(err) + } + payload, err := base64.RawURLEncoding.DecodeString(req.Payload) + if err != nil { + t.Fatal(err) + } + err = json.Unmarshal(payload, v) + if err != nil { + t.Fatal(err) + } +} + +type jwsHead struct { + Alg string + Nonce string + URL string `json:"url"` + KID string `json:"kid"` + JWK map[string]string `json:"jwk"` +} + +func decodeJWSHead(r io.Reader) (*jwsHead, error) { + var req struct{ Protected string } + if err := json.NewDecoder(r).Decode(&req); err != nil { + return nil, err + } + b, err := base64.RawURLEncoding.DecodeString(req.Protected) + if err != nil { + return nil, err + } + var head jwsHead + if err := json.Unmarshal(b, &head); err != nil { + return nil, err + } + return &head, nil +} + +func TestRegisterWithoutKey(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("Replay-Nonce", "test-nonce") + return + } + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, `{}`) + })) + defer ts.Close() + // First verify that using a complete client results in success. + c := Client{ + Key: testKeyEC, + DirectoryURL: ts.URL, + dir: &Directory{RegURL: ts.URL}, + } + if _, err := c.Register(context.Background(), &Account{}, AcceptTOS); err != nil { + t.Fatalf("c.Register() = %v; want success with a complete test client", err) + } + c.Key = nil + if _, err := c.Register(context.Background(), &Account{}, AcceptTOS); err == nil { + t.Error("c.Register() from client without key succeeded, wanted error") + } +} + +func TestAuthorize(t *testing.T) { + tt := []struct{ typ, value string }{ + {"dns", "example.com"}, + {"ip", "1.2.3.4"}, + } + for _, test := range tt { + t.Run(test.typ, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("Replay-Nonce", "test-nonce") + return + } + if r.Method != "POST" { + t.Errorf("r.Method = %q; want POST", r.Method) + } + + var j struct { + Resource string + Identifier struct { + Type string + Value string + } + } + decodeJWSRequest(t, &j, r.Body) + + // Test request + if j.Resource != "new-authz" { + t.Errorf("j.Resource = %q; want new-authz", j.Resource) + } + if j.Identifier.Type != test.typ { + t.Errorf("j.Identifier.Type = %q; want %q", j.Identifier.Type, test.typ) + } + if j.Identifier.Value != test.value { + t.Errorf("j.Identifier.Value = %q; want %q", j.Identifier.Value, test.value) + } + + w.Header().Set("Location", "https://ca.tld/acme/auth/1") + w.WriteHeader(http.StatusCreated) + fmt.Fprintf(w, `{ + "identifier": {"type":%q,"value":%q}, + "status":"pending", + "challenges":[ + { + "type":"http-01", + "status":"pending", + "uri":"https://ca.tld/acme/challenge/publickey/id1", + "token":"token1" + }, + { + "type":"tls-sni-01", + "status":"pending", + "uri":"https://ca.tld/acme/challenge/publickey/id2", + "token":"token2" + } + ], + "combinations":[[0],[1]] + }`, test.typ, test.value) + })) + defer ts.Close() + + var ( + auth *Authorization + err error + ) + cl := Client{ + Key: testKeyEC, + DirectoryURL: ts.URL, + dir: &Directory{AuthzURL: ts.URL}, + } + switch test.typ { + case "dns": + auth, err = cl.Authorize(context.Background(), test.value) + case "ip": + auth, err = cl.AuthorizeIP(context.Background(), test.value) + default: + t.Fatalf("unknown identifier type: %q", test.typ) + } + if err != nil { + t.Fatal(err) + } + + if auth.URI != "https://ca.tld/acme/auth/1" { + t.Errorf("URI = %q; want https://ca.tld/acme/auth/1", auth.URI) + } + if auth.Status != "pending" { + t.Errorf("Status = %q; want pending", auth.Status) + } + if auth.Identifier.Type != test.typ { + t.Errorf("Identifier.Type = %q; want %q", auth.Identifier.Type, test.typ) + } + if auth.Identifier.Value != test.value { + t.Errorf("Identifier.Value = %q; want %q", auth.Identifier.Value, test.value) + } + + if n := len(auth.Challenges); n != 2 { + t.Fatalf("len(auth.Challenges) = %d; want 2", n) + } + + c := auth.Challenges[0] + if c.Type != "http-01" { + t.Errorf("c.Type = %q; want http-01", c.Type) + } + if c.URI != "https://ca.tld/acme/challenge/publickey/id1" { + t.Errorf("c.URI = %q; want https://ca.tld/acme/challenge/publickey/id1", c.URI) + } + if c.Token != "token1" { + t.Errorf("c.Token = %q; want token1", c.Token) + } + + c = auth.Challenges[1] + if c.Type != "tls-sni-01" { + t.Errorf("c.Type = %q; want tls-sni-01", c.Type) + } + if c.URI != "https://ca.tld/acme/challenge/publickey/id2" { + t.Errorf("c.URI = %q; want https://ca.tld/acme/challenge/publickey/id2", c.URI) + } + if c.Token != "token2" { + t.Errorf("c.Token = %q; want token2", c.Token) + } + + combs := [][]int{{0}, {1}} + if !reflect.DeepEqual(auth.Combinations, combs) { + t.Errorf("auth.Combinations: %+v\nwant: %+v\n", auth.Combinations, combs) + } + + }) + } +} + +func TestAuthorizeValid(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("Replay-Nonce", "nonce") + return + } + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"status":"valid"}`)) + })) + defer ts.Close() + client := Client{ + Key: testKey, + DirectoryURL: ts.URL, + dir: &Directory{AuthzURL: ts.URL}, + } + _, err := client.Authorize(context.Background(), "example.com") + if err != nil { + t.Errorf("err = %v", err) + } +} + +func TestWaitAuthorization(t *testing.T) { + t.Run("wait loop", func(t *testing.T) { + var count int + authz, err := runWaitAuthorization(context.Background(), t, func(w http.ResponseWriter, r *http.Request) { + count++ + w.Header().Set("Retry-After", "0") + if count > 1 { + fmt.Fprintf(w, `{"status":"valid"}`) + return + } + fmt.Fprintf(w, `{"status":"pending"}`) + }) + if err != nil { + t.Fatalf("non-nil error: %v", err) + } + if authz == nil { + t.Fatal("authz is nil") + } + }) + t.Run("invalid status", func(t *testing.T) { + _, err := runWaitAuthorization(context.Background(), t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"status":"invalid"}`) + }) + if _, ok := err.(*AuthorizationError); !ok { + t.Errorf("err is %v (%T); want non-nil *AuthorizationError", err, err) + } + }) + t.Run("invalid status with error returns the authorization error", func(t *testing.T) { + _, err := runWaitAuthorization(context.Background(), t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{ + "type": "dns-01", + "status": "invalid", + "error": { + "type": "urn:ietf:params:acme:error:caa", + "detail": "CAA record for prevents issuance", + "status": 403 + }, + "url": "https://acme-v02.api.letsencrypt.org/acme/chall-v3/xxx/xxx", + "token": "xxx", + "validationRecord": [ + { + "hostname": "" + } + ] + }`) + }) + + want := &AuthorizationError{ + Errors: []error{ + (&wireError{ + Status: 403, + Type: "urn:ietf:params:acme:error:caa", + Detail: "CAA record for prevents issuance", + }).error(nil), + }, + } + + _, ok := err.(*AuthorizationError) + if !ok { + t.Errorf("err is %T; want non-nil *AuthorizationError", err) + } + + if err.Error() != want.Error() { + t.Errorf("err is %v; want %v", err, want) + } + }) + t.Run("non-retriable error", func(t *testing.T) { + const code = http.StatusBadRequest + _, err := runWaitAuthorization(context.Background(), t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + }) + res, ok := err.(*Error) + if !ok { + t.Fatalf("err is %v (%T); want a non-nil *Error", err, err) + } + if res.StatusCode != code { + t.Errorf("res.StatusCode = %d; want %d", res.StatusCode, code) + } + }) + for _, code := range []int{http.StatusTooManyRequests, http.StatusInternalServerError} { + t.Run(fmt.Sprintf("retriable %d error", code), func(t *testing.T) { + var count int + authz, err := runWaitAuthorization(context.Background(), t, func(w http.ResponseWriter, r *http.Request) { + count++ + w.Header().Set("Retry-After", "0") + if count > 1 { + fmt.Fprintf(w, `{"status":"valid"}`) + return + } + w.WriteHeader(code) + }) + if err != nil { + t.Fatalf("non-nil error: %v", err) + } + if authz == nil { + t.Fatal("authz is nil") + } + }) + } + t.Run("context cancel", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err := runWaitAuthorization(ctx, t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "60") + fmt.Fprintf(w, `{"status":"pending"}`) + time.AfterFunc(1*time.Millisecond, cancel) + }) + if err == nil { + t.Error("err is nil") + } + }) +} + +func runWaitAuthorization(ctx context.Context, t *testing.T, h http.HandlerFunc) (*Authorization, error) { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Replay-Nonce", fmt.Sprintf("bad-test-nonce-%v", time.Now().UnixNano())) + h(w, r) + })) + defer ts.Close() + + client := &Client{ + Key: testKey, + DirectoryURL: ts.URL, + dir: &Directory{}, + KID: "some-key-id", // set to avoid lookup attempt + } + return client.WaitAuthorization(ctx, ts.URL) +} + +func TestRevokeAuthorization(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("Replay-Nonce", "nonce") + return + } + switch r.URL.Path { + case "/1": + var req struct { + Resource string + Status string + Delete bool + } + decodeJWSRequest(t, &req, r.Body) + if req.Resource != "authz" { + t.Errorf("req.Resource = %q; want authz", req.Resource) + } + if req.Status != "deactivated" { + t.Errorf("req.Status = %q; want deactivated", req.Status) + } + if !req.Delete { + t.Errorf("req.Delete is false") + } + case "/2": + w.WriteHeader(http.StatusBadRequest) + } + })) + defer ts.Close() + client := &Client{ + Key: testKey, + DirectoryURL: ts.URL, // don't dial outside of localhost + dir: &Directory{}, // don't do discovery + } + ctx := context.Background() + if err := client.RevokeAuthorization(ctx, ts.URL+"/1"); err != nil { + t.Errorf("err = %v", err) + } + if client.RevokeAuthorization(ctx, ts.URL+"/2") == nil { + t.Error("nil error") + } +} + +func TestFetchCertCancel(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusBadRequest) + })) + defer ts.Close() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + var err error + go func() { + cl := newTestClient() + _, err = cl.FetchCert(ctx, ts.URL, false) + close(done) + }() + cancel() + <-done + if err != context.Canceled { + t.Errorf("err = %v; want %v", err, context.Canceled) + } +} + +func TestFetchCertDepth(t *testing.T) { + var count byte + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count++ + if count > maxChainLen+1 { + t.Errorf("count = %d; want at most %d", count, maxChainLen+1) + w.WriteHeader(http.StatusInternalServerError) + } + w.Header().Set("Link", fmt.Sprintf("<%s>;rel=up", ts.URL)) + w.Write([]byte{count}) + })) + defer ts.Close() + cl := newTestClient() + _, err := cl.FetchCert(context.Background(), ts.URL, true) + if err == nil { + t.Errorf("err is nil") + } +} + +func TestFetchCertBreadth(t *testing.T) { + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for i := 0; i < maxChainLen+1; i++ { + w.Header().Add("Link", fmt.Sprintf("<%s>;rel=up", ts.URL)) + } + w.Write([]byte{1}) + })) + defer ts.Close() + cl := newTestClient() + _, err := cl.FetchCert(context.Background(), ts.URL, true) + if err == nil { + t.Errorf("err is nil") + } +} + +func TestFetchCertSize(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b := bytes.Repeat([]byte{1}, maxCertSize+1) + w.Write(b) + })) + defer ts.Close() + cl := newTestClient() + _, err := cl.FetchCert(context.Background(), ts.URL, false) + if err == nil { + t.Errorf("err is nil") + } +} + +const ( + leafPEM = `-----BEGIN CERTIFICATE----- +MIIEizCCAvOgAwIBAgIRAITApw7R8HSs7GU7cj8dEyUwDQYJKoZIhvcNAQELBQAw +gYUxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTEtMCsGA1UECwwkY3Bh +bG1lckBwdW1wa2luLmxvY2FsIChDaHJpcyBQYWxtZXIpMTQwMgYDVQQDDCtta2Nl +cnQgY3BhbG1lckBwdW1wa2luLmxvY2FsIChDaHJpcyBQYWxtZXIpMB4XDTIzMDcx +MjE4MjIxNloXDTI1MTAxMjE4MjIxNlowWDEnMCUGA1UEChMebWtjZXJ0IGRldmVs +b3BtZW50IGNlcnRpZmljYXRlMS0wKwYDVQQLDCRjcGFsbWVyQHB1bXBraW4ubG9j +YWwgKENocmlzIFBhbG1lcikwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDNDO8P4MI9jaqVcPtF8C4GgHnTP5EK3U9fgyGApKGxTpicMQkA6z4GXwUP/Fvq +7RuCU9Wg7By5VetKIHF7FxkxWkUMrssr7mV8v6mRCh/a5GqDs14aj5ucjLQAJV74 +tLAdrCiijQ1fkPWc82fob+LkfKWGCWw7Cxf6ZtEyC8jz/DnfQXUvOiZS729ndGF7 +FobKRfIoirD+GI2NTYIp3LAUFSPR6HXTe7HAg8J81VoUKli8z504+FebfMmHePm/ +zIfiI0njAj4czOlZD56/oLsV0WRUizFjafHHUFz1HVdfFw8Qf9IOOTydYOe8M5i0 +lVbVO5G+HP+JDn3cr9MT41B9AgMBAAGjgaEwgZ4wDgYDVR0PAQH/BAQDAgWgMBMG +A1UdJQQMMAoGCCsGAQUFBwMBMB8GA1UdIwQYMBaAFPpL4Q0O7Z7voTkjn2rrFCsf +s8TbMFYGA1UdEQRPME2CC2V4YW1wbGUuY29tgg0qLmV4YW1wbGUuY29tggxleGFt +cGxlLnRlc3SCCWxvY2FsaG9zdIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATANBgkq +hkiG9w0BAQsFAAOCAYEAMlOb7lrHuSxwcnAu7mL1ysTGqKn1d2TyDJAN5W8YFY+4 +XLpofNkK2UzZ0t9LQRnuFUcjmfqmfplh5lpC7pKmtL4G5Qcdc+BczQWcopbxd728 +sht9BKRkH+Bo1I+1WayKKNXW+5bsMv4CH641zxaMBlzjEnPvwKkNaGLMH3x5lIeX +GGgkKNXwVtINmyV+lTNVtu2IlHprxJGCjRfEuX7mEv6uRnqz3Wif+vgyh3MBgM/1 +dUOsTBNH4a6Jl/9VPSOfRdQOStqIlwTa/J1bhTvivsYt1+eWjLnsQJLgZQqwKvYH +BJ30gAk1oNnuSkx9dHbx4mO+4mB9oIYUALXUYakb8JHTOnuMSj9qelVj5vjVxl9q +KRitptU+kLYRA4HSgUXrhDIm4Q6D/w8/ascPqQ3HxPIDFLe+gTofEjqnnsnQB29L +gWpI8l5/MtXAOMdW69eEovnADc2pgaiif0T+v9nNKBc5xfDZHnrnqIqVzQEwL5Qv +niQI8IsWD5LcQ1Eg7kCq +-----END CERTIFICATE-----` +) + +func TestGetRenewalURL(t *testing.T) { + leaf, _ := pem.Decode([]byte(leafPEM)) + + parsedLeaf, err := x509.ParseCertificate(leaf.Bytes) + if err != nil { + t.Fatal(err) + } + + client := newTestClientWithMockDirectory() + urlString, err := client.getRenewalURL(parsedLeaf) + if err != nil { + t.Fatal(err) + } + + parsedURL, err := url.Parse(urlString) + if err != nil { + t.Fatal(err) + } + if scheme := parsedURL.Scheme; scheme == "" { + t.Fatalf("malformed URL scheme: %q from %q", scheme, urlString) + } + if host := parsedURL.Host; host == "" { + t.Fatalf("malformed URL host: %q from %q", host, urlString) + } + if parsedURL.RawQuery != "" { + t.Fatalf("malformed URL: should not have a query") + } + path := parsedURL.EscapedPath() + slash := strings.LastIndex(path, "/") + if slash == -1 { + t.Fatalf("malformed URL path: %q from %q", path, urlString) + } + certID := path[slash+1:] + if certID == "" { + t.Fatalf("missing certificate identifier in URL path: %q from %q", path, urlString) + } + certIDParts := strings.Split(certID, ".") + if len(certIDParts) != 2 { + t.Fatalf("certificate identifier should consist of 2 base64-encoded values separated by a dot: %q from %q", certID, urlString) + } + if _, err := base64.RawURLEncoding.DecodeString(certIDParts[0]); err != nil { + t.Fatalf("malformed AKI part in certificate identifier: %q from %q: %v", certIDParts[0], urlString, err) + } + if _, err := base64.RawURLEncoding.DecodeString(certIDParts[1]); err != nil { + t.Fatalf("malformed Serial part in certificate identifier: %q from %q: %v", certIDParts[1], urlString, err) + } + +} + +func TestUnmarshalRenewalInfo(t *testing.T) { + renewalInfoJSON := `{ + "suggestedWindow": { + "start": "2021-01-03T00:00:00Z", + "end": "2021-01-07T00:00:00Z" + }, + "explanationURL": "https://example.com/docs/example-mass-reissuance-event" + }` + expectedStart := time.Date(2021, time.January, 3, 0, 0, 0, 0, time.UTC) + expectedEnd := time.Date(2021, time.January, 7, 0, 0, 0, 0, time.UTC) + + var info RenewalInfo + if err := json.Unmarshal([]byte(renewalInfoJSON), &info); err != nil { + t.Fatal(err) + } + if _, err := url.Parse(info.ExplanationURL); err != nil { + t.Fatal(err) + } + if !info.SuggestedWindow.Start.Equal(expectedStart) { + t.Fatalf("%v != %v", expectedStart, info.SuggestedWindow.Start) + } + if !info.SuggestedWindow.End.Equal(expectedEnd) { + t.Fatalf("%v != %v", expectedEnd, info.SuggestedWindow.End) + } +} + +func TestNonce_add(t *testing.T) { + var c Client + c.addNonce(http.Header{"Replay-Nonce": {"nonce"}}) + c.addNonce(http.Header{"Replay-Nonce": {}}) + c.addNonce(http.Header{"Replay-Nonce": {"nonce"}}) + + nonces := map[string]struct{}{"nonce": {}} + if !reflect.DeepEqual(c.nonces, nonces) { + t.Errorf("c.nonces = %q; want %q", c.nonces, nonces) + } +} + +func TestNonce_addMax(t *testing.T) { + c := &Client{nonces: make(map[string]struct{})} + for i := 0; i < maxNonces; i++ { + c.nonces[fmt.Sprintf("%d", i)] = struct{}{} + } + c.addNonce(http.Header{"Replay-Nonce": {"nonce"}}) + if n := len(c.nonces); n != maxNonces { + t.Errorf("len(c.nonces) = %d; want %d", n, maxNonces) + } +} + +func TestNonce_fetch(t *testing.T) { + tests := []struct { + code int + nonce string + }{ + {http.StatusOK, "nonce1"}, + {http.StatusBadRequest, "nonce2"}, + {http.StatusOK, ""}, + } + var i int + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "HEAD" { + t.Errorf("%d: r.Method = %q; want HEAD", i, r.Method) + } + w.Header().Set("Replay-Nonce", tests[i].nonce) + w.WriteHeader(tests[i].code) + })) + defer ts.Close() + for ; i < len(tests); i++ { + test := tests[i] + c := newTestClient() + n, err := c.fetchNonce(context.Background(), ts.URL) + if n != test.nonce { + t.Errorf("%d: n=%q; want %q", i, n, test.nonce) + } + switch { + case err == nil && test.nonce == "": + t.Errorf("%d: n=%q, err=%v; want non-nil error", i, n, err) + case err != nil && test.nonce != "": + t.Errorf("%d: n=%q, err=%v; want %q", i, n, err, test.nonce) + } + } +} + +func TestNonce_fetchError(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + })) + defer ts.Close() + c := newTestClient() + _, err := c.fetchNonce(context.Background(), ts.URL) + e, ok := err.(*Error) + if !ok { + t.Fatalf("err is %T; want *Error", err) + } + if e.StatusCode != http.StatusTooManyRequests { + t.Errorf("e.StatusCode = %d; want %d", e.StatusCode, http.StatusTooManyRequests) + } +} + +func TestNonce_popWhenEmpty(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "HEAD" { + t.Errorf("r.Method = %q; want HEAD", r.Method) + } + switch r.URL.Path { + case "/dir-with-nonce": + w.Header().Set("Replay-Nonce", "dirnonce") + case "/new-nonce": + w.Header().Set("Replay-Nonce", "newnonce") + case "/dir-no-nonce", "/empty": + // No nonce in the header. + default: + t.Errorf("Unknown URL: %s", r.URL) + } + })) + defer ts.Close() + ctx := context.Background() + + tt := []struct { + dirURL, popURL, nonce string + wantOK bool + }{ + {ts.URL + "/dir-with-nonce", ts.URL + "/new-nonce", "dirnonce", true}, + {ts.URL + "/dir-no-nonce", ts.URL + "/new-nonce", "newnonce", true}, + {ts.URL + "/dir-no-nonce", ts.URL + "/empty", "", false}, + } + for _, test := range tt { + t.Run(fmt.Sprintf("nonce:%s wantOK:%v", test.nonce, test.wantOK), func(t *testing.T) { + c := Client{DirectoryURL: test.dirURL} + v, err := c.popNonce(ctx, test.popURL) + if !test.wantOK { + if err == nil { + t.Fatalf("c.popNonce(%q) returned nil error", test.popURL) + } + return + } + if err != nil { + t.Fatalf("c.popNonce(%q): %v", test.popURL, err) + } + if v != test.nonce { + t.Errorf("c.popNonce(%q) = %q; want %q", test.popURL, v, test.nonce) + } + }) + } +} + +func TestLinkHeader(t *testing.T) { + h := http.Header{"Link": { + `;rel="next"`, + `; rel=recover`, + `; foo=bar; rel="terms-of-service"`, + `;rel="next"`, + }} + tests := []struct { + rel string + out []string + }{ + {"next", []string{"https://example.com/acme/new-authz", "dup"}}, + {"recover", []string{"https://example.com/acme/recover-reg"}}, + {"terms-of-service", []string{"https://example.com/acme/terms"}}, + {"empty", nil}, + } + for i, test := range tests { + if v := linkHeader(h, test.rel); !reflect.DeepEqual(v, test.out) { + t.Errorf("%d: linkHeader(%q): %v; want %v", i, test.rel, v, test.out) + } + } +} + +func TestTLSSNI01ChallengeCert(t *testing.T) { + const ( + token = "evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA" + // echo -n | shasum -a 256 + san = "dbbd5eefe7b4d06eb9d1d9f5acb4c7cd.a27d320e4b30332f0b6cb441734ad7b0.acme.invalid" + ) + + tlscert, name, err := newTestClient().TLSSNI01ChallengeCert(token) + if err != nil { + t.Fatal(err) + } + + if n := len(tlscert.Certificate); n != 1 { + t.Fatalf("len(tlscert.Certificate) = %d; want 1", n) + } + cert, err := x509.ParseCertificate(tlscert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + if len(cert.DNSNames) != 1 || cert.DNSNames[0] != san { + t.Fatalf("cert.DNSNames = %v; want %q", cert.DNSNames, san) + } + if cert.DNSNames[0] != name { + t.Errorf("cert.DNSNames[0] != name: %q vs %q", cert.DNSNames[0], name) + } + if cn := cert.Subject.CommonName; cn != san { + t.Errorf("cert.Subject.CommonName = %q; want %q", cn, san) + } +} + +func TestTLSSNI02ChallengeCert(t *testing.T) { + const ( + token = "evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA" + // echo -n evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA | shasum -a 256 + sanA = "7ea0aaa69214e71e02cebb18bb867736.09b730209baabf60e43d4999979ff139.token.acme.invalid" + // echo -n | shasum -a 256 + sanB = "dbbd5eefe7b4d06eb9d1d9f5acb4c7cd.a27d320e4b30332f0b6cb441734ad7b0.ka.acme.invalid" + ) + + tlscert, name, err := newTestClient().TLSSNI02ChallengeCert(token) + if err != nil { + t.Fatal(err) + } + + if n := len(tlscert.Certificate); n != 1 { + t.Fatalf("len(tlscert.Certificate) = %d; want 1", n) + } + cert, err := x509.ParseCertificate(tlscert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + names := []string{sanA, sanB} + if !reflect.DeepEqual(cert.DNSNames, names) { + t.Fatalf("cert.DNSNames = %v;\nwant %v", cert.DNSNames, names) + } + sort.Strings(cert.DNSNames) + i := sort.SearchStrings(cert.DNSNames, name) + if i >= len(cert.DNSNames) || cert.DNSNames[i] != name { + t.Errorf("%v doesn't have %q", cert.DNSNames, name) + } + if cn := cert.Subject.CommonName; cn != sanA { + t.Errorf("CommonName = %q; want %q", cn, sanA) + } +} + +func TestTLSALPN01ChallengeCert(t *testing.T) { + const ( + token = "evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA" + keyAuth = "evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA." + testKeyECThumbprint + // echo -n | shasum -a 256 + h = "0420dbbd5eefe7b4d06eb9d1d9f5acb4c7cda27d320e4b30332f0b6cb441734ad7b0" + domain = "example.com" + ) + + extValue, err := hex.DecodeString(h) + if err != nil { + t.Fatal(err) + } + + tlscert, err := newTestClient().TLSALPN01ChallengeCert(token, domain) + if err != nil { + t.Fatal(err) + } + + if n := len(tlscert.Certificate); n != 1 { + t.Fatalf("len(tlscert.Certificate) = %d; want 1", n) + } + cert, err := x509.ParseCertificate(tlscert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + names := []string{domain} + if !reflect.DeepEqual(cert.DNSNames, names) { + t.Fatalf("cert.DNSNames = %v;\nwant %v", cert.DNSNames, names) + } + if cn := cert.Subject.CommonName; cn != domain { + t.Errorf("CommonName = %q; want %q", cn, domain) + } + acmeExts := []pkix.Extension{} + for _, ext := range cert.Extensions { + if idPeACMEIdentifier.Equal(ext.Id) { + acmeExts = append(acmeExts, ext) + } + } + if len(acmeExts) != 1 { + t.Errorf("acmeExts = %v; want exactly one", acmeExts) + } + if !acmeExts[0].Critical { + t.Errorf("acmeExt.Critical = %v; want true", acmeExts[0].Critical) + } + if bytes.Compare(acmeExts[0].Value, extValue) != 0 { + t.Errorf("acmeExt.Value = %v; want %v", acmeExts[0].Value, extValue) + } + +} + +func TestTLSChallengeCertOpt(t *testing.T) { + key, err := rsa.GenerateKey(rand.Reader, 512) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{Organization: []string{"Test"}}, + DNSNames: []string{"should-be-overwritten"}, + } + opts := []CertOption{WithKey(key), WithTemplate(tmpl)} + + client := newTestClient() + cert1, _, err := client.TLSSNI01ChallengeCert("token", opts...) + if err != nil { + t.Fatal(err) + } + cert2, _, err := client.TLSSNI02ChallengeCert("token", opts...) + if err != nil { + t.Fatal(err) + } + + for i, tlscert := range []tls.Certificate{cert1, cert2} { + // verify generated cert private key + tlskey, ok := tlscert.PrivateKey.(*rsa.PrivateKey) + if !ok { + t.Errorf("%d: tlscert.PrivateKey is %T; want *rsa.PrivateKey", i, tlscert.PrivateKey) + continue + } + if tlskey.D.Cmp(key.D) != 0 { + t.Errorf("%d: tlskey.D = %v; want %v", i, tlskey.D, key.D) + } + // verify generated cert public key + x509Cert, err := x509.ParseCertificate(tlscert.Certificate[0]) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + tlspub, ok := x509Cert.PublicKey.(*rsa.PublicKey) + if !ok { + t.Errorf("%d: x509Cert.PublicKey is %T; want *rsa.PublicKey", i, x509Cert.PublicKey) + continue + } + if tlspub.N.Cmp(key.N) != 0 { + t.Errorf("%d: tlspub.N = %v; want %v", i, tlspub.N, key.N) + } + // verify template option + sn := big.NewInt(2) + if x509Cert.SerialNumber.Cmp(sn) != 0 { + t.Errorf("%d: SerialNumber = %v; want %v", i, x509Cert.SerialNumber, sn) + } + org := []string{"Test"} + if !reflect.DeepEqual(x509Cert.Subject.Organization, org) { + t.Errorf("%d: Subject.Organization = %+v; want %+v", i, x509Cert.Subject.Organization, org) + } + for _, v := range x509Cert.DNSNames { + if !strings.HasSuffix(v, ".acme.invalid") { + t.Errorf("%d: invalid DNSNames element: %q", i, v) + } + } + } +} + +func TestHTTP01Challenge(t *testing.T) { + const ( + token = "xxx" + // thumbprint is precomputed for testKeyEC in jws_test.go + value = token + "." + testKeyECThumbprint + urlpath = "/.well-known/acme-challenge/" + token + ) + client := newTestClient() + val, err := client.HTTP01ChallengeResponse(token) + if err != nil { + t.Fatal(err) + } + if val != value { + t.Errorf("val = %q; want %q", val, value) + } + if path := client.HTTP01ChallengePath(token); path != urlpath { + t.Errorf("path = %q; want %q", path, urlpath) + } +} + +func TestDNS01ChallengeRecord(t *testing.T) { + // echo -n xxx. | \ + // openssl dgst -binary -sha256 | \ + // base64 | tr -d '=' | tr '/+' '_-' + const value = "8DERMexQ5VcdJ_prpPiA0mVdp7imgbCgjsG4SqqNMIo" + + val, err := newTestClient().DNS01ChallengeRecord("xxx") + if err != nil { + t.Fatal(err) + } + if val != value { + t.Errorf("val = %q; want %q", val, value) + } +} diff --git a/tempfork/acme/http.go b/tempfork/acme/http.go new file mode 100644 index 0000000000000..58836e5d303e0 --- /dev/null +++ b/tempfork/acme/http.go @@ -0,0 +1,325 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "strconv" + "strings" + "time" +) + +// retryTimer encapsulates common logic for retrying unsuccessful requests. +// It is not safe for concurrent use. +type retryTimer struct { + // backoffFn provides backoff delay sequence for retries. + // See Client.RetryBackoff doc comment. + backoffFn func(n int, r *http.Request, res *http.Response) time.Duration + // n is the current retry attempt. + n int +} + +func (t *retryTimer) inc() { + t.n++ +} + +// backoff pauses the current goroutine as described in Client.RetryBackoff. +func (t *retryTimer) backoff(ctx context.Context, r *http.Request, res *http.Response) error { + d := t.backoffFn(t.n, r, res) + if d <= 0 { + return fmt.Errorf("acme: no more retries for %s; tried %d time(s)", r.URL, t.n) + } + wakeup := time.NewTimer(d) + defer wakeup.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-wakeup.C: + return nil + } +} + +func (c *Client) retryTimer() *retryTimer { + f := c.RetryBackoff + if f == nil { + f = defaultBackoff + } + return &retryTimer{backoffFn: f} +} + +// defaultBackoff provides default Client.RetryBackoff implementation +// using a truncated exponential backoff algorithm, +// as described in Client.RetryBackoff. +// +// The n argument is always bounded between 1 and 30. +// The returned value is always greater than 0. +func defaultBackoff(n int, r *http.Request, res *http.Response) time.Duration { + const max = 10 * time.Second + var jitter time.Duration + if x, err := rand.Int(rand.Reader, big.NewInt(1000)); err == nil { + // Set the minimum to 1ms to avoid a case where + // an invalid Retry-After value is parsed into 0 below, + // resulting in the 0 returned value which would unintentionally + // stop the retries. + jitter = (1 + time.Duration(x.Int64())) * time.Millisecond + } + if v, ok := res.Header["Retry-After"]; ok { + return retryAfter(v[0]) + jitter + } + + if n < 1 { + n = 1 + } + if n > 30 { + n = 30 + } + d := time.Duration(1< max { + return max + } + return d +} + +// retryAfter parses a Retry-After HTTP header value, +// trying to convert v into an int (seconds) or use http.ParseTime otherwise. +// It returns zero value if v cannot be parsed. +func retryAfter(v string) time.Duration { + if i, err := strconv.Atoi(v); err == nil { + return time.Duration(i) * time.Second + } + t, err := http.ParseTime(v) + if err != nil { + return 0 + } + return t.Sub(timeNow()) +} + +// resOkay is a function that reports whether the provided response is okay. +// It is expected to keep the response body unread. +type resOkay func(*http.Response) bool + +// wantStatus returns a function which reports whether the code +// matches the status code of a response. +func wantStatus(codes ...int) resOkay { + return func(res *http.Response) bool { + for _, code := range codes { + if code == res.StatusCode { + return true + } + } + return false + } +} + +// get issues an unsigned GET request to the specified URL. +// It returns a non-error value only when ok reports true. +// +// get retries unsuccessful attempts according to c.RetryBackoff +// until the context is done or a non-retriable error is received. +func (c *Client) get(ctx context.Context, url string, ok resOkay) (*http.Response, error) { + retry := c.retryTimer() + for { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + res, err := c.doNoRetry(ctx, req) + switch { + case err != nil: + return nil, err + case ok(res): + return res, nil + case isRetriable(res.StatusCode): + retry.inc() + resErr := responseError(res) + res.Body.Close() + // Ignore the error value from retry.backoff + // and return the one from last retry, as received from the CA. + if retry.backoff(ctx, req, res) != nil { + return nil, resErr + } + default: + defer res.Body.Close() + return nil, responseError(res) + } + } +} + +// postAsGet is POST-as-GET, a replacement for GET in RFC 8555 +// as described in https://tools.ietf.org/html/rfc8555#section-6.3. +// It makes a POST request in KID form with zero JWS payload. +// See nopayload doc comments in jws.go. +func (c *Client) postAsGet(ctx context.Context, url string, ok resOkay) (*http.Response, error) { + return c.post(ctx, nil, url, noPayload, ok) +} + +// post issues a signed POST request in JWS format using the provided key +// to the specified URL. If key is nil, c.Key is used instead. +// It returns a non-error value only when ok reports true. +// +// post retries unsuccessful attempts according to c.RetryBackoff +// until the context is done or a non-retriable error is received. +// It uses postNoRetry to make individual requests. +func (c *Client) post(ctx context.Context, key crypto.Signer, url string, body interface{}, ok resOkay) (*http.Response, error) { + retry := c.retryTimer() + for { + res, req, err := c.postNoRetry(ctx, key, url, body) + if err != nil { + return nil, err + } + if ok(res) { + return res, nil + } + resErr := responseError(res) + res.Body.Close() + switch { + // Check for bad nonce before isRetriable because it may have been returned + // with an unretriable response code such as 400 Bad Request. + case isBadNonce(resErr): + // Consider any previously stored nonce values to be invalid. + c.clearNonces() + case !isRetriable(res.StatusCode): + return nil, resErr + } + retry.inc() + // Ignore the error value from retry.backoff + // and return the one from last retry, as received from the CA. + if err := retry.backoff(ctx, req, res); err != nil { + return nil, resErr + } + } +} + +// postNoRetry signs the body with the given key and POSTs it to the provided url. +// It is used by c.post to retry unsuccessful attempts. +// The body argument must be JSON-serializable. +// +// If key argument is nil, c.Key is used to sign the request. +// If key argument is nil and c.accountKID returns a non-zero keyID, +// the request is sent in KID form. Otherwise, JWK form is used. +// +// In practice, when interfacing with RFC-compliant CAs most requests are sent in KID form +// and JWK is used only when KID is unavailable: new account endpoint and certificate +// revocation requests authenticated by a cert key. +// See jwsEncodeJSON for other details. +func (c *Client) postNoRetry(ctx context.Context, key crypto.Signer, url string, body interface{}) (*http.Response, *http.Request, error) { + kid := noKeyID + if key == nil { + if c.Key == nil { + return nil, nil, errors.New("acme: Client.Key must be populated to make POST requests") + } + key = c.Key + kid = c.accountKID(ctx) + } + nonce, err := c.popNonce(ctx, url) + if err != nil { + return nil, nil, err + } + b, err := jwsEncodeJSON(body, key, kid, nonce, url) + if err != nil { + return nil, nil, err + } + req, err := http.NewRequest("POST", url, bytes.NewReader(b)) + if err != nil { + return nil, nil, err + } + req.Header.Set("Content-Type", "application/jose+json") + res, err := c.doNoRetry(ctx, req) + if err != nil { + return nil, nil, err + } + c.addNonce(res.Header) + return res, req, nil +} + +// doNoRetry issues a request req, replacing its context (if any) with ctx. +func (c *Client) doNoRetry(ctx context.Context, req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", c.userAgent()) + res, err := c.httpClient().Do(req.WithContext(ctx)) + if err != nil { + select { + case <-ctx.Done(): + // Prefer the unadorned context error. + // (The acme package had tests assuming this, previously from ctxhttp's + // behavior, predating net/http supporting contexts natively) + // TODO(bradfitz): reconsider this in the future. But for now this + // requires no test updates. + return nil, ctx.Err() + default: + return nil, err + } + } + return res, nil +} + +func (c *Client) httpClient() *http.Client { + if c.HTTPClient != nil { + return c.HTTPClient + } + return http.DefaultClient +} + +// packageVersion is the version of the module that contains this package, for +// sending as part of the User-Agent header. It's set in version_go112.go. +var packageVersion string + +// userAgent returns the User-Agent header value. It includes the package name, +// the module version (if available), and the c.UserAgent value (if set). +func (c *Client) userAgent() string { + ua := "golang.org/x/crypto/acme" + if packageVersion != "" { + ua += "@" + packageVersion + } + if c.UserAgent != "" { + ua = c.UserAgent + " " + ua + } + return ua +} + +// isBadNonce reports whether err is an ACME "badnonce" error. +func isBadNonce(err error) bool { + // According to the spec badNonce is urn:ietf:params:acme:error:badNonce. + // However, ACME servers in the wild return their versions of the error. + // See https://tools.ietf.org/html/draft-ietf-acme-acme-02#section-5.4 + // and https://github.com/letsencrypt/boulder/blob/0e07eacb/docs/acme-divergences.md#section-66. + ae, ok := err.(*Error) + return ok && strings.HasSuffix(strings.ToLower(ae.ProblemType), ":badnonce") +} + +// isRetriable reports whether a request can be retried +// based on the response status code. +// +// Note that a "bad nonce" error is returned with a non-retriable 400 Bad Request code. +// Callers should parse the response and check with isBadNonce. +func isRetriable(code int) bool { + return code <= 399 || code >= 500 || code == http.StatusTooManyRequests +} + +// responseError creates an error of Error type from resp. +func responseError(resp *http.Response) error { + // don't care if ReadAll returns an error: + // json.Unmarshal will fail in that case anyway + b, _ := io.ReadAll(resp.Body) + e := &wireError{Status: resp.StatusCode} + if err := json.Unmarshal(b, e); err != nil { + // this is not a regular error response: + // populate detail with anything we received, + // e.Status will already contain HTTP response code value + e.Detail = string(b) + if e.Detail == "" { + e.Detail = resp.Status + } + } + return e.error(resp.Header) +} diff --git a/tempfork/acme/http_test.go b/tempfork/acme/http_test.go new file mode 100644 index 0000000000000..d124e4e219abe --- /dev/null +++ b/tempfork/acme/http_test.go @@ -0,0 +1,255 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +func TestDefaultBackoff(t *testing.T) { + tt := []struct { + nretry int + retryAfter string // Retry-After header + out time.Duration // expected min; max = min + jitter + }{ + {-1, "", time.Second}, // verify the lower bound is 1 + {0, "", time.Second}, // verify the lower bound is 1 + {100, "", 10 * time.Second}, // verify the ceiling + {1, "3600", time.Hour}, // verify the header value is used + {1, "", 1 * time.Second}, + {2, "", 2 * time.Second}, + {3, "", 4 * time.Second}, + {4, "", 8 * time.Second}, + } + for i, test := range tt { + r := httptest.NewRequest("GET", "/", nil) + resp := &http.Response{Header: http.Header{}} + if test.retryAfter != "" { + resp.Header.Set("Retry-After", test.retryAfter) + } + d := defaultBackoff(test.nretry, r, resp) + max := test.out + time.Second // + max jitter + if d < test.out || max < d { + t.Errorf("%d: defaultBackoff(%v) = %v; want between %v and %v", i, test.nretry, d, test.out, max) + } + } +} + +func TestErrorResponse(t *testing.T) { + s := `{ + "status": 400, + "type": "urn:acme:error:xxx", + "detail": "text" + }` + res := &http.Response{ + StatusCode: 400, + Status: "400 Bad Request", + Body: io.NopCloser(strings.NewReader(s)), + Header: http.Header{"X-Foo": {"bar"}}, + } + err := responseError(res) + v, ok := err.(*Error) + if !ok { + t.Fatalf("err = %+v (%T); want *Error type", err, err) + } + if v.StatusCode != 400 { + t.Errorf("v.StatusCode = %v; want 400", v.StatusCode) + } + if v.ProblemType != "urn:acme:error:xxx" { + t.Errorf("v.ProblemType = %q; want urn:acme:error:xxx", v.ProblemType) + } + if v.Detail != "text" { + t.Errorf("v.Detail = %q; want text", v.Detail) + } + if !reflect.DeepEqual(v.Header, res.Header) { + t.Errorf("v.Header = %+v; want %+v", v.Header, res.Header) + } +} + +func TestPostWithRetries(t *testing.T) { + var count int + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count++ + w.Header().Set("Replay-Nonce", fmt.Sprintf("nonce%d", count)) + if r.Method == "HEAD" { + // We expect the client to do 2 head requests to fetch + // nonces, one to start and another after getting badNonce + return + } + + head, err := decodeJWSHead(r.Body) + switch { + case err != nil: + t.Errorf("decodeJWSHead: %v", err) + case head.Nonce == "": + t.Error("head.Nonce is empty") + case head.Nonce == "nonce1": + // Return a badNonce error to force the call to retry. + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"type":"urn:ietf:params:acme:error:badNonce"}`)) + return + } + // Make client.Authorize happy; we're not testing its result. + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"status":"valid"}`)) + })) + defer ts.Close() + + client := &Client{ + Key: testKey, + DirectoryURL: ts.URL, + dir: &Directory{AuthzURL: ts.URL}, + } + // This call will fail with badNonce, causing a retry + if _, err := client.Authorize(context.Background(), "example.com"); err != nil { + t.Errorf("client.Authorize 1: %v", err) + } + if count != 3 { + t.Errorf("total requests count: %d; want 3", count) + } +} + +func TestRetryErrorType(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Replay-Nonce", "nonce") + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte(`{"type":"rateLimited"}`)) + })) + defer ts.Close() + + client := &Client{ + Key: testKey, + RetryBackoff: func(n int, r *http.Request, res *http.Response) time.Duration { + // Do no retries. + return 0 + }, + dir: &Directory{AuthzURL: ts.URL}, + } + + t.Run("post", func(t *testing.T) { + testRetryErrorType(t, func() error { + _, err := client.Authorize(context.Background(), "example.com") + return err + }) + }) + t.Run("get", func(t *testing.T) { + testRetryErrorType(t, func() error { + _, err := client.GetAuthorization(context.Background(), ts.URL) + return err + }) + }) +} + +func testRetryErrorType(t *testing.T, callClient func() error) { + t.Helper() + err := callClient() + if err == nil { + t.Fatal("client.Authorize returned nil error") + } + acmeErr, ok := err.(*Error) + if !ok { + t.Fatalf("err is %v (%T); want *Error", err, err) + } + if acmeErr.StatusCode != http.StatusTooManyRequests { + t.Errorf("acmeErr.StatusCode = %d; want %d", acmeErr.StatusCode, http.StatusTooManyRequests) + } + if acmeErr.ProblemType != "rateLimited" { + t.Errorf("acmeErr.ProblemType = %q; want 'rateLimited'", acmeErr.ProblemType) + } +} + +func TestRetryBackoffArgs(t *testing.T) { + const resCode = http.StatusInternalServerError + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Replay-Nonce", "test-nonce") + w.WriteHeader(resCode) + })) + defer ts.Close() + + // Canceled in backoff. + ctx, cancel := context.WithCancel(context.Background()) + + var nretry int + backoff := func(n int, r *http.Request, res *http.Response) time.Duration { + nretry++ + if n != nretry { + t.Errorf("n = %d; want %d", n, nretry) + } + if nretry == 3 { + cancel() + } + + if r == nil { + t.Error("r is nil") + } + if res.StatusCode != resCode { + t.Errorf("res.StatusCode = %d; want %d", res.StatusCode, resCode) + } + return time.Millisecond + } + + client := &Client{ + Key: testKey, + RetryBackoff: backoff, + dir: &Directory{AuthzURL: ts.URL}, + } + if _, err := client.Authorize(ctx, "example.com"); err == nil { + t.Error("err is nil") + } + if nretry != 3 { + t.Errorf("nretry = %d; want 3", nretry) + } +} + +func TestUserAgent(t *testing.T) { + for _, custom := range []string{"", "CUSTOM_UA"} { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Log(r.UserAgent()) + if s := "golang.org/x/crypto/acme"; !strings.Contains(r.UserAgent(), s) { + t.Errorf("expected User-Agent to contain %q, got %q", s, r.UserAgent()) + } + if !strings.Contains(r.UserAgent(), custom) { + t.Errorf("expected User-Agent to contain %q, got %q", custom, r.UserAgent()) + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"newOrder": "sure"}`)) + })) + defer ts.Close() + + client := &Client{ + Key: testKey, + DirectoryURL: ts.URL, + UserAgent: custom, + } + if _, err := client.Discover(context.Background()); err != nil { + t.Errorf("client.Discover: %v", err) + } + } +} + +func TestAccountKidLoop(t *testing.T) { + // if Client.postNoRetry is called with a nil key argument + // then Client.Key must be set, otherwise we fall into an + // infinite loop (which also causes a deadlock). + client := &Client{dir: &Directory{OrderURL: ":)"}} + _, _, err := client.postNoRetry(context.Background(), nil, "", nil) + if err == nil { + t.Fatal("Client.postNoRetry didn't fail with a nil key") + } + expected := "acme: Client.Key must be populated to make POST requests" + if err.Error() != expected { + t.Fatalf("Unexpected error returned: wanted %q, got %q", expected, err.Error()) + } +} diff --git a/tempfork/acme/jws.go b/tempfork/acme/jws.go new file mode 100644 index 0000000000000..b38828d85935c --- /dev/null +++ b/tempfork/acme/jws.go @@ -0,0 +1,257 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "crypto" + "crypto/ecdsa" + "crypto/hmac" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + _ "crypto/sha512" // need for EC keys + "encoding/asn1" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math/big" +) + +// KeyID is the account key identity provided by a CA during registration. +type KeyID string + +// noKeyID indicates that jwsEncodeJSON should compute and use JWK instead of a KID. +// See jwsEncodeJSON for details. +const noKeyID = KeyID("") + +// noPayload indicates jwsEncodeJSON will encode zero-length octet string +// in a JWS request. This is called POST-as-GET in RFC 8555 and is used to make +// authenticated GET requests via POSTing with an empty payload. +// See https://tools.ietf.org/html/rfc8555#section-6.3 for more details. +const noPayload = "" + +// noNonce indicates that the nonce should be omitted from the protected header. +// See jwsEncodeJSON for details. +const noNonce = "" + +// jsonWebSignature can be easily serialized into a JWS following +// https://tools.ietf.org/html/rfc7515#section-3.2. +type jsonWebSignature struct { + Protected string `json:"protected"` + Payload string `json:"payload"` + Sig string `json:"signature"` +} + +// jwsEncodeJSON signs claimset using provided key and a nonce. +// The result is serialized in JSON format containing either kid or jwk +// fields based on the provided KeyID value. +// +// The claimset is marshalled using json.Marshal unless it is a string. +// In which case it is inserted directly into the message. +// +// If kid is non-empty, its quoted value is inserted in the protected header +// as "kid" field value. Otherwise, JWK is computed using jwkEncode and inserted +// as "jwk" field value. The "jwk" and "kid" fields are mutually exclusive. +// +// If nonce is non-empty, its quoted value is inserted in the protected header. +// +// See https://tools.ietf.org/html/rfc7515#section-7. +func jwsEncodeJSON(claimset interface{}, key crypto.Signer, kid KeyID, nonce, url string) ([]byte, error) { + if key == nil { + return nil, errors.New("nil key") + } + alg, sha := jwsHasher(key.Public()) + if alg == "" || !sha.Available() { + return nil, ErrUnsupportedKey + } + headers := struct { + Alg string `json:"alg"` + KID string `json:"kid,omitempty"` + JWK json.RawMessage `json:"jwk,omitempty"` + Nonce string `json:"nonce,omitempty"` + URL string `json:"url"` + }{ + Alg: alg, + Nonce: nonce, + URL: url, + } + switch kid { + case noKeyID: + jwk, err := jwkEncode(key.Public()) + if err != nil { + return nil, err + } + headers.JWK = json.RawMessage(jwk) + default: + headers.KID = string(kid) + } + phJSON, err := json.Marshal(headers) + if err != nil { + return nil, err + } + phead := base64.RawURLEncoding.EncodeToString([]byte(phJSON)) + var payload string + if val, ok := claimset.(string); ok { + payload = val + } else { + cs, err := json.Marshal(claimset) + if err != nil { + return nil, err + } + payload = base64.RawURLEncoding.EncodeToString(cs) + } + hash := sha.New() + hash.Write([]byte(phead + "." + payload)) + sig, err := jwsSign(key, sha, hash.Sum(nil)) + if err != nil { + return nil, err + } + enc := jsonWebSignature{ + Protected: phead, + Payload: payload, + Sig: base64.RawURLEncoding.EncodeToString(sig), + } + return json.Marshal(&enc) +} + +// jwsWithMAC creates and signs a JWS using the given key and the HS256 +// algorithm. kid and url are included in the protected header. rawPayload +// should not be base64-URL-encoded. +func jwsWithMAC(key []byte, kid, url string, rawPayload []byte) (*jsonWebSignature, error) { + if len(key) == 0 { + return nil, errors.New("acme: cannot sign JWS with an empty MAC key") + } + header := struct { + Algorithm string `json:"alg"` + KID string `json:"kid"` + URL string `json:"url,omitempty"` + }{ + // Only HMAC-SHA256 is supported. + Algorithm: "HS256", + KID: kid, + URL: url, + } + rawProtected, err := json.Marshal(header) + if err != nil { + return nil, err + } + protected := base64.RawURLEncoding.EncodeToString(rawProtected) + payload := base64.RawURLEncoding.EncodeToString(rawPayload) + + h := hmac.New(sha256.New, key) + if _, err := h.Write([]byte(protected + "." + payload)); err != nil { + return nil, err + } + mac := h.Sum(nil) + + return &jsonWebSignature{ + Protected: protected, + Payload: payload, + Sig: base64.RawURLEncoding.EncodeToString(mac), + }, nil +} + +// jwkEncode encodes public part of an RSA or ECDSA key into a JWK. +// The result is also suitable for creating a JWK thumbprint. +// https://tools.ietf.org/html/rfc7517 +func jwkEncode(pub crypto.PublicKey) (string, error) { + switch pub := pub.(type) { + case *rsa.PublicKey: + // https://tools.ietf.org/html/rfc7518#section-6.3.1 + n := pub.N + e := big.NewInt(int64(pub.E)) + // Field order is important. + // See https://tools.ietf.org/html/rfc7638#section-3.3 for details. + return fmt.Sprintf(`{"e":"%s","kty":"RSA","n":"%s"}`, + base64.RawURLEncoding.EncodeToString(e.Bytes()), + base64.RawURLEncoding.EncodeToString(n.Bytes()), + ), nil + case *ecdsa.PublicKey: + // https://tools.ietf.org/html/rfc7518#section-6.2.1 + p := pub.Curve.Params() + n := p.BitSize / 8 + if p.BitSize%8 != 0 { + n++ + } + x := pub.X.Bytes() + if n > len(x) { + x = append(make([]byte, n-len(x)), x...) + } + y := pub.Y.Bytes() + if n > len(y) { + y = append(make([]byte, n-len(y)), y...) + } + // Field order is important. + // See https://tools.ietf.org/html/rfc7638#section-3.3 for details. + return fmt.Sprintf(`{"crv":"%s","kty":"EC","x":"%s","y":"%s"}`, + p.Name, + base64.RawURLEncoding.EncodeToString(x), + base64.RawURLEncoding.EncodeToString(y), + ), nil + } + return "", ErrUnsupportedKey +} + +// jwsSign signs the digest using the given key. +// The hash is unused for ECDSA keys. +func jwsSign(key crypto.Signer, hash crypto.Hash, digest []byte) ([]byte, error) { + switch pub := key.Public().(type) { + case *rsa.PublicKey: + return key.Sign(rand.Reader, digest, hash) + case *ecdsa.PublicKey: + sigASN1, err := key.Sign(rand.Reader, digest, hash) + if err != nil { + return nil, err + } + + var rs struct{ R, S *big.Int } + if _, err := asn1.Unmarshal(sigASN1, &rs); err != nil { + return nil, err + } + + rb, sb := rs.R.Bytes(), rs.S.Bytes() + size := pub.Params().BitSize / 8 + if size%8 > 0 { + size++ + } + sig := make([]byte, size*2) + copy(sig[size-len(rb):], rb) + copy(sig[size*2-len(sb):], sb) + return sig, nil + } + return nil, ErrUnsupportedKey +} + +// jwsHasher indicates suitable JWS algorithm name and a hash function +// to use for signing a digest with the provided key. +// It returns ("", 0) if the key is not supported. +func jwsHasher(pub crypto.PublicKey) (string, crypto.Hash) { + switch pub := pub.(type) { + case *rsa.PublicKey: + return "RS256", crypto.SHA256 + case *ecdsa.PublicKey: + switch pub.Params().Name { + case "P-256": + return "ES256", crypto.SHA256 + case "P-384": + return "ES384", crypto.SHA384 + case "P-521": + return "ES512", crypto.SHA512 + } + } + return "", 0 +} + +// JWKThumbprint creates a JWK thumbprint out of pub +// as specified in https://tools.ietf.org/html/rfc7638. +func JWKThumbprint(pub crypto.PublicKey) (string, error) { + jwk, err := jwkEncode(pub) + if err != nil { + return "", err + } + b := sha256.Sum256([]byte(jwk)) + return base64.RawURLEncoding.EncodeToString(b[:]), nil +} diff --git a/tempfork/acme/jws_test.go b/tempfork/acme/jws_test.go new file mode 100644 index 0000000000000..d5f00ba2d3245 --- /dev/null +++ b/tempfork/acme/jws_test.go @@ -0,0 +1,550 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "math/big" + "testing" +) + +// The following shell command alias is used in the comments +// throughout this file: +// alias b64raw="base64 -w0 | tr -d '=' | tr '/+' '_-'" + +const ( + // Modulus in raw base64: + // 4xgZ3eRPkwoRvy7qeRUbmMDe0V-xH9eWLdu0iheeLlrmD2mqWXfP9IeSKApbn34 + // g8TuAS9g5zhq8ELQ3kmjr-KV86GAMgI6VAcGlq3QrzpTCf_30Ab7-zawrfRaFON + // a1HwEzPY1KHnGVkxJc85gNkwYI9SY2RHXtvln3zs5wITNrdosqEXeaIkVYBEhbh + // Nu54pp3kxo6TuWLi9e6pXeWetEwmlBwtWZlPoib2j3TxLBksKZfoyFyek380mHg + // JAumQ_I2fjj98_97mk3ihOY4AgVdCDj1z_GCoZkG5Rq7nbCGyosyKWyDX00Zs-n + // NqVhoLeIvXC4nnWdJMZ6rogxyQQ + testKeyPEM = ` +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA4xgZ3eRPkwoRvy7qeRUbmMDe0V+xH9eWLdu0iheeLlrmD2mq +WXfP9IeSKApbn34g8TuAS9g5zhq8ELQ3kmjr+KV86GAMgI6VAcGlq3QrzpTCf/30 +Ab7+zawrfRaFONa1HwEzPY1KHnGVkxJc85gNkwYI9SY2RHXtvln3zs5wITNrdosq +EXeaIkVYBEhbhNu54pp3kxo6TuWLi9e6pXeWetEwmlBwtWZlPoib2j3TxLBksKZf +oyFyek380mHgJAumQ/I2fjj98/97mk3ihOY4AgVdCDj1z/GCoZkG5Rq7nbCGyosy +KWyDX00Zs+nNqVhoLeIvXC4nnWdJMZ6rogxyQQIDAQABAoIBACIEZTOI1Kao9nmV +9IeIsuaR1Y61b9neOF/MLmIVIZu+AAJFCMB4Iw11FV6sFodwpEyeZhx2WkpWVN+H +r19eGiLX3zsL0DOdqBJoSIHDWCCMxgnYJ6nvS0nRxX3qVrBp8R2g12Ub+gNPbmFm +ecf/eeERIVxfifd9VsyRu34eDEvcmKFuLYbElFcPh62xE3x12UZvV/sN7gXbawpP +G+w255vbE5MoaKdnnO83cTFlcHvhn24M/78qP7Te5OAeelr1R89kYxQLpuGe4fbS +zc6E3ym5Td6urDetGGrSY1Eu10/8sMusX+KNWkm+RsBRbkyKq72ks/qKpOxOa+c6 +9gm+Y8ECgYEA/iNUyg1ubRdH11p82l8KHtFC1DPE0V1gSZsX29TpM5jS4qv46K+s +8Ym1zmrORM8x+cynfPx1VQZQ34EYeCMIX212ryJ+zDATl4NE0I4muMvSiH9vx6Xc +7FmhNnaYzPsBL5Tm9nmtQuP09YEn8poiOJFiDs/4olnD5ogA5O4THGkCgYEA5MIL +qWYBUuqbEWLRtMruUtpASclrBqNNsJEsMGbeqBJmoMxdHeSZckbLOrqm7GlMyNRJ +Ne/5uWRGSzaMYuGmwsPpERzqEvYFnSrpjW5YtXZ+JtxFXNVfm9Z1gLLgvGpOUCIU +RbpoDckDe1vgUuk3y5+DjZihs+rqIJ45XzXTzBkCgYBWuf3segruJZy5rEKhTv+o +JqeUvRn0jNYYKFpLBeyTVBrbie6GkbUGNIWbrK05pC+c3K9nosvzuRUOQQL1tJbd +4gA3oiD9U4bMFNr+BRTHyZ7OQBcIXdz3t1qhuHVKtnngIAN1p25uPlbRFUNpshnt +jgeVoHlsBhApcs5DUc+pyQKBgDzeHPg/+g4z+nrPznjKnktRY1W+0El93kgi+J0Q +YiJacxBKEGTJ1MKBb8X6sDurcRDm22wMpGfd9I5Cv2v4GsUsF7HD/cx5xdih+G73 +c4clNj/k0Ff5Nm1izPUno4C+0IOl7br39IPmfpSuR6wH/h6iHQDqIeybjxyKvT1G +N0rRAoGBAKGD+4ZI/E1MoJ5CXB8cDDMHagbE3cq/DtmYzE2v1DFpQYu5I4PCm5c7 +EQeIP6dZtv8IMgtGIb91QX9pXvP0aznzQKwYIA8nZgoENCPfiMTPiEDT9e/0lObO +9XWsXpbSTsRPj0sv1rB+UzBJ0PgjK4q2zOF0sNo7b1+6nlM3BWPx +-----END RSA PRIVATE KEY----- +` + + // This thumbprint is for the testKey defined above. + testKeyThumbprint = "6nicxzh6WETQlrvdchkz-U3e3DOQZ4heJKU63rfqMqQ" + + // openssl ecparam -name secp256k1 -genkey -noout + testKeyECPEM = ` +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIK07hGLr0RwyUdYJ8wbIiBS55CjnkMD23DWr+ccnypWLoAoGCCqGSM49 +AwEHoUQDQgAE5lhEug5xK4xBDZ2nAbaxLtaLiv85bxJ7ePd1dkO23HThqIrvawF5 +QAaS/RNouybCiRhRjI3EaxLkQwgrCw0gqQ== +-----END EC PRIVATE KEY----- +` + // openssl ecparam -name secp384r1 -genkey -noout + testKeyEC384PEM = ` +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDAQ4lNtXRORWr1bgKR1CGysr9AJ9SyEk4jiVnlUWWUChmSNL+i9SLSD +Oe/naPqXJ6CgBwYFK4EEACKhZANiAAQzKtj+Ms0vHoTX5dzv3/L5YMXOWuI5UKRj +JigpahYCqXD2BA1j0E/2xt5vlPf+gm0PL+UHSQsCokGnIGuaHCsJAp3ry0gHQEke +WYXapUUFdvaK1R2/2hn5O+eiQM8YzCg= +-----END EC PRIVATE KEY----- +` + // openssl ecparam -name secp521r1 -genkey -noout + testKeyEC512PEM = ` +-----BEGIN EC PRIVATE KEY----- +MIHcAgEBBEIBSNZKFcWzXzB/aJClAb305ibalKgtDA7+70eEkdPt28/3LZMM935Z +KqYHh/COcxuu3Kt8azRAUz3gyr4zZKhlKUSgBwYFK4EEACOhgYkDgYYABAHUNKbx +7JwC7H6pa2sV0tERWhHhB3JmW+OP6SUgMWryvIKajlx73eS24dy4QPGrWO9/ABsD +FqcRSkNVTXnIv6+0mAF25knqIBIg5Q8M9BnOu9GGAchcwt3O7RDHmqewnJJDrbjd +GGnm6rb+NnWR9DIopM0nKNkToWoF/hzopxu4Ae/GsQ== +-----END EC PRIVATE KEY----- +` + // 1. openssl ec -in key.pem -noout -text + // 2. remove first byte, 04 (the header); the rest is X and Y + // 3. convert each with: echo | xxd -r -p | b64raw + testKeyECPubX = "5lhEug5xK4xBDZ2nAbaxLtaLiv85bxJ7ePd1dkO23HQ" + testKeyECPubY = "4aiK72sBeUAGkv0TaLsmwokYUYyNxGsS5EMIKwsNIKk" + testKeyEC384PubX = "MyrY_jLNLx6E1-Xc79_y-WDFzlriOVCkYyYoKWoWAqlw9gQNY9BP9sbeb5T3_oJt" + testKeyEC384PubY = "Dy_lB0kLAqJBpyBrmhwrCQKd68tIB0BJHlmF2qVFBXb2itUdv9oZ-TvnokDPGMwo" + testKeyEC512PubX = "AdQ0pvHsnALsfqlraxXS0RFaEeEHcmZb44_pJSAxavK8gpqOXHvd5Lbh3LhA8atY738AGwMWpxFKQ1VNeci_r7SY" + testKeyEC512PubY = "AXbmSeogEiDlDwz0Gc670YYByFzC3c7tEMeap7CckkOtuN0Yaebqtv42dZH0MiikzSco2ROhagX-HOinG7gB78ax" + + // echo -n '{"crv":"P-256","kty":"EC","x":"","y":""}' | \ + // openssl dgst -binary -sha256 | b64raw + testKeyECThumbprint = "zedj-Bd1Zshp8KLePv2MB-lJ_Hagp7wAwdkA0NUTniU" +) + +var ( + testKey *rsa.PrivateKey + testKeyEC *ecdsa.PrivateKey + testKeyEC384 *ecdsa.PrivateKey + testKeyEC512 *ecdsa.PrivateKey +) + +func init() { + testKey = parseRSA(testKeyPEM, "testKeyPEM") + testKeyEC = parseEC(testKeyECPEM, "testKeyECPEM") + testKeyEC384 = parseEC(testKeyEC384PEM, "testKeyEC384PEM") + testKeyEC512 = parseEC(testKeyEC512PEM, "testKeyEC512PEM") +} + +func decodePEM(s, name string) []byte { + d, _ := pem.Decode([]byte(s)) + if d == nil { + panic("no block found in " + name) + } + return d.Bytes +} + +func parseRSA(s, name string) *rsa.PrivateKey { + b := decodePEM(s, name) + k, err := x509.ParsePKCS1PrivateKey(b) + if err != nil { + panic(fmt.Sprintf("%s: %v", name, err)) + } + return k +} + +func parseEC(s, name string) *ecdsa.PrivateKey { + b := decodePEM(s, name) + k, err := x509.ParseECPrivateKey(b) + if err != nil { + panic(fmt.Sprintf("%s: %v", name, err)) + } + return k +} + +func TestJWSEncodeJSON(t *testing.T) { + claims := struct{ Msg string }{"Hello JWS"} + // JWS signed with testKey and "nonce" as the nonce value + // JSON-serialized JWS fields are split for easier testing + const ( + // {"alg":"RS256","jwk":{"e":"AQAB","kty":"RSA","n":"..."},"nonce":"nonce","url":"url"} + protected = "eyJhbGciOiJSUzI1NiIsImp3ayI6eyJlIjoiQVFBQiIsImt0eSI6" + + "IlJTQSIsIm4iOiI0eGdaM2VSUGt3b1J2eTdxZVJVYm1NRGUwVi14" + + "SDllV0xkdTBpaGVlTGxybUQybXFXWGZQOUllU0tBcGJuMzRnOFR1" + + "QVM5ZzV6aHE4RUxRM2ttanItS1Y4NkdBTWdJNlZBY0dscTNRcnpw" + + "VENmXzMwQWI3LXphd3JmUmFGT05hMUh3RXpQWTFLSG5HVmt4SmM4" + + "NWdOa3dZSTlTWTJSSFh0dmxuM3pzNXdJVE5yZG9zcUVYZWFJa1ZZ" + + "QkVoYmhOdTU0cHAza3hvNlR1V0xpOWU2cFhlV2V0RXdtbEJ3dFda" + + "bFBvaWIyajNUeExCa3NLWmZveUZ5ZWszODBtSGdKQXVtUV9JMmZq" + + "ajk4Xzk3bWszaWhPWTRBZ1ZkQ0RqMXpfR0NvWmtHNVJxN25iQ0d5" + + "b3N5S1d5RFgwMFpzLW5OcVZob0xlSXZYQzRubldkSk1aNnJvZ3h5" + + "UVEifSwibm9uY2UiOiJub25jZSIsInVybCI6InVybCJ9" + // {"Msg":"Hello JWS"} + payload = "eyJNc2ciOiJIZWxsbyBKV1MifQ" + // printf '.' | openssl dgst -binary -sha256 -sign testKey | b64raw + signature = "YFyl_xz1E7TR-3E1bIuASTr424EgCvBHjt25WUFC2VaDjXYV0Rj_" + + "Hd3dJ_2IRqBrXDZZ2n4ZeA_4mm3QFwmwyeDwe2sWElhb82lCZ8iX" + + "uFnjeOmSOjx-nWwPa5ibCXzLq13zZ-OBV1Z4oN_TuailQeRoSfA3" + + "nO8gG52mv1x2OMQ5MAFtt8jcngBLzts4AyhI6mBJ2w7Yaj3ZCriq" + + "DWA3GLFvvHdW1Ba9Z01wtGT2CuZI7DUk_6Qj1b3BkBGcoKur5C9i" + + "bUJtCkABwBMvBQNyD3MmXsrRFRTgvVlyU_yMaucYm7nmzEr_2PaQ" + + "50rFt_9qOfJ4sfbLtG1Wwae57BQx1g" + ) + + b, err := jwsEncodeJSON(claims, testKey, noKeyID, "nonce", "url") + if err != nil { + t.Fatal(err) + } + var jws struct{ Protected, Payload, Signature string } + if err := json.Unmarshal(b, &jws); err != nil { + t.Fatal(err) + } + if jws.Protected != protected { + t.Errorf("protected:\n%s\nwant:\n%s", jws.Protected, protected) + } + if jws.Payload != payload { + t.Errorf("payload:\n%s\nwant:\n%s", jws.Payload, payload) + } + if jws.Signature != signature { + t.Errorf("signature:\n%s\nwant:\n%s", jws.Signature, signature) + } +} + +func TestJWSEncodeNoNonce(t *testing.T) { + kid := KeyID("https://example.org/account/1") + claims := "RawString" + const ( + // {"alg":"ES256","kid":"https://example.org/account/1","nonce":"nonce","url":"url"} + protected = "eyJhbGciOiJFUzI1NiIsImtpZCI6Imh0dHBzOi8vZXhhbXBsZS5vcmcvYWNjb3VudC8xIiwidXJsIjoidXJsIn0" + // "Raw String" + payload = "RawString" + ) + + b, err := jwsEncodeJSON(claims, testKeyEC, kid, "", "url") + if err != nil { + t.Fatal(err) + } + var jws struct{ Protected, Payload, Signature string } + if err := json.Unmarshal(b, &jws); err != nil { + t.Fatal(err) + } + if jws.Protected != protected { + t.Errorf("protected:\n%s\nwant:\n%s", jws.Protected, protected) + } + if jws.Payload != payload { + t.Errorf("payload:\n%s\nwant:\n%s", jws.Payload, payload) + } + + sig, err := base64.RawURLEncoding.DecodeString(jws.Signature) + if err != nil { + t.Fatalf("jws.Signature: %v", err) + } + r, s := big.NewInt(0), big.NewInt(0) + r.SetBytes(sig[:len(sig)/2]) + s.SetBytes(sig[len(sig)/2:]) + h := sha256.Sum256([]byte(protected + "." + payload)) + if !ecdsa.Verify(testKeyEC.Public().(*ecdsa.PublicKey), h[:], r, s) { + t.Error("invalid signature") + } +} + +func TestJWSEncodeKID(t *testing.T) { + kid := KeyID("https://example.org/account/1") + claims := struct{ Msg string }{"Hello JWS"} + // JWS signed with testKeyEC + const ( + // {"alg":"ES256","kid":"https://example.org/account/1","nonce":"nonce","url":"url"} + protected = "eyJhbGciOiJFUzI1NiIsImtpZCI6Imh0dHBzOi8vZXhhbXBsZS5" + + "vcmcvYWNjb3VudC8xIiwibm9uY2UiOiJub25jZSIsInVybCI6InVybCJ9" + // {"Msg":"Hello JWS"} + payload = "eyJNc2ciOiJIZWxsbyBKV1MifQ" + ) + + b, err := jwsEncodeJSON(claims, testKeyEC, kid, "nonce", "url") + if err != nil { + t.Fatal(err) + } + var jws struct{ Protected, Payload, Signature string } + if err := json.Unmarshal(b, &jws); err != nil { + t.Fatal(err) + } + if jws.Protected != protected { + t.Errorf("protected:\n%s\nwant:\n%s", jws.Protected, protected) + } + if jws.Payload != payload { + t.Errorf("payload:\n%s\nwant:\n%s", jws.Payload, payload) + } + + sig, err := base64.RawURLEncoding.DecodeString(jws.Signature) + if err != nil { + t.Fatalf("jws.Signature: %v", err) + } + r, s := big.NewInt(0), big.NewInt(0) + r.SetBytes(sig[:len(sig)/2]) + s.SetBytes(sig[len(sig)/2:]) + h := sha256.Sum256([]byte(protected + "." + payload)) + if !ecdsa.Verify(testKeyEC.Public().(*ecdsa.PublicKey), h[:], r, s) { + t.Error("invalid signature") + } +} + +func TestJWSEncodeJSONEC(t *testing.T) { + tt := []struct { + key *ecdsa.PrivateKey + x, y string + alg, crv string + }{ + {testKeyEC, testKeyECPubX, testKeyECPubY, "ES256", "P-256"}, + {testKeyEC384, testKeyEC384PubX, testKeyEC384PubY, "ES384", "P-384"}, + {testKeyEC512, testKeyEC512PubX, testKeyEC512PubY, "ES512", "P-521"}, + } + for i, test := range tt { + claims := struct{ Msg string }{"Hello JWS"} + b, err := jwsEncodeJSON(claims, test.key, noKeyID, "nonce", "url") + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + var jws struct{ Protected, Payload, Signature string } + if err := json.Unmarshal(b, &jws); err != nil { + t.Errorf("%d: %v", i, err) + continue + } + + b, err = base64.RawURLEncoding.DecodeString(jws.Protected) + if err != nil { + t.Errorf("%d: jws.Protected: %v", i, err) + } + var head struct { + Alg string + Nonce string + URL string `json:"url"` + KID string `json:"kid"` + JWK struct { + Crv string + Kty string + X string + Y string + } `json:"jwk"` + } + if err := json.Unmarshal(b, &head); err != nil { + t.Errorf("%d: jws.Protected: %v", i, err) + } + if head.Alg != test.alg { + t.Errorf("%d: head.Alg = %q; want %q", i, head.Alg, test.alg) + } + if head.Nonce != "nonce" { + t.Errorf("%d: head.Nonce = %q; want nonce", i, head.Nonce) + } + if head.URL != "url" { + t.Errorf("%d: head.URL = %q; want 'url'", i, head.URL) + } + if head.KID != "" { + // We used noKeyID in jwsEncodeJSON: expect no kid value. + t.Errorf("%d: head.KID = %q; want empty", i, head.KID) + } + if head.JWK.Crv != test.crv { + t.Errorf("%d: head.JWK.Crv = %q; want %q", i, head.JWK.Crv, test.crv) + } + if head.JWK.Kty != "EC" { + t.Errorf("%d: head.JWK.Kty = %q; want EC", i, head.JWK.Kty) + } + if head.JWK.X != test.x { + t.Errorf("%d: head.JWK.X = %q; want %q", i, head.JWK.X, test.x) + } + if head.JWK.Y != test.y { + t.Errorf("%d: head.JWK.Y = %q; want %q", i, head.JWK.Y, test.y) + } + } +} + +type customTestSigner struct { + sig []byte + pub crypto.PublicKey +} + +func (s *customTestSigner) Public() crypto.PublicKey { return s.pub } +func (s *customTestSigner) Sign(io.Reader, []byte, crypto.SignerOpts) ([]byte, error) { + return s.sig, nil +} + +func TestJWSEncodeJSONCustom(t *testing.T) { + claims := struct{ Msg string }{"hello"} + const ( + // printf '{"Msg":"hello"}' | b64raw + payload = "eyJNc2ciOiJoZWxsbyJ9" + // printf 'testsig' | b64raw + testsig = "dGVzdHNpZw" + + // the example P256 curve point from https://tools.ietf.org/html/rfc7515#appendix-A.3.1 + // encoded as ASN.1â€Ļ + es256stdsig = "MEUCIA7RIVN5Y2xIPC9/FVgH1AKjsigDOvl8fheBmsMWnqZlAiEA" + + "xQoH04w8cOXY8S2vCEpUgKZlkMXyk1Cajz9/ioOjVNU" + // â€Ļand RFC7518 (https://tools.ietf.org/html/rfc7518#section-3.4) + es256jwsig = "DtEhU3ljbEg8L38VWAfUAqOyKAM6-Xx-F4GawxaepmXFCgfTjDxw" + + "5djxLa8ISlSApmWQxfKTUJqPP3-Kg6NU1Q" + + // printf '{"alg":"ES256","jwk":{"crv":"P-256","kty":"EC","x":,"y":},"nonce":"nonce","url":"url"}' | b64raw + es256phead = "eyJhbGciOiJFUzI1NiIsImp3ayI6eyJjcnYiOiJQLTI1NiIsImt0" + + "eSI6IkVDIiwieCI6IjVsaEV1ZzV4SzR4QkRaMm5BYmF4THRhTGl2" + + "ODVieEo3ZVBkMWRrTzIzSFEiLCJ5IjoiNGFpSzcyc0JlVUFHa3Yw" + + "VGFMc213b2tZVVl5TnhHc1M1RU1JS3dzTklLayJ9LCJub25jZSI6" + + "Im5vbmNlIiwidXJsIjoidXJsIn0" + + // {"alg":"RS256","jwk":{"e":"AQAB","kty":"RSA","n":"..."},"nonce":"nonce","url":"url"} + rs256phead = "eyJhbGciOiJSUzI1NiIsImp3ayI6eyJlIjoiQVFBQiIsImt0eSI6" + + "IlJTQSIsIm4iOiI0eGdaM2VSUGt3b1J2eTdxZVJVYm1NRGUwVi14" + + "SDllV0xkdTBpaGVlTGxybUQybXFXWGZQOUllU0tBcGJuMzRnOFR1" + + "QVM5ZzV6aHE4RUxRM2ttanItS1Y4NkdBTWdJNlZBY0dscTNRcnpw" + + "VENmXzMwQWI3LXphd3JmUmFGT05hMUh3RXpQWTFLSG5HVmt4SmM4" + + "NWdOa3dZSTlTWTJSSFh0dmxuM3pzNXdJVE5yZG9zcUVYZWFJa1ZZ" + + "QkVoYmhOdTU0cHAza3hvNlR1V0xpOWU2cFhlV2V0RXdtbEJ3dFda" + + "bFBvaWIyajNUeExCa3NLWmZveUZ5ZWszODBtSGdKQXVtUV9JMmZq" + + "ajk4Xzk3bWszaWhPWTRBZ1ZkQ0RqMXpfR0NvWmtHNVJxN25iQ0d5" + + "b3N5S1d5RFgwMFpzLW5OcVZob0xlSXZYQzRubldkSk1aNnJvZ3h5" + + "UVEifSwibm9uY2UiOiJub25jZSIsInVybCI6InVybCJ9" + ) + + tt := []struct { + alg, phead string + pub crypto.PublicKey + stdsig, jwsig string + }{ + {"ES256", es256phead, testKeyEC.Public(), es256stdsig, es256jwsig}, + {"RS256", rs256phead, testKey.Public(), testsig, testsig}, + } + for _, tc := range tt { + tc := tc + t.Run(tc.alg, func(t *testing.T) { + stdsig, err := base64.RawStdEncoding.DecodeString(tc.stdsig) + if err != nil { + t.Errorf("couldn't decode test vector: %v", err) + } + signer := &customTestSigner{ + sig: stdsig, + pub: tc.pub, + } + + b, err := jwsEncodeJSON(claims, signer, noKeyID, "nonce", "url") + if err != nil { + t.Fatal(err) + } + var j jsonWebSignature + if err := json.Unmarshal(b, &j); err != nil { + t.Fatal(err) + } + if j.Protected != tc.phead { + t.Errorf("j.Protected = %q\nwant %q", j.Protected, tc.phead) + } + if j.Payload != payload { + t.Errorf("j.Payload = %q\nwant %q", j.Payload, payload) + } + if j.Sig != tc.jwsig { + t.Errorf("j.Sig = %q\nwant %q", j.Sig, tc.jwsig) + } + }) + } +} + +func TestJWSWithMAC(t *testing.T) { + // Example from RFC 7520 Section 4.4.3. + // https://tools.ietf.org/html/rfc7520#section-4.4.3 + b64Key := "hJtXIZ2uSN5kbQfbtTNWbpdmhkV8FJG-Onbc6mxCcYg" + rawPayload := []byte("It\xe2\x80\x99s a dangerous business, Frodo, going out your " + + "door. You step onto the road, and if you don't keep your feet, " + + "there\xe2\x80\x99s no knowing where you might be swept off " + + "to.") + protected := "eyJhbGciOiJIUzI1NiIsImtpZCI6IjAxOGMwYWU1LTRkOWItNDcxYi1iZmQ2LW" + + "VlZjMxNGJjNzAzNyJ9" + payload := "SXTigJlzIGEgZGFuZ2Vyb3VzIGJ1c2luZXNzLCBGcm9kbywg" + + "Z29pbmcgb3V0IHlvdXIgZG9vci4gWW91IHN0ZXAgb250byB0aGUgcm9h" + + "ZCwgYW5kIGlmIHlvdSBkb24ndCBrZWVwIHlvdXIgZmVldCwgdGhlcmXi" + + "gJlzIG5vIGtub3dpbmcgd2hlcmUgeW91IG1pZ2h0IGJlIHN3ZXB0IG9m" + + "ZiB0by4" + sig := "s0h6KThzkfBBBkLspW1h84VsJZFTsPPqMDA7g1Md7p0" + + key, err := base64.RawURLEncoding.DecodeString(b64Key) + if err != nil { + t.Fatalf("unable to decode key: %q", b64Key) + } + got, err := jwsWithMAC(key, "018c0ae5-4d9b-471b-bfd6-eef314bc7037", "", rawPayload) + if err != nil { + t.Fatalf("jwsWithMAC() = %q", err) + } + if got.Protected != protected { + t.Errorf("got.Protected = %q\nwant %q", got.Protected, protected) + } + if got.Payload != payload { + t.Errorf("got.Payload = %q\nwant %q", got.Payload, payload) + } + if got.Sig != sig { + t.Errorf("got.Signature = %q\nwant %q", got.Sig, sig) + } +} + +func TestJWSWithMACError(t *testing.T) { + p := "{}" + if _, err := jwsWithMAC(nil, "", "", []byte(p)); err == nil { + t.Errorf("jwsWithMAC(nil, ...) = success; want err") + } +} + +func TestJWKThumbprintRSA(t *testing.T) { + // Key example from RFC 7638 + const base64N = "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAt" + + "VT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn6" + + "4tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FD" + + "W2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n9" + + "1CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINH" + + "aQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw" + const base64E = "AQAB" + const expected = "NzbLsXh8uDCcd-6MNwXF4W_7noWXFZAfHkxZsRGC9Xs" + + b, err := base64.RawURLEncoding.DecodeString(base64N) + if err != nil { + t.Fatalf("Error parsing example key N: %v", err) + } + n := new(big.Int).SetBytes(b) + + b, err = base64.RawURLEncoding.DecodeString(base64E) + if err != nil { + t.Fatalf("Error parsing example key E: %v", err) + } + e := new(big.Int).SetBytes(b) + + pub := &rsa.PublicKey{N: n, E: int(e.Uint64())} + th, err := JWKThumbprint(pub) + if err != nil { + t.Error(err) + } + if th != expected { + t.Errorf("thumbprint = %q; want %q", th, expected) + } +} + +func TestJWKThumbprintEC(t *testing.T) { + // Key example from RFC 7520 + // expected was computed with + // printf '{"crv":"P-521","kty":"EC","x":"","y":""}' | \ + // openssl dgst -binary -sha256 | b64raw + const ( + base64X = "AHKZLLOsCOzz5cY97ewNUajB957y-C-U88c3v13nmGZx6sYl_oJXu9A5RkT" + + "KqjqvjyekWF-7ytDyRXYgCF5cj0Kt" + base64Y = "AdymlHvOiLxXkEhayXQnNCvDX4h9htZaCJN34kfmC6pV5OhQHiraVySsUda" + + "QkAgDPrwQrJmbnX9cwlGfP-HqHZR1" + expected = "dHri3SADZkrush5HU_50AoRhcKFryN-PI6jPBtPL55M" + ) + + b, err := base64.RawURLEncoding.DecodeString(base64X) + if err != nil { + t.Fatalf("Error parsing example key X: %v", err) + } + x := new(big.Int).SetBytes(b) + + b, err = base64.RawURLEncoding.DecodeString(base64Y) + if err != nil { + t.Fatalf("Error parsing example key Y: %v", err) + } + y := new(big.Int).SetBytes(b) + + pub := &ecdsa.PublicKey{Curve: elliptic.P521(), X: x, Y: y} + th, err := JWKThumbprint(pub) + if err != nil { + t.Error(err) + } + if th != expected { + t.Errorf("thumbprint = %q; want %q", th, expected) + } +} + +func TestJWKThumbprintErrUnsupportedKey(t *testing.T) { + _, err := JWKThumbprint(struct{}{}) + if err != ErrUnsupportedKey { + t.Errorf("err = %q; want %q", err, ErrUnsupportedKey) + } +} diff --git a/tempfork/acme/rfc8555.go b/tempfork/acme/rfc8555.go new file mode 100644 index 0000000000000..3152e531b65cf --- /dev/null +++ b/tempfork/acme/rfc8555.go @@ -0,0 +1,476 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "context" + "crypto" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "time" +) + +// DeactivateReg permanently disables an existing account associated with c.Key. +// A deactivated account can no longer request certificate issuance or access +// resources related to the account, such as orders or authorizations. +// +// It only works with CAs implementing RFC 8555. +func (c *Client) DeactivateReg(ctx context.Context) error { + if _, err := c.Discover(ctx); err != nil { // required by c.accountKID + return err + } + url := string(c.accountKID(ctx)) + if url == "" { + return ErrNoAccount + } + req := json.RawMessage(`{"status": "deactivated"}`) + res, err := c.post(ctx, nil, url, req, wantStatus(http.StatusOK)) + if err != nil { + return err + } + res.Body.Close() + return nil +} + +// registerRFC is equivalent to c.Register but for CAs implementing RFC 8555. +// It expects c.Discover to have already been called. +func (c *Client) registerRFC(ctx context.Context, acct *Account, prompt func(tosURL string) bool) (*Account, error) { + c.cacheMu.Lock() // guard c.kid access + defer c.cacheMu.Unlock() + + req := struct { + TermsAgreed bool `json:"termsOfServiceAgreed,omitempty"` + Contact []string `json:"contact,omitempty"` + ExternalAccountBinding *jsonWebSignature `json:"externalAccountBinding,omitempty"` + }{ + Contact: acct.Contact, + } + if c.dir.Terms != "" { + req.TermsAgreed = prompt(c.dir.Terms) + } + + // set 'externalAccountBinding' field if requested + if acct.ExternalAccountBinding != nil { + eabJWS, err := c.encodeExternalAccountBinding(acct.ExternalAccountBinding) + if err != nil { + return nil, fmt.Errorf("acme: failed to encode external account binding: %v", err) + } + req.ExternalAccountBinding = eabJWS + } + + res, err := c.post(ctx, c.Key, c.dir.RegURL, req, wantStatus( + http.StatusOK, // account with this key already registered + http.StatusCreated, // new account created + )) + if err != nil { + return nil, err + } + + defer res.Body.Close() + a, err := responseAccount(res) + if err != nil { + return nil, err + } + // Cache Account URL even if we return an error to the caller. + // It is by all means a valid and usable "kid" value for future requests. + c.KID = KeyID(a.URI) + if res.StatusCode == http.StatusOK { + return nil, ErrAccountAlreadyExists + } + return a, nil +} + +// encodeExternalAccountBinding will encode an external account binding stanza +// as described in https://tools.ietf.org/html/rfc8555#section-7.3.4. +func (c *Client) encodeExternalAccountBinding(eab *ExternalAccountBinding) (*jsonWebSignature, error) { + jwk, err := jwkEncode(c.Key.Public()) + if err != nil { + return nil, err + } + return jwsWithMAC(eab.Key, eab.KID, c.dir.RegURL, []byte(jwk)) +} + +// updateRegRFC is equivalent to c.UpdateReg but for CAs implementing RFC 8555. +// It expects c.Discover to have already been called. +func (c *Client) updateRegRFC(ctx context.Context, a *Account) (*Account, error) { + url := string(c.accountKID(ctx)) + if url == "" { + return nil, ErrNoAccount + } + req := struct { + Contact []string `json:"contact,omitempty"` + }{ + Contact: a.Contact, + } + res, err := c.post(ctx, nil, url, req, wantStatus(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Body.Close() + return responseAccount(res) +} + +// getRegRFC is equivalent to c.GetReg but for CAs implementing RFC 8555. +// It expects c.Discover to have already been called. +func (c *Client) getRegRFC(ctx context.Context) (*Account, error) { + req := json.RawMessage(`{"onlyReturnExisting": true}`) + res, err := c.post(ctx, c.Key, c.dir.RegURL, req, wantStatus(http.StatusOK)) + if e, ok := err.(*Error); ok && e.ProblemType == "urn:ietf:params:acme:error:accountDoesNotExist" { + return nil, ErrNoAccount + } + if err != nil { + return nil, err + } + + defer res.Body.Close() + return responseAccount(res) +} + +func responseAccount(res *http.Response) (*Account, error) { + var v struct { + Status string + Contact []string + Orders string + } + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("acme: invalid account response: %v", err) + } + return &Account{ + URI: res.Header.Get("Location"), + Status: v.Status, + Contact: v.Contact, + OrdersURL: v.Orders, + }, nil +} + +// accountKeyRollover attempts to perform account key rollover. +// On success it will change client.Key to the new key. +func (c *Client) accountKeyRollover(ctx context.Context, newKey crypto.Signer) error { + dir, err := c.Discover(ctx) // Also required by c.accountKID + if err != nil { + return err + } + kid := c.accountKID(ctx) + if kid == noKeyID { + return ErrNoAccount + } + oldKey, err := jwkEncode(c.Key.Public()) + if err != nil { + return err + } + payload := struct { + Account string `json:"account"` + OldKey json.RawMessage `json:"oldKey"` + }{ + Account: string(kid), + OldKey: json.RawMessage(oldKey), + } + inner, err := jwsEncodeJSON(payload, newKey, noKeyID, noNonce, dir.KeyChangeURL) + if err != nil { + return err + } + + res, err := c.post(ctx, nil, dir.KeyChangeURL, base64.RawURLEncoding.EncodeToString(inner), wantStatus(http.StatusOK)) + if err != nil { + return err + } + defer res.Body.Close() + c.Key = newKey + return nil +} + +// AuthorizeOrder initiates the order-based application for certificate issuance, +// as opposed to pre-authorization in Authorize. +// It is only supported by CAs implementing RFC 8555. +// +// The caller then needs to fetch each authorization with GetAuthorization, +// identify those with StatusPending status and fulfill a challenge using Accept. +// Once all authorizations are satisfied, the caller will typically want to poll +// order status using WaitOrder until it's in StatusReady state. +// To finalize the order and obtain a certificate, the caller submits a CSR with CreateOrderCert. +func (c *Client) AuthorizeOrder(ctx context.Context, id []AuthzID, opt ...OrderOption) (*Order, error) { + dir, err := c.Discover(ctx) + if err != nil { + return nil, err + } + + req := struct { + Identifiers []wireAuthzID `json:"identifiers"` + NotBefore string `json:"notBefore,omitempty"` + NotAfter string `json:"notAfter,omitempty"` + }{} + for _, v := range id { + req.Identifiers = append(req.Identifiers, wireAuthzID{ + Type: v.Type, + Value: v.Value, + }) + } + for _, o := range opt { + switch o := o.(type) { + case orderNotBeforeOpt: + req.NotBefore = time.Time(o).Format(time.RFC3339) + case orderNotAfterOpt: + req.NotAfter = time.Time(o).Format(time.RFC3339) + default: + // Package's fault if we let this happen. + panic(fmt.Sprintf("unsupported order option type %T", o)) + } + } + + res, err := c.post(ctx, nil, dir.OrderURL, req, wantStatus(http.StatusCreated)) + if err != nil { + return nil, err + } + defer res.Body.Close() + return responseOrder(res) +} + +// GetOrder retrives an order identified by the given URL. +// For orders created with AuthorizeOrder, the url value is Order.URI. +// +// If a caller needs to poll an order until its status is final, +// see the WaitOrder method. +func (c *Client) GetOrder(ctx context.Context, url string) (*Order, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + + res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Body.Close() + return responseOrder(res) +} + +// WaitOrder polls an order from the given URL until it is in one of the final states, +// StatusReady, StatusValid or StatusInvalid, the CA responded with a non-retryable error +// or the context is done. +// +// It returns a non-nil Order only if its Status is StatusReady or StatusValid. +// In all other cases WaitOrder returns an error. +// If the Status is StatusInvalid, the returned error is of type *OrderError. +func (c *Client) WaitOrder(ctx context.Context, url string) (*Order, error) { + if _, err := c.Discover(ctx); err != nil { + return nil, err + } + for { + res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK)) + if err != nil { + return nil, err + } + o, err := responseOrder(res) + res.Body.Close() + switch { + case err != nil: + // Skip and retry. + case o.Status == StatusInvalid: + return nil, &OrderError{OrderURL: o.URI, Status: o.Status} + case o.Status == StatusReady || o.Status == StatusValid: + return o, nil + } + + d := retryAfter(res.Header.Get("Retry-After")) + if d == 0 { + // Default retry-after. + // Same reasoning as in WaitAuthorization. + d = time.Second + } + t := time.NewTimer(d) + select { + case <-ctx.Done(): + t.Stop() + return nil, ctx.Err() + case <-t.C: + // Retry. + } + } +} + +func responseOrder(res *http.Response) (*Order, error) { + var v struct { + Status string + Expires time.Time + Identifiers []wireAuthzID + NotBefore time.Time + NotAfter time.Time + Error *wireError + Authorizations []string + Finalize string + Certificate string + } + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return nil, fmt.Errorf("acme: error reading order: %v", err) + } + o := &Order{ + URI: res.Header.Get("Location"), + Status: v.Status, + Expires: v.Expires, + NotBefore: v.NotBefore, + NotAfter: v.NotAfter, + AuthzURLs: v.Authorizations, + FinalizeURL: v.Finalize, + CertURL: v.Certificate, + } + for _, id := range v.Identifiers { + o.Identifiers = append(o.Identifiers, AuthzID{Type: id.Type, Value: id.Value}) + } + if v.Error != nil { + o.Error = v.Error.error(nil /* headers */) + } + return o, nil +} + +// CreateOrderCert submits the CSR (Certificate Signing Request) to a CA at the specified URL. +// The URL is the FinalizeURL field of an Order created with AuthorizeOrder. +// +// If the bundle argument is true, the returned value also contain the CA (issuer) +// certificate chain. Otherwise, only a leaf certificate is returned. +// The returned URL can be used to re-fetch the certificate using FetchCert. +// +// This method is only supported by CAs implementing RFC 8555. See CreateCert for pre-RFC CAs. +// +// CreateOrderCert returns an error if the CA's response is unreasonably large. +// Callers are encouraged to parse the returned value to ensure the certificate is valid and has the expected features. +func (c *Client) CreateOrderCert(ctx context.Context, url string, csr []byte, bundle bool) (der [][]byte, certURL string, err error) { + if _, err := c.Discover(ctx); err != nil { // required by c.accountKID + return nil, "", err + } + + // RFC describes this as "finalize order" request. + req := struct { + CSR string `json:"csr"` + }{ + CSR: base64.RawURLEncoding.EncodeToString(csr), + } + res, err := c.post(ctx, nil, url, req, wantStatus(http.StatusOK)) + if err != nil { + return nil, "", err + } + defer res.Body.Close() + o, err := responseOrder(res) + if err != nil { + return nil, "", err + } + + // Wait for CA to issue the cert if they haven't. + if o.Status != StatusValid { + o, err = c.WaitOrder(ctx, o.URI) + } + if err != nil { + return nil, "", err + } + // The only acceptable status post finalize and WaitOrder is "valid". + if o.Status != StatusValid { + return nil, "", &OrderError{OrderURL: o.URI, Status: o.Status} + } + crt, err := c.fetchCertRFC(ctx, o.CertURL, bundle) + return crt, o.CertURL, err +} + +// fetchCertRFC downloads issued certificate from the given URL. +// It expects the CA to respond with PEM-encoded certificate chain. +// +// The URL argument is the CertURL field of Order. +func (c *Client) fetchCertRFC(ctx context.Context, url string, bundle bool) ([][]byte, error) { + res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Get all the bytes up to a sane maximum. + // Account very roughly for base64 overhead. + const max = maxCertChainSize + maxCertChainSize/33 + b, err := io.ReadAll(io.LimitReader(res.Body, max+1)) + if err != nil { + return nil, fmt.Errorf("acme: fetch cert response stream: %v", err) + } + if len(b) > max { + return nil, errors.New("acme: certificate chain is too big") + } + + // Decode PEM chain. + var chain [][]byte + for { + var p *pem.Block + p, b = pem.Decode(b) + if p == nil { + break + } + if p.Type != "CERTIFICATE" { + return nil, fmt.Errorf("acme: invalid PEM cert type %q", p.Type) + } + + chain = append(chain, p.Bytes) + if !bundle { + return chain, nil + } + if len(chain) > maxChainLen { + return nil, errors.New("acme: certificate chain is too long") + } + } + if len(chain) == 0 { + return nil, errors.New("acme: certificate chain is empty") + } + return chain, nil +} + +// sends a cert revocation request in either JWK form when key is non-nil or KID form otherwise. +func (c *Client) revokeCertRFC(ctx context.Context, key crypto.Signer, cert []byte, reason CRLReasonCode) error { + req := &struct { + Cert string `json:"certificate"` + Reason int `json:"reason"` + }{ + Cert: base64.RawURLEncoding.EncodeToString(cert), + Reason: int(reason), + } + res, err := c.post(ctx, key, c.dir.RevokeURL, req, wantStatus(http.StatusOK)) + if err != nil { + if isAlreadyRevoked(err) { + // Assume it is not an error to revoke an already revoked cert. + return nil + } + return err + } + defer res.Body.Close() + return nil +} + +func isAlreadyRevoked(err error) bool { + e, ok := err.(*Error) + return ok && e.ProblemType == "urn:ietf:params:acme:error:alreadyRevoked" +} + +// ListCertAlternates retrieves any alternate certificate chain URLs for the +// given certificate chain URL. These alternate URLs can be passed to FetchCert +// in order to retrieve the alternate certificate chains. +// +// If there are no alternate issuer certificate chains, a nil slice will be +// returned. +func (c *Client) ListCertAlternates(ctx context.Context, url string) ([]string, error) { + if _, err := c.Discover(ctx); err != nil { // required by c.accountKID + return nil, err + } + + res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // We don't need the body but we need to discard it so we don't end up + // preventing keep-alive + if _, err := io.Copy(io.Discard, res.Body); err != nil { + return nil, fmt.Errorf("acme: cert alternates response stream: %v", err) + } + alts := linkHeader(res.Header, "alternate") + return alts, nil +} diff --git a/tempfork/acme/rfc8555_test.go b/tempfork/acme/rfc8555_test.go new file mode 100644 index 0000000000000..d65720a356e81 --- /dev/null +++ b/tempfork/acme/rfc8555_test.go @@ -0,0 +1,1017 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + "time" +) + +// While contents of this file is pertinent only to RFC8555, +// it is complementary to the tests in the other _test.go files +// many of which are valid for both pre- and RFC8555. +// This will make it easier to clean up the tests once non-RFC compliant +// code is removed. + +func TestRFC_Discover(t *testing.T) { + const ( + nonce = "https://example.com/acme/new-nonce" + reg = "https://example.com/acme/new-acct" + order = "https://example.com/acme/new-order" + authz = "https://example.com/acme/new-authz" + revoke = "https://example.com/acme/revoke-cert" + keychange = "https://example.com/acme/key-change" + metaTerms = "https://example.com/acme/terms/2017-5-30" + metaWebsite = "https://www.example.com/" + metaCAA = "example.com" + ) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{ + "newNonce": %q, + "newAccount": %q, + "newOrder": %q, + "newAuthz": %q, + "revokeCert": %q, + "keyChange": %q, + "meta": { + "termsOfService": %q, + "website": %q, + "caaIdentities": [%q], + "externalAccountRequired": true + } + }`, nonce, reg, order, authz, revoke, keychange, metaTerms, metaWebsite, metaCAA) + })) + defer ts.Close() + c := &Client{DirectoryURL: ts.URL} + dir, err := c.Discover(context.Background()) + if err != nil { + t.Fatal(err) + } + if dir.NonceURL != nonce { + t.Errorf("dir.NonceURL = %q; want %q", dir.NonceURL, nonce) + } + if dir.RegURL != reg { + t.Errorf("dir.RegURL = %q; want %q", dir.RegURL, reg) + } + if dir.OrderURL != order { + t.Errorf("dir.OrderURL = %q; want %q", dir.OrderURL, order) + } + if dir.AuthzURL != authz { + t.Errorf("dir.AuthzURL = %q; want %q", dir.AuthzURL, authz) + } + if dir.RevokeURL != revoke { + t.Errorf("dir.RevokeURL = %q; want %q", dir.RevokeURL, revoke) + } + if dir.KeyChangeURL != keychange { + t.Errorf("dir.KeyChangeURL = %q; want %q", dir.KeyChangeURL, keychange) + } + if dir.Terms != metaTerms { + t.Errorf("dir.Terms = %q; want %q", dir.Terms, metaTerms) + } + if dir.Website != metaWebsite { + t.Errorf("dir.Website = %q; want %q", dir.Website, metaWebsite) + } + if len(dir.CAA) == 0 || dir.CAA[0] != metaCAA { + t.Errorf("dir.CAA = %q; want [%q]", dir.CAA, metaCAA) + } + if !dir.ExternalAccountRequired { + t.Error("dir.Meta.ExternalAccountRequired is false") + } +} + +func TestRFC_popNonce(t *testing.T) { + var count int + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The Client uses only Directory.NonceURL when specified. + // Expect no other URL paths. + if r.URL.Path != "/new-nonce" { + t.Errorf("r.URL.Path = %q; want /new-nonce", r.URL.Path) + } + if count > 0 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + count++ + w.Header().Set("Replay-Nonce", "second") + })) + cl := &Client{ + DirectoryURL: ts.URL, + dir: &Directory{NonceURL: ts.URL + "/new-nonce"}, + } + cl.addNonce(http.Header{"Replay-Nonce": {"first"}}) + + for i, nonce := range []string{"first", "second"} { + v, err := cl.popNonce(context.Background(), "") + if err != nil { + t.Errorf("%d: cl.popNonce: %v", i, err) + } + if v != nonce { + t.Errorf("%d: cl.popNonce = %q; want %q", i, v, nonce) + } + } + // No more nonces and server replies with an error past first nonce fetch. + // Expected to fail. + if _, err := cl.popNonce(context.Background(), ""); err == nil { + t.Error("last cl.popNonce returned nil error") + } +} + +func TestRFC_postKID(t *testing.T) { + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/new-nonce": + w.Header().Set("Replay-Nonce", "nonce") + case "/new-account": + w.Header().Set("Location", "/account-1") + w.Write([]byte(`{"status":"valid"}`)) + case "/post": + b, _ := io.ReadAll(r.Body) // check err later in decodeJWSxxx + head, err := decodeJWSHead(bytes.NewReader(b)) + if err != nil { + t.Errorf("decodeJWSHead: %v", err) + return + } + if head.KID != "/account-1" { + t.Errorf("head.KID = %q; want /account-1", head.KID) + } + if len(head.JWK) != 0 { + t.Errorf("head.JWK = %q; want zero map", head.JWK) + } + if v := ts.URL + "/post"; head.URL != v { + t.Errorf("head.URL = %q; want %q", head.URL, v) + } + + var payload struct{ Msg string } + decodeJWSRequest(t, &payload, bytes.NewReader(b)) + if payload.Msg != "ping" { + t.Errorf("payload.Msg = %q; want ping", payload.Msg) + } + w.Write([]byte("pong")) + default: + t.Errorf("unhandled %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer ts.Close() + + ctx := context.Background() + cl := &Client{ + Key: testKey, + DirectoryURL: ts.URL, + dir: &Directory{ + NonceURL: ts.URL + "/new-nonce", + RegURL: ts.URL + "/new-account", + OrderURL: "/force-rfc-mode", + }, + } + req := json.RawMessage(`{"msg":"ping"}`) + res, err := cl.post(ctx, nil /* use kid */, ts.URL+"/post", req, wantStatus(http.StatusOK)) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + b, _ := io.ReadAll(res.Body) // don't care about err - just checking b + if string(b) != "pong" { + t.Errorf("res.Body = %q; want pong", b) + } +} + +// acmeServer simulates a subset of RFC 8555 compliant CA. +// +// TODO: We also have x/crypto/acme/autocert/acmetest and startACMEServerStub in autocert_test.go. +// It feels like this acmeServer is a sweet spot between usefulness and added complexity. +// Also, acmetest and startACMEServerStub were both written for draft-02, no RFC support. +// The goal is to consolidate all into one ACME test server. +type acmeServer struct { + ts *httptest.Server + handler map[string]http.HandlerFunc // keyed by r.URL.Path + + mu sync.Mutex + nnonce int +} + +func newACMEServer() *acmeServer { + return &acmeServer{handler: make(map[string]http.HandlerFunc)} +} + +func (s *acmeServer) handle(path string, f func(http.ResponseWriter, *http.Request)) { + s.handler[path] = http.HandlerFunc(f) +} + +func (s *acmeServer) start() { + s.ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + // Directory request. + if r.URL.Path == "/" { + fmt.Fprintf(w, `{ + "newNonce": %q, + "newAccount": %q, + "newOrder": %q, + "newAuthz": %q, + "revokeCert": %q, + "keyChange": %q, + "meta": {"termsOfService": %q} + }`, + s.url("/acme/new-nonce"), + s.url("/acme/new-account"), + s.url("/acme/new-order"), + s.url("/acme/new-authz"), + s.url("/acme/revoke-cert"), + s.url("/acme/key-change"), + s.url("/terms"), + ) + return + } + + // All other responses contain a nonce value unconditionally. + w.Header().Set("Replay-Nonce", s.nonce()) + if r.URL.Path == "/acme/new-nonce" { + return + } + + h := s.handler[r.URL.Path] + if h == nil { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, "Unhandled %s", r.URL.Path) + return + } + h.ServeHTTP(w, r) + })) +} + +func (s *acmeServer) close() { + s.ts.Close() +} + +func (s *acmeServer) url(path string) string { + return s.ts.URL + path +} + +func (s *acmeServer) nonce() string { + s.mu.Lock() + defer s.mu.Unlock() + s.nnonce++ + return fmt.Sprintf("nonce%d", s.nnonce) +} + +func (s *acmeServer) error(w http.ResponseWriter, e *wireError) { + w.WriteHeader(e.Status) + json.NewEncoder(w).Encode(e) +} + +func TestRFC_Register(t *testing.T) { + const email = "mailto:user@example.org" + + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusCreated) // 201 means new account created + fmt.Fprintf(w, `{ + "status": "valid", + "contact": [%q], + "orders": %q + }`, email, s.url("/accounts/1/orders")) + + b, _ := io.ReadAll(r.Body) // check err later in decodeJWSxxx + head, err := decodeJWSHead(bytes.NewReader(b)) + if err != nil { + t.Errorf("decodeJWSHead: %v", err) + return + } + if len(head.JWK) == 0 { + t.Error("head.JWK is empty") + } + + var req struct{ Contact []string } + decodeJWSRequest(t, &req, bytes.NewReader(b)) + if len(req.Contact) != 1 || req.Contact[0] != email { + t.Errorf("req.Contact = %q; want [%q]", req.Contact, email) + } + }) + s.start() + defer s.close() + + ctx := context.Background() + cl := &Client{ + Key: testKeyEC, + DirectoryURL: s.url("/"), + } + + var didPrompt bool + a := &Account{Contact: []string{email}} + acct, err := cl.Register(ctx, a, func(tos string) bool { + didPrompt = true + terms := s.url("/terms") + if tos != terms { + t.Errorf("tos = %q; want %q", tos, terms) + } + return true + }) + if err != nil { + t.Fatal(err) + } + okAccount := &Account{ + URI: s.url("/accounts/1"), + Status: StatusValid, + Contact: []string{email}, + OrdersURL: s.url("/accounts/1/orders"), + } + if !reflect.DeepEqual(acct, okAccount) { + t.Errorf("acct = %+v; want %+v", acct, okAccount) + } + if !didPrompt { + t.Error("tos prompt wasn't called") + } + if v := cl.accountKID(ctx); v != KeyID(okAccount.URI) { + t.Errorf("account kid = %q; want %q", v, okAccount.URI) + } +} + +func TestRFC_RegisterExternalAccountBinding(t *testing.T) { + eab := &ExternalAccountBinding{ + KID: "kid-1", + Key: []byte("secret"), + } + + type protected struct { + Algorithm string `json:"alg"` + KID string `json:"kid"` + URL string `json:"url"` + } + const email = "mailto:user@example.org" + + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + if r.Method != "POST" { + t.Errorf("r.Method = %q; want POST", r.Method) + } + + var j struct { + Protected string + Contact []string + TermsOfServiceAgreed bool + ExternalaccountBinding struct { + Protected string + Payload string + Signature string + } + } + decodeJWSRequest(t, &j, r.Body) + protData, err := base64.RawURLEncoding.DecodeString(j.ExternalaccountBinding.Protected) + if err != nil { + t.Fatal(err) + } + + var prot protected + err = json.Unmarshal(protData, &prot) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(j.Contact, []string{email}) { + t.Errorf("j.Contact = %v; want %v", j.Contact, []string{email}) + } + if !j.TermsOfServiceAgreed { + t.Error("j.TermsOfServiceAgreed = false; want true") + } + + // Ensure same KID. + if prot.KID != eab.KID { + t.Errorf("j.ExternalAccountBinding.KID = %s; want %s", prot.KID, eab.KID) + } + // Ensure expected Algorithm. + if prot.Algorithm != "HS256" { + t.Errorf("j.ExternalAccountBinding.Alg = %s; want %s", + prot.Algorithm, "HS256") + } + + // Ensure same URL as outer JWS. + url := fmt.Sprintf("http://%s/acme/new-account", r.Host) + if prot.URL != url { + t.Errorf("j.ExternalAccountBinding.URL = %s; want %s", + prot.URL, url) + } + + // Ensure payload is base64URL encoded string of JWK in outer JWS + jwk, err := jwkEncode(testKeyEC.Public()) + if err != nil { + t.Fatal(err) + } + decodedPayload, err := base64.RawURLEncoding.DecodeString(j.ExternalaccountBinding.Payload) + if err != nil { + t.Fatal(err) + } + if jwk != string(decodedPayload) { + t.Errorf("j.ExternalAccountBinding.Payload = %s; want %s", decodedPayload, jwk) + } + + // Check signature on inner external account binding JWS + hmac := hmac.New(sha256.New, []byte("secret")) + _, err = hmac.Write([]byte(j.ExternalaccountBinding.Protected + "." + j.ExternalaccountBinding.Payload)) + if err != nil { + t.Fatal(err) + } + mac := hmac.Sum(nil) + encodedMAC := base64.RawURLEncoding.EncodeToString(mac) + + if !bytes.Equal([]byte(encodedMAC), []byte(j.ExternalaccountBinding.Signature)) { + t.Errorf("j.ExternalAccountBinding.Signature = %v; want %v", + []byte(j.ExternalaccountBinding.Signature), encodedMAC) + } + + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusCreated) + b, _ := json.Marshal([]string{email}) + fmt.Fprintf(w, `{"status":"valid","orders":"%s","contact":%s}`, s.url("/accounts/1/orders"), b) + }) + s.start() + defer s.close() + + ctx := context.Background() + cl := &Client{ + Key: testKeyEC, + DirectoryURL: s.url("/"), + } + + var didPrompt bool + a := &Account{Contact: []string{email}, ExternalAccountBinding: eab} + acct, err := cl.Register(ctx, a, func(tos string) bool { + didPrompt = true + terms := s.url("/terms") + if tos != terms { + t.Errorf("tos = %q; want %q", tos, terms) + } + return true + }) + if err != nil { + t.Fatal(err) + } + okAccount := &Account{ + URI: s.url("/accounts/1"), + Status: StatusValid, + Contact: []string{email}, + OrdersURL: s.url("/accounts/1/orders"), + } + if !reflect.DeepEqual(acct, okAccount) { + t.Errorf("acct = %+v; want %+v", acct, okAccount) + } + if !didPrompt { + t.Error("tos prompt wasn't called") + } + if v := cl.accountKID(ctx); v != KeyID(okAccount.URI) { + t.Errorf("account kid = %q; want %q", v, okAccount.URI) + } +} + +func TestRFC_RegisterExisting(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) // 200 means account already exists + w.Write([]byte(`{"status": "valid"}`)) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + _, err := cl.Register(context.Background(), &Account{}, AcceptTOS) + if err != ErrAccountAlreadyExists { + t.Errorf("err = %v; want %v", err, ErrAccountAlreadyExists) + } + kid := KeyID(s.url("/accounts/1")) + if v := cl.accountKID(context.Background()); v != kid { + t.Errorf("account kid = %q; want %q", v, kid) + } +} + +func TestRFC_UpdateReg(t *testing.T) { + const email = "mailto:user@example.org" + + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + }) + var didUpdate bool + s.handle("/accounts/1", func(w http.ResponseWriter, r *http.Request) { + didUpdate = true + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + + b, _ := io.ReadAll(r.Body) // check err later in decodeJWSxxx + head, err := decodeJWSHead(bytes.NewReader(b)) + if err != nil { + t.Errorf("decodeJWSHead: %v", err) + return + } + if len(head.JWK) != 0 { + t.Error("head.JWK is non-zero") + } + kid := s.url("/accounts/1") + if head.KID != kid { + t.Errorf("head.KID = %q; want %q", head.KID, kid) + } + + var req struct{ Contact []string } + decodeJWSRequest(t, &req, bytes.NewReader(b)) + if len(req.Contact) != 1 || req.Contact[0] != email { + t.Errorf("req.Contact = %q; want [%q]", req.Contact, email) + } + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + _, err := cl.UpdateReg(context.Background(), &Account{Contact: []string{email}}) + if err != nil { + t.Error(err) + } + if !didUpdate { + t.Error("UpdateReg didn't update the account") + } +} + +func TestRFC_GetReg(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + + head, err := decodeJWSHead(r.Body) + if err != nil { + t.Errorf("decodeJWSHead: %v", err) + return + } + if len(head.JWK) == 0 { + t.Error("head.JWK is empty") + } + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + acct, err := cl.GetReg(context.Background(), "") + if err != nil { + t.Fatal(err) + } + okAccount := &Account{ + URI: s.url("/accounts/1"), + Status: StatusValid, + } + if !reflect.DeepEqual(acct, okAccount) { + t.Errorf("acct = %+v; want %+v", acct, okAccount) + } +} + +func TestRFC_GetRegNoAccount(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + s.error(w, &wireError{ + Status: http.StatusBadRequest, + Type: "urn:ietf:params:acme:error:accountDoesNotExist", + }) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + if _, err := cl.GetReg(context.Background(), ""); err != ErrNoAccount { + t.Errorf("err = %v; want %v", err, ErrNoAccount) + } +} + +func TestRFC_GetRegOtherError(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + if _, err := cl.GetReg(context.Background(), ""); err == nil || err == ErrNoAccount { + t.Errorf("GetReg: %v; want any other non-nil err", err) + } +} + +func TestRFC_AccountKeyRollover(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + }) + s.handle("/acme/key-change", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + if err := cl.AccountKeyRollover(context.Background(), testKeyEC384); err != nil { + t.Errorf("AccountKeyRollover: %v, wanted no error", err) + } else if cl.Key != testKeyEC384 { + t.Error("AccountKeyRollover did not rotate the client key") + } +} + +func TestRFC_DeactivateReg(t *testing.T) { + const email = "mailto:user@example.org" + curStatus := StatusValid + + type account struct { + Status string `json:"status"` + Contact []string `json:"contact"` + AcceptTOS bool `json:"termsOfServiceAgreed"` + Orders string `json:"orders"` + } + + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) // 200 means existing account + json.NewEncoder(w).Encode(account{ + Status: curStatus, + Contact: []string{email}, + AcceptTOS: true, + Orders: s.url("/accounts/1/orders"), + }) + + b, _ := io.ReadAll(r.Body) // check err later in decodeJWSxxx + head, err := decodeJWSHead(bytes.NewReader(b)) + if err != nil { + t.Errorf("decodeJWSHead: %v", err) + return + } + if len(head.JWK) == 0 { + t.Error("head.JWK is empty") + } + + var req struct { + Status string `json:"status"` + Contact []string `json:"contact"` + AcceptTOS bool `json:"termsOfServiceAgreed"` + OnlyExisting bool `json:"onlyReturnExisting"` + } + decodeJWSRequest(t, &req, bytes.NewReader(b)) + if !req.OnlyExisting { + t.Errorf("req.OnlyReturnExisting = %t; want = %t", req.OnlyExisting, true) + } + }) + s.handle("/accounts/1", func(w http.ResponseWriter, r *http.Request) { + if curStatus == StatusValid { + curStatus = StatusDeactivated + w.WriteHeader(http.StatusOK) + } else { + s.error(w, &wireError{ + Status: http.StatusUnauthorized, + Type: "urn:ietf:params:acme:error:unauthorized", + }) + } + var req account + b, _ := io.ReadAll(r.Body) // check err later in decodeJWSxxx + head, err := decodeJWSHead(bytes.NewReader(b)) + if err != nil { + t.Errorf("decodeJWSHead: %v", err) + return + } + if len(head.JWK) != 0 { + t.Error("head.JWK is not empty") + } + if !strings.HasSuffix(head.KID, "/accounts/1") { + t.Errorf("head.KID = %q; want suffix /accounts/1", head.KID) + } + + decodeJWSRequest(t, &req, bytes.NewReader(b)) + if req.Status != StatusDeactivated { + t.Errorf("req.Status = %q; want = %q", req.Status, StatusDeactivated) + } + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + if err := cl.DeactivateReg(context.Background()); err != nil { + t.Errorf("DeactivateReg: %v, wanted no error", err) + } + if err := cl.DeactivateReg(context.Background()); err == nil { + t.Errorf("DeactivateReg: %v, wanted error for unauthorized", err) + } +} + +func TestRF_DeactivateRegNoAccount(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + s.error(w, &wireError{ + Status: http.StatusBadRequest, + Type: "urn:ietf:params:acme:error:accountDoesNotExist", + }) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + if err := cl.DeactivateReg(context.Background()); !errors.Is(err, ErrNoAccount) { + t.Errorf("DeactivateReg: %v, wanted ErrNoAccount", err) + } +} + +func TestRFC_AuthorizeOrder(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + }) + s.handle("/acme/new-order", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/orders/1")) + w.WriteHeader(http.StatusCreated) + fmt.Fprintf(w, `{ + "status": "pending", + "expires": "2019-09-01T00:00:00Z", + "notBefore": "2019-08-31T00:00:00Z", + "notAfter": "2019-09-02T00:00:00Z", + "identifiers": [{"type":"dns", "value":"example.org"}], + "authorizations": [%q] + }`, s.url("/authz/1")) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + o, err := cl.AuthorizeOrder(context.Background(), DomainIDs("example.org"), + WithOrderNotBefore(time.Date(2019, 8, 31, 0, 0, 0, 0, time.UTC)), + WithOrderNotAfter(time.Date(2019, 9, 2, 0, 0, 0, 0, time.UTC)), + ) + if err != nil { + t.Fatal(err) + } + okOrder := &Order{ + URI: s.url("/orders/1"), + Status: StatusPending, + Expires: time.Date(2019, 9, 1, 0, 0, 0, 0, time.UTC), + NotBefore: time.Date(2019, 8, 31, 0, 0, 0, 0, time.UTC), + NotAfter: time.Date(2019, 9, 2, 0, 0, 0, 0, time.UTC), + Identifiers: []AuthzID{AuthzID{Type: "dns", Value: "example.org"}}, + AuthzURLs: []string{s.url("/authz/1")}, + } + if !reflect.DeepEqual(o, okOrder) { + t.Errorf("AuthorizeOrder = %+v; want %+v", o, okOrder) + } +} + +func TestRFC_GetOrder(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + }) + s.handle("/orders/1", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/orders/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{ + "status": "invalid", + "expires": "2019-09-01T00:00:00Z", + "notBefore": "2019-08-31T00:00:00Z", + "notAfter": "2019-09-02T00:00:00Z", + "identifiers": [{"type":"dns", "value":"example.org"}], + "authorizations": ["/authz/1"], + "finalize": "/orders/1/fin", + "certificate": "/orders/1/cert", + "error": {"type": "badRequest"} + }`)) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + o, err := cl.GetOrder(context.Background(), s.url("/orders/1")) + if err != nil { + t.Fatal(err) + } + okOrder := &Order{ + URI: s.url("/orders/1"), + Status: StatusInvalid, + Expires: time.Date(2019, 9, 1, 0, 0, 0, 0, time.UTC), + NotBefore: time.Date(2019, 8, 31, 0, 0, 0, 0, time.UTC), + NotAfter: time.Date(2019, 9, 2, 0, 0, 0, 0, time.UTC), + Identifiers: []AuthzID{AuthzID{Type: "dns", Value: "example.org"}}, + AuthzURLs: []string{"/authz/1"}, + FinalizeURL: "/orders/1/fin", + CertURL: "/orders/1/cert", + Error: &Error{ProblemType: "badRequest"}, + } + if !reflect.DeepEqual(o, okOrder) { + t.Errorf("GetOrder = %+v\nwant %+v", o, okOrder) + } +} + +func TestRFC_WaitOrder(t *testing.T) { + for _, st := range []string{StatusReady, StatusValid} { + t.Run(st, func(t *testing.T) { + testWaitOrderStatus(t, st) + }) + } +} + +func testWaitOrderStatus(t *testing.T, okStatus string) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + }) + var count int + s.handle("/orders/1", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/orders/1")) + w.WriteHeader(http.StatusOK) + s := StatusPending + if count > 0 { + s = okStatus + } + fmt.Fprintf(w, `{"status": %q}`, s) + count++ + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + order, err := cl.WaitOrder(context.Background(), s.url("/orders/1")) + if err != nil { + t.Fatalf("WaitOrder: %v", err) + } + if order.Status != okStatus { + t.Errorf("order.Status = %q; want %q", order.Status, okStatus) + } +} + +func TestRFC_WaitOrderError(t *testing.T) { + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "valid"}`)) + }) + var count int + s.handle("/orders/1", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/orders/1")) + w.WriteHeader(http.StatusOK) + s := StatusPending + if count > 0 { + s = StatusInvalid + } + fmt.Fprintf(w, `{"status": %q}`, s) + count++ + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + _, err := cl.WaitOrder(context.Background(), s.url("/orders/1")) + if err == nil { + t.Fatal("WaitOrder returned nil error") + } + e, ok := err.(*OrderError) + if !ok { + t.Fatalf("err = %v (%T); want OrderError", err, err) + } + if e.OrderURL != s.url("/orders/1") { + t.Errorf("e.OrderURL = %q; want %q", e.OrderURL, s.url("/orders/1")) + } + if e.Status != StatusInvalid { + t.Errorf("e.Status = %q; want %q", e.Status, StatusInvalid) + } +} + +func TestRFC_CreateOrderCert(t *testing.T) { + q := &x509.CertificateRequest{ + Subject: pkix.Name{CommonName: "example.org"}, + } + csr, err := x509.CreateCertificateRequest(rand.Reader, q, testKeyEC) + if err != nil { + t.Fatal(err) + } + + tmpl := &x509.Certificate{SerialNumber: big.NewInt(1)} + leaf, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &testKeyEC.PublicKey, testKeyEC) + if err != nil { + t.Fatal(err) + } + + s := newACMEServer() + s.handle("/acme/new-account", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/accounts/1")) + w.Write([]byte(`{"status": "valid"}`)) + }) + var count int + s.handle("/pleaseissue", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", s.url("/pleaseissue")) + st := StatusProcessing + if count > 0 { + st = StatusValid + } + fmt.Fprintf(w, `{"status":%q, "certificate":%q}`, st, s.url("/crt")) + count++ + }) + s.handle("/crt", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/pem-certificate-chain") + pem.Encode(w, &pem.Block{Type: "CERTIFICATE", Bytes: leaf}) + }) + s.start() + defer s.close() + ctx := context.Background() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + cert, curl, err := cl.CreateOrderCert(ctx, s.url("/pleaseissue"), csr, true) + if err != nil { + t.Fatalf("CreateOrderCert: %v", err) + } + if _, err := x509.ParseCertificate(cert[0]); err != nil { + t.Errorf("ParseCertificate: %v", err) + } + if !reflect.DeepEqual(cert[0], leaf) { + t.Errorf("cert and leaf bytes don't match") + } + if u := s.url("/crt"); curl != u { + t.Errorf("curl = %q; want %q", curl, u) + } +} + +func TestRFC_AlreadyRevokedCert(t *testing.T) { + s := newACMEServer() + s.handle("/acme/revoke-cert", func(w http.ResponseWriter, r *http.Request) { + s.error(w, &wireError{ + Status: http.StatusBadRequest, + Type: "urn:ietf:params:acme:error:alreadyRevoked", + }) + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + err := cl.RevokeCert(context.Background(), testKeyEC, []byte{0}, CRLReasonUnspecified) + if err != nil { + t.Fatalf("RevokeCert: %v", err) + } +} + +func TestRFC_ListCertAlternates(t *testing.T) { + s := newACMEServer() + s.handle("/crt", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/pem-certificate-chain") + w.Header().Add("Link", `;rel="alternate"`) + w.Header().Add("Link", `; rel="alternate"`) + w.Header().Add("Link", `; rel="index"`) + }) + s.handle("/crt2", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/pem-certificate-chain") + }) + s.start() + defer s.close() + + cl := &Client{Key: testKeyEC, DirectoryURL: s.url("/")} + crts, err := cl.ListCertAlternates(context.Background(), s.url("/crt")) + if err != nil { + t.Fatalf("ListCertAlternates: %v", err) + } + want := []string{"https://example.com/crt/2", "https://example.com/crt/3"} + if !reflect.DeepEqual(crts, want) { + t.Errorf("ListCertAlternates(/crt): %v; want %v", crts, want) + } + crts, err = cl.ListCertAlternates(context.Background(), s.url("/crt2")) + if err != nil { + t.Fatalf("ListCertAlternates: %v", err) + } + if crts != nil { + t.Errorf("ListCertAlternates(/crt2): %v; want nil", crts) + } +} diff --git a/tempfork/acme/sync_to_upstream_test.go b/tempfork/acme/sync_to_upstream_test.go new file mode 100644 index 0000000000000..d6bea7a11320a --- /dev/null +++ b/tempfork/acme/sync_to_upstream_test.go @@ -0,0 +1,70 @@ +package acme + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + _ "github.com/tailscale/golang-x-crypto/acme" // so it's on disk for the test +) + +// Verify that the files tempfork/acme/*.go (other than this test file) match the +// files in "github.com/tailscale/golang-x-crypto/acme" which is where we develop +// our fork of golang.org/x/crypto/acme and merge with upstream, but then we vendor +// just its acme package into tailscale.com/tempfork/acme. +// +// Development workflow: +// +// - make a change in github.com/tailscale/golang-x-crypto/acme +// - merge it (ideally with golang.org/x/crypto/acme too) +// - rebase github.com/tailscale/golang-x-crypto/acme with upstream x/crypto/acme +// as needed +// - in the tailscale.com repo, run "go get github.com/tailscale/golang-x-crypto/acme@main" +// - run go test ./tempfork/acme to watch it fail; the failure includes +// a shell command you should run to copy the *.go files from tailscale/golang-x-crypto +// to tailscale.com. +// - watch tests pass. git add it all. +// - send PR to tailscale.com +func TestSyncedToUpstream(t *testing.T) { + const pkg = "github.com/tailscale/golang-x-crypto/acme" + out, err := exec.Command("go", "list", "-f", "{{.Dir}}", pkg).Output() + if err != nil { + t.Fatalf("failed to find %s's location o disk: %v", pkg, err) + } + xDir := strings.TrimSpace(string(out)) + + t.Logf("at %s", xDir) + scanDir := func(dir string) map[string]string { + m := map[string]string{} // filename => Go contents + ents, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, de := range ents { + name := de.Name() + if name == "sync_to_upstream_test.go" { + continue + } + if !strings.HasSuffix(name, ".go") { + continue + } + b, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + m[name] = string(b) + } + + return m + } + + want := scanDir(xDir) + got := scanDir(".") + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("files differ (-want +got):\n%s", diff) + t.Errorf("to fix, run from module root:\n\ncp %s/*.go ./tempfork/acme && ./tool/go mod tidy\n", xDir) + } +} diff --git a/tempfork/acme/types.go b/tempfork/acme/types.go new file mode 100644 index 0000000000000..9fad800b4ab70 --- /dev/null +++ b/tempfork/acme/types.go @@ -0,0 +1,632 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "crypto" + "crypto/x509" + "errors" + "fmt" + "net/http" + "strings" + "time" +) + +// ACME status values of Account, Order, Authorization and Challenge objects. +// See https://tools.ietf.org/html/rfc8555#section-7.1.6 for details. +const ( + StatusDeactivated = "deactivated" + StatusExpired = "expired" + StatusInvalid = "invalid" + StatusPending = "pending" + StatusProcessing = "processing" + StatusReady = "ready" + StatusRevoked = "revoked" + StatusUnknown = "unknown" + StatusValid = "valid" +) + +// CRLReasonCode identifies the reason for a certificate revocation. +type CRLReasonCode int + +// CRL reason codes as defined in RFC 5280. +const ( + CRLReasonUnspecified CRLReasonCode = 0 + CRLReasonKeyCompromise CRLReasonCode = 1 + CRLReasonCACompromise CRLReasonCode = 2 + CRLReasonAffiliationChanged CRLReasonCode = 3 + CRLReasonSuperseded CRLReasonCode = 4 + CRLReasonCessationOfOperation CRLReasonCode = 5 + CRLReasonCertificateHold CRLReasonCode = 6 + CRLReasonRemoveFromCRL CRLReasonCode = 8 + CRLReasonPrivilegeWithdrawn CRLReasonCode = 9 + CRLReasonAACompromise CRLReasonCode = 10 +) + +var ( + // ErrUnsupportedKey is returned when an unsupported key type is encountered. + ErrUnsupportedKey = errors.New("acme: unknown key type; only RSA and ECDSA are supported") + + // ErrAccountAlreadyExists indicates that the Client's key has already been registered + // with the CA. It is returned by Register method. + ErrAccountAlreadyExists = errors.New("acme: account already exists") + + // ErrNoAccount indicates that the Client's key has not been registered with the CA. + ErrNoAccount = errors.New("acme: account does not exist") +) + +// A Subproblem describes an ACME subproblem as reported in an Error. +type Subproblem struct { + // Type is a URI reference that identifies the problem type, + // typically in a "urn:acme:error:xxx" form. + Type string + // Detail is a human-readable explanation specific to this occurrence of the problem. + Detail string + // Instance indicates a URL that the client should direct a human user to visit + // in order for instructions on how to agree to the updated Terms of Service. + // In such an event CA sets StatusCode to 403, Type to + // "urn:ietf:params:acme:error:userActionRequired", and adds a Link header with relation + // "terms-of-service" containing the latest TOS URL. + Instance string + // Identifier may contain the ACME identifier that the error is for. + Identifier *AuthzID +} + +func (sp Subproblem) String() string { + str := fmt.Sprintf("%s: ", sp.Type) + if sp.Identifier != nil { + str += fmt.Sprintf("[%s: %s] ", sp.Identifier.Type, sp.Identifier.Value) + } + str += sp.Detail + return str +} + +// Error is an ACME error, defined in Problem Details for HTTP APIs doc +// http://tools.ietf.org/html/draft-ietf-appsawg-http-problem. +type Error struct { + // StatusCode is The HTTP status code generated by the origin server. + StatusCode int + // ProblemType is a URI reference that identifies the problem type, + // typically in a "urn:acme:error:xxx" form. + ProblemType string + // Detail is a human-readable explanation specific to this occurrence of the problem. + Detail string + // Instance indicates a URL that the client should direct a human user to visit + // in order for instructions on how to agree to the updated Terms of Service. + // In such an event CA sets StatusCode to 403, ProblemType to + // "urn:ietf:params:acme:error:userActionRequired" and a Link header with relation + // "terms-of-service" containing the latest TOS URL. + Instance string + // Header is the original server error response headers. + // It may be nil. + Header http.Header + // Subproblems may contain more detailed information about the individual problems + // that caused the error. This field is only sent by RFC 8555 compatible ACME + // servers. Defined in RFC 8555 Section 6.7.1. + Subproblems []Subproblem +} + +func (e *Error) Error() string { + str := fmt.Sprintf("%d %s: %s", e.StatusCode, e.ProblemType, e.Detail) + if len(e.Subproblems) > 0 { + str += fmt.Sprintf("; subproblems:") + for _, sp := range e.Subproblems { + str += fmt.Sprintf("\n\t%s", sp) + } + } + return str +} + +// AuthorizationError indicates that an authorization for an identifier +// did not succeed. +// It contains all errors from Challenge items of the failed Authorization. +type AuthorizationError struct { + // URI uniquely identifies the failed Authorization. + URI string + + // Identifier is an AuthzID.Value of the failed Authorization. + Identifier string + + // Errors is a collection of non-nil error values of Challenge items + // of the failed Authorization. + Errors []error +} + +func (a *AuthorizationError) Error() string { + e := make([]string, len(a.Errors)) + for i, err := range a.Errors { + e[i] = err.Error() + } + + if a.Identifier != "" { + return fmt.Sprintf("acme: authorization error for %s: %s", a.Identifier, strings.Join(e, "; ")) + } + + return fmt.Sprintf("acme: authorization error: %s", strings.Join(e, "; ")) +} + +// OrderError is returned from Client's order related methods. +// It indicates the order is unusable and the clients should start over with +// AuthorizeOrder. +// +// The clients can still fetch the order object from CA using GetOrder +// to inspect its state. +type OrderError struct { + OrderURL string + Status string +} + +func (oe *OrderError) Error() string { + return fmt.Sprintf("acme: order %s status: %s", oe.OrderURL, oe.Status) +} + +// RateLimit reports whether err represents a rate limit error and +// any Retry-After duration returned by the server. +// +// See the following for more details on rate limiting: +// https://tools.ietf.org/html/draft-ietf-acme-acme-05#section-5.6 +func RateLimit(err error) (time.Duration, bool) { + e, ok := err.(*Error) + if !ok { + return 0, false + } + // Some CA implementations may return incorrect values. + // Use case-insensitive comparison. + if !strings.HasSuffix(strings.ToLower(e.ProblemType), ":ratelimited") { + return 0, false + } + if e.Header == nil { + return 0, true + } + return retryAfter(e.Header.Get("Retry-After")), true +} + +// Account is a user account. It is associated with a private key. +// Non-RFC 8555 fields are empty when interfacing with a compliant CA. +type Account struct { + // URI is the account unique ID, which is also a URL used to retrieve + // account data from the CA. + // When interfacing with RFC 8555-compliant CAs, URI is the "kid" field + // value in JWS signed requests. + URI string + + // Contact is a slice of contact info used during registration. + // See https://tools.ietf.org/html/rfc8555#section-7.3 for supported + // formats. + Contact []string + + // Status indicates current account status as returned by the CA. + // Possible values are StatusValid, StatusDeactivated, and StatusRevoked. + Status string + + // OrdersURL is a URL from which a list of orders submitted by this account + // can be fetched. + OrdersURL string + + // The terms user has agreed to. + // A value not matching CurrentTerms indicates that the user hasn't agreed + // to the actual Terms of Service of the CA. + // + // It is non-RFC 8555 compliant. Package users can store the ToS they agree to + // during Client's Register call in the prompt callback function. + AgreedTerms string + + // Actual terms of a CA. + // + // It is non-RFC 8555 compliant. Use Directory's Terms field. + // When a CA updates their terms and requires an account agreement, + // a URL at which instructions to do so is available in Error's Instance field. + CurrentTerms string + + // Authz is the authorization URL used to initiate a new authz flow. + // + // It is non-RFC 8555 compliant. Use Directory's AuthzURL or OrderURL. + Authz string + + // Authorizations is a URI from which a list of authorizations + // granted to this account can be fetched via a GET request. + // + // It is non-RFC 8555 compliant and is obsoleted by OrdersURL. + Authorizations string + + // Certificates is a URI from which a list of certificates + // issued for this account can be fetched via a GET request. + // + // It is non-RFC 8555 compliant and is obsoleted by OrdersURL. + Certificates string + + // ExternalAccountBinding represents an arbitrary binding to an account of + // the CA which the ACME server is tied to. + // See https://tools.ietf.org/html/rfc8555#section-7.3.4 for more details. + ExternalAccountBinding *ExternalAccountBinding +} + +// ExternalAccountBinding contains the data needed to form a request with +// an external account binding. +// See https://tools.ietf.org/html/rfc8555#section-7.3.4 for more details. +type ExternalAccountBinding struct { + // KID is the Key ID of the symmetric MAC key that the CA provides to + // identify an external account from ACME. + KID string + + // Key is the bytes of the symmetric key that the CA provides to identify + // the account. Key must correspond to the KID. + Key []byte +} + +func (e *ExternalAccountBinding) String() string { + return fmt.Sprintf("&{KID: %q, Key: redacted}", e.KID) +} + +// Directory is ACME server discovery data. +// See https://tools.ietf.org/html/rfc8555#section-7.1.1 for more details. +type Directory struct { + // NonceURL indicates an endpoint where to fetch fresh nonce values from. + NonceURL string + + // RegURL is an account endpoint URL, allowing for creating new accounts. + // Pre-RFC 8555 CAs also allow modifying existing accounts at this URL. + RegURL string + + // OrderURL is used to initiate the certificate issuance flow + // as described in RFC 8555. + OrderURL string + + // AuthzURL is used to initiate identifier pre-authorization flow. + // Empty string indicates the flow is unsupported by the CA. + AuthzURL string + + // CertURL is a new certificate issuance endpoint URL. + // It is non-RFC 8555 compliant and is obsoleted by OrderURL. + CertURL string + + // RevokeURL is used to initiate a certificate revocation flow. + RevokeURL string + + // KeyChangeURL allows to perform account key rollover flow. + KeyChangeURL string + + // RenewalInfoURL allows to perform certificate renewal using the ACME + // Renewal Information (ARI) Extension. + RenewalInfoURL string + + // Term is a URI identifying the current terms of service. + Terms string + + // Website is an HTTP or HTTPS URL locating a website + // providing more information about the ACME server. + Website string + + // CAA consists of lowercase hostname elements, which the ACME server + // recognises as referring to itself for the purposes of CAA record validation + // as defined in RFC 6844. + CAA []string + + // ExternalAccountRequired indicates that the CA requires for all account-related + // requests to include external account binding information. + ExternalAccountRequired bool +} + +// Order represents a client's request for a certificate. +// It tracks the request flow progress through to issuance. +type Order struct { + // URI uniquely identifies an order. + URI string + + // Status represents the current status of the order. + // It indicates which action the client should take. + // + // Possible values are StatusPending, StatusReady, StatusProcessing, StatusValid and StatusInvalid. + // Pending means the CA does not believe that the client has fulfilled the requirements. + // Ready indicates that the client has fulfilled all the requirements and can submit a CSR + // to obtain a certificate. This is done with Client's CreateOrderCert. + // Processing means the certificate is being issued. + // Valid indicates the CA has issued the certificate. It can be downloaded + // from the Order's CertURL. This is done with Client's FetchCert. + // Invalid means the certificate will not be issued. Users should consider this order + // abandoned. + Status string + + // Expires is the timestamp after which CA considers this order invalid. + Expires time.Time + + // Identifiers contains all identifier objects which the order pertains to. + Identifiers []AuthzID + + // NotBefore is the requested value of the notBefore field in the certificate. + NotBefore time.Time + + // NotAfter is the requested value of the notAfter field in the certificate. + NotAfter time.Time + + // AuthzURLs represents authorizations to complete before a certificate + // for identifiers specified in the order can be issued. + // It also contains unexpired authorizations that the client has completed + // in the past. + // + // Authorization objects can be fetched using Client's GetAuthorization method. + // + // The required authorizations are dictated by CA policies. + // There may not be a 1:1 relationship between the identifiers and required authorizations. + // Required authorizations can be identified by their StatusPending status. + // + // For orders in the StatusValid or StatusInvalid state these are the authorizations + // which were completed. + AuthzURLs []string + + // FinalizeURL is the endpoint at which a CSR is submitted to obtain a certificate + // once all the authorizations are satisfied. + FinalizeURL string + + // CertURL points to the certificate that has been issued in response to this order. + CertURL string + + // The error that occurred while processing the order as received from a CA, if any. + Error *Error +} + +// OrderOption allows customizing Client.AuthorizeOrder call. +type OrderOption interface { + privateOrderOpt() +} + +// WithOrderNotBefore sets order's NotBefore field. +func WithOrderNotBefore(t time.Time) OrderOption { + return orderNotBeforeOpt(t) +} + +// WithOrderNotAfter sets order's NotAfter field. +func WithOrderNotAfter(t time.Time) OrderOption { + return orderNotAfterOpt(t) +} + +type orderNotBeforeOpt time.Time + +func (orderNotBeforeOpt) privateOrderOpt() {} + +type orderNotAfterOpt time.Time + +func (orderNotAfterOpt) privateOrderOpt() {} + +// Authorization encodes an authorization response. +type Authorization struct { + // URI uniquely identifies a authorization. + URI string + + // Status is the current status of an authorization. + // Possible values are StatusPending, StatusValid, StatusInvalid, StatusDeactivated, + // StatusExpired and StatusRevoked. + Status string + + // Identifier is what the account is authorized to represent. + Identifier AuthzID + + // The timestamp after which the CA considers the authorization invalid. + Expires time.Time + + // Wildcard is true for authorizations of a wildcard domain name. + Wildcard bool + + // Challenges that the client needs to fulfill in order to prove possession + // of the identifier (for pending authorizations). + // For valid authorizations, the challenge that was validated. + // For invalid authorizations, the challenge that was attempted and failed. + // + // RFC 8555 compatible CAs require users to fuflfill only one of the challenges. + Challenges []*Challenge + + // A collection of sets of challenges, each of which would be sufficient + // to prove possession of the identifier. + // Clients must complete a set of challenges that covers at least one set. + // Challenges are identified by their indices in the challenges array. + // If this field is empty, the client needs to complete all challenges. + // + // This field is unused in RFC 8555. + Combinations [][]int +} + +// AuthzID is an identifier that an account is authorized to represent. +type AuthzID struct { + Type string // The type of identifier, "dns" or "ip". + Value string // The identifier itself, e.g. "example.org". +} + +// DomainIDs creates a slice of AuthzID with "dns" identifier type. +func DomainIDs(names ...string) []AuthzID { + a := make([]AuthzID, len(names)) + for i, v := range names { + a[i] = AuthzID{Type: "dns", Value: v} + } + return a +} + +// IPIDs creates a slice of AuthzID with "ip" identifier type. +// Each element of addr is textual form of an address as defined +// in RFC 1123 Section 2.1 for IPv4 and in RFC 5952 Section 4 for IPv6. +func IPIDs(addr ...string) []AuthzID { + a := make([]AuthzID, len(addr)) + for i, v := range addr { + a[i] = AuthzID{Type: "ip", Value: v} + } + return a +} + +// wireAuthzID is ACME JSON representation of authorization identifier objects. +type wireAuthzID struct { + Type string `json:"type"` + Value string `json:"value"` +} + +// wireAuthz is ACME JSON representation of Authorization objects. +type wireAuthz struct { + Identifier wireAuthzID + Status string + Expires time.Time + Wildcard bool + Challenges []wireChallenge + Combinations [][]int + Error *wireError +} + +func (z *wireAuthz) authorization(uri string) *Authorization { + a := &Authorization{ + URI: uri, + Status: z.Status, + Identifier: AuthzID{Type: z.Identifier.Type, Value: z.Identifier.Value}, + Expires: z.Expires, + Wildcard: z.Wildcard, + Challenges: make([]*Challenge, len(z.Challenges)), + Combinations: z.Combinations, // shallow copy + } + for i, v := range z.Challenges { + a.Challenges[i] = v.challenge() + } + return a +} + +func (z *wireAuthz) error(uri string) *AuthorizationError { + err := &AuthorizationError{ + URI: uri, + Identifier: z.Identifier.Value, + } + + if z.Error != nil { + err.Errors = append(err.Errors, z.Error.error(nil)) + } + + for _, raw := range z.Challenges { + if raw.Error != nil { + err.Errors = append(err.Errors, raw.Error.error(nil)) + } + } + + return err +} + +// Challenge encodes a returned CA challenge. +// Its Error field may be non-nil if the challenge is part of an Authorization +// with StatusInvalid. +type Challenge struct { + // Type is the challenge type, e.g. "http-01", "tls-alpn-01", "dns-01". + Type string + + // URI is where a challenge response can be posted to. + URI string + + // Token is a random value that uniquely identifies the challenge. + Token string + + // Status identifies the status of this challenge. + // In RFC 8555, possible values are StatusPending, StatusProcessing, StatusValid, + // and StatusInvalid. + Status string + + // Validated is the time at which the CA validated this challenge. + // Always zero value in pre-RFC 8555. + Validated time.Time + + // Error indicates the reason for an authorization failure + // when this challenge was used. + // The type of a non-nil value is *Error. + Error error +} + +// wireChallenge is ACME JSON challenge representation. +type wireChallenge struct { + URL string `json:"url"` // RFC + URI string `json:"uri"` // pre-RFC + Type string + Token string + Status string + Validated time.Time + Error *wireError +} + +func (c *wireChallenge) challenge() *Challenge { + v := &Challenge{ + URI: c.URL, + Type: c.Type, + Token: c.Token, + Status: c.Status, + } + if v.URI == "" { + v.URI = c.URI // c.URL was empty; use legacy + } + if v.Status == "" { + v.Status = StatusPending + } + if c.Error != nil { + v.Error = c.Error.error(nil) + } + return v +} + +// wireError is a subset of fields of the Problem Details object +// as described in https://tools.ietf.org/html/rfc7807#section-3.1. +type wireError struct { + Status int + Type string + Detail string + Instance string + Subproblems []Subproblem +} + +func (e *wireError) error(h http.Header) *Error { + err := &Error{ + StatusCode: e.Status, + ProblemType: e.Type, + Detail: e.Detail, + Instance: e.Instance, + Header: h, + Subproblems: e.Subproblems, + } + return err +} + +// CertOption is an optional argument type for the TLS ChallengeCert methods for +// customizing a temporary certificate for TLS-based challenges. +type CertOption interface { + privateCertOpt() +} + +// WithKey creates an option holding a private/public key pair. +// The private part signs a certificate, and the public part represents the signee. +func WithKey(key crypto.Signer) CertOption { + return &certOptKey{key} +} + +type certOptKey struct { + key crypto.Signer +} + +func (*certOptKey) privateCertOpt() {} + +// WithTemplate creates an option for specifying a certificate template. +// See x509.CreateCertificate for template usage details. +// +// In TLS ChallengeCert methods, the template is also used as parent, +// resulting in a self-signed certificate. +// The DNSNames field of t is always overwritten for tls-sni challenge certs. +func WithTemplate(t *x509.Certificate) CertOption { + return (*certOptTemplate)(t) +} + +type certOptTemplate x509.Certificate + +func (*certOptTemplate) privateCertOpt() {} + +// RenewalInfoWindow describes the time frame during which the ACME client +// should attempt to renew, using the ACME Renewal Info Extension. +type RenewalInfoWindow struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` +} + +// RenewalInfo describes the suggested renewal window for a given certificate, +// returned from an ACME server, using the ACME Renewal Info Extension. +type RenewalInfo struct { + SuggestedWindow RenewalInfoWindow `json:"suggestedWindow"` + ExplanationURL string `json:"explanationURL"` +} diff --git a/tempfork/acme/types_test.go b/tempfork/acme/types_test.go new file mode 100644 index 0000000000000..59ce7e7602ca3 --- /dev/null +++ b/tempfork/acme/types_test.go @@ -0,0 +1,219 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package acme + +import ( + "errors" + "net/http" + "reflect" + "testing" + "time" +) + +func TestExternalAccountBindingString(t *testing.T) { + eab := ExternalAccountBinding{ + KID: "kid", + Key: []byte("key"), + } + got := eab.String() + want := `&{KID: "kid", Key: redacted}` + if got != want { + t.Errorf("eab.String() = %q, want: %q", got, want) + } +} + +func TestRateLimit(t *testing.T) { + now := time.Date(2017, 04, 27, 10, 0, 0, 0, time.UTC) + f := timeNow + defer func() { timeNow = f }() + timeNow = func() time.Time { return now } + + h120, hTime := http.Header{}, http.Header{} + h120.Set("Retry-After", "120") + hTime.Set("Retry-After", "Tue Apr 27 11:00:00 2017") + + err1 := &Error{ + ProblemType: "urn:ietf:params:acme:error:nolimit", + Header: h120, + } + err2 := &Error{ + ProblemType: "urn:ietf:params:acme:error:rateLimited", + Header: h120, + } + err3 := &Error{ + ProblemType: "urn:ietf:params:acme:error:rateLimited", + Header: nil, + } + err4 := &Error{ + ProblemType: "urn:ietf:params:acme:error:rateLimited", + Header: hTime, + } + + tt := []struct { + err error + res time.Duration + ok bool + }{ + {nil, 0, false}, + {errors.New("dummy"), 0, false}, + {err1, 0, false}, + {err2, 2 * time.Minute, true}, + {err3, 0, true}, + {err4, time.Hour, true}, + } + for i, test := range tt { + res, ok := RateLimit(test.err) + if ok != test.ok { + t.Errorf("%d: RateLimit(%+v): ok = %v; want %v", i, test.err, ok, test.ok) + continue + } + if res != test.res { + t.Errorf("%d: RateLimit(%+v) = %v; want %v", i, test.err, res, test.res) + } + } +} + +func TestAuthorizationError(t *testing.T) { + tests := []struct { + desc string + err *AuthorizationError + msg string + }{ + { + desc: "when auth error identifier is set", + err: &AuthorizationError{ + Identifier: "domain.com", + Errors: []error{ + (&wireError{ + Status: 403, + Type: "urn:ietf:params:acme:error:caa", + Detail: "CAA record for domain.com prevents issuance", + }).error(nil), + }, + }, + msg: "acme: authorization error for domain.com: 403 urn:ietf:params:acme:error:caa: CAA record for domain.com prevents issuance", + }, + + { + desc: "when auth error identifier is unset", + err: &AuthorizationError{ + Errors: []error{ + (&wireError{ + Status: 403, + Type: "urn:ietf:params:acme:error:caa", + Detail: "CAA record for domain.com prevents issuance", + }).error(nil), + }, + }, + msg: "acme: authorization error: 403 urn:ietf:params:acme:error:caa: CAA record for domain.com prevents issuance", + }, + } + + for _, tt := range tests { + if tt.err.Error() != tt.msg { + t.Errorf("got: %s\nwant: %s", tt.err, tt.msg) + } + } +} + +func TestSubproblems(t *testing.T) { + tests := []struct { + wire wireError + expectedOut Error + }{ + { + wire: wireError{ + Status: 1, + Type: "urn:error", + Detail: "it's an error", + }, + expectedOut: Error{ + StatusCode: 1, + ProblemType: "urn:error", + Detail: "it's an error", + }, + }, + { + wire: wireError{ + Status: 1, + Type: "urn:error", + Detail: "it's an error", + Subproblems: []Subproblem{ + { + Type: "urn:error:sub", + Detail: "it's a subproblem", + }, + }, + }, + expectedOut: Error{ + StatusCode: 1, + ProblemType: "urn:error", + Detail: "it's an error", + Subproblems: []Subproblem{ + { + Type: "urn:error:sub", + Detail: "it's a subproblem", + }, + }, + }, + }, + { + wire: wireError{ + Status: 1, + Type: "urn:error", + Detail: "it's an error", + Subproblems: []Subproblem{ + { + Type: "urn:error:sub", + Detail: "it's a subproblem", + Identifier: &AuthzID{Type: "dns", Value: "example"}, + }, + }, + }, + expectedOut: Error{ + StatusCode: 1, + ProblemType: "urn:error", + Detail: "it's an error", + Subproblems: []Subproblem{ + { + Type: "urn:error:sub", + Detail: "it's a subproblem", + Identifier: &AuthzID{Type: "dns", Value: "example"}, + }, + }, + }, + }, + } + + for _, tc := range tests { + out := tc.wire.error(nil) + if !reflect.DeepEqual(*out, tc.expectedOut) { + t.Errorf("Unexpected error: wanted %v, got %v", tc.expectedOut, *out) + } + } +} + +func TestErrorStringerWithSubproblems(t *testing.T) { + err := Error{ + StatusCode: 1, + ProblemType: "urn:error", + Detail: "it's an error", + Subproblems: []Subproblem{ + { + Type: "urn:error:sub", + Detail: "it's a subproblem", + }, + { + Type: "urn:error:sub", + Detail: "it's a subproblem", + Identifier: &AuthzID{Type: "dns", Value: "example"}, + }, + }, + } + expectedStr := "1 urn:error: it's an error; subproblems:\n\turn:error:sub: it's a subproblem\n\turn:error:sub: [dns: example] it's a subproblem" + if err.Error() != expectedStr { + t.Errorf("Unexpected error string: wanted %q, got %q", expectedStr, err.Error()) + } +} diff --git a/tempfork/acme/version_go112.go b/tempfork/acme/version_go112.go new file mode 100644 index 0000000000000..cc5fab604b8d0 --- /dev/null +++ b/tempfork/acme/version_go112.go @@ -0,0 +1,27 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.12 + +package acme + +import "runtime/debug" + +func init() { + // Set packageVersion if the binary was built in modules mode and x/crypto + // was not replaced with a different module. + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + for _, m := range info.Deps { + if m.Path != "golang.org/x/crypto" { + continue + } + if m.Replace == nil { + packageVersion = m.Version + } + break + } +} From ba1f9a3918f092e5428620640e571ba36ab646e9 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 24 Jan 2025 19:56:33 -0800 Subject: [PATCH 205/223] types/persist: remove Persist.LegacyFrontendPrivateMachineKey It was a temporary migration over four years ago. It's no longer relevant. Updates #610 Change-Id: I1f00c9485fab13ede6f77603f7d4235222c2a481 Signed-off-by: Brad Fitzpatrick --- ipn/ipnlocal/local.go | 17 ++--------------- ipn/ipnlocal/state_test.go | 6 ------ ipn/prefs_test.go | 9 +-------- types/persist/persist.go | 22 +++------------------- types/persist/persist_clone.go | 15 +++++++-------- types/persist/persist_test.go | 14 +------------- types/persist/persist_view.go | 18 +++++++----------- 7 files changed, 21 insertions(+), 80 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 58cd4025f44a9..a6e3f19528ebf 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1082,7 +1082,6 @@ func stripKeysFromPrefs(p ipn.PrefsView) ipn.PrefsView { } p2 := p.AsStruct() - p2.Persist.LegacyFrontendPrivateMachineKey = key.MachinePrivate{} p2.Persist.PrivateNodeKey = key.NodePrivate{} p2.Persist.OldPrivateNodeKey = key.NodePrivate{} p2.Persist.NetworkLockKey = key.NLPrivate{} @@ -3343,11 +3342,6 @@ func (b *LocalBackend) initMachineKeyLocked() (err error) { return nil } - var legacyMachineKey key.MachinePrivate - if p := b.pm.CurrentPrefs().Persist(); p.Valid() { - legacyMachineKey = p.LegacyFrontendPrivateMachineKey() - } - keyText, err := b.store.ReadState(ipn.MachineKeyStateKey) if err == nil { if err := b.machinePrivKey.UnmarshalText(keyText); err != nil { @@ -3356,9 +3350,6 @@ func (b *LocalBackend) initMachineKeyLocked() (err error) { if b.machinePrivKey.IsZero() { return fmt.Errorf("invalid zero key stored in %v key of %v", ipn.MachineKeyStateKey, b.store) } - if !legacyMachineKey.IsZero() && !legacyMachineKey.Equal(b.machinePrivKey) { - b.logf("frontend-provided legacy machine key ignored; used value from server state") - } return nil } if err != ipn.ErrStateNotExist { @@ -3368,12 +3359,8 @@ func (b *LocalBackend) initMachineKeyLocked() (err error) { // If we didn't find one already on disk and the prefs already // have a legacy machine key, use that. Otherwise generate a // new one. - if !legacyMachineKey.IsZero() { - b.machinePrivKey = legacyMachineKey - } else { - b.logf("generating new machine key") - b.machinePrivKey = key.NewMachine() - } + b.logf("generating new machine key") + b.machinePrivKey = key.NewMachine() keyText, _ = b.machinePrivKey.MarshalText() if err := ipn.WriteState(b.store, ipn.MachineKeyStateKey, keyText); err != nil { diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index ef4b0ed62809f..1b3b43af66e38 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -949,8 +949,6 @@ func TestEditPrefsHasNoKeys(t *testing.T) { Persist: &persist.Persist{ PrivateNodeKey: key.NewNode(), OldPrivateNodeKey: key.NewNode(), - - LegacyFrontendPrivateMachineKey: key.NewMachine(), }, }).View(), ipn.NetworkProfile{}) if p := b.pm.CurrentPrefs().Persist(); !p.Valid() || p.PrivateNodeKey().IsZero() { @@ -977,10 +975,6 @@ func TestEditPrefsHasNoKeys(t *testing.T) { t.Errorf("OldPrivateNodeKey = %v; want zero", p.Persist().OldPrivateNodeKey()) } - if !p.Persist().LegacyFrontendPrivateMachineKey().IsZero() { - t.Errorf("LegacyFrontendPrivateMachineKey = %v; want zero", p.Persist().LegacyFrontendPrivateMachineKey()) - } - if !p.Persist().NetworkLockKey().IsZero() { t.Errorf("NetworkLockKey= %v; want zero", p.Persist().NetworkLockKey()) } diff --git a/ipn/prefs_test.go b/ipn/prefs_test.go index 31671c0f8e4ef..91b835e3e3e19 100644 --- a/ipn/prefs_test.go +++ b/ipn/prefs_test.go @@ -467,13 +467,6 @@ func TestPrefsPretty(t *testing.T) { "darwin", `Prefs{ra=false dns=false want=true tags=tag:foo,tag:bar url="http://localhost:1234" update=off Persist=nil}`, }, - { - Prefs{ - Persist: &persist.Persist{}, - }, - "linux", - `Prefs{ra=false dns=false want=false routes=[] nf=off update=off Persist{lm=, o=, n= u=""}}`, - }, { Prefs{ Persist: &persist.Persist{ @@ -481,7 +474,7 @@ func TestPrefsPretty(t *testing.T) { }, }, "linux", - `Prefs{ra=false dns=false want=false routes=[] nf=off update=off Persist{lm=, o=, n=[B1VKl] u=""}}`, + `Prefs{ra=false dns=false want=false routes=[] nf=off update=off Persist{o=, n=[B1VKl] u=""}}`, }, { Prefs{ diff --git a/types/persist/persist.go b/types/persist/persist.go index 8b555abd42c1e..d888a6afb6af5 100644 --- a/types/persist/persist.go +++ b/types/persist/persist.go @@ -21,17 +21,6 @@ import ( type Persist struct { _ structs.Incomparable - // LegacyFrontendPrivateMachineKey is here temporarily - // (starting 2020-09-28) during migration of Windows users' - // machine keys from frontend storage to the backend. On the - // first LocalBackend.Start call, the backend will initialize - // the real (backend-owned) machine key from the frontend's - // provided value (if non-zero), picking a new random one if - // needed. This field should be considered read-only from GUI - // frontends. The real value should not be written back in - // this field, lest the frontend persist it to disk. - LegacyFrontendPrivateMachineKey key.MachinePrivate `json:"PrivateMachineKey"` - PrivateNodeKey key.NodePrivate OldPrivateNodeKey key.NodePrivate // needed to request key rotation UserProfile tailcfg.UserProfile @@ -95,8 +84,7 @@ func (p *Persist) Equals(p2 *Persist) bool { return false } - return p.LegacyFrontendPrivateMachineKey.Equal(p2.LegacyFrontendPrivateMachineKey) && - p.PrivateNodeKey.Equal(p2.PrivateNodeKey) && + return p.PrivateNodeKey.Equal(p2.PrivateNodeKey) && p.OldPrivateNodeKey.Equal(p2.OldPrivateNodeKey) && p.UserProfile.Equal(&p2.UserProfile) && p.NetworkLockKey.Equal(p2.NetworkLockKey) && @@ -106,18 +94,14 @@ func (p *Persist) Equals(p2 *Persist) bool { func (p *Persist) Pretty() string { var ( - mk key.MachinePublic ok, nk key.NodePublic ) - if !p.LegacyFrontendPrivateMachineKey.IsZero() { - mk = p.LegacyFrontendPrivateMachineKey.Public() - } if !p.OldPrivateNodeKey.IsZero() { ok = p.OldPrivateNodeKey.Public() } if !p.PrivateNodeKey.IsZero() { nk = p.PublicNodeKey() } - return fmt.Sprintf("Persist{lm=%v, o=%v, n=%v u=%#v}", - mk.ShortString(), ok.ShortString(), nk.ShortString(), p.UserProfile.LoginName) + return fmt.Sprintf("Persist{o=%v, n=%v u=%#v}", + ok.ShortString(), nk.ShortString(), p.UserProfile.LoginName) } diff --git a/types/persist/persist_clone.go b/types/persist/persist_clone.go index 95dd65ac18e67..680419ff2f30b 100644 --- a/types/persist/persist_clone.go +++ b/types/persist/persist_clone.go @@ -25,12 +25,11 @@ func (src *Persist) Clone() *Persist { // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _PersistCloneNeedsRegeneration = Persist(struct { - _ structs.Incomparable - LegacyFrontendPrivateMachineKey key.MachinePrivate - PrivateNodeKey key.NodePrivate - OldPrivateNodeKey key.NodePrivate - UserProfile tailcfg.UserProfile - NetworkLockKey key.NLPrivate - NodeID tailcfg.StableNodeID - DisallowedTKAStateIDs []string + _ structs.Incomparable + PrivateNodeKey key.NodePrivate + OldPrivateNodeKey key.NodePrivate + UserProfile tailcfg.UserProfile + NetworkLockKey key.NLPrivate + NodeID tailcfg.StableNodeID + DisallowedTKAStateIDs []string }{}) diff --git a/types/persist/persist_test.go b/types/persist/persist_test.go index 6b159573d4302..dbf2a6d8c7662 100644 --- a/types/persist/persist_test.go +++ b/types/persist/persist_test.go @@ -21,13 +21,12 @@ func fieldsOf(t reflect.Type) (fields []string) { } func TestPersistEqual(t *testing.T) { - persistHandles := []string{"LegacyFrontendPrivateMachineKey", "PrivateNodeKey", "OldPrivateNodeKey", "UserProfile", "NetworkLockKey", "NodeID", "DisallowedTKAStateIDs"} + persistHandles := []string{"PrivateNodeKey", "OldPrivateNodeKey", "UserProfile", "NetworkLockKey", "NodeID", "DisallowedTKAStateIDs"} if have := fieldsOf(reflect.TypeFor[Persist]()); !reflect.DeepEqual(have, persistHandles) { t.Errorf("Persist.Equal check might be out of sync\nfields: %q\nhandled: %q\n", have, persistHandles) } - m1 := key.NewMachine() k1 := key.NewNode() nl1 := key.NewNLPrivate() tests := []struct { @@ -39,17 +38,6 @@ func TestPersistEqual(t *testing.T) { {&Persist{}, nil, false}, {&Persist{}, &Persist{}, true}, - { - &Persist{LegacyFrontendPrivateMachineKey: m1}, - &Persist{LegacyFrontendPrivateMachineKey: key.NewMachine()}, - false, - }, - { - &Persist{LegacyFrontendPrivateMachineKey: m1}, - &Persist{LegacyFrontendPrivateMachineKey: m1}, - true, - }, - { &Persist{PrivateNodeKey: k1}, &Persist{PrivateNodeKey: key.NewNode()}, diff --git a/types/persist/persist_view.go b/types/persist/persist_view.go index ce600be3e0753..55eb40c51ac47 100644 --- a/types/persist/persist_view.go +++ b/types/persist/persist_view.go @@ -62,9 +62,6 @@ func (v *PersistView) UnmarshalJSON(b []byte) error { return nil } -func (v PersistView) LegacyFrontendPrivateMachineKey() key.MachinePrivate { - return v.Đļ.LegacyFrontendPrivateMachineKey -} func (v PersistView) PrivateNodeKey() key.NodePrivate { return v.Đļ.PrivateNodeKey } func (v PersistView) OldPrivateNodeKey() key.NodePrivate { return v.Đļ.OldPrivateNodeKey } func (v PersistView) UserProfile() tailcfg.UserProfile { return v.Đļ.UserProfile } @@ -76,12 +73,11 @@ func (v PersistView) DisallowedTKAStateIDs() views.Slice[string] { // A compilation failure here means this code must be regenerated, with the command at the top of this file. var _PersistViewNeedsRegeneration = Persist(struct { - _ structs.Incomparable - LegacyFrontendPrivateMachineKey key.MachinePrivate - PrivateNodeKey key.NodePrivate - OldPrivateNodeKey key.NodePrivate - UserProfile tailcfg.UserProfile - NetworkLockKey key.NLPrivate - NodeID tailcfg.StableNodeID - DisallowedTKAStateIDs []string + _ structs.Incomparable + PrivateNodeKey key.NodePrivate + OldPrivateNodeKey key.NodePrivate + UserProfile tailcfg.UserProfile + NetworkLockKey key.NLPrivate + NodeID tailcfg.StableNodeID + DisallowedTKAStateIDs []string }{}) From 079973de8280c23b7a3f74f6c2d0ac5b7d963d9d Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 27 Jan 2025 22:03:22 +0000 Subject: [PATCH 206/223] tempfork/acme: fix TestSyncedToUpstream with Windows line endings Updates #10238 Change-Id: Ic85811c267679a9f79377f376d77dee3a9d92ce7 Signed-off-by: Brad Fitzpatrick --- tempfork/acme/sync_to_upstream_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tempfork/acme/sync_to_upstream_test.go b/tempfork/acme/sync_to_upstream_test.go index d6bea7a11320a..e22c8c1a86d01 100644 --- a/tempfork/acme/sync_to_upstream_test.go +++ b/tempfork/acme/sync_to_upstream_test.go @@ -55,7 +55,7 @@ func TestSyncedToUpstream(t *testing.T) { if err != nil { t.Fatal(err) } - m[name] = string(b) + m[name] = strings.ReplaceAll(string(b), "\r", "") } return m From 6f10fe8ab1f7d4f212610719a02c5b612575b858 Mon Sep 17 00:00:00 2001 From: yejingchen Date: Tue, 28 Jan 2025 18:05:49 +0800 Subject: [PATCH 207/223] cmd/tailscale: add warning to help text of `--force-reauth` (#14778) The warning text is adapted from https://tailscale.com/kb/1028/key-expiry#renewing-keys-for-an-expired-device . There is already https://github.com/tailscale/tailscale/pull/7575 which presents a warning when connected over Tailscale, however the detection is done by checking SSH environment variables, which are absent within systemd's run0*. That means `--force-reauth` will happily bring down Tailscale connection, leaving the user in despair. Changing only the help text is by no means a complete solution, but hopefully it will stop users from blindly trying it out, and motivate them to search for a proper solution. *: https://www.freedesktop.org/software/systemd/man/devel/run0.html Updates #3849 Signed-off-by: yejingchen --- cmd/tailscale/cli/up.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index 4af264d73a991..da3780e39c31f 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -139,7 +139,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { // Some flags are only for "up", not "login". upf.BoolVar(&upArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)") upf.BoolVar(&upArgs.reset, "reset", false, "reset unspecified settings to their default values") - upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication") + upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication (WARNING: this will bring down the Tailscale connection and thus should not be done remotely over SSH or RDP)") registerAcceptRiskFlag(upf, &upArgs.acceptedRisks) } From 3abfbf50aebbe3ba57dc749165edb56be6715c0a Mon Sep 17 00:00:00 2001 From: Anton Tolchanov Date: Tue, 28 Jan 2025 12:10:28 +0000 Subject: [PATCH 208/223] tsnet: return from Accept when the listener gets closed Fixes #14808 Signed-off-by: Anton Tolchanov --- tsnet/tsnet.go | 7 ++++--- tsnet/tsnet_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 3505c94539d48..23a9f9a981cd0 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -1286,11 +1286,12 @@ type listener struct { } func (ln *listener) Accept() (net.Conn, error) { - c, ok := <-ln.conn - if !ok { + select { + case c := <-ln.conn: + return c, nil + case <-ln.closedc: return nil, fmt.Errorf("tsnet: %w", net.ErrClosed) } - return c, nil } func (ln *listener) Addr() net.Addr { return addr{ln} } diff --git a/tsnet/tsnet_test.go b/tsnet/tsnet_test.go index 552e8dbee390a..0f245b0156730 100644 --- a/tsnet/tsnet_test.go +++ b/tsnet/tsnet_test.go @@ -667,6 +667,37 @@ func TestFunnel(t *testing.T) { } } +func TestListenerClose(t *testing.T) { + ctx := context.Background() + controlURL, _ := startControl(t) + + s1, _, _ := startServer(t, ctx, controlURL, "s1") + + ln, err := s1.Listen("tcp", ":8080") + if err != nil { + t.Fatal(err) + } + + errc := make(chan error, 1) + go func() { + c, err := ln.Accept() + if c != nil { + c.Close() + } + errc <- err + }() + + ln.Close() + select { + case err := <-errc: + if !errors.Is(err, net.ErrClosed) { + t.Errorf("unexpected error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for Accept to return") + } +} + func dialIngressConn(from, to *Server, target string) (net.Conn, error) { toLC := must.Get(to.LocalClient()) toStatus := must.Get(toLC.StatusWithoutPeers(context.Background())) From 46fd4e58a27495263336b86ee961ee28d8c332b7 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Mon, 27 Jan 2025 13:05:27 -0600 Subject: [PATCH 209/223] ssh,tempfork/gliderlabs/ssh: replace github.com/tailscale/golang-x-crypto/ssh with golang.org/x/crypto/ssh The upstream crypto package now supports sending banners at any time during authentication, so the Tailscale fork of crypto/ssh is no longer necessary. github.com/tailscale/golang-x-crypto is still needed for some custom ACME autocert functionality. tempfork/gliderlabs is still necessary because of a few other customizations, mostly related to TTY handling. Updates #8593 Signed-off-by: Percy Wegmann --- cmd/k8s-operator/depaware.txt | 11 +- cmd/ssh-auth-none-demo/ssh-auth-none-demo.go | 24 +- cmd/tailscaled/depaware.txt | 7 +- cmd/tailscaled/deps_test.go | 1 - go.mod | 2 +- go.sum | 4 +- ipn/ipnlocal/ssh.go | 2 +- ssh/tailssh/tailssh.go | 310 ++++++++----------- ssh/tailssh/tailssh_integration_test.go | 2 +- ssh/tailssh/tailssh_test.go | 5 +- tempfork/gliderlabs/ssh/agent.go | 2 +- tempfork/gliderlabs/ssh/context.go | 11 +- tempfork/gliderlabs/ssh/options.go | 2 +- tempfork/gliderlabs/ssh/options_test.go | 2 +- tempfork/gliderlabs/ssh/server.go | 2 +- tempfork/gliderlabs/ssh/session.go | 2 +- tempfork/gliderlabs/ssh/session_test.go | 2 +- tempfork/gliderlabs/ssh/ssh.go | 4 +- tempfork/gliderlabs/ssh/tcpip.go | 2 +- tempfork/gliderlabs/ssh/tcpip_test.go | 2 +- tempfork/gliderlabs/ssh/util.go | 2 +- tempfork/gliderlabs/ssh/wrap.go | 2 +- 22 files changed, 172 insertions(+), 231 deletions(-) diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index e32fd4a2b0d8f..972dbfc2c2fe5 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -197,9 +197,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - LD github.com/tailscale/golang-x-crypto/internal/poly1305 from github.com/tailscale/golang-x-crypto/ssh - LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal - LD github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ @@ -986,12 +983,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ - LD golang.org/x/crypto/blowfish from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf - golang.org/x/crypto/chacha20 from github.com/tailscale/golang-x-crypto/ssh+ + LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf + golang.org/x/crypto/chacha20 from golang.org/x/crypto/ssh+ golang.org/x/crypto/chacha20poly1305 from crypto/tls+ golang.org/x/crypto/cryptobyte from crypto/ecdsa+ golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ - golang.org/x/crypto/curve25519 from github.com/tailscale/golang-x-crypto/ssh+ + golang.org/x/crypto/curve25519 from golang.org/x/crypto/ssh+ golang.org/x/crypto/hkdf from crypto/tls+ golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ @@ -1000,6 +997,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/sha3 from crypto/internal/mlkem768+ + LD golang.org/x/crypto/ssh from tailscale.com/ipn/ipnlocal + LD golang.org/x/crypto/ssh/internal/bcrypt_pbkdf from golang.org/x/crypto/ssh golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ golang.org/x/exp/maps from sigs.k8s.io/controller-runtime/pkg/cache+ golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator+ diff --git a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go index ee929299a4273..39af584ecd481 100644 --- a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go +++ b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go @@ -6,6 +6,9 @@ // highlight the unique parts of the Tailscale SSH server so SSH // client authors can hit it easily and fix their SSH clients without // needing to set up Tailscale and Tailscale SSH. +// +// Connections are allowed using any username except for "denyme". Connecting as +// "denyme" will result in an authentication failure with error message. package main import ( @@ -16,6 +19,7 @@ import ( "crypto/rsa" "crypto/x509" "encoding/pem" + "errors" "flag" "fmt" "io" @@ -24,7 +28,7 @@ import ( "path/filepath" "time" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" "tailscale.com/tempfork/gliderlabs/ssh" ) @@ -62,13 +66,21 @@ func main() { Handler: handleSessionPostSSHAuth, ServerConfigCallback: func(ctx ssh.Context) *gossh.ServerConfig { start := time.Now() + var spac gossh.ServerPreAuthConn return &gossh.ServerConfig{ - NextAuthMethodCallback: func(conn gossh.ConnMetadata, prevErrors []error) []string { - return []string{"tailscale"} + PreAuthConnCallback: func(conn gossh.ServerPreAuthConn) { + spac = conn }, NoClientAuth: true, // required for the NoClientAuthCallback to run NoClientAuthCallback: func(cm gossh.ConnMetadata) (*gossh.Permissions, error) { - cm.SendAuthBanner(fmt.Sprintf("# Banner: doing none auth at %v\r\n", time.Since(start))) + spac.SendAuthBanner(fmt.Sprintf("# Banner: doing none auth at %v\r\n", time.Since(start))) + + if cm.User() == "denyme" { + return nil, &gossh.BannerError{ + Err: errors.New("denying access"), + Message: "denyme is not allowed to access this machine\n", + } + } totalBanners := 2 if cm.User() == "banners" { @@ -77,9 +89,9 @@ func main() { for banner := 2; banner <= totalBanners; banner++ { time.Sleep(time.Second) if banner == totalBanners { - cm.SendAuthBanner(fmt.Sprintf("# Banner%d: access granted at %v\r\n", banner, time.Since(start))) + spac.SendAuthBanner(fmt.Sprintf("# Banner%d: access granted at %v\r\n", banner, time.Since(start))) } else { - cm.SendAuthBanner(fmt.Sprintf("# Banner%d at %v\r\n", banner, time.Since(start))) + spac.SendAuthBanner(fmt.Sprintf("# Banner%d at %v\r\n", banner, time.Since(start))) } } return nil, nil diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index a7ad83818d317..a6fae54ffc237 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -152,9 +152,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - LD github.com/tailscale/golang-x-crypto/internal/poly1305 from github.com/tailscale/golang-x-crypto/ssh - LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal+ - LD github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ @@ -439,12 +436,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ - LD golang.org/x/crypto/blowfish from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf+ + LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/chacha20poly1305 from crypto/tls+ golang.org/x/crypto/cryptobyte from crypto/ecdsa+ golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ - golang.org/x/crypto/curve25519 from github.com/tailscale/golang-x-crypto/ssh+ + golang.org/x/crypto/curve25519 from golang.org/x/crypto/ssh+ golang.org/x/crypto/hkdf from crypto/tls+ golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ diff --git a/cmd/tailscaled/deps_test.go b/cmd/tailscaled/deps_test.go index 2b4bc280d26cf..7f06abc6c5ba1 100644 --- a/cmd/tailscaled/deps_test.go +++ b/cmd/tailscaled/deps_test.go @@ -17,7 +17,6 @@ func TestOmitSSH(t *testing.T) { Tags: "ts_omit_ssh", BadDeps: map[string]string{ "tailscale.com/ssh/tailssh": msg, - "golang.org/x/crypto/ssh": msg, "tailscale.com/sessionrecording": msg, "github.com/anmitsu/go-shlex": msg, "github.com/creack/pty": msg, diff --git a/go.mod b/go.mod index 8e52a9ab337b0..2489e34d711d0 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( go.uber.org/zap v1.27.0 go4.org/mem v0.0.0-20240501181205-ae6ca9944745 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba - golang.org/x/crypto v0.32.0 + golang.org/x/crypto v0.32.1-0.20250118192723-a8ea4be81f07 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 golang.org/x/mod v0.22.0 golang.org/x/net v0.34.0 diff --git a/go.sum b/go.sum index c1c82ad7794c7..b10e98da2b84d 100644 --- a/go.sum +++ b/go.sum @@ -1058,8 +1058,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.32.1-0.20250118192723-a8ea4be81f07 h1:Z+Zg+aXJYq6f4TK2E4H+vZkQ4dJAWnInXDR6hM9znxo= +golang.org/x/crypto v0.32.1-0.20250118192723-a8ea4be81f07/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/ipn/ipnlocal/ssh.go b/ipn/ipnlocal/ssh.go index 383d03f5aa9be..47a74e2820905 100644 --- a/ipn/ipnlocal/ssh.go +++ b/ipn/ipnlocal/ssh.go @@ -24,8 +24,8 @@ import ( "strings" "sync" - "github.com/tailscale/golang-x-crypto/ssh" "go4.org/mem" + "golang.org/x/crypto/ssh" "tailscale.com/tailcfg" "tailscale.com/util/lineiter" "tailscale.com/util/mak" diff --git a/ssh/tailssh/tailssh.go b/ssh/tailssh/tailssh.go index 7f21ccd1182ee..638ff99b8c188 100644 --- a/ssh/tailssh/tailssh.go +++ b/ssh/tailssh/tailssh.go @@ -29,7 +29,7 @@ import ( "syscall" "time" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" "tailscale.com/envknob" "tailscale.com/ipn/ipnlocal" "tailscale.com/logtail/backoff" @@ -198,8 +198,11 @@ func (srv *server) OnPolicyChange() { // Setup and discover server info // - ServerConfigCallback // -// Do the user auth -// - NoClientAuthHandler +// Get access to a ServerPreAuthConn (useful for sending banners) +// +// Do the user auth with a NoClientAuthCallback. If user specified +// a username ending in "+password", follow this with password auth +// (to work around buggy SSH clients that don't work with noauth). // // Once auth is done, the conn can be multiplexed with multiple sessions and // channels concurrently. At which point any of the following can be called @@ -219,15 +222,12 @@ type conn struct { idH string connID string // ID that's shared with control - // anyPasswordIsOkay is whether the client is authorized but has requested - // password-based auth to work around their buggy SSH client. When set, we - // accept any password in the PasswordHandler. - anyPasswordIsOkay bool // set by NoClientAuthCallback + // spac is a [gossh.ServerPreAuthConn] used for sending auth banners. + // Banners cannot be sent after auth completes. + spac gossh.ServerPreAuthConn - action0 *tailcfg.SSHAction // set by doPolicyAuth; first matching action - currentAction *tailcfg.SSHAction // set by doPolicyAuth, updated by resolveNextAction - finalAction *tailcfg.SSHAction // set by doPolicyAuth or resolveNextAction - finalActionErr error // set by doPolicyAuth or resolveNextAction + action0 *tailcfg.SSHAction // set by clientAuth + finalAction *tailcfg.SSHAction // set by clientAuth info *sshConnInfo // set by setInfo localUser *userMeta // set by doPolicyAuth @@ -254,141 +254,142 @@ func (c *conn) vlogf(format string, args ...any) { } } -// isAuthorized walks through the action chain and returns nil if the connection -// is authorized. If the connection is not authorized, it returns -// errDenied. If the action chain resolution fails, it returns the -// resolution error. -func (c *conn) isAuthorized(ctx ssh.Context) error { - action := c.currentAction - for { - if action.Accept { - return nil - } - if action.Reject || action.HoldAndDelegate == "" { - return errDenied - } - var err error - action, err = c.resolveNextAction(ctx) - if err != nil { - return err - } - if action.Message != "" { - if err := ctx.SendAuthBanner(action.Message); err != nil { - return err - } - } +// errDenied is returned by auth callbacks when a connection is denied by the +// policy. It returns a gossh.BannerError to make sure the message gets +// displayed as an auth banner. +func errDenied(message string) error { + if message == "" { + message = "tailscale: access denied" + } + return &gossh.BannerError{ + Message: message, } } -// errDenied is returned by auth callbacks when a connection is denied by the -// policy. -var errDenied = errors.New("ssh: access denied") +// bannerError creates a gossh.BannerError that will result in the given +// message being displayed to the client. If err != nil, this also logs +// message:error. The contents of err is not leaked to clients in the banner. +func (c *conn) bannerError(message string, err error) error { + if err != nil { + c.logf("%s: %s", message, err) + } + return &gossh.BannerError{ + Err: err, + Message: fmt.Sprintf("tailscale: %s", message), + } +} -// NoClientAuthCallback implements gossh.NoClientAuthCallback and is called by -// the ssh.Server when the client first connects with the "none" -// authentication method. +// clientAuth is responsible for performing client authentication. // -// It is responsible for continuing policy evaluation from BannerCallback (or -// starting it afresh). It returns an error if the policy evaluation fails, or -// if the decision is "reject" -// -// It either returns nil (accept) or errDenied (reject). The errors may be wrapped. -func (c *conn) NoClientAuthCallback(ctx ssh.Context) error { +// If policy evaluation fails, it returns an error. +// If access is denied, it returns an error. +func (c *conn) clientAuth(cm gossh.ConnMetadata) (*gossh.Permissions, error) { if c.insecureSkipTailscaleAuth { - return nil - } - if err := c.doPolicyAuth(ctx); err != nil { - return err - } - if err := c.isAuthorized(ctx); err != nil { - return err + return &gossh.Permissions{}, nil } - // Let users specify a username ending in +password to force password auth. - // This exists for buggy SSH clients that get confused by success from - // "none" auth. - if strings.HasSuffix(ctx.User(), forcePasswordSuffix) { - c.anyPasswordIsOkay = true - return errors.New("any password please") // not shown to users + if err := c.setInfo(cm); err != nil { + return nil, c.bannerError("failed to get connection info", err) } - return nil -} -func (c *conn) nextAuthMethodCallback(cm gossh.ConnMetadata, prevErrors []error) (nextMethod []string) { - switch { - case c.anyPasswordIsOkay: - nextMethod = append(nextMethod, "password") + action, localUser, acceptEnv, err := c.evaluatePolicy() + if err != nil { + return nil, c.bannerError("failed to evaluate SSH policy", err) } - // The fake "tailscale" method is always appended to next so OpenSSH renders - // that in parens as the final failure. (It also shows up in "ssh -v", etc) - nextMethod = append(nextMethod, "tailscale") - return -} - -// fakePasswordHandler is our implementation of the PasswordHandler hook that -// checks whether the user's password is correct. But we don't actually use -// passwords. This exists only for when the user's username ends in "+password" -// to signal that their SSH client is buggy and gets confused by auth type -// "none" succeeding and they want our SSH server to require a dummy password -// prompt instead. We then accept any password since we've already authenticated -// & authorized them. -func (c *conn) fakePasswordHandler(ctx ssh.Context, password string) bool { - return c.anyPasswordIsOkay -} + c.action0 = action -// doPolicyAuth verifies that conn can proceed. -// It returns nil if the matching policy action is Accept or -// HoldAndDelegate. Otherwise, it returns errDenied. -func (c *conn) doPolicyAuth(ctx ssh.Context) error { - if err := c.setInfo(ctx); err != nil { - c.logf("failed to get conninfo: %v", err) - return errDenied - } - a, localUser, acceptEnv, err := c.evaluatePolicy() - if err != nil { - return fmt.Errorf("%w: %v", errDenied, err) - } - c.action0 = a - c.currentAction = a - c.acceptEnv = acceptEnv - if a.Message != "" { - if err := ctx.SendAuthBanner(a.Message); err != nil { - return fmt.Errorf("SendBanner: %w", err) - } - } - if a.Accept || a.HoldAndDelegate != "" { - if a.Accept { - c.finalAction = a - } + if action.Accept || action.HoldAndDelegate != "" { + // Immediately look up user information for purposes of generating + // hold and delegate URL (if necessary). lu, err := userLookup(localUser) if err != nil { - c.logf("failed to look up %v: %v", localUser, err) - ctx.SendAuthBanner(fmt.Sprintf("failed to look up %v\r\n", localUser)) - return err + return nil, c.bannerError(fmt.Sprintf("failed to look up local user %q ", localUser), err) } gids, err := lu.GroupIds() if err != nil { - c.logf("failed to look up local user's group IDs: %v", err) - return err + return nil, c.bannerError("failed to look up local user's group IDs", err) } c.userGroupIDs = gids c.localUser = lu - return nil + c.acceptEnv = acceptEnv } - if a.Reject { - c.finalAction = a - return errDenied + + for { + switch { + case action.Accept: + metricTerminalAccept.Add(1) + if action.Message != "" { + if err := c.spac.SendAuthBanner(action.Message); err != nil { + return nil, fmt.Errorf("error sending auth welcome message: %w", err) + } + } + c.finalAction = action + return &gossh.Permissions{}, nil + case action.Reject: + metricTerminalReject.Add(1) + c.finalAction = action + return nil, errDenied(action.Message) + case action.HoldAndDelegate != "": + if action.Message != "" { + if err := c.spac.SendAuthBanner(action.Message); err != nil { + return nil, fmt.Errorf("error sending hold and delegate message: %w", err) + } + } + + url := action.HoldAndDelegate + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + metricHolds.Add(1) + url = c.expandDelegateURLLocked(url) + + var err error + action, err = c.fetchSSHAction(ctx, url) + if err != nil { + metricTerminalFetchError.Add(1) + return nil, c.bannerError("failed to fetch next SSH action", fmt.Errorf("fetch failed from %s: %w", url, err)) + } + default: + metricTerminalMalformed.Add(1) + return nil, c.bannerError("reached Action that had neither Accept, Reject, nor HoldAndDelegate", nil) + } } - // Shouldn't get here, but: - return errDenied } // ServerConfig implements ssh.ServerConfigCallback. func (c *conn) ServerConfig(ctx ssh.Context) *gossh.ServerConfig { return &gossh.ServerConfig{ - NoClientAuth: true, // required for the NoClientAuthCallback to run - NextAuthMethodCallback: c.nextAuthMethodCallback, + PreAuthConnCallback: func(spac gossh.ServerPreAuthConn) { + c.spac = spac + }, + NoClientAuth: true, // required for the NoClientAuthCallback to run + NoClientAuthCallback: func(cm gossh.ConnMetadata) (*gossh.Permissions, error) { + // First perform client authentication, which can potentially + // involve multiple steps (for example prompting user to log in to + // Tailscale admin panel to confirm identity). + perms, err := c.clientAuth(cm) + if err != nil { + return nil, err + } + + // Authentication succeeded. Buggy SSH clients get confused by + // success from the "none" auth method. As a workaround, let users + // specify a username ending in "+password" to force password auth. + // The actual value of the password doesn't matter. + if strings.HasSuffix(cm.User(), forcePasswordSuffix) { + return nil, &gossh.PartialSuccessError{ + Next: gossh.ServerAuthCallbacks{ + PasswordCallback: func(_ gossh.ConnMetadata, password []byte) (*gossh.Permissions, error) { + return &gossh.Permissions{}, nil + }, + }, + } + } + + return perms, nil + }, } } @@ -399,7 +400,7 @@ func (srv *server) newConn() (*conn, error) { // Stop accepting new connections. // Connections in the auth phase are handled in handleConnPostSSHAuth. // Existing sessions are terminated by Shutdown. - return nil, errDenied + return nil, errDenied("tailscale: server is shutting down") } srv.mu.Unlock() c := &conn{srv: srv} @@ -410,9 +411,6 @@ func (srv *server) newConn() (*conn, error) { Version: "Tailscale", ServerConfigCallback: c.ServerConfig, - NoClientAuthHandler: c.NoClientAuthCallback, - PasswordHandler: c.fakePasswordHandler, - Handler: c.handleSessionPostSSHAuth, LocalPortForwardingCallback: c.mayForwardLocalPortTo, ReversePortForwardingCallback: c.mayReversePortForwardTo, @@ -523,16 +521,16 @@ func toIPPort(a net.Addr) (ipp netip.AddrPort) { return netip.AddrPortFrom(tanetaddr.Unmap(), uint16(ta.Port)) } -// connInfo returns a populated sshConnInfo from the provided arguments, +// connInfo populates the sshConnInfo from the provided arguments, // validating only that they represent a known Tailscale identity. -func (c *conn) setInfo(ctx ssh.Context) error { +func (c *conn) setInfo(cm gossh.ConnMetadata) error { if c.info != nil { return nil } ci := &sshConnInfo{ - sshUser: strings.TrimSuffix(ctx.User(), forcePasswordSuffix), - src: toIPPort(ctx.RemoteAddr()), - dst: toIPPort(ctx.LocalAddr()), + sshUser: strings.TrimSuffix(cm.User(), forcePasswordSuffix), + src: toIPPort(cm.RemoteAddr()), + dst: toIPPort(cm.LocalAddr()), } if !tsaddr.IsTailscaleIP(ci.dst.Addr()) { return fmt.Errorf("tailssh: rejecting non-Tailscale local address %v", ci.dst) @@ -547,7 +545,7 @@ func (c *conn) setInfo(ctx ssh.Context) error { ci.node = node ci.uprof = uprof - c.idH = ctx.SessionID() + c.idH = string(cm.SessionID()) c.info = ci c.logf("handling conn: %v", ci.String()) return nil @@ -594,62 +592,6 @@ func (c *conn) handleSessionPostSSHAuth(s ssh.Session) { ss.run() } -// resolveNextAction starts at c.currentAction and makes it way through the -// action chain one step at a time. An action without a HoldAndDelegate is -// considered the final action. Once a final action is reached, this function -// will keep returning that action. It updates c.currentAction to the next -// action in the chain. When the final action is reached, it also sets -// c.finalAction to the final action. -func (c *conn) resolveNextAction(sctx ssh.Context) (action *tailcfg.SSHAction, err error) { - if c.finalAction != nil || c.finalActionErr != nil { - return c.finalAction, c.finalActionErr - } - - defer func() { - if action != nil { - c.currentAction = action - if action.Accept || action.Reject { - c.finalAction = action - } - } - if err != nil { - c.finalActionErr = err - } - }() - - ctx, cancel := context.WithCancel(sctx) - defer cancel() - - // Loop processing/fetching Actions until one reaches a - // terminal state (Accept, Reject, or invalid Action), or - // until fetchSSHAction times out due to the context being - // done (client disconnect) or its 30 minute timeout passes. - // (Which is a long time for somebody to see login - // instructions and go to a URL to do something.) - action = c.currentAction - if action.Accept || action.Reject { - if action.Reject { - metricTerminalReject.Add(1) - } else { - metricTerminalAccept.Add(1) - } - return action, nil - } - url := action.HoldAndDelegate - if url == "" { - metricTerminalMalformed.Add(1) - return nil, errors.New("reached Action that lacked Accept, Reject, and HoldAndDelegate") - } - metricHolds.Add(1) - url = c.expandDelegateURLLocked(url) - nextAction, err := c.fetchSSHAction(ctx, url) - if err != nil { - metricTerminalFetchError.Add(1) - return nil, fmt.Errorf("fetching SSHAction from %s: %w", url, err) - } - return nextAction, nil -} - func (c *conn) expandDelegateURLLocked(actionURL string) string { nm := c.srv.lb.NetMap() ci := c.info diff --git a/ssh/tailssh/tailssh_integration_test.go b/ssh/tailssh/tailssh_integration_test.go index 1799d340019cb..5c4f533b11c00 100644 --- a/ssh/tailssh/tailssh_integration_test.go +++ b/ssh/tailssh/tailssh_integration_test.go @@ -32,8 +32,8 @@ import ( "github.com/bramvdbogaerde/go-scp" "github.com/google/go-cmp/cmp" "github.com/pkg/sftp" - gossh "github.com/tailscale/golang-x-crypto/ssh" "golang.org/x/crypto/ssh" + gossh "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" "tailscale.com/net/tsdial" "tailscale.com/tailcfg" diff --git a/ssh/tailssh/tailssh_test.go b/ssh/tailssh/tailssh_test.go index 9f3616d8ca8ab..2071366599028 100644 --- a/ssh/tailssh/tailssh_test.go +++ b/ssh/tailssh/tailssh_test.go @@ -31,7 +31,7 @@ import ( "testing" "time" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" "tailscale.com/ipn/ipnlocal" @@ -805,7 +805,8 @@ func TestSSHAuthFlow(t *testing.T) { state: &localState{ sshEnabled: true, }, - authErr: true, + authErr: true, + wantBanners: []string{"tailscale: failed to evaluate SSH policy"}, }, { name: "accept", diff --git a/tempfork/gliderlabs/ssh/agent.go b/tempfork/gliderlabs/ssh/agent.go index 86a5bce7f8ebc..99e84c1e5c64c 100644 --- a/tempfork/gliderlabs/ssh/agent.go +++ b/tempfork/gliderlabs/ssh/agent.go @@ -7,7 +7,7 @@ import ( "path" "sync" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) const ( diff --git a/tempfork/gliderlabs/ssh/context.go b/tempfork/gliderlabs/ssh/context.go index d43de6f09c8a5..505a43dbf3ffe 100644 --- a/tempfork/gliderlabs/ssh/context.go +++ b/tempfork/gliderlabs/ssh/context.go @@ -6,7 +6,7 @@ import ( "net" "sync" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) // contextKey is a value for use with context.WithValue. It's used as @@ -55,8 +55,6 @@ var ( // ContextKeyPublicKey is a context key for use with Contexts in this package. // The associated value will be of type PublicKey. ContextKeyPublicKey = &contextKey{"public-key"} - - ContextKeySendAuthBanner = &contextKey{"send-auth-banner"} ) // Context is a package specific context interface. It exposes connection @@ -91,8 +89,6 @@ type Context interface { // SetValue allows you to easily write new values into the underlying context. SetValue(key, value interface{}) - - SendAuthBanner(banner string) error } type sshContext struct { @@ -121,7 +117,6 @@ func applyConnMetadata(ctx Context, conn gossh.ConnMetadata) { ctx.SetValue(ContextKeyUser, conn.User()) ctx.SetValue(ContextKeyLocalAddr, conn.LocalAddr()) ctx.SetValue(ContextKeyRemoteAddr, conn.RemoteAddr()) - ctx.SetValue(ContextKeySendAuthBanner, conn.SendAuthBanner) } func (ctx *sshContext) SetValue(key, value interface{}) { @@ -158,7 +153,3 @@ func (ctx *sshContext) LocalAddr() net.Addr { func (ctx *sshContext) Permissions() *Permissions { return ctx.Value(ContextKeyPermissions).(*Permissions) } - -func (ctx *sshContext) SendAuthBanner(msg string) error { - return ctx.Value(ContextKeySendAuthBanner).(func(string) error)(msg) -} diff --git a/tempfork/gliderlabs/ssh/options.go b/tempfork/gliderlabs/ssh/options.go index aa87a4f39db9e..29c8ef141842b 100644 --- a/tempfork/gliderlabs/ssh/options.go +++ b/tempfork/gliderlabs/ssh/options.go @@ -3,7 +3,7 @@ package ssh import ( "os" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) // PasswordAuth returns a functional option that sets PasswordHandler on the server. diff --git a/tempfork/gliderlabs/ssh/options_test.go b/tempfork/gliderlabs/ssh/options_test.go index 7cf6f376c6a88..47342b0f67923 100644 --- a/tempfork/gliderlabs/ssh/options_test.go +++ b/tempfork/gliderlabs/ssh/options_test.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "testing" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) func newTestSessionWithOptions(t *testing.T, srv *Server, cfg *gossh.ClientConfig, options ...Option) (*gossh.Session, *gossh.Client, func()) { diff --git a/tempfork/gliderlabs/ssh/server.go b/tempfork/gliderlabs/ssh/server.go index 1086a72caf0e5..473e5fbd6fc8f 100644 --- a/tempfork/gliderlabs/ssh/server.go +++ b/tempfork/gliderlabs/ssh/server.go @@ -8,7 +8,7 @@ import ( "sync" "time" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) // ErrServerClosed is returned by the Server's Serve, ListenAndServe, diff --git a/tempfork/gliderlabs/ssh/session.go b/tempfork/gliderlabs/ssh/session.go index 0a4a21e534401..a7a9a3eebd96f 100644 --- a/tempfork/gliderlabs/ssh/session.go +++ b/tempfork/gliderlabs/ssh/session.go @@ -9,7 +9,7 @@ import ( "sync" "github.com/anmitsu/go-shlex" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) // Session provides access to information about an SSH session and methods diff --git a/tempfork/gliderlabs/ssh/session_test.go b/tempfork/gliderlabs/ssh/session_test.go index a60be5ec12d4e..fe61a9d96be9b 100644 --- a/tempfork/gliderlabs/ssh/session_test.go +++ b/tempfork/gliderlabs/ssh/session_test.go @@ -9,7 +9,7 @@ import ( "net" "testing" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) func (srv *Server) serveOnce(l net.Listener) error { diff --git a/tempfork/gliderlabs/ssh/ssh.go b/tempfork/gliderlabs/ssh/ssh.go index 644cb257d9afa..54bd31ec2fcb4 100644 --- a/tempfork/gliderlabs/ssh/ssh.go +++ b/tempfork/gliderlabs/ssh/ssh.go @@ -4,7 +4,7 @@ import ( "crypto/subtle" "net" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) type Signal string @@ -105,7 +105,7 @@ type Pty struct { // requested by the client as part of the pty-req. These are outlined as // part of https://datatracker.ietf.org/doc/html/rfc4254#section-8. // - // The opcodes are defined as constants in github.com/tailscale/golang-x-crypto/ssh (VINTR,VQUIT,etc.). + // The opcodes are defined as constants in golang.org/x/crypto/ssh (VINTR,VQUIT,etc.). // Boolean opcodes have values 0 or 1. Modes gossh.TerminalModes } diff --git a/tempfork/gliderlabs/ssh/tcpip.go b/tempfork/gliderlabs/ssh/tcpip.go index 056a0c7343daf..335fda65754ea 100644 --- a/tempfork/gliderlabs/ssh/tcpip.go +++ b/tempfork/gliderlabs/ssh/tcpip.go @@ -7,7 +7,7 @@ import ( "strconv" "sync" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) const ( diff --git a/tempfork/gliderlabs/ssh/tcpip_test.go b/tempfork/gliderlabs/ssh/tcpip_test.go index 118b5d53ac4a1..b3ba60a9bb6b8 100644 --- a/tempfork/gliderlabs/ssh/tcpip_test.go +++ b/tempfork/gliderlabs/ssh/tcpip_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - gossh "github.com/tailscale/golang-x-crypto/ssh" + gossh "golang.org/x/crypto/ssh" ) var sampleServerResponse = []byte("Hello world") diff --git a/tempfork/gliderlabs/ssh/util.go b/tempfork/gliderlabs/ssh/util.go index e3b5716a3ab55..3bee06dcdef39 100644 --- a/tempfork/gliderlabs/ssh/util.go +++ b/tempfork/gliderlabs/ssh/util.go @@ -5,7 +5,7 @@ import ( "crypto/rsa" "encoding/binary" - "github.com/tailscale/golang-x-crypto/ssh" + "golang.org/x/crypto/ssh" ) func generateSigner() (ssh.Signer, error) { diff --git a/tempfork/gliderlabs/ssh/wrap.go b/tempfork/gliderlabs/ssh/wrap.go index 17867d7518dd1..d1f2b161e6932 100644 --- a/tempfork/gliderlabs/ssh/wrap.go +++ b/tempfork/gliderlabs/ssh/wrap.go @@ -1,6 +1,6 @@ package ssh -import gossh "github.com/tailscale/golang-x-crypto/ssh" +import gossh "golang.org/x/crypto/ssh" // PublicKey is an abstraction of different types of public keys. type PublicKey interface { From f1514a944a167fde05888273440342598112050c Mon Sep 17 00:00:00 2001 From: Mario Minardi Date: Tue, 28 Jan 2025 14:35:24 -0700 Subject: [PATCH 210/223] go.toolchain.rev: bump from Go 1.23.3 to 1.23.5 (#14814) Update Go toolchain to 1.23.5. Updates #cleanup Signed-off-by: Mario Minardi --- go.toolchain.rev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.toolchain.rev b/go.toolchain.rev index e90440d41701f..900450dcaec92 100644 --- a/go.toolchain.rev +++ b/go.toolchain.rev @@ -1 +1 @@ -161c3b79ed91039e65eb148f2547dea6b91e2247 +64f7854906c3121fe3ada3d05f1936d3420d6ffa From 0aa54151f290df4675714a338c0e067bf5fd050c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:03:13 -0700 Subject: [PATCH 211/223] .github: Bump actions/checkout from 3.6.0 to 4.2.2 (#14139) Bumps [actions/checkout](https://github.com/actions/checkout) from 3.6.0 to 4.2.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.6.0...11bd71901bbe5b1630ceea73d27597364c9af683) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checklocks.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/docker-file-build.yml | 2 +- .github/workflows/flakehub-publish-tagged.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/govulncheck.yml | 2 +- .github/workflows/installer.yml | 6 +--- .github/workflows/kubemanifests.yaml | 2 +- .github/workflows/ssh-integrationtest.yml | 2 +- .github/workflows/test.yml | 34 +++++++++---------- .github/workflows/update-flake.yml | 2 +- .../workflows/update-webclient-prebuilt.yml | 2 +- .github/workflows/webclient.yml | 2 +- 13 files changed, 29 insertions(+), 33 deletions(-) diff --git a/.github/workflows/checklocks.yml b/.github/workflows/checklocks.yml index 064797c884a60..7464524ce99e2 100644 --- a/.github/workflows/checklocks.yml +++ b/.github/workflows/checklocks.yml @@ -18,7 +18,7 @@ jobs: runs-on: [ ubuntu-latest ] steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Build checklocks run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 928240c5375d5..4251752180470 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Install a more recent Go that understands modern go.mod content. - name: Install Go diff --git a/.github/workflows/docker-file-build.yml b/.github/workflows/docker-file-build.yml index c535755724391..04611e172bbea 100644 --- a/.github/workflows/docker-file-build.yml +++ b/.github/workflows/docker-file-build.yml @@ -10,6 +10,6 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: "Build Docker image" run: docker build . diff --git a/.github/workflows/flakehub-publish-tagged.yml b/.github/workflows/flakehub-publish-tagged.yml index 60fdba91c1247..9ff12c6a3fd14 100644 --- a/.github/workflows/flakehub-publish-tagged.yml +++ b/.github/workflows/flakehub-publish-tagged.yml @@ -17,7 +17,7 @@ jobs: id-token: "write" contents: "read" steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}" - uses: "DeterminateSystems/nix-installer-action@main" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 58e6115918050..b9a9eb33d5c9d 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -23,7 +23,7 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 989e55fb112ea..47d278e1c93d4 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install govulncheck run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 1c39e4d743a0d..adc4a0a607ad2 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -36,7 +36,6 @@ jobs: - "ubuntu:24.04" - "elementary/docker:stable" - "elementary/docker:unstable" - - "parrotsec/core:lts-amd64" - "parrotsec/core:latest" - "kalilinux/kali-rolling" - "kalilinux/kali-dev" @@ -92,10 +91,7 @@ jobs: || contains(matrix.image, 'parrotsec') || contains(matrix.image, 'kalilinux') - name: checkout - # We cannot use v4, as it requires a newer glibc version than some of the - # tested images provide. See - # https://github.com/actions/checkout/issues/1487 - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: run installer run: scripts/installer.sh # Package installation can fail in docker because systemd is not running diff --git a/.github/workflows/kubemanifests.yaml b/.github/workflows/kubemanifests.yaml index f943ccb524f35..5b100a2763e3b 100644 --- a/.github/workflows/kubemanifests.yaml +++ b/.github/workflows/kubemanifests.yaml @@ -17,7 +17,7 @@ jobs: runs-on: [ ubuntu-latest ] steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Build and lint Helm chart run: | eval `./tool/go run ./cmd/mkversion` diff --git a/.github/workflows/ssh-integrationtest.yml b/.github/workflows/ssh-integrationtest.yml index a82696307ea4b..829d10ab8c2c8 100644 --- a/.github/workflows/ssh-integrationtest.yml +++ b/.github/workflows/ssh-integrationtest.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Run SSH integration tests run: | make sshintegrationtest \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a6ef6c36ec133..a368afc67a290 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,7 +50,7 @@ jobs: - shard: '4/4' steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: build test wrapper run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper - name: integration tests as root @@ -78,7 +78,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Restore Cache uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: @@ -150,7 +150,7 @@ jobs: runs-on: windows-2022 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 @@ -190,7 +190,7 @@ jobs: options: --privileged steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: chown run: chown -R $(id -u):$(id -g) $PWD - name: privileged tests @@ -202,7 +202,7 @@ jobs: if: github.repository == 'tailscale/tailscale' steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Run VM tests run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004 env: @@ -214,7 +214,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: build all run: ./tool/go install -race ./cmd/... - name: build tests @@ -258,7 +258,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Restore Cache uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: @@ -295,7 +295,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: build some run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient env: @@ -323,7 +323,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Restore Cache uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: @@ -356,7 +356,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch # some Android breakages early. @@ -371,7 +371,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Restore Cache uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: @@ -405,7 +405,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: test tailscale_go run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/... @@ -477,7 +477,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: check depaware run: | export PATH=$(./tool/go env GOROOT)/bin:$PATH @@ -487,7 +487,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: check that 'go generate' is clean run: | pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp') @@ -500,7 +500,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: check that 'go mod tidy' is clean run: | ./tool/go mod tidy @@ -512,7 +512,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: check licenses run: ./scripts/check_license_headers.sh . @@ -528,7 +528,7 @@ jobs: goarch: "386" steps: - name: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: install staticcheck run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck - name: run staticcheck diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index 151ed6bab4b9c..4d9db490b3273 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Run update-flakes run: ./update-flake.sh diff --git a/.github/workflows/update-webclient-prebuilt.yml b/.github/workflows/update-webclient-prebuilt.yml index 11665460be8ec..f2d1e65a55427 100644 --- a/.github/workflows/update-webclient-prebuilt.yml +++ b/.github/workflows/update-webclient-prebuilt.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Run go get run: | diff --git a/.github/workflows/webclient.yml b/.github/workflows/webclient.yml index 9afb7730d9a56..b1cfb7620f97d 100644 --- a/.github/workflows/webclient.yml +++ b/.github/workflows/webclient.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install deps run: ./tool/yarn --cwd client/web - name: Run lint From eb299302ba454e8b7e2fc65a972d9710090bbea8 Mon Sep 17 00:00:00 2001 From: Andrew Dunham Date: Fri, 24 Jan 2025 13:26:08 -0500 Subject: [PATCH 212/223] types/views: fix SliceEqualAnyOrderFunc short optimization This was flagged by @tkhattra on the merge commit; thanks! Updates tailscale/corp#25479 Signed-off-by: Andrew Dunham Change-Id: Ia8045640f02bd4dcc0fe7433249fd72ac6b9cf52 --- types/views/views.go | 38 ++++++++++++++++++++++++++++++++------ types/views/views_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/types/views/views.go b/types/views/views.go index d8acf27ce4fca..ae776c3b22331 100644 --- a/types/views/views.go +++ b/types/views/views.go @@ -386,14 +386,32 @@ func SliceEqualAnyOrderFunc[T any, V comparable](a, b Slice[T], cmp func(T) V) b // do the quadratic thing. We can also only check the items between // diffStart and the end. nRemain := a.Len() - diffStart - if nRemain <= 5 { - maxLen := a.Len() // same as b.Len() - for i := diffStart; i < maxLen; i++ { - av := cmp(a.At(i)) + const shortOptLen = 5 + if nRemain <= shortOptLen { + // These track which elements in a and b have been matched, so + // that we don't treat arrays with differing number of + // duplicate elements as equal (e.g. [1, 1, 2] and [1, 2, 2]). + var aMatched, bMatched [shortOptLen]bool + + // Compare each element in a to each element in b + for i := range nRemain { + av := cmp(a.At(i + diffStart)) found := false - for j := diffStart; j < maxLen; j++ { - bv := cmp(b.At(j)) + for j := range nRemain { + // Skip elements in b that have already been + // used to match an item in a. + if bMatched[j] { + continue + } + + bv := cmp(b.At(j + diffStart)) if av == bv { + // Mark these elements as already + // matched, so that a future loop + // iteration (of a duplicate element) + // doesn't match it again. + aMatched[i] = true + bMatched[j] = true found = true break } @@ -402,6 +420,14 @@ func SliceEqualAnyOrderFunc[T any, V comparable](a, b Slice[T], cmp func(T) V) b return false } } + + // Verify all elements were matched exactly once. + for i := range nRemain { + if !aMatched[i] || !bMatched[i] { + return false + } + } + return true } diff --git a/types/views/views_test.go b/types/views/views_test.go index 70e021aa4f8bf..7837a89d6ed54 100644 --- a/types/views/views_test.go +++ b/types/views/views_test.go @@ -197,6 +197,38 @@ func TestSliceEqualAnyOrderFunc(t *testing.T) { // Long difference; past the quadratic limit longDiff := ncFrom("b", "a", "c", "d", "e", "f", "g", "h", "i", "k") // differs at end c.Check(SliceEqualAnyOrderFunc(longSlice, longDiff, cmp), qt.Equals, false) + + // The short slice optimization had a bug where it wouldn't handle + // duplicate elements; test various cases here driven by code coverage. + shortTestCases := []struct { + name string + s1, s2 Slice[nc] + want bool + }{ + { + name: "duplicates_same_length", + s1: ncFrom("a", "a", "b"), + s2: ncFrom("a", "b", "b"), + want: false, + }, + { + name: "duplicates_different_matched", + s1: ncFrom("x", "y", "a", "a", "b"), + s2: ncFrom("x", "y", "b", "a", "a"), + want: true, + }, + { + name: "item_in_a_not_b", + s1: ncFrom("x", "y", "a", "b", "c"), + s2: ncFrom("x", "y", "b", "c", "q"), + want: false, + }, + } + for _, tc := range shortTestCases { + t.Run("short_"+tc.name, func(t *testing.T) { + c.Check(SliceEqualAnyOrderFunc(tc.s1, tc.s2, cmp), qt.Equals, tc.want) + }) + } } func TestSliceEqual(t *testing.T) { From b406f209c380f4c20cca5709a01f66275143b867 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Wed, 29 Jan 2025 09:35:50 +0200 Subject: [PATCH 213/223] cmd/{k8s-operator,containerboot},kube: ensure egress ProxyGroup proxies don't terminate while cluster traffic is still routed to them (#14436) cmd/{containerboot,k8s-operator},kube: add preshutdown hook for egress PG proxies This change is part of work towards minimizing downtime during update rollouts of egress ProxyGroup replicas. This change: - updates the containerboot health check logic to return Pod IP in headers, if set - always runs the health check for egress PG proxies - updates ClusterIP Services created for PG egress endpoints to include the health check endpoint - implements preshutdown endpoint in proxies. The preshutdown endpoint logic waits till, for all currently configured egress services, the ClusterIP Service health check endpoint is no longer returned by the shutting-down Pod (by looking at the new Pod IP header). - ensures that kubelet is configured to call the preshutdown endpoint This reduces the possibility that, as replicas are terminated during an update, a replica gets terminated to which cluster traffic is still being routed via the ClusterIP Service because kube proxy has not yet updated routig rules. This is not a perfect check as in practice, it only checks that the kube proxy on the node on which the proxy runs has updated rules. However, overall this might be good enough. The preshutdown logic is disabled if users have configured a custom health check port via TS_LOCAL_ADDR_PORT env var. This change throws a warnign if so and in future setting of that env var for operator proxies might be disallowed (as users shouldn't need to configure this for a Pod directly). This is backwards compatible with earlier proxy versions. Updates tailscale/tailscale#14326 Signed-off-by: Irbe Krumina --- cmd/containerboot/healthz.go | 13 +- cmd/containerboot/main.go | 22 ++- cmd/containerboot/main_test.go | 127 ++++++++---- cmd/containerboot/services.go | 212 +++++++++++++++++++-- cmd/containerboot/services_test.go | 149 +++++++++++++++ cmd/containerboot/settings.go | 34 ++-- cmd/k8s-operator/egress-eps.go | 13 +- cmd/k8s-operator/egress-services.go | 94 +++++++-- cmd/k8s-operator/egress-services_test.go | 54 +++--- cmd/k8s-operator/operator.go | 10 +- cmd/k8s-operator/proxygroup.go | 32 +++- cmd/k8s-operator/proxygroup_specs.go | 75 +++++++- cmd/k8s-operator/proxygroup_test.go | 99 ++++++++-- cmd/k8s-operator/sts.go | 3 + k8s-operator/conditions.go | 10 - kube/egressservices/egressservices.go | 13 +- kube/egressservices/egressservices_test.go | 2 +- kube/kubetypes/types.go | 5 + 18 files changed, 791 insertions(+), 176 deletions(-) diff --git a/cmd/containerboot/healthz.go b/cmd/containerboot/healthz.go index 895290733cf5f..6d03bd6d32b12 100644 --- a/cmd/containerboot/healthz.go +++ b/cmd/containerboot/healthz.go @@ -6,9 +6,12 @@ package main import ( + "fmt" "log" "net/http" "sync" + + "tailscale.com/kube/kubetypes" ) // healthz is a simple health check server, if enabled it returns 200 OK if @@ -17,6 +20,7 @@ import ( type healthz struct { sync.Mutex hasAddrs bool + podIPv4 string } func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -24,7 +28,10 @@ func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer h.Unlock() if h.hasAddrs { - w.Write([]byte("ok")) + w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4) + if _, err := w.Write([]byte("ok")); err != nil { + http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError) + } } else { http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable) } @@ -43,8 +50,8 @@ func (h *healthz) update(healthy bool) { // healthHandlers registers a simple health handler at /healthz. // A containerized tailscale instance is considered healthy if // it has at least one tailnet IP address. -func healthHandlers(mux *http.ServeMux) *healthz { - h := &healthz{} +func healthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { + h := &healthz{podIPv4: podIPv4} mux.Handle("GET /healthz", h) return h } diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 895be108b0090..0aca27f5fdd77 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -191,17 +191,18 @@ func main() { defer killTailscaled() var healthCheck *healthz + ep := &egressProxy{} if cfg.HealthCheckAddrPort != "" { mux := http.NewServeMux() log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort) - healthCheck = healthHandlers(mux) + healthCheck = healthHandlers(mux, cfg.PodIPv4) close := runHTTPServer(mux, cfg.HealthCheckAddrPort) defer close() } - if cfg.localMetricsEnabled() || cfg.localHealthEnabled() { + if cfg.localMetricsEnabled() || cfg.localHealthEnabled() || cfg.egressSvcsTerminateEPEnabled() { mux := http.NewServeMux() if cfg.localMetricsEnabled() { @@ -211,7 +212,11 @@ func main() { if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) - healthCheck = healthHandlers(mux) + healthCheck = healthHandlers(mux, cfg.PodIPv4) + } + if cfg.EgressProxiesCfgPath != "" { + log.Printf("Running preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP) + ep.registerHandlers(mux) } close := runHTTPServer(mux, cfg.LocalAddrPort) @@ -639,20 +644,21 @@ runLoop: // will then continuously monitor the config file and netmap updates and // reconfigure the firewall rules as needed. If any of its operations fail, it // will crash this node. - if cfg.EgressSvcsCfgPath != "" { - log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressSvcsCfgPath) + if cfg.EgressProxiesCfgPath != "" { + log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressProxiesCfgPath) egressSvcsNotify = make(chan ipn.Notify) - ep := egressProxy{ - cfgPath: cfg.EgressSvcsCfgPath, + opts := egressProxyRunOpts{ + cfgPath: cfg.EgressProxiesCfgPath, nfr: nfr, kc: kc, + tsClient: client, stateSecret: cfg.KubeSecret, netmapChan: egressSvcsNotify, podIPv4: cfg.PodIPv4, tailnetAddrs: addrs, } go func() { - if err := ep.run(ctx, n); err != nil { + if err := ep.run(ctx, n, opts); err != nil { egressSvcsErrorChan <- err } }() diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index dacfb5bc687b1..c8066f2c13edd 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -32,6 +32,8 @@ import ( "golang.org/x/sys/unix" "tailscale.com/ipn" "tailscale.com/kube/egressservices" + "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/types/netmap" @@ -54,20 +56,9 @@ func TestContainerBoot(t *testing.T) { defer kube.Close() tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"} - tailscaledConfBytes, err := json.Marshal(tailscaledConf) - if err != nil { - t.Fatalf("error unmarshaling tailscaled config: %v", err) - } serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} - serveConfBytes, err := json.Marshal(serveConf) - if err != nil { - t.Fatalf("error unmarshaling serve config: %v", err) - } - egressSvcsCfg := egressservices.Configs{"foo": {TailnetTarget: egressservices.TailnetTarget{FQDN: "foo.tailnetxyx.ts.net"}}} - egressSvcsCfgBytes, err := json.Marshal(egressSvcsCfg) - if err != nil { - t.Fatalf("error unmarshaling egress services config: %v", err) - } + egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net") + egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net") dirs := []string{ "var/lib", @@ -84,16 +75,17 @@ func TestContainerBoot(t *testing.T) { } } files := map[string][]byte{ - "usr/bin/tailscaled": fakeTailscaled, - "usr/bin/tailscale": fakeTailscale, - "usr/bin/iptables": fakeTailscale, - "usr/bin/ip6tables": fakeTailscale, - "dev/net/tun": []byte(""), - "proc/sys/net/ipv4/ip_forward": []byte("0"), - "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), - "etc/tailscaled/cap-95.hujson": tailscaledConfBytes, - "etc/tailscaled/serve-config.json": serveConfBytes, - "etc/tailscaled/egress-services-config.json": egressSvcsCfgBytes, + "usr/bin/tailscaled": fakeTailscaled, + "usr/bin/tailscale": fakeTailscale, + "usr/bin/iptables": fakeTailscale, + "usr/bin/ip6tables": fakeTailscale, + "dev/net/tun": []byte(""), + "proc/sys/net/ipv4/ip_forward": []byte("0"), + "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), + "etc/tailscaled/cap-95.hujson": mustJSON(t, tailscaledConf), + "etc/tailscaled/serve-config.json": mustJSON(t, serveConf), + filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg), + filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings): []byte("4"), } resetFiles := func() { for path, content := range files { @@ -132,6 +124,9 @@ func TestContainerBoot(t *testing.T) { healthURL := func(port int) string { return fmt.Sprintf("http://127.0.0.1:%d/healthz", port) } + egressSvcTerminateURL := func(port int) string { + return fmt.Sprintf("http://127.0.0.1:%d%s", port, kubetypes.EgessServicesPreshutdownEP) + } capver := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion) @@ -896,9 +891,10 @@ func TestContainerBoot(t *testing.T) { { Name: "egress_svcs_config_kube", Env: map[string]string{ - "KUBERNETES_SERVICE_HOST": kube.Host, - "KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, - "TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"), + "KUBERNETES_SERVICE_HOST": kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, + "TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"), + "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort), }, KubeSecret: map[string]string{ "authkey": "tskey-key", @@ -912,28 +908,35 @@ func TestContainerBoot(t *testing.T) { WantKubeSecret: map[string]string{ "authkey": "tskey-key", }, + EndpointStatuses: map[string]int{ + egressSvcTerminateURL(localAddrPort): 200, + }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ + "egress-services": mustBase64(t, egressStatus), "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net", "device_id": "myID", "device_ips": `["100.64.0.1"]`, "tailscale_capver": capver, }, + EndpointStatuses: map[string]int{ + egressSvcTerminateURL(localAddrPort): 200, + }, }, }, }, { Name: "egress_svcs_config_no_kube", Env: map[string]string{ - "TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"), - "TS_AUTHKEY": "tskey-key", + "TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"), + "TS_AUTHKEY": "tskey-key", }, Phases: []phase{ { - WantFatalLog: "TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes", + WantFatalLog: "TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes", }, }, }, @@ -1394,13 +1397,31 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } for _, op := range req { - if op.Op != "remove" { + if op.Op == "remove" { + if !strings.HasPrefix(op.Path, "/data/") { + panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) + } + delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) + } else if op.Op == "replace" { + path, ok := strings.CutPrefix(op.Path, "/data/") + if !ok { + panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) + } + req := make([]kubeclient.JSONPatch, 0) + if err := json.Unmarshal(bs, &req); err != nil { + panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) + } + + for _, patch := range req { + val, ok := patch.Value.(string) + if !ok { + panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", patch.Value)) + } + k.secret[path] = val + } + } else { panic(fmt.Sprintf("unsupported json-patch op %q", op.Op)) } - if !strings.HasPrefix(op.Path, "/data/") { - panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) - } - delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) } case "application/strategic-merge-patch+json": req := struct { @@ -1419,3 +1440,41 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("unhandled HTTP method %q", r.Method)) } } + +func mustBase64(t *testing.T, v any) string { + b := mustJSON(t, v) + s := base64.StdEncoding.WithPadding('=').EncodeToString(b) + return s +} + +func mustJSON(t *testing.T, v any) []byte { + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("error converting %v to json: %v", v, err) + } + return b +} + +// egress services status given one named tailnet target specified by FQDN. As written by the proxy to its state Secret. +func egressSvcStatus(name, fqdn string) egressservices.Status { + return egressservices.Status{ + Services: map[string]*egressservices.ServiceStatus{ + name: { + TailnetTarget: egressservices.TailnetTarget{ + FQDN: fqdn, + }, + }, + }, + } +} + +// egress config given one named tailnet target specified by FQDN. +func egressSvcConfig(name, fqdn string) egressservices.Configs { + return egressservices.Configs{ + name: egressservices.Config{ + TailnetTarget: egressservices.TailnetTarget{ + FQDN: fqdn, + }, + }, + } +} diff --git a/cmd/containerboot/services.go b/cmd/containerboot/services.go index aed00250d001e..177cb2d50c06a 100644 --- a/cmd/containerboot/services.go +++ b/cmd/containerboot/services.go @@ -11,18 +11,24 @@ import ( "errors" "fmt" "log" + "net/http" "net/netip" "os" "path/filepath" "reflect" + "strconv" "strings" "time" "github.com/fsnotify/fsnotify" + "tailscale.com/client/tailscale" "tailscale.com/ipn" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/syncs" "tailscale.com/tailcfg" + "tailscale.com/util/httpm" "tailscale.com/util/linuxfw" "tailscale.com/util/mak" ) @@ -37,13 +43,15 @@ const tailscaleTunInterface = "tailscale0" // egressProxy knows how to configure firewall rules to route cluster traffic to // one or more tailnet services. type egressProxy struct { - cfgPath string // path to egress service config file + cfgPath string // path to a directory with egress services config files nfr linuxfw.NetfilterRunner // never nil kc kubeclient.Client // never nil stateSecret string // name of the kube state Secret + tsClient *tailscale.LocalClient // never nil + netmapChan chan ipn.Notify // chan to receive netmap updates on podIPv4 string // never empty string, currently only IPv4 is supported @@ -55,15 +63,29 @@ type egressProxy struct { // memory at all. targetFQDNs map[string][]netip.Prefix - // used to configure firewall rules. - tailnetAddrs []netip.Prefix + tailnetAddrs []netip.Prefix // tailnet IPs of this tailnet device + + // shortSleep is the backoff sleep between healthcheck endpoint calls - can be overridden in tests. + shortSleep time.Duration + // longSleep is the time to sleep after the routing rules are updated to increase the chance that kube + // proxies on all nodes have updated their routing configuration. It can be configured to 0 in + // tests. + longSleep time.Duration + // client is a client that can send HTTP requests. + client httpClient +} + +// httpClient is a client that can send HTTP requests and can be mocked in tests. +type httpClient interface { + Do(*http.Request) (*http.Response, error) } // run configures egress proxy firewall rules and ensures that the firewall rules are reconfigured when: // - the mounted egress config has changed // - the proxy's tailnet IP addresses have changed // - tailnet IPs have changed for any backend targets specified by tailnet FQDN -func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error { +func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRunOpts) error { + ep.configure(opts) var tickChan <-chan time.Time var eventChan <-chan fsnotify.Event // TODO (irbekrm): take a look if this can be pulled into a single func @@ -75,7 +97,7 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error { tickChan = ticker.C } else { defer w.Close() - if err := w.Add(filepath.Dir(ep.cfgPath)); err != nil { + if err := w.Add(ep.cfgPath); err != nil { return fmt.Errorf("failed to add fsnotify watch: %w", err) } eventChan = w.Events @@ -85,28 +107,52 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error { return err } for { - var err error select { case <-ctx.Done(): return nil case <-tickChan: - err = ep.sync(ctx, n) + log.Printf("periodic sync, ensuring firewall config is up to date...") case <-eventChan: log.Printf("config file change detected, ensuring firewall config is up to date...") - err = ep.sync(ctx, n) case n = <-ep.netmapChan: shouldResync := ep.shouldResync(n) - if shouldResync { - log.Printf("netmap change detected, ensuring firewall config is up to date...") - err = ep.sync(ctx, n) + if !shouldResync { + continue } + log.Printf("netmap change detected, ensuring firewall config is up to date...") } - if err != nil { + if err := ep.sync(ctx, n); err != nil { return fmt.Errorf("error syncing egress service config: %w", err) } } } +type egressProxyRunOpts struct { + cfgPath string + nfr linuxfw.NetfilterRunner + kc kubeclient.Client + tsClient *tailscale.LocalClient + stateSecret string + netmapChan chan ipn.Notify + podIPv4 string + tailnetAddrs []netip.Prefix +} + +// applyOpts configures egress proxy using the provided options. +func (ep *egressProxy) configure(opts egressProxyRunOpts) { + ep.cfgPath = opts.cfgPath + ep.nfr = opts.nfr + ep.kc = opts.kc + ep.tsClient = opts.tsClient + ep.stateSecret = opts.stateSecret + ep.netmapChan = opts.netmapChan + ep.podIPv4 = opts.podIPv4 + ep.tailnetAddrs = opts.tailnetAddrs + ep.client = &http.Client{} // default HTTP client + ep.shortSleep = time.Second + ep.longSleep = time.Second * 10 +} + // sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if // any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current // firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such @@ -327,7 +373,8 @@ func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, s // getConfigs gets the mounted egress service configuration. func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) { - j, err := os.ReadFile(ep.cfgPath) + svcsCfg := filepath.Join(ep.cfgPath, egressservices.KeyEgressServices) + j, err := os.ReadFile(svcsCfg) if os.IsNotExist(err) { return nil, nil } @@ -569,3 +616,142 @@ func servicesStatusIsEqual(st, st1 *egressservices.Status) bool { st1.PodIPv4 = "" return reflect.DeepEqual(*st, *st1) } + +// registerHandlers adds a new handler to the provided ServeMux that can be called as a Kubernetes prestop hook to +// delay shutdown till it's safe to do so. +func (ep *egressProxy) registerHandlers(mux *http.ServeMux) { + mux.Handle(fmt.Sprintf("GET %s", kubetypes.EgessServicesPreshutdownEP), ep) +} + +// ServeHTTP serves /internal-egress-services-preshutdown endpoint, when it receives a request, it periodically polls +// the configured health check endpoint for each egress service till it the health check endpoint no longer hits this +// proxy Pod. It uses the Pod-IPv4 header to verify if health check response is received from this Pod. +func (ep *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + cfgs, err := ep.getConfigs() + if err != nil { + http.Error(w, fmt.Sprintf("error retrieving egress services configs: %v", err), http.StatusInternalServerError) + return + } + if cfgs == nil { + if _, err := w.Write([]byte("safe to terminate")); err != nil { + http.Error(w, fmt.Sprintf("error writing termination status: %v", err), http.StatusInternalServerError) + return + } + } + hp, err := ep.getHEPPings() + if err != nil { + http.Error(w, fmt.Sprintf("error determining the number of times health check endpoint should be pinged: %v", err), http.StatusInternalServerError) + return + } + ep.waitTillSafeToShutdown(r.Context(), cfgs, hp) +} + +// waitTillSafeToShutdown looks up all egress targets configured to be proxied via this instance and, for each target +// whose configuration includes a healthcheck endpoint, pings the endpoint till none of the responses +// are returned by this instance or till the HTTP request times out. In practice, the endpoint will be a Kubernetes Service for whom one of the backends +// would normally be this Pod. When this Pod is being deleted, the operator should have removed it from the Service +// backends and eventually kube proxy routing rules should be updated to no longer route traffic for the Service to this +// Pod. +func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs *egressservices.Configs, hp int) { + if cfgs == nil || len(*cfgs) == 0 { // avoid sleeping if no services are configured + return + } + log.Printf("Ensuring that cluster traffic for egress targets is no longer routed via this Pod...") + wg := syncs.WaitGroup{} + + for s, cfg := range *cfgs { + hep := cfg.HealthCheckEndpoint + if hep == "" { + log.Printf("Tailnet target %q does not have a cluster healthcheck specified, unable to verify if cluster traffic for the target is still routed via this Pod", s) + continue + } + svc := s + wg.Go(func() { + log.Printf("Ensuring that cluster traffic is no longer routed to %q via this Pod...", svc) + for { + if ctx.Err() != nil { // kubelet's HTTP request timeout + log.Printf("Cluster traffic for %s did not stop being routed to this Pod.", svc) + return + } + found, err := lookupPodRoute(ctx, hep, ep.podIPv4, hp, ep.client) + if err != nil { + log.Printf("unable to reach endpoint %q, assuming the routing rules for this Pod have been deleted: %v", hep, err) + break + } + if !found { + log.Printf("service %q is no longer routed through this Pod", svc) + break + } + log.Printf("service %q is still routed through this Pod, waiting...", svc) + time.Sleep(ep.shortSleep) + } + }) + } + wg.Wait() + // The check above really only checked that the routing rules are updated on this node. Sleep for a bit to + // ensure that the routing rules are updated on other nodes. TODO(irbekrm): this may or may not be good enough. + // If it's not good enough, we'd probably want to do something more complex, where the proxies check each other. + log.Printf("Sleeping for %s before shutdown to ensure that kube proxies on all nodes have updated routing configuration", ep.longSleep) + time.Sleep(ep.longSleep) +} + +// lookupPodRoute calls the healthcheck endpoint repeat times and returns true if the endpoint returns with the podIP +// header at least once. +func lookupPodRoute(ctx context.Context, hep, podIP string, repeat int, client httpClient) (bool, error) { + for range repeat { + f, err := lookup(ctx, hep, podIP, client) + if err != nil { + return false, err + } + if f { + return true, nil + } + } + return false, nil +} + +// lookup calls the healthcheck endpoint and returns true if the response contains the podIP header. +func lookup(ctx context.Context, hep, podIP string, client httpClient) (bool, error) { + req, err := http.NewRequestWithContext(ctx, httpm.GET, hep, nil) + if err != nil { + return false, fmt.Errorf("error creating new HTTP request: %v", err) + } + + // Close the TCP connection to ensure that the next request is routed to a different backend. + req.Close = true + + resp, err := client.Do(req) + if err != nil { + log.Printf("Endpoint %q can not be reached: %v, likely because there are no (more) healthy backends", hep, err) + return true, nil + } + defer resp.Body.Close() + gotIP := resp.Header.Get(kubetypes.PodIPv4Header) + return strings.EqualFold(podIP, gotIP), nil +} + +// getHEPPings gets the number of pings that should be sent to a health check endpoint to ensure that each configured +// backend is hit. This assumes that a health check endpoint is a Kubernetes Service and traffic to backend Pods is +// round robin load balanced. +func (ep *egressProxy) getHEPPings() (int, error) { + hepPingsPath := filepath.Join(ep.cfgPath, egressservices.KeyHEPPings) + j, err := os.ReadFile(hepPingsPath) + if os.IsNotExist(err) { + return 0, nil + } + if err != nil { + return -1, err + } + if len(j) == 0 || string(j) == "" { + return 0, nil + } + hp, err := strconv.Atoi(string(j)) + if err != nil { + return -1, fmt.Errorf("error parsing hep pings as int: %v", err) + } + if hp < 0 { + log.Printf("[unexpected] hep pings is negative: %d", hp) + return 0, nil + } + return hp, nil +} diff --git a/cmd/containerboot/services_test.go b/cmd/containerboot/services_test.go index 46f6db1cf6d0e..724626b072c2b 100644 --- a/cmd/containerboot/services_test.go +++ b/cmd/containerboot/services_test.go @@ -6,11 +6,18 @@ package main import ( + "context" + "fmt" + "io" + "net/http" "net/netip" "reflect" + "strings" + "sync" "testing" "tailscale.com/kube/egressservices" + "tailscale.com/kube/kubetypes" ) func Test_updatesForSvc(t *testing.T) { @@ -173,3 +180,145 @@ func Test_updatesForSvc(t *testing.T) { }) } } + +// A failure of this test will most likely look like a timeout. +func TestWaitTillSafeToShutdown(t *testing.T) { + podIP := "10.0.0.1" + anotherIP := "10.0.0.2" + + tests := []struct { + name string + // services is a map of service name to the number of calls to make to the healthcheck endpoint before + // returning a response that does NOT contain this Pod's IP in headers. + services map[string]int + replicas int + healthCheckSet bool + }{ + { + name: "no_configs", + }, + { + name: "one_service_immediately_safe_to_shutdown", + services: map[string]int{ + "svc1": 0, + }, + replicas: 2, + healthCheckSet: true, + }, + { + name: "multiple_services_immediately_safe_to_shutdown", + services: map[string]int{ + "svc1": 0, + "svc2": 0, + "svc3": 0, + }, + replicas: 2, + healthCheckSet: true, + }, + { + name: "multiple_services_no_healthcheck_endpoints", + services: map[string]int{ + "svc1": 0, + "svc2": 0, + "svc3": 0, + }, + replicas: 2, + }, + { + name: "one_service_eventually_safe_to_shutdown", + services: map[string]int{ + "svc1": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP + }, + replicas: 2, + healthCheckSet: true, + }, + { + name: "multiple_services_eventually_safe_to_shutdown", + services: map[string]int{ + "svc1": 1, // After 1 call to health check endpoint, no longer returns this Pod's IP + "svc2": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP + "svc3": 5, // After 5 calls to the health check endpoint, no longer returns this Pod's IP + }, + replicas: 2, + healthCheckSet: true, + }, + { + name: "multiple_services_eventually_safe_to_shutdown_with_higher_replica_count", + services: map[string]int{ + "svc1": 7, + "svc2": 10, + }, + replicas: 5, + healthCheckSet: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfgs := &egressservices.Configs{} + switches := make(map[string]int) + + for svc, callsToSwitch := range tt.services { + endpoint := fmt.Sprintf("http://%s.local", svc) + if tt.healthCheckSet { + (*cfgs)[svc] = egressservices.Config{ + HealthCheckEndpoint: endpoint, + } + } + switches[endpoint] = callsToSwitch + } + + ep := &egressProxy{ + podIPv4: podIP, + client: &mockHTTPClient{ + podIP: podIP, + anotherIP: anotherIP, + switches: switches, + }, + } + + ep.waitTillSafeToShutdown(context.Background(), cfgs, tt.replicas) + }) + } +} + +// mockHTTPClient is a client that receives an HTTP call for an egress service endpoint and returns a response with an +// IP address in a 'Pod-IPv4' header. It can be configured to return one IP address for N calls, then switch to another +// IP address to simulate a scenario where an IP is eventually no longer a backend for an endpoint. +// TODO(irbekrm): to test this more thoroughly, we should have the client take into account the number of replicas and +// return as if traffic was round robin load balanced across different Pods. +type mockHTTPClient struct { + // podIP - initial IP address to return, that matches the current proxy's IP address. + podIP string + anotherIP string + // after how many calls to an endpoint, the client should start returning 'anotherIP' instead of 'podIP. + switches map[string]int + mu sync.Mutex // protects the following + // calls tracks the number of calls received. + calls map[string]int +} + +func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { + m.mu.Lock() + if m.calls == nil { + m.calls = make(map[string]int) + } + + endpoint := req.URL.String() + m.calls[endpoint]++ + calls := m.calls[endpoint] + m.mu.Unlock() + + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("")), + } + + if calls <= m.switches[endpoint] { + resp.Header.Set(kubetypes.PodIPv4Header, m.podIP) // Pod is still routable + } else { + resp.Header.Set(kubetypes.PodIPv4Header, m.anotherIP) // Pod is no longer routable + } + return resp, nil +} diff --git a/cmd/containerboot/settings.go b/cmd/containerboot/settings.go index 5fc6cc3f06d15..0da18e52c0312 100644 --- a/cmd/containerboot/settings.go +++ b/cmd/containerboot/settings.go @@ -64,16 +64,16 @@ type settings struct { // when setting up rules to proxy cluster traffic to cluster ingress // target. // Deprecated: use PodIPv4, PodIPv6 instead to support dual stack clusters - PodIP string - PodIPv4 string - PodIPv6 string - PodUID string - HealthCheckAddrPort string - LocalAddrPort string - MetricsEnabled bool - HealthCheckEnabled bool - DebugAddrPort string - EgressSvcsCfgPath string + PodIP string + PodIPv4 string + PodIPv6 string + PodUID string + HealthCheckAddrPort string + LocalAddrPort string + MetricsEnabled bool + HealthCheckEnabled bool + DebugAddrPort string + EgressProxiesCfgPath string } func configFromEnv() (*settings, error) { @@ -107,7 +107,7 @@ func configFromEnv() (*settings, error) { MetricsEnabled: defaultBool("TS_ENABLE_METRICS", false), HealthCheckEnabled: defaultBool("TS_ENABLE_HEALTH_CHECK", false), DebugAddrPort: defaultEnv("TS_DEBUG_ADDR_PORT", ""), - EgressSvcsCfgPath: defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""), + EgressProxiesCfgPath: defaultEnv("TS_EGRESS_PROXIES_CONFIG_PATH", ""), PodUID: defaultEnv("POD_UID", ""), } podIPs, ok := os.LookupEnv("POD_IPS") @@ -186,7 +186,7 @@ func (s *settings) validate() error { return fmt.Errorf("error parsing TS_HEALTHCHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err) } } - if s.localMetricsEnabled() || s.localHealthEnabled() { + if s.localMetricsEnabled() || s.localHealthEnabled() || s.EgressProxiesCfgPath != "" { if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil { return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err) } @@ -199,8 +199,8 @@ func (s *settings) validate() error { if s.HealthCheckEnabled && s.HealthCheckAddrPort != "" { return errors.New("TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0, use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT") } - if s.EgressSvcsCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") { - return errors.New("TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes") + if s.EgressProxiesCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") { + return errors.New("TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes") } return nil } @@ -291,7 +291,7 @@ func isOneStepConfig(cfg *settings) bool { // as an L3 proxy, proxying to an endpoint provided via one of the config env // vars. func isL3Proxy(cfg *settings) bool { - return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressSvcsCfgPath != "" + return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressProxiesCfgPath != "" } // hasKubeStateStore returns true if the state must be stored in a Kubernetes @@ -308,6 +308,10 @@ func (cfg *settings) localHealthEnabled() bool { return cfg.LocalAddrPort != "" && cfg.HealthCheckEnabled } +func (cfg *settings) egressSvcsTerminateEPEnabled() bool { + return cfg.LocalAddrPort != "" && cfg.EgressProxiesCfgPath != "" +} + // defaultEnv returns the value of the given envvar name, or defVal if // unset. func defaultEnv(name, defVal string) string { diff --git a/cmd/k8s-operator/egress-eps.go b/cmd/k8s-operator/egress-eps.go index 85992abed9e37..3441e12ba93ec 100644 --- a/cmd/k8s-operator/egress-eps.go +++ b/cmd/k8s-operator/egress-eps.go @@ -20,7 +20,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - tsoperator "tailscale.com/k8s-operator" "tailscale.com/kube/egressservices" "tailscale.com/types/ptr" ) @@ -71,25 +70,27 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ if err != nil { return res, fmt.Errorf("error retrieving ExternalName Service: %w", err) } - if !tsoperator.EgressServiceIsValidAndConfigured(svc) { - l.Infof("Cluster resources for ExternalName Service %s/%s are not yet configured", svc.Namespace, svc.Name) - return res, nil - } // TODO(irbekrm): currently this reconcile loop runs all the checks every time it's triggered, which is // wasteful. Once we have a Ready condition for ExternalName Services for ProxyGroup, use the condition to // determine if a reconcile is needed. oldEps := eps.DeepCopy() - proxyGroupName := eps.Labels[labelProxyGroup] tailnetSvc := tailnetSvcName(svc) l = l.With("tailnet-service-name", tailnetSvc) // Retrieve the desired tailnet service configuration from the ConfigMap. + proxyGroupName := eps.Labels[labelProxyGroup] _, cfgs, err := egressSvcsConfigs(ctx, er.Client, proxyGroupName, er.tsNamespace) if err != nil { return res, fmt.Errorf("error retrieving tailnet services configuration: %w", err) } + if cfgs == nil { + // TODO(irbekrm): this path would be hit if egress service was once exposed on a ProxyGroup that later + // got deleted. Probably the EndpointSlices then need to be deleted too- need to rethink this flow. + l.Debugf("No egress config found, likely because ProxyGroup has not been created") + return res, nil + } cfg, ok := (*cfgs)[tailnetSvc] if !ok { l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc) diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 55003ee91de1b..cf218ba4fb0af 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -59,6 +59,8 @@ const ( maxPorts = 1000 indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group" + + tsHealthCheckPortName = "tailscale-health-check" ) var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount) @@ -229,15 +231,16 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s found := false for _, wantsPM := range svc.Spec.Ports { if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) { - // We don't use the port name to distinguish this port internally, but Kubernetes - // require that, for Service ports with more than one name each port is uniquely named. - // So we can always pick the port name from the ExternalName Service as at this point we - // know that those are valid names because Kuberentes already validated it once. Note - // that users could have changed an unnamed port to a named port and might have changed - // port names- this should still work. + // We want to both preserve the user set port names for ease of debugging, but also + // ensure that we name all unnamed ports as the ClusterIP Service that we create will + // always have at least two ports. // https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services // See also https://github.com/tailscale/tailscale/issues/13406#issuecomment-2507230388 - clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name + if wantsPM.Name != "" { + clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name + } else { + clusterIPSvc.Spec.Ports[i].Name = "tailscale-unnamed" + } found = true break } @@ -252,6 +255,12 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s // ClusterIP Service produce new target port and add a portmapping to // the ClusterIP Service. for _, wantsPM := range svc.Spec.Ports { + // Because we add a healthcheck port of our own, we will always have at least two ports. That + // means that we cannot have ports with name not set. + // https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + if wantsPM.Name == "" { + wantsPM.Name = "tailscale-unnamed" + } found := false for _, gotPM := range clusterIPSvc.Spec.Ports { if wantsPM.Port == gotPM.Port && strings.EqualFold(string(wantsPM.Protocol), string(gotPM.Protocol)) { @@ -278,6 +287,25 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s }) } } + var healthCheckPort int32 = defaultLocalAddrPort + + for { + if !slices.ContainsFunc(svc.Spec.Ports, func(p corev1.ServicePort) bool { + return p.Port == healthCheckPort + }) { + break + } + healthCheckPort++ + if healthCheckPort > 10002 { + return nil, false, fmt.Errorf("unable to find a free port for internal health check in range [9002, 10002]") + } + } + clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{ + Name: tsHealthCheckPortName, + Port: healthCheckPort, + TargetPort: intstr.FromInt(defaultLocalAddrPort), + Protocol: "TCP", + }) if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) { if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) { svc.Labels = clusterIPSvc.Labels @@ -320,7 +348,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s } tailnetSvc := tailnetSvcName(svc) gotCfg := (*cfgs)[tailnetSvc] - wantsCfg := egressSvcCfg(svc, clusterIPSvc) + wantsCfg := egressSvcCfg(svc, clusterIPSvc, esr.tsNamespace, l) if !reflect.DeepEqual(gotCfg, wantsCfg) { l.Debugf("updating egress services ConfigMap %s", cm.Name) mak.Set(cfgs, tailnetSvc, wantsCfg) @@ -504,10 +532,8 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s return false, nil } if !tsoperator.ProxyGroupIsReady(pg) { - l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) - return false, nil } l.Debugf("egress service is valid") @@ -515,6 +541,24 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s return true, nil } +func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, l *zap.SugaredLogger) egressservices.Config { + d := retrieveClusterDomain(ns, l) + tt := tailnetTargetFromSvc(externalNameSvc) + hep := healthCheckForSvc(clusterIPSvc, d) + cfg := egressservices.Config{ + TailnetTarget: tt, + HealthCheckEndpoint: hep, + } + for _, svcPort := range clusterIPSvc.Spec.Ports { + if svcPort.Name == tsHealthCheckPortName { + continue // exclude healthcheck from egress svcs configs + } + pm := portMap(svcPort) + mak.Set(&cfg.Ports, pm, struct{}{}) + } + return cfg +} + func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string { violations := validateService(svc) @@ -584,16 +628,6 @@ func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget { } } -func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service) egressservices.Config { - tt := tailnetTargetFromSvc(externalNameSvc) - cfg := egressservices.Config{TailnetTarget: tt} - for _, svcPort := range clusterIPSvc.Spec.Ports { - pm := portMap(svcPort) - mak.Set(&cfg.Ports, pm, struct{}{}) - } - return cfg -} - func portMap(p corev1.ServicePort) egressservices.PortMap { // TODO (irbekrm): out of bounds check? return egressservices.PortMap{Protocol: string(p.Protocol), MatchPort: uint16(p.TargetPort.IntVal), TargetPort: uint16(p.Port)} @@ -618,7 +652,11 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts Namespace: tsNamespace, }, } - if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { + err = cl.Get(ctx, client.ObjectKeyFromObject(cm), cm) + if apierrors.IsNotFound(err) { // ProxyGroup resources have not been created (yet) + return nil, nil, nil + } + if err != nil { return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err) } cfgs = &egressservices.Configs{} @@ -740,3 +778,17 @@ func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1. svc.Status = *st return err } + +// healthCheckForSvc return the URL of the containerboot's health check endpoint served by this Service or empty string. +func healthCheckForSvc(svc *corev1.Service, clusterDomain string) string { + // This version of the operator always sets health check port on the egress Services. However, it is possible + // that this reconcile loops runs during a proxy upgrade from a version that did not set the health check port + // and parses a Service that does not have the port set yet. + i := slices.IndexFunc(svc.Spec.Ports, func(port corev1.ServicePort) bool { + return port.Name == tsHealthCheckPortName + }) + if i == -1 { + return "" + } + return fmt.Sprintf("http://%s.%s.svc.%s:%d/healthz", svc.Name, svc.Namespace, clusterDomain, svc.Spec.Ports[i].Port) +} diff --git a/cmd/k8s-operator/egress-services_test.go b/cmd/k8s-operator/egress-services_test.go index ab0008ca0af25..d8a5dfd32c1c2 100644 --- a/cmd/k8s-operator/egress-services_test.go +++ b/cmd/k8s-operator/egress-services_test.go @@ -18,6 +18,7 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" @@ -78,42 +79,16 @@ func TestTailscaleEgressServices(t *testing.T) { Selector: nil, Ports: []corev1.ServicePort{ { - Name: "http", Protocol: "TCP", Port: 80, }, - { - Name: "https", - Protocol: "TCP", - Port: 443, - }, }, }, } - t.Run("proxy_group_not_ready", func(t *testing.T) { + t.Run("service_one_unnamed_port", func(t *testing.T) { mustCreate(t, fc, svc) expectReconciled(t, esr, "default", "test") - // Service should have EgressSvcValid condition set to Unknown. - svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)} - expectEqual(t, fc, svc) - }) - - t.Run("proxy_group_ready", func(t *testing.T) { - mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) { - pg.Status.Conditions = []metav1.Condition{ - condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock), - } - }) - expectReconciled(t, esr, "default", "test") - validateReadyService(t, fc, esr, svc, clock, zl, cm) - }) - t.Run("service_retain_one_unnamed_port", func(t *testing.T) { - svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80}} - mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { - s.Spec.Ports = svc.Spec.Ports - }) - expectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("service_add_two_named_ports", func(t *testing.T) { @@ -164,7 +139,7 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco // Verify that an EndpointSlice has been created. expectEqual(t, fc, endpointSlice(name, svc, clusterSvc)) // Verify that ConfigMap contains configuration for the new egress service. - mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm) + mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl) r := svcConfiguredReason(svc, true, zl.Sugar()) // Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the // CluterIP Service. @@ -203,6 +178,23 @@ func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *c func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service { labels := egressSvcChildResourceLabels(extNSvc) + ports := make([]corev1.ServicePort, len(extNSvc.Spec.Ports)) + for i, port := range extNSvc.Spec.Ports { + ports[i] = corev1.ServicePort{ // Copy the port to avoid modifying the original. + Name: port.Name, + Port: port.Port, + Protocol: port.Protocol, + } + if port.Name == "" { + ports[i].Name = "tailscale-unnamed" + } + } + ports = append(ports, corev1.ServicePort{ + Name: "tailscale-health-check", + Port: 9002, + TargetPort: intstr.FromInt(9002), + Protocol: "TCP", + }) return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -212,7 +204,7 @@ func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service { }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, - Ports: extNSvc.Spec.Ports, + Ports: ports, }, } } @@ -257,9 +249,9 @@ func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort { return ports } -func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) { +func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap, l *zap.Logger) { t.Helper() - wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc) + wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc, clusterIPSvc.Namespace, l.Sugar()) if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil { t.Fatalf("Error retrieving ConfigMap: %v", err) } diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index f349e7848ee27..6631c4f98f24f 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -777,7 +777,7 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger) } } -// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass, +// proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass, // returns a list of reconcile requests for all Connectors that have // .spec.proxyClass set. func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { @@ -998,7 +998,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile. // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all // user-created ExternalName Services that should be exposed on this ProxyGroup. func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { - return func(_ context.Context, o client.Object) []reconcile.Request { + return func(ctx context.Context, o client.Object) []reconcile.Request { pg, ok := o.(*tsapi.ProxyGroup) if !ok { logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup") @@ -1008,7 +1008,7 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) return nil } svcList := &corev1.ServiceList{} - if err := cl.List(context.Background(), svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil { + if err := cl.List(ctx, svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil { logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name) return nil } @@ -1028,7 +1028,7 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) // epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that // should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service. func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { - return func(_ context.Context, o client.Object) []reconcile.Request { + return func(ctx context.Context, o client.Object) []reconcile.Request { svc, ok := o.(*corev1.Service) if !ok { logger.Infof("[unexpected] Service handler triggered for an object that is not a Service") @@ -1038,7 +1038,7 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns return nil } epsList := &discoveryv1.EndpointSliceList{} - if err := cl.List(context.Background(), epsList, client.InNamespace(ns), + if err := cl.List(ctx, epsList, client.InNamespace(ns), client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil { logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name) return nil diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index f6de31727311e..4b17d3470bbeb 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -32,6 +32,7 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/egressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstime" @@ -166,6 +167,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error()) } + validateProxyClassForPG(logger, pg, proxyClass) if !tsoperator.ProxyClassIsReady(proxyClass) { message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName) logger.Info(message) @@ -204,6 +206,31 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady) } +// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup. +func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) { + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { + return + } + // Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check + // beig accessible on the replica Pod IP:9002. This address can also be modified by users, via + // TS_LOCAL_ADDR_PORT env var. + // + // Currently TS_LOCAL_ADDR_PORT controls Pod's health check and metrics address. _Probably_ there is no need for + // users to set this to a custom value. Users who want to consume metrics, should integrate with the metrics + // Service and/or ServiceMonitor, rather than Pods directly. The health check is likely not useful to integrate + // directly with for operator proxies (and we should aim for unified lifecycle logic in the operator, users + // shouldn't need to set their own). + // + // TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later). + if hasLocalAddrPortSet(pc) { + msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+ + "This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+ + "In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+ + "Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name) + logger.Warn(msg) + } +} + func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error { logger := r.logger(pg.Name) r.mu.Lock() @@ -253,10 +280,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return fmt.Errorf("error provisioning RoleBinding: %w", err) } if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - cm := pgEgressCM(pg, r.tsNamespace) + cm, hp := pgEgressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { existing.ObjectMeta.Labels = cm.ObjectMeta.Labels existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences + mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp) }); err != nil { return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err) } @@ -270,7 +298,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } - ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode) + ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, proxyClass) if err != nil { return fmt.Errorf("error generating StatefulSet spec: %w", err) } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 556a2ed7690b4..1ea91004b4b61 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -7,11 +7,14 @@ package main import ( "fmt" + "slices" + "strconv" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/yaml" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" @@ -19,9 +22,12 @@ import ( "tailscale.com/types/ptr" ) +// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. +const deletionGracePeriodSeconds int64 = 360 + // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string) (*appsv1.StatefulSet, error) { +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) @@ -145,15 +151,25 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string } if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - envs = append(envs, corev1.EnvVar{ - Name: "TS_EGRESS_SERVICES_CONFIG_PATH", - Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices), - }, + envs = append(envs, + // TODO(irbekrm): in 1.80 we deprecated TS_EGRESS_SERVICES_CONFIG_PATH in favour of + // TS_EGRESS_PROXIES_CONFIG_PATH. Remove it in 1.84. + corev1.EnvVar{ + Name: "TS_EGRESS_SERVICES_CONFIG_PATH", + Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices), + }, + corev1.EnvVar{ + Name: "TS_EGRESS_PROXIES_CONFIG_PATH", + Value: "/etc/proxies", + }, corev1.EnvVar{ Name: "TS_INTERNAL_APP", Value: kubetypes.AppProxyGroupEgress, }, - ) + corev1.EnvVar{ + Name: "TS_ENABLE_HEALTH_CHECK", + Value: "true", + }) } else { // ingress envs = append(envs, corev1.EnvVar{ Name: "TS_INTERNAL_APP", @@ -167,6 +183,25 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string return append(c.Env, envs...) }() + // The pre-stop hook is used to ensure that a replica does not get terminated while cluster traffic for egress + // services is still being routed to it. + // + // This mechanism currently (2025-01-26) rely on the local health check being accessible on the Pod's + // IP, so they are not supported for ProxyGroups where users have configured TS_LOCAL_ADDR_PORT to a custom + // value. + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && !hasLocalAddrPortSet(proxyClass) { + c.Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: kubetypes.EgessServicesPreshutdownEP, + Port: intstr.FromInt(defaultLocalAddrPort), + }, + }, + } + // Set the deletion grace period to 6 minutes to ensure that the pre-stop hook has enough time to terminate + // gracefully. + ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds) + } return ss, nil } @@ -258,7 +293,9 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S return secrets } -func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { +func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) (*corev1.ConfigMap, []byte) { + hp := hepPings(pg) + hpBs := []byte(strconv.Itoa(hp)) return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: pgEgressCMName(pg.Name), @@ -266,8 +303,10 @@ func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { Labels: pgLabels(pg.Name, nil), OwnerReferences: pgOwnerReference(pg), }, - } + BinaryData: map[string][]byte{egressservices.KeyHEPPings: hpBs}, + }, hpBs } + func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -313,3 +352,23 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 { func pgEgressCMName(pg string) string { return fmt.Sprintf("%s-egress-config", pg) } + +// hasLocalAddrPortSet returns true if the proxyclass has the TS_LOCAL_ADDR_PORT env var set. For egress ProxyGroups, +// currently (2025-01-26) this means that the ProxyGroup does not support graceful failover. +func hasLocalAddrPortSet(proxyClass *tsapi.ProxyClass) bool { + if proxyClass == nil || proxyClass.Spec.StatefulSet == nil || proxyClass.Spec.StatefulSet.Pod == nil || proxyClass.Spec.StatefulSet.Pod.TailscaleContainer == nil { + return false + } + return slices.ContainsFunc(proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env, func(env tsapi.Env) bool { + return env.Name == envVarTSLocalAddrPort + }) +} + +// hepPings returns the number of times a health check endpoint exposed by a Service fronting ProxyGroup replicas should +// be pinged to ensure that all currently configured backend replicas are hit. +func hepPings(pg *tsapi.ProxyGroup) int { + rc := pgReplicas(pg) + // Assuming a Service implemented using round robin load balancing, number-of-replica-times should be enough, but in + // practice, we cannot assume that the requests will be load balanced perfectly. + return int(rc) * 3 +} diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index e7c85d3871d5d..29100de1decd0 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -19,13 +19,13 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/client/tailscale" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/kube/egressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/types/ptr" @@ -97,7 +97,7 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, false, "") + expectProxyGroupResources(t, fc, pg, false, "", pc) }) t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) { @@ -118,11 +118,11 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "") + expectProxyGroupResources(t, fc, pg, true, "", pc) if expected := 1; reconciler.egressProxyGroups.Len() != expected { t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } - expectProxyGroupResources(t, fc, pg, true, "") + expectProxyGroupResources(t, fc, pg, true, "", pc) keyReq := tailscale.KeyCapabilities{ Devices: tailscale.KeyDeviceCapabilities{ Create: tailscale.KeyDeviceCreateCapabilities{ @@ -154,7 +154,7 @@ func TestProxyGroup(t *testing.T) { } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash) + expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) }) t.Run("scale_up_to_3", func(t *testing.T) { @@ -165,7 +165,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash) + expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) @@ -175,7 +175,7 @@ func TestProxyGroup(t *testing.T) { TailnetIPs: []string{"1.2.3.4", "::1"}, }) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash) + expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) }) t.Run("scale_down_to_1", func(t *testing.T) { @@ -188,7 +188,7 @@ func TestProxyGroup(t *testing.T) { pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash) + expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) }) t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) { @@ -202,7 +202,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74") + expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74", pc) }) t.Run("enable_metrics", func(t *testing.T) { @@ -246,12 +246,29 @@ func TestProxyGroup(t *testing.T) { // The fake client does not clean up objects whose owner has been // deleted, so we can't test for the owned resources getting deleted. }) + } func TestProxyGroupTypes(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Generation: 1, + }, + Spec: tsapi.ProxyClassSpec{}, + } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). + WithObjects(pc). + WithStatusSubresource(pc). Build() + mustUpdateStatus(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { + p.Status.Conditions = []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + }} + }) zl, _ := zap.NewDevelopment() reconciler := &ProxyGroupReconciler{ @@ -274,9 +291,7 @@ func TestProxyGroupTypes(t *testing.T) { Replicas: ptr.To[int32](0), }, } - if err := fc.Create(context.Background(), pg); err != nil { - t.Fatal(err) - } + mustCreate(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) verifyProxyGroupCounts(t, reconciler, 0, 1) @@ -286,7 +301,8 @@ func TestProxyGroupTypes(t *testing.T) { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress) - verifyEnvVar(t, sts, "TS_EGRESS_SERVICES_CONFIG_PATH", fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices)) + verifyEnvVar(t, sts, "TS_EGRESS_PROXIES_CONFIG_PATH", "/etc/proxies") + verifyEnvVar(t, sts, "TS_ENABLE_HEALTH_CHECK", "true") // Verify that egress configuration has been set up. cm := &corev1.ConfigMap{} @@ -323,6 +339,57 @@ func TestProxyGroupTypes(t *testing.T) { if diff := cmp.Diff(expectedVolumeMounts, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" { t.Errorf("unexpected volume mounts (-want +got):\n%s", diff) } + + expectedLifecycle := corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: kubetypes.EgessServicesPreshutdownEP, + Port: intstr.FromInt(defaultLocalAddrPort), + }, + }, + } + if diff := cmp.Diff(expectedLifecycle, *sts.Spec.Template.Spec.Containers[0].Lifecycle); diff != "" { + t.Errorf("unexpected lifecycle (-want +got):\n%s", diff) + } + if *sts.Spec.Template.DeletionGracePeriodSeconds != deletionGracePeriodSeconds { + t.Errorf("unexpected deletion grace period seconds %d, want %d", *sts.Spec.Template.DeletionGracePeriodSeconds, deletionGracePeriodSeconds) + } + }) + t.Run("egress_type_no_lifecycle_hook_when_local_addr_port_set", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-egress-no-lifecycle", + UID: "test-egress-no-lifecycle-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + Replicas: ptr.To[int32](0), + ProxyClass: "test", + }, + } + mustCreate(t, fc, pg) + mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { + p.Spec.StatefulSet = &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &tsapi.Container{ + Env: []tsapi.Env{{ + Name: "TS_LOCAL_ADDR_PORT", + Value: "127.0.0.1:8080", + }}, + }, + }, + } + }) + expectReconciled(t, reconciler, "", pg.Name) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + if sts.Spec.Template.Spec.Containers[0].Lifecycle != nil { + t.Error("lifecycle hook was set when TS_LOCAL_ADDR_PORT was configured via ProxyClass") + } }) t.Run("ingress_type", func(t *testing.T) { @@ -341,7 +408,7 @@ func TestProxyGroupTypes(t *testing.T) { } expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 1, 1) + verifyProxyGroupCounts(t, reconciler, 1, 2) sts := &appsv1.StatefulSet{} if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -402,13 +469,13 @@ func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue str t.Errorf("%s environment variable not found", name) } -func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) { +func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string, proxyClass *tsapi.ProxyClass) { t.Helper() role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto") + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", proxyClass) if err != nil { t.Fatal(err) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index fce6bfdd73b69..c1d13f33dc443 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -101,6 +101,9 @@ const ( proxyTypeIngressResource = "ingress_resource" proxyTypeConnector = "connector" proxyTypeProxyGroup = "proxygroup" + + envVarTSLocalAddrPort = "TS_LOCAL_ADDR_PORT" + defaultLocalAddrPort = 9002 // metrics and health check port ) var ( diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index 1ecedfc0751aa..abe8f7f9cc6fa 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -75,16 +75,6 @@ func RemoveServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionTy }) } -func EgressServiceIsValidAndConfigured(svc *corev1.Service) bool { - for _, typ := range []tsapi.ConditionType{tsapi.EgressSvcValid, tsapi.EgressSvcConfigured} { - cond := GetServiceCondition(svc, typ) - if cond == nil || cond.Status != metav1.ConditionTrue { - return false - } - } - return true -} - // SetRecorderCondition ensures that Recorder status has a condition with the // given attributes. LastTransitionTime gets set every time condition's status // changes. diff --git a/kube/egressservices/egressservices.go b/kube/egressservices/egressservices.go index 04a1c362b00c4..2515f1bf3a476 100644 --- a/kube/egressservices/egressservices.go +++ b/kube/egressservices/egressservices.go @@ -13,9 +13,15 @@ import ( "net/netip" ) -// KeyEgressServices is name of the proxy state Secret field that contains the -// currently applied egress proxy config. -const KeyEgressServices = "egress-services" +const ( + // KeyEgressServices is name of the proxy state Secret field that contains the + // currently applied egress proxy config. + KeyEgressServices = "egress-services" + + // KeyHEPPings is the number of times an egress service health check endpoint needs to be pinged to ensure that + // each currently configured backend is hit. In practice, it depends on the number of ProxyGroup replicas. + KeyHEPPings = "hep-pings" +) // Configs contains the desired configuration for egress services keyed by // service name. @@ -24,6 +30,7 @@ type Configs map[string]Config // Config is an egress service configuration. // TODO(irbekrm): version this? type Config struct { + HealthCheckEndpoint string `json:"healthCheckEndpoint"` // TailnetTarget is the target to which cluster traffic for this service // should be proxied. TailnetTarget TailnetTarget `json:"tailnetTarget"` diff --git a/kube/egressservices/egressservices_test.go b/kube/egressservices/egressservices_test.go index d6f952ea0a463..806ad91be61cd 100644 --- a/kube/egressservices/egressservices_test.go +++ b/kube/egressservices/egressservices_test.go @@ -55,7 +55,7 @@ func Test_jsonMarshalConfig(t *testing.T) { protocol: "tcp", matchPort: 4003, targetPort: 80, - wantsBs: []byte(`{"tailnetTarget":{"ip":"","fqdn":""},"ports":[{"protocol":"tcp","matchPort":4003,"targetPort":80}]}`), + wantsBs: []byte(`{"healthCheckEndpoint":"","tailnetTarget":{"ip":"","fqdn":""},"ports":[{"protocol":"tcp","matchPort":4003,"targetPort":80}]}`), }, } for _, tt := range tests { diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index afc4890188ea8..894cbb41d539d 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -43,4 +43,9 @@ const ( // that cluster workloads behind the Ingress can now be accessed via the given DNS name over HTTPS. KeyHTTPSEndpoint string = "https_endpoint" ValueNoHTTPS string = "no-https" + + // Pod's IPv4 address header key as returned by containerboot health check endpoint. + PodIPv4Header string = "Pod-IPv4" + + EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown" ) From 52f88f782a45652d9db25b1563e5defae1e42897 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Wed, 29 Jan 2025 17:48:05 +0200 Subject: [PATCH 214/223] cmd/k8s-operator: don't set deprecated configfile hash on new proxies (#14817) Fixes the configfile reload logic- if the tailscale capver can not yet be determined because the device info is not yet written to the state Secret, don't assume that the proxy is pre-110. Updates tailscale/tailscale#13032 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/operator_test.go | 65 ------------------------------- cmd/k8s-operator/sts.go | 2 +- 2 files changed, 1 insertion(+), 66 deletions(-) diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 2fa14e33b8ecb..73c795bb3a3ca 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -1339,71 +1339,6 @@ func TestProxyFirewallMode(t *testing.T) { expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) } -func TestTailscaledConfigfileHash(t *testing.T) { - fc := fake.NewFakeClient() - ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } - clock := tstest.NewClock(tstest.ClockOpts{}) - sr := &ServiceReconciler{ - Client: fc, - ssr: &tailscaleSTSReconciler{ - Client: fc, - tsClient: ft, - defaultTags: []string{"tag:k8s"}, - operatorNamespace: "operator-ns", - proxyImage: "tailscale/tailscale", - }, - logger: zl.Sugar(), - clock: clock, - isDefaultLoadBalancer: true, - } - - // Create a service that we should manage, and check that the initial round - // of objects looks right. - mustCreate(t, fc, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - // The apiserver is supposed to set the UID, but the fake client - // doesn't. So, set it explicitly because other code later depends - // on it being set. - UID: types.UID("1234-UID"), - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.20.30.40", - Type: corev1.ServiceTypeLoadBalancer, - }, - }) - - expectReconciled(t, sr, "default", "test") - expectReconciled(t, sr, "default", "test") - - fullName, shortName := findGenName(t, fc, "default", "test", "svc") - o := configOpts{ - stsName: shortName, - secretName: fullName, - namespace: "default", - parentType: "svc", - hostname: "default-test", - clusterTargetIP: "10.20.30.40", - confFileHash: "848bff4b5ba83ac999e6984c8464e597156daba961ae045e7dbaef606d54ab5e", - app: kubetypes.AppIngressProxy, - } - expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) - - // 2. Hostname gets changed, configfile is updated and a new hash value - // is produced. - mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { - mak.Set(&svc.Annotations, AnnotationHostname, "another-test") - }) - o.hostname = "another-test" - o.confFileHash = "d4cc13f09f55f4f6775689004f9a466723325b84d2b590692796bfe22aeaa389" - expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) -} func Test_isMagicDNSName(t *testing.T) { tests := []struct { in string diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index c1d13f33dc443..0bc9d6fb9c890 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -697,7 +697,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S // being created, there is no need for a restart. // TODO(irbekrm): remove this in 1.84. hash := tsConfigHash - if dev != nil && dev.capver >= 110 { + if dev == nil || dev.capver >= 110 { hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] } s.Spec = ss.Spec From b60f6b849af1fae1cf343be98f7fb1714c9ea165 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Wed, 29 Jan 2025 10:25:50 -0600 Subject: [PATCH 215/223] Revert "ssh,tempfork/gliderlabs/ssh: replace github.com/tailscale/golang-x-crypto/ssh with golang.org/x/crypto/ssh" This reverts commit 46fd4e58a27495263336b86ee961ee28d8c332b7. We don't want to include this in 1.80 yet, but can add it back post 1.80. Updates #8593 Signed-off-by: Percy Wegmann --- cmd/k8s-operator/depaware.txt | 11 +- cmd/ssh-auth-none-demo/ssh-auth-none-demo.go | 24 +- cmd/tailscaled/depaware.txt | 7 +- cmd/tailscaled/deps_test.go | 1 + go.mod | 2 +- go.sum | 4 +- ipn/ipnlocal/ssh.go | 2 +- ssh/tailssh/tailssh.go | 310 +++++++++++-------- ssh/tailssh/tailssh_integration_test.go | 2 +- ssh/tailssh/tailssh_test.go | 5 +- tempfork/gliderlabs/ssh/agent.go | 2 +- tempfork/gliderlabs/ssh/context.go | 11 +- tempfork/gliderlabs/ssh/options.go | 2 +- tempfork/gliderlabs/ssh/options_test.go | 2 +- tempfork/gliderlabs/ssh/server.go | 2 +- tempfork/gliderlabs/ssh/session.go | 2 +- tempfork/gliderlabs/ssh/session_test.go | 2 +- tempfork/gliderlabs/ssh/ssh.go | 4 +- tempfork/gliderlabs/ssh/tcpip.go | 2 +- tempfork/gliderlabs/ssh/tcpip_test.go | 2 +- tempfork/gliderlabs/ssh/util.go | 2 +- tempfork/gliderlabs/ssh/wrap.go | 2 +- 22 files changed, 231 insertions(+), 172 deletions(-) diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 972dbfc2c2fe5..e32fd4a2b0d8f 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -197,6 +197,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ + LD github.com/tailscale/golang-x-crypto/internal/poly1305 from github.com/tailscale/golang-x-crypto/ssh + LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal + LD github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ @@ -983,12 +986,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ - LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf - golang.org/x/crypto/chacha20 from golang.org/x/crypto/ssh+ + LD golang.org/x/crypto/blowfish from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf + golang.org/x/crypto/chacha20 from github.com/tailscale/golang-x-crypto/ssh+ golang.org/x/crypto/chacha20poly1305 from crypto/tls+ golang.org/x/crypto/cryptobyte from crypto/ecdsa+ golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ - golang.org/x/crypto/curve25519 from golang.org/x/crypto/ssh+ + golang.org/x/crypto/curve25519 from github.com/tailscale/golang-x-crypto/ssh+ golang.org/x/crypto/hkdf from crypto/tls+ golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ @@ -997,8 +1000,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/sha3 from crypto/internal/mlkem768+ - LD golang.org/x/crypto/ssh from tailscale.com/ipn/ipnlocal - LD golang.org/x/crypto/ssh/internal/bcrypt_pbkdf from golang.org/x/crypto/ssh golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ golang.org/x/exp/maps from sigs.k8s.io/controller-runtime/pkg/cache+ golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator+ diff --git a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go index 39af584ecd481..ee929299a4273 100644 --- a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go +++ b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go @@ -6,9 +6,6 @@ // highlight the unique parts of the Tailscale SSH server so SSH // client authors can hit it easily and fix their SSH clients without // needing to set up Tailscale and Tailscale SSH. -// -// Connections are allowed using any username except for "denyme". Connecting as -// "denyme" will result in an authentication failure with error message. package main import ( @@ -19,7 +16,6 @@ import ( "crypto/rsa" "crypto/x509" "encoding/pem" - "errors" "flag" "fmt" "io" @@ -28,7 +24,7 @@ import ( "path/filepath" "time" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" "tailscale.com/tempfork/gliderlabs/ssh" ) @@ -66,21 +62,13 @@ func main() { Handler: handleSessionPostSSHAuth, ServerConfigCallback: func(ctx ssh.Context) *gossh.ServerConfig { start := time.Now() - var spac gossh.ServerPreAuthConn return &gossh.ServerConfig{ - PreAuthConnCallback: func(conn gossh.ServerPreAuthConn) { - spac = conn + NextAuthMethodCallback: func(conn gossh.ConnMetadata, prevErrors []error) []string { + return []string{"tailscale"} }, NoClientAuth: true, // required for the NoClientAuthCallback to run NoClientAuthCallback: func(cm gossh.ConnMetadata) (*gossh.Permissions, error) { - spac.SendAuthBanner(fmt.Sprintf("# Banner: doing none auth at %v\r\n", time.Since(start))) - - if cm.User() == "denyme" { - return nil, &gossh.BannerError{ - Err: errors.New("denying access"), - Message: "denyme is not allowed to access this machine\n", - } - } + cm.SendAuthBanner(fmt.Sprintf("# Banner: doing none auth at %v\r\n", time.Since(start))) totalBanners := 2 if cm.User() == "banners" { @@ -89,9 +77,9 @@ func main() { for banner := 2; banner <= totalBanners; banner++ { time.Sleep(time.Second) if banner == totalBanners { - spac.SendAuthBanner(fmt.Sprintf("# Banner%d: access granted at %v\r\n", banner, time.Since(start))) + cm.SendAuthBanner(fmt.Sprintf("# Banner%d: access granted at %v\r\n", banner, time.Since(start))) } else { - spac.SendAuthBanner(fmt.Sprintf("# Banner%d at %v\r\n", banner, time.Since(start))) + cm.SendAuthBanner(fmt.Sprintf("# Banner%d at %v\r\n", banner, time.Since(start))) } } return nil, nil diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index a6fae54ffc237..a7ad83818d317 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -152,6 +152,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ + LD github.com/tailscale/golang-x-crypto/internal/poly1305 from github.com/tailscale/golang-x-crypto/ssh + LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal+ + LD github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ @@ -436,12 +439,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ - LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf + LD golang.org/x/crypto/blowfish from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf+ golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/chacha20poly1305 from crypto/tls+ golang.org/x/crypto/cryptobyte from crypto/ecdsa+ golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ - golang.org/x/crypto/curve25519 from golang.org/x/crypto/ssh+ + golang.org/x/crypto/curve25519 from github.com/tailscale/golang-x-crypto/ssh+ golang.org/x/crypto/hkdf from crypto/tls+ golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ diff --git a/cmd/tailscaled/deps_test.go b/cmd/tailscaled/deps_test.go index 7f06abc6c5ba1..2b4bc280d26cf 100644 --- a/cmd/tailscaled/deps_test.go +++ b/cmd/tailscaled/deps_test.go @@ -17,6 +17,7 @@ func TestOmitSSH(t *testing.T) { Tags: "ts_omit_ssh", BadDeps: map[string]string{ "tailscale.com/ssh/tailssh": msg, + "golang.org/x/crypto/ssh": msg, "tailscale.com/sessionrecording": msg, "github.com/anmitsu/go-shlex": msg, "github.com/creack/pty": msg, diff --git a/go.mod b/go.mod index 2489e34d711d0..8e52a9ab337b0 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( go.uber.org/zap v1.27.0 go4.org/mem v0.0.0-20240501181205-ae6ca9944745 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba - golang.org/x/crypto v0.32.1-0.20250118192723-a8ea4be81f07 + golang.org/x/crypto v0.32.0 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 golang.org/x/mod v0.22.0 golang.org/x/net v0.34.0 diff --git a/go.sum b/go.sum index b10e98da2b84d..c1c82ad7794c7 100644 --- a/go.sum +++ b/go.sum @@ -1058,8 +1058,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.32.1-0.20250118192723-a8ea4be81f07 h1:Z+Zg+aXJYq6f4TK2E4H+vZkQ4dJAWnInXDR6hM9znxo= -golang.org/x/crypto v0.32.1-0.20250118192723-a8ea4be81f07/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/ipn/ipnlocal/ssh.go b/ipn/ipnlocal/ssh.go index 47a74e2820905..383d03f5aa9be 100644 --- a/ipn/ipnlocal/ssh.go +++ b/ipn/ipnlocal/ssh.go @@ -24,8 +24,8 @@ import ( "strings" "sync" + "github.com/tailscale/golang-x-crypto/ssh" "go4.org/mem" - "golang.org/x/crypto/ssh" "tailscale.com/tailcfg" "tailscale.com/util/lineiter" "tailscale.com/util/mak" diff --git a/ssh/tailssh/tailssh.go b/ssh/tailssh/tailssh.go index 638ff99b8c188..7f21ccd1182ee 100644 --- a/ssh/tailssh/tailssh.go +++ b/ssh/tailssh/tailssh.go @@ -29,7 +29,7 @@ import ( "syscall" "time" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" "tailscale.com/envknob" "tailscale.com/ipn/ipnlocal" "tailscale.com/logtail/backoff" @@ -198,11 +198,8 @@ func (srv *server) OnPolicyChange() { // Setup and discover server info // - ServerConfigCallback // -// Get access to a ServerPreAuthConn (useful for sending banners) -// -// Do the user auth with a NoClientAuthCallback. If user specified -// a username ending in "+password", follow this with password auth -// (to work around buggy SSH clients that don't work with noauth). +// Do the user auth +// - NoClientAuthHandler // // Once auth is done, the conn can be multiplexed with multiple sessions and // channels concurrently. At which point any of the following can be called @@ -222,12 +219,15 @@ type conn struct { idH string connID string // ID that's shared with control - // spac is a [gossh.ServerPreAuthConn] used for sending auth banners. - // Banners cannot be sent after auth completes. - spac gossh.ServerPreAuthConn + // anyPasswordIsOkay is whether the client is authorized but has requested + // password-based auth to work around their buggy SSH client. When set, we + // accept any password in the PasswordHandler. + anyPasswordIsOkay bool // set by NoClientAuthCallback - action0 *tailcfg.SSHAction // set by clientAuth - finalAction *tailcfg.SSHAction // set by clientAuth + action0 *tailcfg.SSHAction // set by doPolicyAuth; first matching action + currentAction *tailcfg.SSHAction // set by doPolicyAuth, updated by resolveNextAction + finalAction *tailcfg.SSHAction // set by doPolicyAuth or resolveNextAction + finalActionErr error // set by doPolicyAuth or resolveNextAction info *sshConnInfo // set by setInfo localUser *userMeta // set by doPolicyAuth @@ -254,142 +254,141 @@ func (c *conn) vlogf(format string, args ...any) { } } -// errDenied is returned by auth callbacks when a connection is denied by the -// policy. It returns a gossh.BannerError to make sure the message gets -// displayed as an auth banner. -func errDenied(message string) error { - if message == "" { - message = "tailscale: access denied" - } - return &gossh.BannerError{ - Message: message, +// isAuthorized walks through the action chain and returns nil if the connection +// is authorized. If the connection is not authorized, it returns +// errDenied. If the action chain resolution fails, it returns the +// resolution error. +func (c *conn) isAuthorized(ctx ssh.Context) error { + action := c.currentAction + for { + if action.Accept { + return nil + } + if action.Reject || action.HoldAndDelegate == "" { + return errDenied + } + var err error + action, err = c.resolveNextAction(ctx) + if err != nil { + return err + } + if action.Message != "" { + if err := ctx.SendAuthBanner(action.Message); err != nil { + return err + } + } } } -// bannerError creates a gossh.BannerError that will result in the given -// message being displayed to the client. If err != nil, this also logs -// message:error. The contents of err is not leaked to clients in the banner. -func (c *conn) bannerError(message string, err error) error { - if err != nil { - c.logf("%s: %s", message, err) - } - return &gossh.BannerError{ - Err: err, - Message: fmt.Sprintf("tailscale: %s", message), - } -} +// errDenied is returned by auth callbacks when a connection is denied by the +// policy. +var errDenied = errors.New("ssh: access denied") -// clientAuth is responsible for performing client authentication. +// NoClientAuthCallback implements gossh.NoClientAuthCallback and is called by +// the ssh.Server when the client first connects with the "none" +// authentication method. // -// If policy evaluation fails, it returns an error. -// If access is denied, it returns an error. -func (c *conn) clientAuth(cm gossh.ConnMetadata) (*gossh.Permissions, error) { +// It is responsible for continuing policy evaluation from BannerCallback (or +// starting it afresh). It returns an error if the policy evaluation fails, or +// if the decision is "reject" +// +// It either returns nil (accept) or errDenied (reject). The errors may be wrapped. +func (c *conn) NoClientAuthCallback(ctx ssh.Context) error { if c.insecureSkipTailscaleAuth { - return &gossh.Permissions{}, nil + return nil + } + if err := c.doPolicyAuth(ctx); err != nil { + return err + } + if err := c.isAuthorized(ctx); err != nil { + return err } - if err := c.setInfo(cm); err != nil { - return nil, c.bannerError("failed to get connection info", err) + // Let users specify a username ending in +password to force password auth. + // This exists for buggy SSH clients that get confused by success from + // "none" auth. + if strings.HasSuffix(ctx.User(), forcePasswordSuffix) { + c.anyPasswordIsOkay = true + return errors.New("any password please") // not shown to users } + return nil +} - action, localUser, acceptEnv, err := c.evaluatePolicy() - if err != nil { - return nil, c.bannerError("failed to evaluate SSH policy", err) +func (c *conn) nextAuthMethodCallback(cm gossh.ConnMetadata, prevErrors []error) (nextMethod []string) { + switch { + case c.anyPasswordIsOkay: + nextMethod = append(nextMethod, "password") } - c.action0 = action + // The fake "tailscale" method is always appended to next so OpenSSH renders + // that in parens as the final failure. (It also shows up in "ssh -v", etc) + nextMethod = append(nextMethod, "tailscale") + return +} + +// fakePasswordHandler is our implementation of the PasswordHandler hook that +// checks whether the user's password is correct. But we don't actually use +// passwords. This exists only for when the user's username ends in "+password" +// to signal that their SSH client is buggy and gets confused by auth type +// "none" succeeding and they want our SSH server to require a dummy password +// prompt instead. We then accept any password since we've already authenticated +// & authorized them. +func (c *conn) fakePasswordHandler(ctx ssh.Context, password string) bool { + return c.anyPasswordIsOkay +} - if action.Accept || action.HoldAndDelegate != "" { - // Immediately look up user information for purposes of generating - // hold and delegate URL (if necessary). +// doPolicyAuth verifies that conn can proceed. +// It returns nil if the matching policy action is Accept or +// HoldAndDelegate. Otherwise, it returns errDenied. +func (c *conn) doPolicyAuth(ctx ssh.Context) error { + if err := c.setInfo(ctx); err != nil { + c.logf("failed to get conninfo: %v", err) + return errDenied + } + a, localUser, acceptEnv, err := c.evaluatePolicy() + if err != nil { + return fmt.Errorf("%w: %v", errDenied, err) + } + c.action0 = a + c.currentAction = a + c.acceptEnv = acceptEnv + if a.Message != "" { + if err := ctx.SendAuthBanner(a.Message); err != nil { + return fmt.Errorf("SendBanner: %w", err) + } + } + if a.Accept || a.HoldAndDelegate != "" { + if a.Accept { + c.finalAction = a + } lu, err := userLookup(localUser) if err != nil { - return nil, c.bannerError(fmt.Sprintf("failed to look up local user %q ", localUser), err) + c.logf("failed to look up %v: %v", localUser, err) + ctx.SendAuthBanner(fmt.Sprintf("failed to look up %v\r\n", localUser)) + return err } gids, err := lu.GroupIds() if err != nil { - return nil, c.bannerError("failed to look up local user's group IDs", err) + c.logf("failed to look up local user's group IDs: %v", err) + return err } c.userGroupIDs = gids c.localUser = lu - c.acceptEnv = acceptEnv + return nil } - - for { - switch { - case action.Accept: - metricTerminalAccept.Add(1) - if action.Message != "" { - if err := c.spac.SendAuthBanner(action.Message); err != nil { - return nil, fmt.Errorf("error sending auth welcome message: %w", err) - } - } - c.finalAction = action - return &gossh.Permissions{}, nil - case action.Reject: - metricTerminalReject.Add(1) - c.finalAction = action - return nil, errDenied(action.Message) - case action.HoldAndDelegate != "": - if action.Message != "" { - if err := c.spac.SendAuthBanner(action.Message); err != nil { - return nil, fmt.Errorf("error sending hold and delegate message: %w", err) - } - } - - url := action.HoldAndDelegate - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - metricHolds.Add(1) - url = c.expandDelegateURLLocked(url) - - var err error - action, err = c.fetchSSHAction(ctx, url) - if err != nil { - metricTerminalFetchError.Add(1) - return nil, c.bannerError("failed to fetch next SSH action", fmt.Errorf("fetch failed from %s: %w", url, err)) - } - default: - metricTerminalMalformed.Add(1) - return nil, c.bannerError("reached Action that had neither Accept, Reject, nor HoldAndDelegate", nil) - } + if a.Reject { + c.finalAction = a + return errDenied } + // Shouldn't get here, but: + return errDenied } // ServerConfig implements ssh.ServerConfigCallback. func (c *conn) ServerConfig(ctx ssh.Context) *gossh.ServerConfig { return &gossh.ServerConfig{ - PreAuthConnCallback: func(spac gossh.ServerPreAuthConn) { - c.spac = spac - }, - NoClientAuth: true, // required for the NoClientAuthCallback to run - NoClientAuthCallback: func(cm gossh.ConnMetadata) (*gossh.Permissions, error) { - // First perform client authentication, which can potentially - // involve multiple steps (for example prompting user to log in to - // Tailscale admin panel to confirm identity). - perms, err := c.clientAuth(cm) - if err != nil { - return nil, err - } - - // Authentication succeeded. Buggy SSH clients get confused by - // success from the "none" auth method. As a workaround, let users - // specify a username ending in "+password" to force password auth. - // The actual value of the password doesn't matter. - if strings.HasSuffix(cm.User(), forcePasswordSuffix) { - return nil, &gossh.PartialSuccessError{ - Next: gossh.ServerAuthCallbacks{ - PasswordCallback: func(_ gossh.ConnMetadata, password []byte) (*gossh.Permissions, error) { - return &gossh.Permissions{}, nil - }, - }, - } - } - - return perms, nil - }, + NoClientAuth: true, // required for the NoClientAuthCallback to run + NextAuthMethodCallback: c.nextAuthMethodCallback, } } @@ -400,7 +399,7 @@ func (srv *server) newConn() (*conn, error) { // Stop accepting new connections. // Connections in the auth phase are handled in handleConnPostSSHAuth. // Existing sessions are terminated by Shutdown. - return nil, errDenied("tailscale: server is shutting down") + return nil, errDenied } srv.mu.Unlock() c := &conn{srv: srv} @@ -411,6 +410,9 @@ func (srv *server) newConn() (*conn, error) { Version: "Tailscale", ServerConfigCallback: c.ServerConfig, + NoClientAuthHandler: c.NoClientAuthCallback, + PasswordHandler: c.fakePasswordHandler, + Handler: c.handleSessionPostSSHAuth, LocalPortForwardingCallback: c.mayForwardLocalPortTo, ReversePortForwardingCallback: c.mayReversePortForwardTo, @@ -521,16 +523,16 @@ func toIPPort(a net.Addr) (ipp netip.AddrPort) { return netip.AddrPortFrom(tanetaddr.Unmap(), uint16(ta.Port)) } -// connInfo populates the sshConnInfo from the provided arguments, +// connInfo returns a populated sshConnInfo from the provided arguments, // validating only that they represent a known Tailscale identity. -func (c *conn) setInfo(cm gossh.ConnMetadata) error { +func (c *conn) setInfo(ctx ssh.Context) error { if c.info != nil { return nil } ci := &sshConnInfo{ - sshUser: strings.TrimSuffix(cm.User(), forcePasswordSuffix), - src: toIPPort(cm.RemoteAddr()), - dst: toIPPort(cm.LocalAddr()), + sshUser: strings.TrimSuffix(ctx.User(), forcePasswordSuffix), + src: toIPPort(ctx.RemoteAddr()), + dst: toIPPort(ctx.LocalAddr()), } if !tsaddr.IsTailscaleIP(ci.dst.Addr()) { return fmt.Errorf("tailssh: rejecting non-Tailscale local address %v", ci.dst) @@ -545,7 +547,7 @@ func (c *conn) setInfo(cm gossh.ConnMetadata) error { ci.node = node ci.uprof = uprof - c.idH = string(cm.SessionID()) + c.idH = ctx.SessionID() c.info = ci c.logf("handling conn: %v", ci.String()) return nil @@ -592,6 +594,62 @@ func (c *conn) handleSessionPostSSHAuth(s ssh.Session) { ss.run() } +// resolveNextAction starts at c.currentAction and makes it way through the +// action chain one step at a time. An action without a HoldAndDelegate is +// considered the final action. Once a final action is reached, this function +// will keep returning that action. It updates c.currentAction to the next +// action in the chain. When the final action is reached, it also sets +// c.finalAction to the final action. +func (c *conn) resolveNextAction(sctx ssh.Context) (action *tailcfg.SSHAction, err error) { + if c.finalAction != nil || c.finalActionErr != nil { + return c.finalAction, c.finalActionErr + } + + defer func() { + if action != nil { + c.currentAction = action + if action.Accept || action.Reject { + c.finalAction = action + } + } + if err != nil { + c.finalActionErr = err + } + }() + + ctx, cancel := context.WithCancel(sctx) + defer cancel() + + // Loop processing/fetching Actions until one reaches a + // terminal state (Accept, Reject, or invalid Action), or + // until fetchSSHAction times out due to the context being + // done (client disconnect) or its 30 minute timeout passes. + // (Which is a long time for somebody to see login + // instructions and go to a URL to do something.) + action = c.currentAction + if action.Accept || action.Reject { + if action.Reject { + metricTerminalReject.Add(1) + } else { + metricTerminalAccept.Add(1) + } + return action, nil + } + url := action.HoldAndDelegate + if url == "" { + metricTerminalMalformed.Add(1) + return nil, errors.New("reached Action that lacked Accept, Reject, and HoldAndDelegate") + } + metricHolds.Add(1) + url = c.expandDelegateURLLocked(url) + nextAction, err := c.fetchSSHAction(ctx, url) + if err != nil { + metricTerminalFetchError.Add(1) + return nil, fmt.Errorf("fetching SSHAction from %s: %w", url, err) + } + return nextAction, nil +} + func (c *conn) expandDelegateURLLocked(actionURL string) string { nm := c.srv.lb.NetMap() ci := c.info diff --git a/ssh/tailssh/tailssh_integration_test.go b/ssh/tailssh/tailssh_integration_test.go index 5c4f533b11c00..1799d340019cb 100644 --- a/ssh/tailssh/tailssh_integration_test.go +++ b/ssh/tailssh/tailssh_integration_test.go @@ -32,8 +32,8 @@ import ( "github.com/bramvdbogaerde/go-scp" "github.com/google/go-cmp/cmp" "github.com/pkg/sftp" + gossh "github.com/tailscale/golang-x-crypto/ssh" "golang.org/x/crypto/ssh" - gossh "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" "tailscale.com/net/tsdial" "tailscale.com/tailcfg" diff --git a/ssh/tailssh/tailssh_test.go b/ssh/tailssh/tailssh_test.go index 2071366599028..9f3616d8ca8ab 100644 --- a/ssh/tailssh/tailssh_test.go +++ b/ssh/tailssh/tailssh_test.go @@ -31,7 +31,7 @@ import ( "testing" "time" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" "tailscale.com/ipn/ipnlocal" @@ -805,8 +805,7 @@ func TestSSHAuthFlow(t *testing.T) { state: &localState{ sshEnabled: true, }, - authErr: true, - wantBanners: []string{"tailscale: failed to evaluate SSH policy"}, + authErr: true, }, { name: "accept", diff --git a/tempfork/gliderlabs/ssh/agent.go b/tempfork/gliderlabs/ssh/agent.go index 99e84c1e5c64c..86a5bce7f8ebc 100644 --- a/tempfork/gliderlabs/ssh/agent.go +++ b/tempfork/gliderlabs/ssh/agent.go @@ -7,7 +7,7 @@ import ( "path" "sync" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) const ( diff --git a/tempfork/gliderlabs/ssh/context.go b/tempfork/gliderlabs/ssh/context.go index 505a43dbf3ffe..d43de6f09c8a5 100644 --- a/tempfork/gliderlabs/ssh/context.go +++ b/tempfork/gliderlabs/ssh/context.go @@ -6,7 +6,7 @@ import ( "net" "sync" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) // contextKey is a value for use with context.WithValue. It's used as @@ -55,6 +55,8 @@ var ( // ContextKeyPublicKey is a context key for use with Contexts in this package. // The associated value will be of type PublicKey. ContextKeyPublicKey = &contextKey{"public-key"} + + ContextKeySendAuthBanner = &contextKey{"send-auth-banner"} ) // Context is a package specific context interface. It exposes connection @@ -89,6 +91,8 @@ type Context interface { // SetValue allows you to easily write new values into the underlying context. SetValue(key, value interface{}) + + SendAuthBanner(banner string) error } type sshContext struct { @@ -117,6 +121,7 @@ func applyConnMetadata(ctx Context, conn gossh.ConnMetadata) { ctx.SetValue(ContextKeyUser, conn.User()) ctx.SetValue(ContextKeyLocalAddr, conn.LocalAddr()) ctx.SetValue(ContextKeyRemoteAddr, conn.RemoteAddr()) + ctx.SetValue(ContextKeySendAuthBanner, conn.SendAuthBanner) } func (ctx *sshContext) SetValue(key, value interface{}) { @@ -153,3 +158,7 @@ func (ctx *sshContext) LocalAddr() net.Addr { func (ctx *sshContext) Permissions() *Permissions { return ctx.Value(ContextKeyPermissions).(*Permissions) } + +func (ctx *sshContext) SendAuthBanner(msg string) error { + return ctx.Value(ContextKeySendAuthBanner).(func(string) error)(msg) +} diff --git a/tempfork/gliderlabs/ssh/options.go b/tempfork/gliderlabs/ssh/options.go index 29c8ef141842b..aa87a4f39db9e 100644 --- a/tempfork/gliderlabs/ssh/options.go +++ b/tempfork/gliderlabs/ssh/options.go @@ -3,7 +3,7 @@ package ssh import ( "os" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) // PasswordAuth returns a functional option that sets PasswordHandler on the server. diff --git a/tempfork/gliderlabs/ssh/options_test.go b/tempfork/gliderlabs/ssh/options_test.go index 47342b0f67923..7cf6f376c6a88 100644 --- a/tempfork/gliderlabs/ssh/options_test.go +++ b/tempfork/gliderlabs/ssh/options_test.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "testing" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) func newTestSessionWithOptions(t *testing.T, srv *Server, cfg *gossh.ClientConfig, options ...Option) (*gossh.Session, *gossh.Client, func()) { diff --git a/tempfork/gliderlabs/ssh/server.go b/tempfork/gliderlabs/ssh/server.go index 473e5fbd6fc8f..1086a72caf0e5 100644 --- a/tempfork/gliderlabs/ssh/server.go +++ b/tempfork/gliderlabs/ssh/server.go @@ -8,7 +8,7 @@ import ( "sync" "time" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) // ErrServerClosed is returned by the Server's Serve, ListenAndServe, diff --git a/tempfork/gliderlabs/ssh/session.go b/tempfork/gliderlabs/ssh/session.go index a7a9a3eebd96f..0a4a21e534401 100644 --- a/tempfork/gliderlabs/ssh/session.go +++ b/tempfork/gliderlabs/ssh/session.go @@ -9,7 +9,7 @@ import ( "sync" "github.com/anmitsu/go-shlex" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) // Session provides access to information about an SSH session and methods diff --git a/tempfork/gliderlabs/ssh/session_test.go b/tempfork/gliderlabs/ssh/session_test.go index fe61a9d96be9b..a60be5ec12d4e 100644 --- a/tempfork/gliderlabs/ssh/session_test.go +++ b/tempfork/gliderlabs/ssh/session_test.go @@ -9,7 +9,7 @@ import ( "net" "testing" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) func (srv *Server) serveOnce(l net.Listener) error { diff --git a/tempfork/gliderlabs/ssh/ssh.go b/tempfork/gliderlabs/ssh/ssh.go index 54bd31ec2fcb4..644cb257d9afa 100644 --- a/tempfork/gliderlabs/ssh/ssh.go +++ b/tempfork/gliderlabs/ssh/ssh.go @@ -4,7 +4,7 @@ import ( "crypto/subtle" "net" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) type Signal string @@ -105,7 +105,7 @@ type Pty struct { // requested by the client as part of the pty-req. These are outlined as // part of https://datatracker.ietf.org/doc/html/rfc4254#section-8. // - // The opcodes are defined as constants in golang.org/x/crypto/ssh (VINTR,VQUIT,etc.). + // The opcodes are defined as constants in github.com/tailscale/golang-x-crypto/ssh (VINTR,VQUIT,etc.). // Boolean opcodes have values 0 or 1. Modes gossh.TerminalModes } diff --git a/tempfork/gliderlabs/ssh/tcpip.go b/tempfork/gliderlabs/ssh/tcpip.go index 335fda65754ea..056a0c7343daf 100644 --- a/tempfork/gliderlabs/ssh/tcpip.go +++ b/tempfork/gliderlabs/ssh/tcpip.go @@ -7,7 +7,7 @@ import ( "strconv" "sync" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) const ( diff --git a/tempfork/gliderlabs/ssh/tcpip_test.go b/tempfork/gliderlabs/ssh/tcpip_test.go index b3ba60a9bb6b8..118b5d53ac4a1 100644 --- a/tempfork/gliderlabs/ssh/tcpip_test.go +++ b/tempfork/gliderlabs/ssh/tcpip_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - gossh "golang.org/x/crypto/ssh" + gossh "github.com/tailscale/golang-x-crypto/ssh" ) var sampleServerResponse = []byte("Hello world") diff --git a/tempfork/gliderlabs/ssh/util.go b/tempfork/gliderlabs/ssh/util.go index 3bee06dcdef39..e3b5716a3ab55 100644 --- a/tempfork/gliderlabs/ssh/util.go +++ b/tempfork/gliderlabs/ssh/util.go @@ -5,7 +5,7 @@ import ( "crypto/rsa" "encoding/binary" - "golang.org/x/crypto/ssh" + "github.com/tailscale/golang-x-crypto/ssh" ) func generateSigner() (ssh.Signer, error) { diff --git a/tempfork/gliderlabs/ssh/wrap.go b/tempfork/gliderlabs/ssh/wrap.go index d1f2b161e6932..17867d7518dd1 100644 --- a/tempfork/gliderlabs/ssh/wrap.go +++ b/tempfork/gliderlabs/ssh/wrap.go @@ -1,6 +1,6 @@ package ssh -import gossh "golang.org/x/crypto/ssh" +import gossh "github.com/tailscale/golang-x-crypto/ssh" // PublicKey is an abstraction of different types of public keys. type PublicKey interface { From 8bd04bdd3a6ceca64dfd04b49035cc16cbe2b2e1 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 29 Jan 2025 20:44:01 +0000 Subject: [PATCH 216/223] go.mod: bump gorilla/csrf for security fix (#14822) For https://github.com/gorilla/csrf/commit/9dd6af1f6d30fc79fb0d972394deebdabad6b5eb Update client/web and safeweb to correctly signal to the csrf middleware whether the request is being served over TLS. This determines whether Origin and Referer header checks are strictly enforced. The gorilla library previously did not enforce these checks due to a logic bug based on erroneous use of the net/http.Request API. The patch to fix this also inverts the library behavior to presume that every request is being served over TLS, necessitating these changes. Updates tailscale/corp#25340 Signed-off-by: Patrick O'Doherty Co-authored-by: Patrick O'Doherty --- client/web/web.go | 16 +++++++++++++--- go.mod | 2 +- go.sum | 4 ++-- safeweb/http.go | 6 ++++++ 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/client/web/web.go b/client/web/web.go index 4e48669230db3..3a7feea40c398 100644 --- a/client/web/web.go +++ b/client/web/web.go @@ -211,15 +211,25 @@ func NewServer(opts ServerOpts) (s *Server, err error) { // The client is secured by limiting the interface it listens on, // or by authenticating requests before they reach the web client. csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false)) + + // signal to the CSRF middleware that the request is being served over + // plaintext HTTP to skip TLS-only header checks. + withSetPlaintext := func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r = csrf.PlaintextHTTPRequest(r) + h.ServeHTTP(w, r) + }) + } + switch s.mode { case LoginServerMode: - s.apiHandler = csrfProtect(http.HandlerFunc(s.serveLoginAPI)) + s.apiHandler = csrfProtect(withSetPlaintext(http.HandlerFunc(s.serveLoginAPI))) metric = "web_login_client_initialization" case ReadOnlyServerMode: - s.apiHandler = csrfProtect(http.HandlerFunc(s.serveLoginAPI)) + s.apiHandler = csrfProtect(withSetPlaintext(http.HandlerFunc(s.serveLoginAPI))) metric = "web_readonly_client_initialization" case ManageServerMode: - s.apiHandler = csrfProtect(http.HandlerFunc(s.serveAPI)) + s.apiHandler = csrfProtect(withSetPlaintext(http.HandlerFunc(s.serveAPI))) metric = "web_client_initialization" } diff --git a/go.mod b/go.mod index 8e52a9ab337b0..6a50805854d1e 100644 --- a/go.mod +++ b/go.mod @@ -265,7 +265,7 @@ require ( github.com/gordonklaus/ineffassign v0.1.0 // indirect github.com/goreleaser/chglog v0.5.0 // indirect github.com/goreleaser/fileglob v1.3.0 // indirect - github.com/gorilla/csrf v1.7.2 + github.com/gorilla/csrf v1.7.3-0.20250123201450-9dd6af1f6d30 github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect diff --git a/go.sum b/go.sum index c1c82ad7794c7..c38c960291dc1 100644 --- a/go.sum +++ b/go.sum @@ -529,8 +529,8 @@ github.com/goreleaser/fileglob v1.3.0 h1:/X6J7U8lbDpQtBvGcwwPS6OpzkNVlVEsFUVRx9+ github.com/goreleaser/fileglob v1.3.0/go.mod h1:Jx6BoXv3mbYkEzwm9THo7xbr5egkAraxkGorbJb4RxU= github.com/goreleaser/nfpm/v2 v2.33.1 h1:EkdAzZyVhAI9JC1vjmjjbmnNzyH1J6Cu4JCsA7YcQuc= github.com/goreleaser/nfpm/v2 v2.33.1/go.mod h1:8wwWWvJWmn84xo/Sqiv0aMvEGTHlHZTXTEuVSgQpkIM= -github.com/gorilla/csrf v1.7.2 h1:oTUjx0vyf2T+wkrx09Trsev1TE+/EbDAeHtSTbtC2eI= -github.com/gorilla/csrf v1.7.2/go.mod h1:F1Fj3KG23WYHE6gozCmBAezKookxbIvUJT+121wTuLk= +github.com/gorilla/csrf v1.7.3-0.20250123201450-9dd6af1f6d30 h1:fiJdrgVBkjZ5B1HJ2WQwNOaXB+QyYcNXTA3t1XYLz0M= +github.com/gorilla/csrf v1.7.3-0.20250123201450-9dd6af1f6d30/go.mod h1:F1Fj3KG23WYHE6gozCmBAezKookxbIvUJT+121wTuLk= github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= diff --git a/safeweb/http.go b/safeweb/http.go index 983ff2fad8031..143c4dceea856 100644 --- a/safeweb/http.go +++ b/safeweb/http.go @@ -318,6 +318,12 @@ func checkHandlerType(apiPattern, browserPattern string) handlerType { } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // if we are not in a secure context, signal to the CSRF middleware that + // TLS-only header checks should be skipped + if !s.Config.SecureContext { + r = csrf.PlaintextHTTPRequest(r) + } + _, bp := s.BrowserMux.Handler(r) _, ap := s.APIMux.Handler(r) switch { From 3f39211f987c4127b447be0c29e2e4ab08176b11 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Thu, 30 Jan 2025 10:47:45 +0200 Subject: [PATCH 217/223] cmd/k8s-operator: check that cluster traffic is routed to egress ProxyGroup Pod before marking it as ready (#14792) This change builds on top of #14436 to ensure minimum downtime during egress ProxyGroup update rollouts: - adds a readiness gate for ProxyGroup replicas that prevents kubelet from marking the replica Pod as ready before a corresponding readiness condition has been added to the Pod - adds a reconciler that reconciles egress ProxyGroup Pods and, for each that is not ready, if cluster traffic for relevant egress endpoints is routed via this Pod- if so add the readiness condition to allow kubelet to mark the Pod as ready. During the sequenced StatefulSet update rollouts kubelet does not restart a Pod before the previous replica has been updated and marked as ready, so ensuring that a replica is not marked as ready allows to avoid a temporary post-update situation where all replicas have been restarted, but none of the new ones are yet set up as an endpoint for the egress service, so cluster traffic is dropped. Updates tailscale/tailscale#14326 Signed-off-by: Irbe Krumina --- .../deploy/chart/templates/operator-rbac.yaml | 5 +- .../deploy/manifests/operator.yaml | 7 + cmd/k8s-operator/egress-pod-readiness.go | 274 +++++++++ cmd/k8s-operator/egress-pod-readiness_test.go | 525 ++++++++++++++++++ cmd/k8s-operator/egress-services-readiness.go | 15 +- cmd/k8s-operator/operator.go | 70 +++ cmd/k8s-operator/testutils_test.go | 28 + 7 files changed, 916 insertions(+), 8 deletions(-) create mode 100644 cmd/k8s-operator/egress-pod-readiness.go create mode 100644 cmd/k8s-operator/egress-pod-readiness_test.go diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 637bdf793c2b9..7056ef42fc802 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -63,7 +63,10 @@ rules: verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] - apiGroups: [""] resources: ["pods"] - verbs: ["get","list","watch"] + verbs: ["get","list","watch", "update"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] - apiGroups: ["apps"] resources: ["statefulsets", "deployments"] verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index def5716f6ce5f..e966ef5593149 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -4854,6 +4854,13 @@ rules: - get - list - watch + - update + - apiGroups: + - "" + resources: + - pods/status + verbs: + - update - apiGroups: - apps resources: diff --git a/cmd/k8s-operator/egress-pod-readiness.go b/cmd/k8s-operator/egress-pod-readiness.go new file mode 100644 index 0000000000000..a6c57bf9dd09b --- /dev/null +++ b/cmd/k8s-operator/egress-pod-readiness.go @@ -0,0 +1,274 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "slices" + "strings" + "sync/atomic" + "time" + + "go.uber.org/zap" + xslices "golang.org/x/exp/slices" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" + "tailscale.com/logtail/backoff" + "tailscale.com/tstime" + "tailscale.com/util/httpm" +) + +const tsEgressReadinessGate = "tailscale.com/egress-services" + +// egressPodsReconciler is responsible for setting tailscale.com/egress-services condition on egress ProxyGroup Pods. +// The condition is used as a readiness gate for the Pod, meaning that kubelet will not mark the Pod as ready before the +// condition is set. The ProxyGroup StatefulSet updates are rolled out in such a way that no Pod is restarted, before +// the previous Pod is marked as ready, so ensuring that the Pod does not get marked as ready when it is not yet able to +// route traffic for egress service prevents downtime during restarts caused by no available endpoints left because +// every Pod has been recreated and is not yet added to endpoints. +// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate +type egressPodsReconciler struct { + client.Client + logger *zap.SugaredLogger + tsNamespace string + clock tstime.Clock + httpClient doer // http client that can be set to a mock client in tests + maxBackoff time.Duration // max backoff period between health check calls +} + +// Reconcile reconciles an egress ProxyGroup Pods on changes to those Pods and ProxyGroup EndpointSlices. It ensures +// that for each Pod who is ready to route traffic to all egress services for the ProxyGroup, the Pod has a +// tailscale.com/egress-services condition to set, so that kubelet will mark the Pod as ready. +// +// For the Pod to be ready +// to route traffic to the egress service, the kube proxy needs to have set up the Pod's IP as an endpoint for the +// ClusterIP Service corresponding to the egress service. +// +// Note that the endpoints for the ClusterIP Service are configured by the operator itself using custom +// EndpointSlices(egress-eps-reconciler), so the routing is not blocked on Pod's readiness. +// +// Each egress service has a corresponding ClusterIP Service, that exposes all user configured +// tailnet ports, as well as a health check port for the proxy. +// +// The reconciler calls the health check endpoint of each Service up to N number of times, where N is the number of +// replicas for the ProxyGroup x 3, and checks if the received response is healthy response from the Pod being reconciled. +// +// The health check response contains a header with the +// Pod's IP address- this is used to determine whether the response is received from this Pod. +// +// If the Pod does not appear to be serving the health check endpoint (pre-v1.80 proxies), the reconciler just sets the +// readiness condition for backwards compatibility reasons. +func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + l := er.logger.With("Pod", req.NamespacedName) + l.Debugf("starting reconcile") + defer l.Debugf("reconcile finished") + + pod := new(corev1.Pod) + err = er.Get(ctx, req.NamespacedName, pod) + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to get Pod: %w", err) + } + if !pod.DeletionTimestamp.IsZero() { + l.Debugf("Pod is being deleted, do nothing") + return res, nil + } + if pod.Labels[LabelParentType] != proxyTypeProxyGroup { + l.Infof("[unexpected] reconciler called for a Pod that is not a ProxyGroup Pod") + return res, nil + } + + // If the Pod does not have the readiness gate set, there is no need to add the readiness condition. In practice + // this will happen if the user has configured custom TS_LOCAL_ADDR_PORT, thus disabling the graceful failover. + if !slices.ContainsFunc(pod.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool { + return r.ConditionType == tsEgressReadinessGate + }) { + l.Debug("Pod does not have egress readiness gate set, skipping") + return res, nil + } + + proxyGroupName := pod.Labels[LabelParentName] + pg := new(tsapi.ProxyGroup) + if err := er.Get(ctx, types.NamespacedName{Name: proxyGroupName}, pg); err != nil { + return res, fmt.Errorf("error getting ProxyGroup %q: %w", proxyGroupName, err) + } + if pg.Spec.Type != typeEgress { + l.Infof("[unexpected] reconciler called for %q ProxyGroup Pod", pg.Spec.Type) + return res, nil + } + // Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup. + lbls := map[string]string{ + LabelManaged: "true", + labelProxyGroup: proxyGroupName, + labelSvcType: typeEgress, + } + svcs := &corev1.ServiceList{} + if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil { + return res, fmt.Errorf("error listing ClusterIP Services") + } + + idx := xslices.IndexFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool { + return c.Type == tsEgressReadinessGate + }) + if idx != -1 { + l.Debugf("Pod is already ready, do nothing") + return res, nil + } + + var routesMissing atomic.Bool + errChan := make(chan error, len(svcs.Items)) + for _, svc := range svcs.Items { + s := svc + go func() { + ll := l.With("service_name", s.Name) + d := retrieveClusterDomain(er.tsNamespace, ll) + healthCheckAddr := healthCheckForSvc(&s, d) + if healthCheckAddr == "" { + ll.Debugf("ClusterIP Service does not expose a health check endpoint, unable to verify if routing is set up") + errChan <- nil + return + } + + var routesSetup bool + bo := backoff.NewBackoff(s.Name, ll.Infof, er.maxBackoff) + for range numCalls(pgReplicas(pg)) { + if ctx.Err() != nil { + errChan <- nil + return + } + state, err := er.lookupPodRouteViaSvc(ctx, pod, healthCheckAddr, ll) + if err != nil { + errChan <- fmt.Errorf("error validating if routing has been set up for Pod: %w", err) + return + } + if state == healthy || state == cannotVerify { + routesSetup = true + break + } + if state == unreachable || state == unhealthy || state == podNotReady { + bo.BackOff(ctx, errors.New("backoff")) + } + } + if !routesSetup { + ll.Debugf("Pod is not yet configured as Service endpoint") + routesMissing.Store(true) + } + errChan <- nil + }() + } + for range len(svcs.Items) { + e := <-errChan + err = errors.Join(err, e) + } + if err != nil { + return res, fmt.Errorf("error verifying conectivity: %w", err) + } + if rm := routesMissing.Load(); rm { + l.Info("Pod is not yet added as an endpoint for all egress targets, waiting...") + return reconcile.Result{RequeueAfter: shortRequeue}, nil + } + if err := er.setPodReady(ctx, pod, l); err != nil { + return res, fmt.Errorf("error setting Pod as ready: %w", err) + } + return res, nil +} + +func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, l *zap.SugaredLogger) error { + if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool { + return c.Type == tsEgressReadinessGate + }) { + return nil + } + l.Infof("Pod is ready to route traffic to all egress targets") + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: tsEgressReadinessGate, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: er.clock.Now()}, + }) + return er.Status().Update(ctx, pod) +} + +// healthCheckState is the result of a single request to an egress Service health check endpoint with a goal to hit a +// specific backend Pod. +type healthCheckState int8 + +const ( + cannotVerify healthCheckState = iota // not verifiable for this setup (i.e earlier proxy version) + unreachable // no backends or another network error + notFound // hit another backend + unhealthy // not 200 + podNotReady // Pod is not ready, i.e does not have an IP address yet + healthy // 200 +) + +// lookupPodRouteViaSvc attempts to reach a Pod using a health check endpoint served by a Service and returns the state of the health check. +func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, l *zap.SugaredLogger) (healthCheckState, error) { + if !slices.ContainsFunc(pod.Spec.Containers[0].Env, func(e corev1.EnvVar) bool { + return e.Name == "TS_ENABLE_HEALTH_CHECK" && e.Value == "true" + }) { + l.Debugf("Pod does not have health check enabled, unable to verify if it is currently routable via Service") + return cannotVerify, nil + } + wantsIP, err := podIPv4(pod) + if err != nil { + return -1, fmt.Errorf("error determining Pod's IP address: %w", err) + } + if wantsIP == "" { + return podNotReady, nil + } + + ctx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + req, err := http.NewRequestWithContext(ctx, httpm.GET, healthCheckAddr, nil) + if err != nil { + return -1, fmt.Errorf("error creating new HTTP request: %w", err) + } + // Do not re-use the same connection for the next request so to maximize the chance of hitting all backends equally. + req.Close = true + resp, err := er.httpClient.Do(req) + if err != nil { + // This is most likely because this is the first Pod and is not yet added to Service endoints. Other + // error types are possible, but checking for those would likely make the system too fragile. + return unreachable, nil + } + defer resp.Body.Close() + gotIP := resp.Header.Get(kubetypes.PodIPv4Header) + if gotIP == "" { + l.Debugf("Health check does not return Pod's IP header, unable to verify if Pod is currently routable via Service") + return cannotVerify, nil + } + if !strings.EqualFold(wantsIP, gotIP) { + return notFound, nil + } + if resp.StatusCode != http.StatusOK { + return unhealthy, nil + } + return healthy, nil +} + +// numCalls return the number of times an endpoint on a ProxyGroup Service should be called till it can be safely +// assumed that, if none of the responses came back from a specific Pod then traffic for the Service is currently not +// being routed to that Pod. This assumes that traffic for the Service is routed via round robin, so +// InternalTrafficPolicy must be 'Cluster' and session affinity must be None. +func numCalls(replicas int32) int32 { + return replicas * 3 +} + +// doer is an interface for HTTP client that can be set to a mock client in tests. +type doer interface { + Do(*http.Request) (*http.Response, error) +} diff --git a/cmd/k8s-operator/egress-pod-readiness_test.go b/cmd/k8s-operator/egress-pod-readiness_test.go new file mode 100644 index 0000000000000..5e6fa2bb4c9fa --- /dev/null +++ b/cmd/k8s-operator/egress-pod-readiness_test.go @@ -0,0 +1,525 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "bytes" + "errors" + "fmt" + "io" + "log" + "net/http" + "sync" + "testing" + "time" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" + "tailscale.com/tstest" + "tailscale.com/types/ptr" +) + +func TestEgressPodReadiness(t *testing.T) { + // We need to pass a Pod object to WithStatusSubresource because of some quirks in how the fake client + // works. Without this code we would not be able to update Pod's status further down. + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&corev1.Pod{}). + Build() + zl, _ := zap.NewDevelopment() + cl := tstest.NewClock(tstest.ClockOpts{}) + rec := &egressPodsReconciler{ + tsNamespace: "operator-ns", + Client: fc, + logger: zl.Sugar(), + clock: cl, + } + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dev", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: "egress", + Replicas: ptr.To(int32(3)), + }, + } + mustCreate(t, fc, pg) + podIP := "10.0.0.2" + podTemplate := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "operator-ns", + Name: "pod", + Labels: map[string]string{ + LabelParentType: "proxygroup", + LabelParentName: "dev", + }, + }, + Spec: corev1.PodSpec{ + ReadinessGates: []corev1.PodReadinessGate{{ + ConditionType: tsEgressReadinessGate, + }}, + Containers: []corev1.Container{{ + Name: "tailscale", + Env: []corev1.EnvVar{{ + Name: "TS_ENABLE_HEALTH_CHECK", + Value: "true", + }}, + }}, + }, + Status: corev1.PodStatus{ + PodIPs: []corev1.PodIP{{IP: podIP}}, + }, + } + + t.Run("no_egress_services", func(t *testing.T) { + pod := podTemplate.DeepCopy() + mustCreate(t, fc, pod) + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod) + }) + t.Run("one_svc_already_routed_to", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + mustCreateAll(t, fc, svc, pod) + resp := readyResps(podIP, 1) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{hep: resp}, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + + // A subsequent reconcile should not change the Pod. + expectReconciled(t, rec, "operator-ns", pod.Name) + expectEqual(t, fc, pod) + + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("one_svc_many_backends_eventually_routed_to", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + mustCreateAll(t, fc, svc, pod) + // For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only + // return with the right Pod IP. + resps := append(readyResps("10.0.0.3", 4), append(readyResps("10.0.0.4", 4), readyResps(podIP, 1)...)...) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{hep: resps}, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("one_svc_one_backend_eventually_healthy", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + mustCreateAll(t, fc, svc, pod) + // For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only + // return with 200 status code. + resps := append(unreadyResps(podIP, 8), readyResps(podIP, 1)...) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{hep: resps}, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("one_svc_one_backend_never_routable", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + mustCreateAll(t, fc, svc, pod) + // For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times and Pod should be + // requeued if neither of those succeed. + resps := readyResps("10.0.0.3", 9) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{hep: resps}, + } + rec.httpClient = &httpCl + expectRequeue(t, rec, "operator-ns", pod.Name) + + // Pod should not have readiness gate condition set. + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("one_svc_many_backends_already_routable", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + svc2, hep2 := newSvc("svc-2", 9002) + svc3, hep3 := newSvc("svc-3", 9002) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + resps := readyResps(podIP, 1) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + hep2: resps, + hep3: resps, + }, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should not have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) + t.Run("one_svc_many_backends_eventually_routable_and_healthy", func(t *testing.T) { + pod := podTemplate.DeepCopy() + svc, hep := newSvc("svc", 9002) + svc2, hep2 := newSvc("svc-2", 9002) + svc3, hep3 := newSvc("svc-3", 9002) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...) + resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...) + resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + hep2: resps2, + hep3: resps3, + }, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) + t.Run("one_svc_many_backends_never_routable_and_healthy", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + svc2, hep2 := newSvc("svc-2", 9002) + svc3, hep3 := newSvc("svc-3", 9002) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod + // will be requeued if neither succeeds. + resps := readyResps("10.0.0.3", 9) + resps2 := append(readyResps("10.0.0.3", 5), readyResps("10.0.0.4", 4)...) + resps3 := unreadyResps(podIP, 9) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + hep2: resps2, + hep3: resps3, + }, + } + rec.httpClient = &httpCl + expectRequeue(t, rec, "operator-ns", pod.Name) + + // Pod should not have readiness gate condition set. + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) + t.Run("one_svc_many_backends_one_never_routable", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + svc2, hep2 := newSvc("svc-2", 9002) + svc3, hep3 := newSvc("svc-3", 9002) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod + // will be requeued if any one never succeeds. + resps := readyResps(podIP, 9) + resps2 := readyResps(podIP, 9) + resps3 := append(readyResps("10.0.0.3", 5), readyResps("10.0.0.4", 4)...) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + hep2: resps2, + hep3: resps3, + }, + } + rec.httpClient = &httpCl + expectRequeue(t, rec, "operator-ns", pod.Name) + + // Pod should not have readiness gate condition set. + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) + t.Run("one_svc_many_backends_one_never_healthy", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + svc2, hep2 := newSvc("svc-2", 9002) + svc3, hep3 := newSvc("svc-3", 9002) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod + // will be requeued if any one never succeeds. + resps := readyResps(podIP, 9) + resps2 := unreadyResps(podIP, 9) + resps3 := readyResps(podIP, 9) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + hep2: resps2, + hep3: resps3, + }, + } + rec.httpClient = &httpCl + expectRequeue(t, rec, "operator-ns", pod.Name) + + // Pod should not have readiness gate condition set. + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) + t.Run("one_svc_many_backends_different_ports_eventually_healthy_and_routable", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9003) + svc2, hep2 := newSvc("svc-2", 9004) + svc3, hep3 := newSvc("svc-3", 9010) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried up to 9 times and + // marked as success as soon as one try succeeds. + resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...) + resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...) + resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + hep2: resps2, + hep3: resps3, + }, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) + // Proxies of 1.78 and earlier did not set the Pod IP header. + t.Run("pod_does_not_return_ip_header", func(t *testing.T) { + pod := podTemplate.DeepCopy() + pod.Name = "foo-bar" + + svc, hep := newSvc("foo-bar", 9002) + mustCreateAll(t, fc, svc, pod) + // If a response does not contain Pod IP header, we assume that this is an earlier proxy version, + // readiness cannot be verified so the readiness gate is just set to true. + resps := unreadyResps("", 1) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + }, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("one_svc_one_backend_eventually_healthy_and_routable", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + svc, hep := newSvc("svc", 9002) + mustCreateAll(t, fc, svc, pod) + // If a response errors, it is probably because the Pod is not yet properly running, so retry. + resps := append(erroredResps(8), readyResps(podIP, 1)...) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{ + hep: resps, + }, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("one_svc_one_backend_svc_does_not_have_health_port", func(t *testing.T) { + pod := podTemplate.DeepCopy() + + // If a Service does not have health port set, we assume that it is not possible to determine Pod's + // readiness and set it to ready. + svc, _ := newSvc("svc", -1) + mustCreateAll(t, fc, svc, pod) + rec.httpClient = nil + expectReconciled(t, rec, "operator-ns", pod.Name) + + // Pod should have readiness gate condition set. + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("error_setting_up_healthcheck", func(t *testing.T) { + pod := podTemplate.DeepCopy() + // This is not a realistic reason for error, but we are just testing the behaviour of a healthcheck + // lookup failing. + pod.Status.PodIPs = []corev1.PodIP{{IP: "not-an-ip"}} + + svc, _ := newSvc("svc", 9002) + svc2, _ := newSvc("svc-2", 9002) + svc3, _ := newSvc("svc-3", 9002) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + rec.httpClient = nil + expectError(t, rec, "operator-ns", pod.Name) + + // Pod should not have readiness gate condition set. + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) + t.Run("pod_does_not_have_an_ip_address", func(t *testing.T) { + pod := podTemplate.DeepCopy() + pod.Status.PodIPs = nil + + svc, _ := newSvc("svc", 9002) + svc2, _ := newSvc("svc-2", 9002) + svc3, _ := newSvc("svc-3", 9002) + mustCreateAll(t, fc, svc, svc2, svc3, pod) + rec.httpClient = nil + expectRequeue(t, rec, "operator-ns", pod.Name) + + // Pod should not have readiness gate condition set. + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc, svc2, svc3) + }) +} + +func readyResps(ip string, num int) (resps []fakeResponse) { + for range num { + resps = append(resps, fakeResponse{statusCode: 200, podIP: ip}) + } + return resps +} + +func unreadyResps(ip string, num int) (resps []fakeResponse) { + for range num { + resps = append(resps, fakeResponse{statusCode: 503, podIP: ip}) + } + return resps +} + +func erroredResps(num int) (resps []fakeResponse) { + for range num { + resps = append(resps, fakeResponse{err: errors.New("timeout")}) + } + return resps +} + +func newSvc(name string, port int32) (*corev1.Service, string) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "operator-ns", + Name: name, + Labels: map[string]string{ + LabelManaged: "true", + labelProxyGroup: "dev", + labelSvcType: typeEgress, + }, + }, + Spec: corev1.ServiceSpec{}, + } + if port != -1 { + svc.Spec.Ports = []corev1.ServicePort{ + { + Name: tsHealthCheckPortName, + Port: port, + TargetPort: intstr.FromInt(9002), + Protocol: "TCP", + }, + } + } + return svc, fmt.Sprintf("http://%s.operator-ns.svc.cluster.local:%d/healthz", name, port) +} + +func podSetReady(pod *corev1.Pod, cl *tstest.Clock) { + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: tsEgressReadinessGate, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }) +} + +// fakeHTTPClient is a mock HTTP client with a preset map of request URLs to list of responses. When it receives a +// request for a specific URL, it returns the preset response for that URL. It errors if an unexpected request is +// received. +type fakeHTTPClient struct { + t *testing.T + mu sync.Mutex // protects following + state map[string][]fakeResponse +} + +func (f *fakeHTTPClient) Do(req *http.Request) (*http.Response, error) { + f.mu.Lock() + resps := f.state[req.URL.String()] + if len(resps) == 0 { + f.mu.Unlock() + log.Printf("\n\n\nURL %q\n\n\n", req.URL) + f.t.Fatalf("fakeHTTPClient received an unexpected request for %q", req.URL) + } + defer func() { + if len(resps) == 1 { + delete(f.state, req.URL.String()) + f.mu.Unlock() + return + } + f.state[req.URL.String()] = f.state[req.URL.String()][1:] + f.mu.Unlock() + }() + + resp := resps[0] + if resp.err != nil { + return nil, resp.err + } + r := http.Response{ + StatusCode: resp.statusCode, + Header: make(http.Header), + Body: io.NopCloser(bytes.NewReader([]byte{})), + } + r.Header.Add(kubetypes.PodIPv4Header, resp.podIP) + return &r, nil +} + +type fakeResponse struct { + err error + statusCode int + podIP string // for the Pod IP header +} diff --git a/cmd/k8s-operator/egress-services-readiness.go b/cmd/k8s-operator/egress-services-readiness.go index f1964d452633c..5e95a52790395 100644 --- a/cmd/k8s-operator/egress-services-readiness.go +++ b/cmd/k8s-operator/egress-services-readiness.go @@ -48,11 +48,12 @@ type egressSvcsReadinessReconciler struct { // service to determine how many replicas are currently able to route traffic. func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { l := esrr.logger.With("Service", req.NamespacedName) - defer l.Info("reconcile finished") + l.Debugf("starting reconcile") + defer l.Debugf("reconcile finished") svc := new(corev1.Service) if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) { - l.Info("Service not found") + l.Debugf("Service not found") return res, nil } else if err != nil { return res, fmt.Errorf("failed to get Service: %w", err) @@ -127,16 +128,16 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re return res, err } if pod == nil { - l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i) + l.Warnf("[unexpected] ProxyGroup is ready, but replica %d was not found", i) reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady return res, nil } - l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs) + l.Debugf("looking at Pod with IPs %v", pod.Status.PodIPs) ready := false for _, ep := range eps.Endpoints { - l.Infof("looking at endpoint with addresses %v", ep.Addresses) + l.Debugf("looking at endpoint with addresses %v", ep.Addresses) if endpointReadyForPod(&ep, pod, l) { - l.Infof("endpoint is ready for Pod") + l.Debugf("endpoint is ready for Pod") ready = true break } @@ -165,7 +166,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool { podIP, err := podIPv4(pod) if err != nil { - l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err) + l.Warnf("[unexpected] error retrieving Pod's IPv4 address: %v", err) return false } // Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this. diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 6631c4f98f24f..8fa979094d9fa 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -9,6 +9,7 @@ package main import ( "context" + "net/http" "os" "regexp" "strconv" @@ -453,6 +454,24 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err) } + podsForEps := handler.EnqueueRequestsFromMapFunc(podsFromEgressEps(mgr.GetClient(), opts.log, opts.tailscaleNamespace)) + podsER := handler.EnqueueRequestsFromMapFunc(egressPodsHandler) + err = builder. + ControllerManagedBy(mgr). + Named("egress-pods-readiness-reconciler"). + Watches(&discoveryv1.EndpointSlice{}, podsForEps). + Watches(&corev1.Pod{}, podsER). + Complete(&egressPodsReconciler{ + Client: mgr.GetClient(), + tsNamespace: opts.tailscaleNamespace, + clock: tstime.DefaultClock{}, + logger: opts.log.Named("egress-pods-readiness-reconciler"), + httpClient: http.DefaultClient, + }) + if err != nil { + startlog.Fatalf("could not create egress Pods readiness reconciler: %v", err) + } + // ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that // define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get // reconciled if the CRD is applied at a later point. @@ -906,6 +925,20 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request { } } +func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request { + if typ := o.GetLabels()[LabelParentType]; typ != proxyTypeProxyGroup { + return nil + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: o.GetNamespace(), + Name: o.GetName(), + }, + }, + } +} + // egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is, // returns reconciler requests for all egress EndpointSlices for that ProxyGroup. func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc { @@ -1056,6 +1089,43 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns } } +func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + eps, ok := o.(*discoveryv1.EndpointSlice) + if !ok { + logger.Infof("[unexpected] EndpointSlice handler triggered for an object that is not a EndpointSlice") + return nil + } + if eps.Labels[labelProxyGroup] == "" { + return nil + } + if eps.Labels[labelSvcType] != "egress" { + return nil + } + podLabels := map[string]string{ + LabelManaged: "true", + LabelParentType: "proxygroup", + LabelParentName: eps.Labels[labelProxyGroup], + } + podList := &corev1.PodList{} + if err := cl.List(ctx, podList, client.InNamespace(ns), + client.MatchingLabels(podLabels)); err != nil { + logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on EndpointSlice %s", err, eps.Name) + return nil + } + reqs := make([]reconcile.Request, 0) + for _, pod := range podList.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: pod.Namespace, + Name: pod.Name, + }, + }) + } + return reqs + } +} + // proxyClassesWithServiceMonitor returns an event handler that, given that the event is for the Prometheus // ServiceMonitor CRD, returns all ProxyClasses that define that a ServiceMonitor should be created. func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 160f24ec90fa1..83c42cb7633a6 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -583,6 +583,21 @@ func mustCreate(t *testing.T, client client.Client, obj client.Object) { t.Fatalf("creating %q: %v", obj.GetName(), err) } } +func mustCreateAll(t *testing.T, client client.Client, objs ...client.Object) { + t.Helper() + for _, obj := range objs { + mustCreate(t, client, obj) + } +} + +func mustDeleteAll(t *testing.T, client client.Client, objs ...client.Object) { + t.Helper() + for _, obj := range objs { + if err := client.Delete(context.Background(), obj); err != nil { + t.Fatalf("deleting %q: %v", obj.GetName(), err) + } + } +} func mustUpdate[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string, update func(O)) { t.Helper() @@ -706,6 +721,19 @@ func expectRequeue(t *testing.T, sr reconcile.Reconciler, ns, name string) { t.Fatalf("expected timed requeue, got success") } } +func expectError(t *testing.T, sr reconcile.Reconciler, ns, name string) { + t.Helper() + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: ns, + }, + } + _, err := sr.Reconcile(context.Background(), req) + if err == nil { + t.Error("Reconcile: expected error but did not get one") + } +} // expectEvents accepts a test recorder and a list of events, tests that expected // events are sent down the recorder's channel. Waits for 5s for each event. From ed8bb3b56438a10e9f7bf4b3bb20e0c8ebf6dffb Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 30 Jan 2025 07:22:52 +0000 Subject: [PATCH 218/223] control/controlclient: add missing word in comment Found by review.ai. Updates #cleanup Change-Id: Ib9126de7327527b8b3818d92cc774bb1c7b6f974 Signed-off-by: Brad Fitzpatrick --- control/controlclient/auto.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control/controlclient/auto.go b/control/controlclient/auto.go index a5397594e864f..92db9382ea04b 100644 --- a/control/controlclient/auto.go +++ b/control/controlclient/auto.go @@ -625,7 +625,7 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM // We CAS here because the caller goroutine is // doing a Store which we want to want to win // a race. This is only a memory optimization - // and is for correctness: + // and is not for correctness: c.lastStatus.CompareAndSwap(newSt, nil) }) } From 0ed4aa028fbd3f0cd4a5a2d86a962f354ae954e4 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 30 Jan 2025 07:23:36 +0000 Subject: [PATCH 219/223] control/controlclient: flesh out a recently added comment Updates tailscale/corp#26058 Change-Id: Ib46161fbb2e79c080f886083665961f02cbf5949 --- control/controlclient/auto.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/control/controlclient/auto.go b/control/controlclient/auto.go index 92db9382ea04b..da123f8c457c4 100644 --- a/control/controlclient/auto.go +++ b/control/controlclient/auto.go @@ -620,12 +620,17 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM } } c.observer.SetControlClientStatus(c, *newSt) - // Best effort stop retaining the memory now that - // we've sent it to the observer (LocalBackend). - // We CAS here because the caller goroutine is - // doing a Store which we want to want to win - // a race. This is only a memory optimization - // and is not for correctness: + + // Best effort stop retaining the memory now that we've sent it to the + // observer (LocalBackend). We CAS here because the caller goroutine is + // doing a Store which we want to win a race. This is only a memory + // optimization and is not for correctness. + // + // If the CAS fails, that means somebody else's Store replaced our + // pointer (so mission accomplished: our netmap is no longer retained in + // any case) and that Store caller will be responsible for removing + // their own netmap (or losing their race too, down the chain). + // Eventually the last caller will win this CAS and zero lastStatus. c.lastStatus.CompareAndSwap(newSt, nil) }) } From a49af98b3167d325bc9c0d43e61d6dc6c494c544 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Thu, 30 Jan 2025 13:36:33 +0200 Subject: [PATCH 220/223] cmd/k8s-operator: temporarily disable HA Ingress controller (#14833) The HA Ingress functionality is not actually doing anything valuable yet, so don't run the controller in 1.80 release yet. Updates tailscale/tailscale#24795 Signed-off-by: Irbe Krumina --- .../crds/tailscale.com_proxygroups.yaml | 2 +- .../deploy/manifests/operator.yaml | 2 +- cmd/k8s-operator/operator.go | 58 ------------------- k8s-operator/api.md | 2 +- .../apis/v1alpha1/types_proxygroup.go | 2 +- 5 files changed, 4 insertions(+), 62 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 86e74e441bc52..e101c201f35f4 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -103,7 +103,7 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Currently the only supported type is egress. Type is immutable once a ProxyGroup is created. type: string enum: diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index e966ef5593149..54b32bef0a917 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2860,7 +2860,7 @@ spec: type: array type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Currently the only supported type is egress. Type is immutable once a ProxyGroup is created. enum: - egress diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 8fa979094d9fa..8fcd1342c4c05 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -331,28 +331,6 @@ func runReconcilers(opts reconcilerOpts) { if err != nil { startlog.Fatalf("could not create ingress reconciler: %v", err) } - lc, err := opts.tsServer.LocalClient() - if err != nil { - startlog.Fatalf("could not get local client: %v", err) - } - err = builder. - ControllerManagedBy(mgr). - For(&networkingv1.Ingress{}). - Named("ingress-pg-reconciler"). - Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))). - Complete(&IngressPGReconciler{ - recorder: eventRecorder, - tsClient: opts.tsClient, - tsnetServer: opts.tsServer, - defaultTags: strings.Split(opts.proxyTags, ","), - Client: mgr.GetClient(), - logger: opts.log.Named("ingress-pg-reconciler"), - lc: lc, - tsNamespace: opts.tailscaleNamespace, - }) - if err != nil { - startlog.Fatalf("could not create ingress-pg-reconciler: %v", err) - } connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector")) // If a ProxyClassChanges, enqueue all Connectors that have @@ -1178,42 +1156,6 @@ func indexEgressServices(o client.Object) []string { return []string{o.GetAnnotations()[AnnotationProxyGroup]} } -// serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service -// associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation, -// the associated Ingress gets reconciled. -func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { - return func(ctx context.Context, o client.Object) []reconcile.Request { - ingList := networkingv1.IngressList{} - if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil { - logger.Debugf("error listing Ingresses: %v", err) - return nil - } - reqs := make([]reconcile.Request, 0) - for _, ing := range ingList.Items { - if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { - continue - } - if !hasProxyGroupAnnotation(&ing) { - continue - } - if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() { - reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) - } - for _, rule := range ing.Spec.Rules { - if rule.HTTP == nil { - continue - } - for _, path := range rule.HTTP.Paths { - if path.Backend.Service != nil && path.Backend.Service.Name == o.GetName() { - reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) - } - } - } - } - return reqs - } -} - func hasProxyGroupAnnotation(obj client.Object) bool { ing := obj.(*networkingv1.Ingress) return ing.Annotations[AnnotationProxyGroup] != "" diff --git a/k8s-operator/api.md b/k8s-operator/api.md index fae25b1f61708..64756c8f1bfd4 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -599,7 +599,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress]
Type: string
| +| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress.
Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress]
Type: string
| | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
If you specify custom tags here, make sure you also make the operator
an owner of these tags.
See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
Tags cannot be changed once a ProxyGroup device has been created.
Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
Type: string
| | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2. | | Minimum: 0
| | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
by the ProxyGroup. Each device will have the integer number from its
StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index f95fc58d0f35c..cb9f678f8c2d4 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -48,7 +48,7 @@ type ProxyGroupList struct { } type ProxyGroupSpec struct { - // Type of the ProxyGroup proxies. Supported types are egress and ingress. + // Type of the ProxyGroup proxies. Currently the only supported type is egress. // Type is immutable once a ProxyGroup is created. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable" Type ProxyGroupType `json:"type"` From c2af1cd9e347abbe7fa7ef52ca21df3230abbfe1 Mon Sep 17 00:00:00 2001 From: Anton Tolchanov Date: Wed, 29 Jan 2025 15:35:37 +0000 Subject: [PATCH 221/223] prober: support multiple probes running concurrently Some probes might need to run for longer than their scheduling interval, so this change relaxes the 1-at-a-time restriction, allowing us to configure probe concurrency and timeout separately. The default values remain the same (concurrency of 1; timeout of 80% of interval). Updates tailscale/corp#25479 Signed-off-by: Anton Tolchanov --- prober/prober.go | 48 ++++++++++++++++++++--------- prober/prober_test.go | 72 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/prober/prober.go b/prober/prober.go index e3860e7b9ba57..d80db773a8448 100644 --- a/prober/prober.go +++ b/prober/prober.go @@ -7,6 +7,7 @@ package prober import ( + "cmp" "container/ring" "context" "encoding/json" @@ -20,6 +21,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "tailscale.com/syncs" "tailscale.com/tsweb" ) @@ -44,6 +46,14 @@ type ProbeClass struct { // exposed by this probe class. Labels Labels + // Timeout is the maximum time the probe function is allowed to run before + // its context is cancelled. Defaults to 80% of the scheduling interval. + Timeout time.Duration + + // Concurrency is the maximum number of concurrent probe executions + // allowed for this probe class. Defaults to 1. + Concurrency int + // Metrics allows a probe class to export custom Metrics. Can be nil. Metrics func(prometheus.Labels) []prometheus.Metric } @@ -131,9 +141,12 @@ func newProbe(p *Prober, name string, interval time.Duration, l prometheus.Label cancel: cancel, stopped: make(chan struct{}), + runSema: syncs.NewSemaphore(cmp.Or(pc.Concurrency, 1)), + name: name, probeClass: pc, interval: interval, + timeout: cmp.Or(pc.Timeout, time.Duration(float64(interval)*0.8)), initialDelay: initialDelay(name, interval), successHist: ring.New(recentHistSize), latencyHist: ring.New(recentHistSize), @@ -226,11 +239,12 @@ type Probe struct { ctx context.Context cancel context.CancelFunc // run to initiate shutdown stopped chan struct{} // closed when shutdown is complete - runMu sync.Mutex // ensures only one probe runs at a time + runSema syncs.Semaphore // restricts concurrency per probe name string probeClass ProbeClass interval time.Duration + timeout time.Duration initialDelay time.Duration tick ticker @@ -282,17 +296,15 @@ func (p *Probe) loop() { t := p.prober.newTicker(p.initialDelay) select { case <-t.Chan(): - p.run() case <-p.ctx.Done(): t.Stop() return } t.Stop() - } else { - p.run() } if p.prober.once { + p.run() return } @@ -315,9 +327,12 @@ func (p *Probe) loop() { p.tick = p.prober.newTicker(p.interval) defer p.tick.Stop() for { + // Run the probe in a new goroutine every tick. Default concurrency & timeout + // settings will ensure that only one probe is running at a time. + go p.run() + select { case <-p.tick.Chan(): - p.run() case <-p.ctx.Done(): return } @@ -331,8 +346,13 @@ func (p *Probe) loop() { // that the probe either succeeds or fails before the next cycle is scheduled to // start. func (p *Probe) run() (pi ProbeInfo, err error) { - p.runMu.Lock() - defer p.runMu.Unlock() + // Probes are scheduled each p.interval, so we don't wait longer than that. + semaCtx, cancel := context.WithTimeout(p.ctx, p.interval) + defer cancel() + if !p.runSema.AcquireContext(semaCtx) { + return pi, fmt.Errorf("probe %s: context cancelled", p.name) + } + defer p.runSema.Release() p.recordStart() defer func() { @@ -344,19 +364,21 @@ func (p *Probe) run() (pi ProbeInfo, err error) { if r := recover(); r != nil { log.Printf("probe %s panicked: %v", p.name, r) err = fmt.Errorf("panic: %v", r) - p.recordEnd(err) + p.recordEndLocked(err) } }() ctx := p.ctx if !p.IsContinuous() { - timeout := time.Duration(float64(p.interval) * 0.8) var cancel func() - ctx, cancel = context.WithTimeout(ctx, timeout) + ctx, cancel = context.WithTimeout(ctx, p.timeout) defer cancel() } err = p.probeClass.Probe(ctx) - p.recordEnd(err) + + p.mu.Lock() + defer p.mu.Unlock() + p.recordEndLocked(err) if err != nil { log.Printf("probe %s: %v", p.name, err) } @@ -370,10 +392,8 @@ func (p *Probe) recordStart() { p.mu.Unlock() } -func (p *Probe) recordEnd(err error) { +func (p *Probe) recordEndLocked(err error) { end := p.prober.now() - p.mu.Lock() - defer p.mu.Unlock() p.end = end p.succeeded = err == nil p.lastErr = err diff --git a/prober/prober_test.go b/prober/prober_test.go index 3905bfbc91576..109953b65c121 100644 --- a/prober/prober_test.go +++ b/prober/prober_test.go @@ -149,6 +149,74 @@ func TestProberTimingSpread(t *testing.T) { notCalled() } +func TestProberTimeout(t *testing.T) { + clk := newFakeTime() + p := newForTest(clk.Now, clk.NewTicker) + + var done sync.WaitGroup + done.Add(1) + pfunc := FuncProbe(func(ctx context.Context) error { + defer done.Done() + select { + case <-ctx.Done(): + return ctx.Err() + } + }) + pfunc.Timeout = time.Microsecond + probe := p.Run("foo", 30*time.Second, nil, pfunc) + waitActiveProbes(t, p, clk, 1) + done.Wait() + probe.mu.Lock() + info := probe.probeInfoLocked() + probe.mu.Unlock() + wantInfo := ProbeInfo{ + Name: "foo", + Interval: 30 * time.Second, + Labels: map[string]string{"class": "", "name": "foo"}, + Status: ProbeStatusFailed, + Error: "context deadline exceeded", + RecentResults: []bool{false}, + RecentLatencies: nil, + } + if diff := cmp.Diff(wantInfo, info, cmpopts.IgnoreFields(ProbeInfo{}, "Start", "End", "Latency")); diff != "" { + t.Fatalf("unexpected ProbeInfo (-want +got):\n%s", diff) + } + if got := info.Latency; got > time.Second { + t.Errorf("info.Latency = %v, want at most 1s", got) + } +} + +func TestProberConcurrency(t *testing.T) { + clk := newFakeTime() + p := newForTest(clk.Now, clk.NewTicker) + + var ran atomic.Int64 + stopProbe := make(chan struct{}) + pfunc := FuncProbe(func(ctx context.Context) error { + ran.Add(1) + <-stopProbe + return nil + }) + pfunc.Timeout = time.Hour + pfunc.Concurrency = 3 + p.Run("foo", time.Second, nil, pfunc) + waitActiveProbes(t, p, clk, 1) + + for range 50 { + clk.Advance(time.Second) + } + + if err := tstest.WaitFor(convergenceTimeout, func() error { + if got, want := ran.Load(), int64(3); got != want { + return fmt.Errorf("expected %d probes to run concurrently, got %d", want, got) + } + return nil + }); err != nil { + t.Fatal(err) + } + close(stopProbe) +} + func TestProberRun(t *testing.T) { clk := newFakeTime() p := newForTest(clk.Now, clk.NewTicker) @@ -450,9 +518,11 @@ func TestProbeInfoRecent(t *testing.T) { for _, r := range tt.results { probe.recordStart() clk.Advance(r.latency) - probe.recordEnd(r.err) + probe.recordEndLocked(r.err) } + probe.mu.Lock() info := probe.probeInfoLocked() + probe.mu.Unlock() if diff := cmp.Diff(tt.wantProbeInfo, info, cmpopts.IgnoreFields(ProbeInfo{}, "Start", "End", "Interval")); diff != "" { t.Fatalf("unexpected ProbeInfo (-want +got):\n%s", diff) } From 138a83efe11659220643f5975b28c4a030fefe9a Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Thu, 30 Jan 2025 13:51:10 +0000 Subject: [PATCH 222/223] cmd/containerboot: wait for consistent state on shutdown (#14263) tailscaled's ipn package writes a collection of keys to state after authenticating to control, but one at a time. If containerboot happens to send a SIGTERM signal to tailscaled in the middle of writing those keys, it may shut down with an inconsistent state Secret and never recover. While we can't durably fix this with our current single-use auth keys (no atomic operation to auth + write state), we can reduce the window for this race condition by checking for partial state before sending SIGTERM to tailscaled. Best effort only. Updates #14080 Change-Id: I0532d51b6f0b7d391e538468bd6a0a80dbe1d9f7 Signed-off-by: Tom Proctor --- cmd/containerboot/kube.go | 66 +++++++++++++++ cmd/containerboot/kube_test.go | 33 ++++++++ cmd/containerboot/main.go | 124 ++++++++++++++++++---------- cmd/containerboot/main_test.go | 142 +++++++++++++++++++++++++------- cmd/containerboot/tailscaled.go | 4 +- 5 files changed, 294 insertions(+), 75 deletions(-) diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 4d00687ee4566..0a2dfa1bf342f 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -8,15 +8,22 @@ package main import ( "context" "encoding/json" + "errors" "fmt" + "log" "net/http" "net/netip" "os" + "strings" + "time" + "tailscale.com/ipn" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" + "tailscale.com/logtail/backoff" "tailscale.com/tailcfg" + "tailscale.com/types/logger" ) // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use @@ -126,3 +133,62 @@ func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error { } return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") } + +// waitForConsistentState waits for tailscaled to finish writing state if it +// looks like it's started. It is designed to reduce the likelihood that +// tailscaled gets shut down in the window between authenticating to control +// and finishing writing state. However, it's not bullet proof because we can't +// atomically authenticate and write state. +func (kc *kubeClient) waitForConsistentState(ctx context.Context) error { + var logged bool + + bo := backoff.NewBackoff("", logger.Discard, 2*time.Second) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + secret, err := kc.GetSecret(ctx, kc.stateSecret) + if ctx.Err() != nil || kubeclient.IsNotFoundErr(err) { + return nil + } + if err != nil { + return fmt.Errorf("getting Secret %q: %v", kc.stateSecret, err) + } + + if hasConsistentState(secret.Data) { + return nil + } + + if !logged { + log.Printf("Waiting for tailscaled to finish writing state to Secret %q", kc.stateSecret) + logged = true + } + bo.BackOff(ctx, errors.New("")) // Fake error to trigger actual sleep. + } +} + +// hasConsistentState returns true is there is either no state or the full set +// of expected keys are present. +func hasConsistentState(d map[string][]byte) bool { + var ( + _, hasCurrent = d[string(ipn.CurrentProfileStateKey)] + _, hasKnown = d[string(ipn.KnownProfilesStateKey)] + _, hasMachine = d[string(ipn.MachineKeyStateKey)] + hasProfile bool + ) + + for k := range d { + if strings.HasPrefix(k, "profile-") { + if hasProfile { + return false // We only expect one profile. + } + hasProfile = true + } + } + + // Approximate check, we don't want to reimplement all of profileManager. + return (hasCurrent && hasKnown && hasMachine && hasProfile) || + (!hasCurrent && !hasKnown && !hasMachine && !hasProfile) +} diff --git a/cmd/containerboot/kube_test.go b/cmd/containerboot/kube_test.go index 2ba69af7c0f57..413971bc6df23 100644 --- a/cmd/containerboot/kube_test.go +++ b/cmd/containerboot/kube_test.go @@ -9,8 +9,10 @@ import ( "context" "errors" "testing" + "time" "github.com/google/go-cmp/cmp" + "tailscale.com/ipn" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" ) @@ -205,3 +207,34 @@ func TestSetupKube(t *testing.T) { }) } } + +func TestWaitForConsistentState(t *testing.T) { + data := map[string][]byte{ + // Missing _current-profile. + string(ipn.KnownProfilesStateKey): []byte(""), + string(ipn.MachineKeyStateKey): []byte(""), + "profile-foo": []byte(""), + } + kc := &kubeClient{ + Client: &kubeclient.FakeClient{ + GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) { + return &kubeapi.Secret{ + Data: data, + }, nil + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := kc.waitForConsistentState(ctx); err != context.DeadlineExceeded { + t.Fatalf("expected DeadlineExceeded, got %v", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + data[string(ipn.CurrentProfileStateKey)] = []byte("") + if err := kc.waitForConsistentState(ctx); err != nil { + t.Fatalf("expected nil, got %v", err) + } +} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 0aca27f5fdd77..cf4bd86201d65 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -137,53 +137,83 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) { } func main() { + if err := run(); err != nil && !errors.Is(err, context.Canceled) { + log.Fatal(err) + } +} + +func run() error { log.SetPrefix("boot: ") tailscale.I_Acknowledge_This_API_Is_Unstable = true cfg, err := configFromEnv() if err != nil { - log.Fatalf("invalid configuration: %v", err) + return fmt.Errorf("invalid configuration: %w", err) } if !cfg.UserspaceMode { if err := ensureTunFile(cfg.Root); err != nil { - log.Fatalf("Unable to create tuntap device file: %v", err) + return fmt.Errorf("unable to create tuntap device file: %w", err) } if cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.Routes != nil || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" { if err := ensureIPForwarding(cfg.Root, cfg.ProxyTargetIP, cfg.TailnetTargetIP, cfg.TailnetTargetFQDN, cfg.Routes); err != nil { log.Printf("Failed to enable IP forwarding: %v", err) log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.") if cfg.InKubernetes { - log.Fatalf("You can either set the sysctls as a privileged initContainer, or run the tailscale container with privileged=true.") + return fmt.Errorf("you can either set the sysctls as a privileged initContainer, or run the tailscale container with privileged=true.") } else { - log.Fatalf("You can fix this by running the container with privileged=true, or the equivalent in your container runtime that permits access to sysctls.") + return fmt.Errorf("you can fix this by running the container with privileged=true, or the equivalent in your container runtime that permits access to sysctls.") } } } } - // Context is used for all setup stuff until we're in steady + // Root context for the whole containerboot process, used to make sure + // shutdown signals are promptly and cleanly handled. + ctx, cancel := contextWithExitSignalWatch() + defer cancel() + + // bootCtx is used for all setup stuff until we're in steady // state, so that if something is hanging we eventually time out // and crashloop the container. - bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + bootCtx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() var kc *kubeClient if cfg.InKubernetes { kc, err = newKubeClient(cfg.Root, cfg.KubeSecret) if err != nil { - log.Fatalf("error initializing kube client: %v", err) + return fmt.Errorf("error initializing kube client: %w", err) } if err := cfg.setupKube(bootCtx, kc); err != nil { - log.Fatalf("error setting up for running on Kubernetes: %v", err) + return fmt.Errorf("error setting up for running on Kubernetes: %w", err) } } client, daemonProcess, err := startTailscaled(bootCtx, cfg) if err != nil { - log.Fatalf("failed to bring up tailscale: %v", err) + return fmt.Errorf("failed to bring up tailscale: %w", err) } killTailscaled := func() { + if hasKubeStateStore(cfg) { + // Check we're not shutting tailscaled down while it's still writing + // state. If we authenticate and fail to write all the state, we'll + // never recover automatically. + // + // The default termination grace period for a Pod is 30s. We wait 25s at + // most so that we still reserve some of that budget for tailscaled + // to receive and react to a SIGTERM before the SIGKILL that k8s + // will send at the end of the grace period. + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + defer cancel() + + log.Printf("Checking for consistent state") + err := kc.waitForConsistentState(ctx) + if err != nil { + log.Printf("Error waiting for consistent state on shutdown: %v", err) + } + } + log.Printf("Sending SIGTERM to tailscaled") if err := daemonProcess.Signal(unix.SIGTERM); err != nil { log.Fatalf("error shutting tailscaled down: %v", err) } @@ -231,7 +261,7 @@ func main() { w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState) if err != nil { - log.Fatalf("failed to watch tailscaled for updates: %v", err) + return fmt.Errorf("failed to watch tailscaled for updates: %w", err) } // Now that we've started tailscaled, we can symlink the socket to the @@ -267,18 +297,18 @@ func main() { didLogin = true w.Close() if err := tailscaleUp(bootCtx, cfg); err != nil { - return fmt.Errorf("failed to auth tailscale: %v", err) + return fmt.Errorf("failed to auth tailscale: %w", err) } w, err = client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) if err != nil { - return fmt.Errorf("rewatching tailscaled for updates after auth: %v", err) + return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) } return nil } if isTwoStepConfigAlwaysAuth(cfg) { if err := authTailscale(); err != nil { - log.Fatalf("failed to auth tailscale: %v", err) + return fmt.Errorf("failed to auth tailscale: %w", err) } } @@ -286,7 +316,7 @@ authLoop: for { n, err := w.Next() if err != nil { - log.Fatalf("failed to read from tailscaled: %v", err) + return fmt.Errorf("failed to read from tailscaled: %w", err) } if n.State != nil { @@ -295,10 +325,10 @@ authLoop: if isOneStepConfig(cfg) { // This could happen if this is the first time tailscaled was run for this // device and the auth key was not passed via the configfile. - log.Fatalf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.") + return fmt.Errorf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.") } if err := authTailscale(); err != nil { - log.Fatalf("failed to auth tailscale: %v", err) + return fmt.Errorf("failed to auth tailscale: %w", err) } case ipn.NeedsMachineAuth: log.Printf("machine authorization required, please visit the admin panel") @@ -318,14 +348,11 @@ authLoop: w.Close() - ctx, cancel := contextWithExitSignalWatch() - defer cancel() - if isTwoStepConfigAuthOnce(cfg) { // Now that we are authenticated, we can set/reset any of the // settings that we need to. if err := tailscaleSet(ctx, cfg); err != nil { - log.Fatalf("failed to auth tailscale: %v", err) + return fmt.Errorf("failed to auth tailscale: %w", err) } } @@ -334,11 +361,11 @@ authLoop: if cfg.ServeConfigPath != "" { log.Printf("serve proxy: unsetting previous config") if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil { - log.Fatalf("failed to unset serve config: %v", err) + return fmt.Errorf("failed to unset serve config: %w", err) } if hasKubeStateStore(cfg) { if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil { - log.Fatalf("failed to update HTTPS endpoint in tailscale state: %v", err) + return fmt.Errorf("failed to update HTTPS endpoint in tailscale state: %w", err) } } } @@ -349,19 +376,19 @@ authLoop: // wipe it, but it's good hygiene. log.Printf("Deleting authkey from kube secret") if err := kc.deleteAuthKey(ctx); err != nil { - log.Fatalf("deleting authkey from kube secret: %v", err) + return fmt.Errorf("deleting authkey from kube secret: %w", err) } } if hasKubeStateStore(cfg) { if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil { - log.Fatalf("storing capability version and UID: %v", err) + return fmt.Errorf("storing capability version and UID: %w", err) } } w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) if err != nil { - log.Fatalf("rewatching tailscaled for updates after auth: %v", err) + return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) } // If tailscaled config was read from a mounted file, watch the file for updates and reload. @@ -391,7 +418,7 @@ authLoop: if isL3Proxy(cfg) { nfr, err = newNetfilterRunner(log.Printf) if err != nil { - log.Fatalf("error creating new netfilter runner: %v", err) + return fmt.Errorf("error creating new netfilter runner: %w", err) } } @@ -462,9 +489,9 @@ runLoop: killTailscaled() break runLoop case err := <-errChan: - log.Fatalf("failed to read from tailscaled: %v", err) + return fmt.Errorf("failed to read from tailscaled: %w", err) case err := <-cfgWatchErrChan: - log.Fatalf("failed to watch tailscaled config: %v", err) + return fmt.Errorf("failed to watch tailscaled config: %w", err) case n := <-notifyChan: if n.State != nil && *n.State != ipn.Running { // Something's gone wrong and we've left the authenticated state. @@ -472,7 +499,7 @@ runLoop: // control flow required to make it work now is hard. So, just crash // the container and rely on the container runtime to restart us, // whereupon we'll go through initial auth again. - log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State) + return fmt.Errorf("tailscaled left running state (now in state %q), exiting", *n.State) } if n.NetMap != nil { addrs = n.NetMap.SelfNode.Addresses().AsSlice() @@ -490,7 +517,7 @@ runLoop: deviceID := n.NetMap.SelfNode.StableID() if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceID, &deviceID) { if err := kc.storeDeviceID(ctx, n.NetMap.SelfNode.StableID()); err != nil { - log.Fatalf("storing device ID in Kubernetes Secret: %v", err) + return fmt.Errorf("storing device ID in Kubernetes Secret: %w", err) } } if cfg.TailnetTargetFQDN != "" { @@ -527,12 +554,12 @@ runLoop: rulesInstalled = true log.Printf("Installing forwarding rules for destination %v", ea.String()) if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil { - log.Fatalf("installing egress proxy rules for destination %s: %v", ea.String(), err) + return fmt.Errorf("installing egress proxy rules for destination %s: %v", ea.String(), err) } } } if !rulesInstalled { - log.Fatalf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT()) + return fmt.Errorf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT()) } } currentEgressIPs = newCurentEgressIPs @@ -540,7 +567,7 @@ runLoop: if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged { log.Printf("Installing proxy rules") if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil { - log.Fatalf("installing ingress proxy rules: %v", err) + return fmt.Errorf("installing ingress proxy rules: %w", err) } } if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged { @@ -556,7 +583,7 @@ runLoop: if backendsHaveChanged { log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs) if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { - log.Fatalf("error installing ingress proxy rules: %v", err) + return fmt.Errorf("error installing ingress proxy rules: %w", err) } } resetTimer(false) @@ -578,7 +605,7 @@ runLoop: if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 { log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP) if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil { - log.Fatalf("installing egress proxy rules: %v", err) + return fmt.Errorf("installing egress proxy rules: %w", err) } } // If this is a L7 cluster ingress proxy (set up @@ -590,7 +617,7 @@ runLoop: if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 { log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP) if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil { - log.Fatalf("installing rules to forward traffic to node's tailnet IP: %v", err) + return fmt.Errorf("installing rules to forward traffic to node's tailnet IP: %w", err) } } currentIPs = newCurrentIPs @@ -609,7 +636,7 @@ runLoop: deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()} if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceEndpoints, &deviceEndpoints) { if err := kc.storeDeviceEndpoints(ctx, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil { - log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err) + return fmt.Errorf("storing device IPs and FQDN in Kubernetes Secret: %w", err) } } @@ -700,16 +727,18 @@ runLoop: if backendsHaveChanged && len(addrs) != 0 { log.Printf("Backend address change detected, installing proxy rules for backends %v", newBackendAddrs) if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { - log.Fatalf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err) + return fmt.Errorf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err) } } backendAddrs = newBackendAddrs resetTimer(false) case e := <-egressSvcsErrorChan: - log.Fatalf("egress proxy failed: %v", e) + return fmt.Errorf("egress proxy failed: %v", e) } } wg.Wait() + + return nil } // ensureTunFile checks that /dev/net/tun exists, creating it if @@ -738,13 +767,13 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) { ip4s, err := net.DefaultResolver.LookupIP(ctx, "ip4", name) if err != nil { if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) { - return nil, fmt.Errorf("error looking up IPv4 addresses: %v", err) + return nil, fmt.Errorf("error looking up IPv4 addresses: %w", err) } } ip6s, err := net.DefaultResolver.LookupIP(ctx, "ip6", name) if err != nil { if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) { - return nil, fmt.Errorf("error looking up IPv6 addresses: %v", err) + return nil, fmt.Errorf("error looking up IPv6 addresses: %w", err) } } if len(ip4s) == 0 && len(ip6s) == 0 { @@ -757,7 +786,7 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) { // context that gets cancelled when a signal is received and a cancel function // that can be called to free the resources when the watch should be stopped. func contextWithExitSignalWatch() (context.Context, func()) { - closeChan := make(chan string) + closeChan := make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) @@ -769,8 +798,11 @@ func contextWithExitSignalWatch() (context.Context, func()) { return } }() + closeOnce := sync.Once{} f := func() { - closeChan <- "goodbye" + closeOnce.Do(func() { + close(closeChan) + }) } return ctx, f } @@ -823,7 +855,11 @@ func runHTTPServer(mux *http.ServeMux, addr string) (close func() error) { go func() { if err := srv.Serve(ln); err != nil { - log.Fatalf("failed running server: %v", err) + if err != http.ErrServerClosed { + log.Fatalf("failed running server: %v", err) + } else { + log.Printf("HTTP server at %s closed", addr) + } } }() diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index c8066f2c13edd..bc158dac555e3 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" "sync" + "syscall" "testing" "time" @@ -50,9 +51,7 @@ func TestContainerBoot(t *testing.T) { defer lapi.Close() kube := kubeServer{FSRoot: d} - if err := kube.Start(); err != nil { - t.Fatal(err) - } + kube.Start(t) defer kube.Close() tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"} @@ -138,15 +137,29 @@ func TestContainerBoot(t *testing.T) { // WantCmds is the commands that containerboot should run in this phase. WantCmds []string + // WantKubeSecret is the secret keys/values that should exist in the // kube secret. WantKubeSecret map[string]string + + // Update the kube secret with these keys/values at the beginning of the + // phase (simulates our fake tailscaled doing it). + UpdateKubeSecret map[string]string + // WantFiles files that should exist in the container and their // contents. WantFiles map[string]string - // WantFatalLog is the fatal log message we expect from containerboot. - // If set for a phase, the test will finish on that phase. - WantFatalLog string + + // WantLog is a log message we expect from containerboot. + WantLog string + + // If set for a phase, the test will expect containerboot to exit with + // this error code, and the test will finish on that phase without + // waiting for the successful startup log message. + WantExitCode *int + + // The signal to send to containerboot at the start of the phase. + Signal *syscall.Signal EndpointStatuses map[string]int } @@ -434,7 +447,8 @@ func TestContainerBoot(t *testing.T) { }, }, }, - WantFatalLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false", + WantLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false", + WantExitCode: ptr.To(1), }, }, }, @@ -936,7 +950,64 @@ func TestContainerBoot(t *testing.T) { }, Phases: []phase{ { - WantFatalLog: "TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes", + WantLog: "TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes", + WantExitCode: ptr.To(1), + }, + }, + }, + { + Name: "kube_shutdown_during_state_write", + Env: map[string]string{ + "KUBERNETES_SERVICE_HOST": kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, + "TS_ENABLE_HEALTH_CHECK": "true", + }, + KubeSecret: map[string]string{ + "authkey": "tskey-key", + }, + Phases: []phase{ + { + // Normal startup. + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", + }, + WantKubeSecret: map[string]string{ + "authkey": "tskey-key", + }, + }, + { + // SIGTERM before state is finished writing, should wait for + // consistent state before propagating SIGTERM to tailscaled. + Signal: ptr.To(unix.SIGTERM), + UpdateKubeSecret: map[string]string{ + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + // Missing "_current-profile" key. + }, + WantKubeSecret: map[string]string{ + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + }, + WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"", + }, + { + // tailscaled has finished writing state, should propagate SIGTERM. + UpdateKubeSecret: map[string]string{ + "_current-profile": "foo", + }, + WantKubeSecret: map[string]string{ + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + "_current-profile": "foo", + }, + WantLog: "HTTP server at [::]:9002 closed", + WantExitCode: ptr.To(0), }, }, }, @@ -984,26 +1055,36 @@ func TestContainerBoot(t *testing.T) { var wantCmds []string for i, p := range test.Phases { + for k, v := range p.UpdateKubeSecret { + kube.SetSecret(k, v) + } lapi.Notify(p.Notify) - if p.WantFatalLog != "" { + if p.Signal != nil { + cmd.Process.Signal(*p.Signal) + } + if p.WantLog != "" { err := tstest.WaitFor(2*time.Second, func() error { - state, err := cmd.Process.Wait() - if err != nil { - return err - } - if state.ExitCode() != 1 { - return fmt.Errorf("process exited with code %d but wanted %d", state.ExitCode(), 1) - } - waitLogLine(t, time.Second, cbOut, p.WantFatalLog) + waitLogLine(t, time.Second, cbOut, p.WantLog) return nil }) if err != nil { t.Fatal(err) } + } + + if p.WantExitCode != nil { + state, err := cmd.Process.Wait() + if err != nil { + t.Fatal(err) + } + if state.ExitCode() != *p.WantExitCode { + t.Fatalf("phase %d: want exit code %d, got %d", i, *p.WantExitCode, state.ExitCode()) + } // Early test return, we don't expect the successful startup log message. return } + wantCmds = append(wantCmds, p.WantCmds...) waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n")) err := tstest.WaitFor(2*time.Second, func() error { @@ -1059,6 +1140,9 @@ func TestContainerBoot(t *testing.T) { } } waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal") + if cmd.ProcessState != nil { + t.Fatalf("containerboot should be running but exited with exit code %d", cmd.ProcessState.ExitCode()) + } }) } } @@ -1290,18 +1374,18 @@ func (k *kubeServer) Reset() { k.secret = map[string]string{} } -func (k *kubeServer) Start() error { +func (k *kubeServer) Start(t *testing.T) { root := filepath.Join(k.FSRoot, "var/run/secrets/kubernetes.io/serviceaccount") if err := os.MkdirAll(root, 0700); err != nil { - return err + t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "namespace"), []byte("default"), 0600); err != nil { - return err + t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "token"), []byte("bearer_token"), 0600); err != nil { - return err + t.Fatal(err) } k.srv = httptest.NewTLSServer(k) @@ -1310,13 +1394,11 @@ func (k *kubeServer) Start() error { var cert bytes.Buffer if err := pem.Encode(&cert, &pem.Block{Type: "CERTIFICATE", Bytes: k.srv.Certificate().Raw}); err != nil { - return err + t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "ca.crt"), cert.Bytes(), 0600); err != nil { - return err + t.Fatal(err) } - - return nil } func (k *kubeServer) Close() { @@ -1365,6 +1447,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("reading request body: %v", err), http.StatusInternalServerError) return } + defer r.Body.Close() switch r.Method { case "GET": @@ -1397,12 +1480,13 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } for _, op := range req { - if op.Op == "remove" { + switch op.Op { + case "remove": if !strings.HasPrefix(op.Path, "/data/") { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) - } else if op.Op == "replace" { + case "replace": path, ok := strings.CutPrefix(op.Path, "/data/") if !ok { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) @@ -1419,7 +1503,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { } k.secret[path] = val } - } else { + default: panic(fmt.Sprintf("unsupported json-patch op %q", op.Op)) } } @@ -1437,7 +1521,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("unknown content type %q", r.Header.Get("Content-Type"))) } default: - panic(fmt.Sprintf("unhandled HTTP method %q", r.Method)) + panic(fmt.Sprintf("unhandled HTTP request %s %s", r.Method, r.URL)) } } diff --git a/cmd/containerboot/tailscaled.go b/cmd/containerboot/tailscaled.go index fc209247723df..1ff068b974938 100644 --- a/cmd/containerboot/tailscaled.go +++ b/cmd/containerboot/tailscaled.go @@ -42,14 +42,14 @@ func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient log.Printf("Waiting for tailscaled socket") for { if ctx.Err() != nil { - log.Fatalf("Timed out waiting for tailscaled socket") + return nil, nil, errors.New("timed out waiting for tailscaled socket") } _, err := os.Stat(cfg.Socket) if errors.Is(err, fs.ErrNotExist) { time.Sleep(100 * time.Millisecond) continue } else if err != nil { - log.Fatalf("Waiting for tailscaled socket: %v", err) + return nil, nil, fmt.Errorf("error waiting for tailscaled socket: %w", err) } break } From 649a71f8acaad5b91f9cd5a4c1fa164c780bac7e Mon Sep 17 00:00:00 2001 From: Andrea Gottardo Date: Thu, 30 Jan 2025 12:52:55 -0800 Subject: [PATCH 223/223] VERSION.txt: this is v1.80.0 (#14837) Signed-off-by: Andrea Gottardo --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index b3a8c61e6a864..aaceec04e0401 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.79.0 +1.80.0