From 82b851e984613ca24efca16f632c98712014f493 Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Fri, 24 Mar 2023 18:08:43 +0100 Subject: [PATCH 1/9] feat: Add native image handler backend This change adds a new native backend to remove eventually the need to depend on external skopeo binary. Using the native library allows much better control and error handling. It also enables us to add more easily new features like signing for example. * Migrate image inspect and copy into a interface * Migrate skopeo based calls into new package * Remove code duplication in registries * Add cache layer backend * Add new optional alternative native image backend --- cmd/root.go | 15 +++++- go.mod | 43 ++++++++++++--- go.sum | 102 +++++++++++++++++++++++++++++------ pkg/backend/backend.go | 20 +++++++ pkg/backend/cached.go | 44 +++++++++++++++ pkg/backend/native.go | 97 +++++++++++++++++++++++++++++++++ pkg/backend/skopeo.go | 98 +++++++++++++++++++++++++++++++++ pkg/config/config.go | 2 + pkg/registry/client.go | 21 ++++++-- pkg/registry/ecr.go | 119 ++++++++++++++++------------------------- pkg/registry/gar.go | 106 +++++++++--------------------------- 11 files changed, 482 insertions(+), 185 deletions(-) create mode 100644 pkg/backend/backend.go create mode 100644 pkg/backend/cached.go create mode 100644 pkg/backend/native.go create mode 100644 pkg/backend/skopeo.go diff --git a/cmd/root.go b/cmd/root.go index 7d2121b4..49c86a8f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -30,6 +30,7 @@ import ( "syscall" "time" + "github.com/estahn/k8s-image-swapper/pkg/backend" "github.com/estahn/k8s-image-swapper/pkg/config" "github.com/estahn/k8s-image-swapper/pkg/registry" "github.com/estahn/k8s-image-swapper/pkg/secrets" @@ -63,10 +64,18 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`, //metricsRec := metrics.NewPrometheus(promReg) log.Trace().Interface("config", cfg).Msg("config") + var imageBackend backend.Backend = backend.NewSkopeo() + + if cfg.NativeBackend { + log.Warn().Msg("experimental native image backend enabled") + imageBackend = backend.NewNative() + + } + // Create registry clients for source registries sourceRegistryClients := []registry.Client{} for _, reg := range cfg.Source.Registries { - sourceRegistryClient, err := registry.NewClient(reg) + sourceRegistryClient, err := registry.NewClient(reg, imageBackend) if err != nil { log.Err(err).Msgf("error connecting to source registry at %s", reg.Domain()) os.Exit(1) @@ -75,7 +84,7 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`, } // Create a registry client for private target registry - targetRegistryClient, err := registry.NewClient(cfg.Target) + targetRegistryClient, err := registry.NewClient(cfg.Target, imageBackend) if err != nil { log.Err(err).Msgf("error connecting to target registry at %s", cfg.Target.Domain()) os.Exit(1) @@ -207,6 +216,7 @@ func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.k8s-image-swapper.yaml)") rootCmd.PersistentFlags().StringVar(&cfg.LogLevel, "log-level", "info", "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]") rootCmd.PersistentFlags().StringVar(&cfg.LogFormat, "log-format", "json", "Format of the log messages. Valid levels: [json, console]") + rootCmd.PersistentFlags().BoolVar(&cfg.NativeBackend, "native-backend", false, "Native backend is a experimental image handler to replace the skopeo based binary execution") // Cobra also supports local flags, which will only run // when this action is called directly. @@ -215,6 +225,7 @@ func init() { rootCmd.Flags().StringVar(&cfg.TLSCertFile, "tls-cert-file", "", "File containing the TLS certificate") rootCmd.Flags().StringVar(&cfg.TLSKeyFile, "tls-key-file", "", "File containing the TLS private key") rootCmd.Flags().BoolVar(&cfg.DryRun, "dry-run", true, "If true, print the action taken without taking it") + } // initConfig reads in config file and ENV variables if set. diff --git a/go.mod b/go.mod index b0f91541..4635c4a3 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,8 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) +require github.com/containers/common v0.51.1 + require ( cloud.google.com/go v0.115.1 // indirect cloud.google.com/go/auth v0.9.5 // indirect @@ -40,12 +42,16 @@ require ( cloud.google.com/go/longrunning v0.6.0 // indirect cloud.google.com/go/storage v1.43.0 // indirect dario.cat/mergo v1.0.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/BurntSushi/toml v1.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.12.5 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect @@ -57,6 +63,7 @@ require ( github.com/containers/ocicrypt v1.2.0 // indirect github.com/containers/storage v1.55.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f // indirect github.com/cyphar/filepath-securejoin v0.3.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect @@ -65,19 +72,27 @@ require ( github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect + github.com/go-jose/go-jose/v4 v4.0.2 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-sql-driver/mysql v1.5.0 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -110,18 +125,22 @@ require ( github.com/homeport/dyff v1.6.0 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect + github.com/jinzhu/copier v0.3.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/pgzip v1.2.6 // indirect + github.com/letsencrypt/boulder v0.0.0-20240418210053-89b07f4543e0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -136,6 +155,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect @@ -145,42 +165,51 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.2.0 // indirect + github.com/proglottis/gpgme v0.1.3 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect + github.com/sigstore/fulcio v1.4.5 // indirect + github.com/sigstore/rekor v1.3.6 // indirect + github.com/sigstore/sigstore v1.8.4 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/sylabs/sif/v2 v2.18.0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect + github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect github.com/tmccombs/hcl2json v0.3.3 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/urfave/cli v1.22.14 // indirect github.com/vbatts/tar-split v0.11.5 // indirect + github.com/vbauerster/mpb/v8 v8.7.5 // indirect github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zclconf/go-cty v1.9.1 // indirect + go.mongodb.org/mongo-driver v1.14.0 // indirect + go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect - go.opentelemetry.io/proto/otlp v1.0.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/net v0.29.0 // indirect diff --git a/go.sum b/go.sum index 8b673ef9..af6aab03 100644 --- a/go.sum +++ b/go.sum @@ -193,6 +193,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= @@ -207,6 +209,10 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/Microsoft/hcsshim v0.12.5 h1:bpTInLlDy/nDRWFVcefDZZ1+U8tS+rz3MxjKgu9boo0= github.com/Microsoft/hcsshim v0.12.5/go.mod h1:tIUGego4G1EN5Hb6KC90aDYiUI2dqLSTTOCjVNpOgZ8= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -219,6 +225,8 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= @@ -258,6 +266,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= +github.com/containers/common v0.51.1 h1:xMQpFSuUovZgPRWJu1+uBixKsPcRrD4aHADkouz2K9Q= +github.com/containers/common v0.51.1/go.mod h1:3W2WIdalgQfrsX/T5tjX+6CxgT3ThJVN2G9sNuFjuCM= github.com/containers/image/v5 v5.32.2 h1:SzNE2Y6sf9b1GJoC8qjCuMBXwQrACFp4p0RK15+4gmQ= github.com/containers/image/v5 v5.32.2/go.mod h1:v1l73VeMugfj/QtKI+jhYbwnwFCFnNGckvbST3rQ5Hk= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= @@ -272,6 +282,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f h1:eHnXnuK47UlSTOQexbzxAZfekVz6i+LKRdj1CU5DPaM= +github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= github.com/cyphar/filepath-securejoin v0.3.1 h1:1V7cHiaW+C+39wEfpH6XlLBQo3j/PciWFrgfCLS8XrE= github.com/cyphar/filepath-securejoin v0.3.1/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -298,8 +310,9 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -335,25 +348,42 @@ github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0/go.mod h1:f4zRH github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.0.2 h1:R3l3kkBds16bO7ZFAEEcofK0MkrAJt3jlJznWZG0nvk= +github.com/go-jose/go-jose/v4 v4.0.2/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -490,8 +520,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRaxEM6G0ro= github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78= github.com/gruntwork-io/terratest v0.47.2 h1:t6iWwsqJH7Gx0RwXleU/vjc+2c0JXRMdj3DxYXTBssQ= @@ -526,12 +556,14 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= -github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= +github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -558,6 +590,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/letsencrypt/boulder v0.0.0-20240418210053-89b07f4543e0 h1:aiPrFdHDCCvigNBCkOWj2lv9Bx5xDp210OANZEoiP0I= +github.com/letsencrypt/boulder v0.0.0-20240418210053-89b07f4543e0/go.mod h1:srVwm2N3DC/tWqQ+igZXDrmKlNRN8X/dmJ1wEZrv760= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -577,9 +611,15 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= +github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs/v3 v3.0.1 h1:YaoXgBePoMA12+S1u/ddkv+QqxcfiZK4prI6HPnkFiU= github.com/mistifyio/go-zfs/v3 v3.0.1/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -618,8 +658,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= @@ -646,6 +688,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= +github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -655,6 +699,9 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -677,10 +724,18 @@ github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA= +github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigstore/fulcio v1.4.5 h1:WWNnrOknD0DbruuZWCbN+86WRROpEl3Xts+WT2Ek1yc= +github.com/sigstore/fulcio v1.4.5/go.mod h1:oz3Qwlma8dWcSS/IENR/6SjbW4ipN0cxpRVfgdsjMU8= +github.com/sigstore/rekor v1.3.6 h1:QvpMMJVWAp69a3CHzdrLelqEqpTM3ByQRt5B5Kspbi8= +github.com/sigstore/rekor v1.3.6/go.mod h1:JDTSNNMdQ/PxdsS49DJkJ+pRJCO/83nbR5p3aZQteXc= +github.com/sigstore/sigstore v1.8.4 h1:g4ICNpiENFnWxjmBzBDWUn62rNFeny/P77HUC8da32w= +github.com/sigstore/sigstore v1.8.4/go.mod h1:1jIKtkTFEeISen7en+ZPWdDHazqhxco/+v9CNjc7oNg= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -700,6 +755,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -729,6 +786,8 @@ github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BG github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U= github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -739,6 +798,8 @@ github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= +github.com/vbauerster/mpb/v8 v8.7.5 h1:hUF3zaNsuaBBwzEFoCvfuX3cpesQXZC0Phm/JcHZQ+c= +github.com/vbauerster/mpb/v8 v8.7.5/go.mod h1:bRCnR7K+mj5WXKsy0NWB6Or+wctYGvVwKn6huwvxKa0= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -746,6 +807,12 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -758,6 +825,10 @@ github.com/zclconf/go-cty v1.8.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUA github.com/zclconf/go-cty v1.9.1 h1:viqrgQwFl5UpSxc046qblj78wZXVDFnSOufaOTER+cc= github.com/zclconf/go-cty v1.9.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= +go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= +go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -773,8 +844,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+n go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= @@ -786,10 +857,11 @@ go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+M go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go new file mode 100644 index 00000000..af18692f --- /dev/null +++ b/pkg/backend/backend.go @@ -0,0 +1,20 @@ +package backend + +import ( + "context" + + ctypes "github.com/containers/image/v5/types" +) + +type Credentials struct { + // AuthFile is the optional path of the containers authentication file + AuthFile string + // Creds optional USERNAME[:PASSWORD] for accessing the registry + Creds string +} + +// Backend describes a image handler +type Backend interface { + Exists(ctx context.Context, imageRef ctypes.ImageReference, srcCreds Credentials) (bool, error) + Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCreds Credentials, destRef ctypes.ImageReference, destCreds Credentials) error +} diff --git a/pkg/backend/cached.go b/pkg/backend/cached.go new file mode 100644 index 00000000..7c19cb39 --- /dev/null +++ b/pkg/backend/cached.go @@ -0,0 +1,44 @@ +package backend + +import ( + "context" + + ctypes "github.com/containers/image/v5/types" + "github.com/dgraph-io/ristretto" + "github.com/rs/zerolog/log" +) + +// Cached backend adds a cache layer in front of a backend +type Cached struct { + Cache *ristretto.Cache + Backend Backend +} + +func NewCached(cache *ristretto.Cache, backend Backend) *Cached { + return &Cached{ + Backend: backend, + Cache: cache, + } +} + +func (c *Cached) Exists(ctx context.Context, imageRef ctypes.ImageReference, creds Credentials) (bool, error) { + ref := imageRef.DockerReference().String() + if _, found := c.Cache.Get(ref); found { + log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in cache") + return true, nil + } + + exists, err := c.Backend.Exists(ctx, imageRef, creds) + if err != nil { + return false, err + } + + if exists { + c.Cache.Set(ref, "", 1) + } + return exists, nil +} + +func (c *Cached) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCreds Credentials, destRef ctypes.ImageReference, destCreds Credentials) error { + return c.Backend.Copy(ctx, srcRef, srcCreds, destRef, destCreds) +} diff --git a/pkg/backend/native.go b/pkg/backend/native.go new file mode 100644 index 00000000..cdcc6125 --- /dev/null +++ b/pkg/backend/native.go @@ -0,0 +1,97 @@ +package backend + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/containers/common/pkg/retry" + "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/signature" + ctypes "github.com/containers/image/v5/types" +) + +type Native struct { + retryOpts retry.Options +} + +func NewNative() *Native { + return &Native{ + retryOpts: retry.Options{ + MaxRetry: 3, + Delay: time.Millisecond * 100, + }, + } +} + +func (n *Native) newContext(creds Credentials) *ctypes.SystemContext { + // default is no creds + dockerAuth := &ctypes.DockerAuthConfig{} + + if creds.Creds != "" { + username, password, _ := strings.Cut(creds.Creds, ":") + dockerAuth = &ctypes.DockerAuthConfig{ + Username: username, + Password: password, + } + } + + return &ctypes.SystemContext{ + AuthFilePath: creds.AuthFile, + DockerAuthConfig: dockerAuth, + + // It actually defaults to the current runtime, ao we may not need to override it + // OSChoice: "linux", + } +} + +func (n *Native) Exists(ctx context.Context, imageRef ctypes.ImageReference, creds Credentials) (bool, error) { + srcImage, err := imageRef.NewImageSource(ctx, n.newContext(creds)) + if err != nil { + return false, err + } + defer srcImage.Close() + + var rawManifest []byte + if err := retry.IfNecessary(ctx, func() error { + rawManifest, _, err = srcImage.GetManifest(ctx, nil) + return err + }, &n.retryOpts); err != nil { + // TODO: check if error is only client errors or also not found? + return false, fmt.Errorf("Error retrieving manifest for image: %w", err) + } + + exists := len(rawManifest) > 0 + + return exists, nil + +} + +func (n *Native) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCreds Credentials, destRef ctypes.ImageReference, destCreds Credentials) error { + policy, err := signature.DefaultPolicy(nil) + if err != nil { + return fmt.Errorf("unable to get image copy policy: %q", err) + } + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + return fmt.Errorf("unable to get image copy policy context: %q", err) + } + defer policyContext.Destroy() + + opts := ©.Options{ + SourceCtx: n.newContext(srcCreds), + DestinationCtx: n.newContext(destCreds), + ImageListSelection: copy.CopyAllImages, // multi-arch + } + + _, err = copy.Image(ctx, policyContext, destRef, srcRef, opts) + + return retry.IfNecessary(ctx, func() error { + _, err := copy.Image(ctx, policyContext, destRef, srcRef, opts) + if err != nil { + return fmt.Errorf("failed to copy image: %q", err) + } + return nil + }, &n.retryOpts) +} diff --git a/pkg/backend/skopeo.go b/pkg/backend/skopeo.go new file mode 100644 index 00000000..01d9a470 --- /dev/null +++ b/pkg/backend/skopeo.go @@ -0,0 +1,98 @@ +package backend + +import ( + "context" + "fmt" + "os/exec" + + ctypes "github.com/containers/image/v5/types" + "github.com/rs/zerolog/log" +) + +// Skopeo is the legacy Backend by leveraging execution of the skopeo binary +type Skopeo struct { + retries int +} + +func NewSkopeo() *Skopeo { + return &Skopeo{ + retries: 3, + } +} + +func (s *Skopeo) credArgs(creds Credentials, prefix string) []string { + args := make([]string, 0) + + if creds.AuthFile != "" { + args = append(args, fmt.Sprintf("--%sauthfile", prefix), creds.AuthFile) + } + + if creds.Creds != "" { + args = append(args, fmt.Sprintf("--%screds", prefix), creds.Creds) + } + + if len(args) == 0 { + args = append(args, fmt.Sprintf("--%sno-creds", prefix), creds.Creds) + } + + return args +} + +func (s *Skopeo) Exists(ctx context.Context, imageRef ctypes.ImageReference, srcCreds Credentials) (bool, error) { + ref := imageRef.DockerReference().String() + + app := "skopeo" + args := []string{ + "inspect", + "--retry-times", "3", + "docker://" + ref, + } + + args = append(args, s.credArgs(srcCreds, "")...) + + log.Ctx(ctx).Trace().Str("app", app).Strs("args", args).Msg("executing command to inspect image") + if err := exec.CommandContext(ctx, app, args...).Run(); err != nil { + log.Ctx(ctx).Trace().Str("ref", ref).Msg("not found in target repository") + return false, nil + } + + log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in target repository") + return true, nil +} + +func (s *Skopeo) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCreds Credentials, destRef ctypes.ImageReference, destCreds Credentials) error { + src := srcRef.DockerReference().String() + dest := destRef.DockerReference().String() + app := "skopeo" + args := []string{ + "--override-os", "linux", + "copy", + "--multi-arch", "all", + "--retry-times", "3", + "docker://" + src, + "docker://" + dest, + } + + args = append(args, s.credArgs(srcCreds, "src-")...) + args = append(args, s.credArgs(destCreds, "dst-")...) + + log.Ctx(ctx). + Trace(). + Str("app", app). + Strs("args", args). + Msg("execute command to copy image") + + output, cmdErr := exec.CommandContext(ctx, app, args...).CombinedOutput() + + // check if the command timed out during execution for proper logging + if err := ctx.Err(); err != nil { + return err + } + + // enrich error with output from the command which may contain the actual reason + if cmdErr != nil { + return fmt.Errorf("command error, stderr: %q, stdout: %q", cmdErr.Error(), string(output)) + } + + return nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 3dabc717..a15420c3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -36,6 +36,8 @@ type Config struct { LogLevel string `yaml:"logLevel" validate:"oneof=trace debug info warn error fatal"` LogFormat string `yaml:"logFormat" validate:"oneof=json console"` + NativeBackend bool `yaml:"nativeBackend"` + ListenAddress string DryRun bool `yaml:"dryRun"` diff --git a/pkg/registry/client.go b/pkg/registry/client.go index da373e2c..bc4e3eca 100644 --- a/pkg/registry/client.go +++ b/pkg/registry/client.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" + "github.com/dgraph-io/ristretto" + "github.com/estahn/k8s-image-swapper/pkg/backend" "github.com/estahn/k8s-image-swapper/pkg/config" "github.com/estahn/k8s-image-swapper/pkg/types" @@ -17,8 +19,6 @@ type Client interface { CreateRepository(ctx context.Context, name string) error RepositoryExists() bool CopyImage(ctx context.Context, src ctypes.ImageReference, srcCreds string, dest ctypes.ImageReference, destCreds string) error - PullImage() error - PutImage() error ImageExists(ctx context.Context, ref ctypes.ImageReference) bool // Endpoint returns the domain of the registry @@ -38,7 +38,7 @@ type AuthConfig struct { } // NewClient returns a registry client ready for use without the need to specify an implementation -func NewClient(r config.Registry) (Client, error) { +func NewClient(r config.Registry, imageBackend backend.Backend) (Client, error) { if err := config.CheckRegistryConfiguration(r); err != nil { return nil, err } @@ -48,11 +48,22 @@ func NewClient(r config.Registry) (Client, error) { return nil, err } + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, // number of keys to track frequency of (10M). + MaxCost: 1 << 30, // maximum cost of cache (1GB). + BufferItems: 64, // number of keys per Get buffer. + }) + if err != nil { + return nil, err + } + + cachedBackend := backend.NewCached(cache, imageBackend) + switch registry { case types.RegistryAWS: - return NewECRClient(r.AWS) + return NewECRClient(r.AWS, cachedBackend, cache) case types.RegistryGCP: - return NewGARClient(r.GCP) + return NewGARClient(r.GCP, cachedBackend) default: return nil, fmt.Errorf(`registry of type "%s" is not supported`, r.Type) } diff --git a/pkg/registry/ecr.go b/pkg/registry/ecr.go index e651159d..f91f78b4 100644 --- a/pkg/registry/ecr.go +++ b/pkg/registry/ecr.go @@ -4,9 +4,11 @@ import ( "context" "encoding/base64" "fmt" +<<<<<<< HEAD "math/rand" +======= +>>>>>>> 359ee16 (feat: Add native image handler backend) "net/http" - "os/exec" "time" "github.com/containers/image/v5/docker/reference" @@ -19,12 +21,14 @@ import ( "github.com/aws/aws-sdk-go/service/ecr/ecriface" ctypes "github.com/containers/image/v5/types" "github.com/dgraph-io/ristretto" + "github.com/estahn/k8s-image-swapper/pkg/backend" "github.com/estahn/k8s-image-swapper/pkg/config" "github.com/go-co-op/gocron" "github.com/rs/zerolog/log" ) type ECRClient struct { +<<<<<<< HEAD client ecriface.ECRAPI ecrDomain string authToken []byte @@ -32,9 +36,21 @@ type ECRClient struct { scheduler *gocron.Scheduler targetAccount string options config.ECROptions +======= + client ecriface.ECRAPI + ecrDomain string + authToken []byte + cache *ristretto.Cache + scheduler *gocron.Scheduler + targetAccount string + accessPolicy string + lifecyclePolicy string + tags []config.Tag + backend backend.Backend +>>>>>>> 359ee16 (feat: Add native image handler backend) } -func NewECRClient(clientConfig config.AWS) (*ECRClient, error) { +func NewECRClient(clientConfig config.AWS, imageBackend backend.Backend, cache *ristretto.Cache) (*ECRClient, error) { ecrDomain := clientConfig.EcrDomain() var sess *session.Session @@ -65,25 +81,28 @@ func NewECRClient(clientConfig config.AWS) (*ECRClient, error) { })) ecrClient := ecr.New(sess, cfg) - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e7, // number of keys to track frequency of (10M). - MaxCost: 1 << 30, // maximum cost of cache (1GB). - BufferItems: 64, // number of keys per Get buffer. - }) - if err != nil { - panic(err) - } - scheduler := gocron.NewScheduler(time.UTC) scheduler.StartAsync() client := &ECRClient{ +<<<<<<< HEAD client: ecrClient, ecrDomain: ecrDomain, cache: cache, scheduler: scheduler, targetAccount: clientConfig.AccountID, options: clientConfig.ECROptions, +======= + client: ecrClient, + ecrDomain: ecrDomain, + cache: cache, + scheduler: scheduler, + targetAccount: clientConfig.AccountID, + accessPolicy: clientConfig.ECROptions.AccessPolicy, + lifecyclePolicy: clientConfig.ECROptions.LifecyclePolicy, + tags: clientConfig.ECROptions.Tags, + backend: imageBackend, +>>>>>>> 359ee16 (feat: Add native image handler backend) } if err := client.scheduleTokenRenewal(); err != nil { @@ -178,85 +197,35 @@ func (e *ECRClient) RepositoryExists() bool { } func (e *ECRClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, srcCreds string, destRef ctypes.ImageReference, destCreds string) error { - src := srcRef.DockerReference().String() - dest := destRef.DockerReference().String() - app := "skopeo" - args := []string{ - "--override-os", "linux", - "copy", - "--multi-arch", "all", - "--retry-times", "3", - "docker://" + src, - "docker://" + dest, + srcCredentials := backend.Credentials{ + AuthFile: srcCreds, } - - if len(srcCreds) > 0 { - args = append(args, "--src-authfile", srcCreds) - } else { - args = append(args, "--src-no-creds") + dstCredentials := backend.Credentials{ + Creds: destCreds, } - if len(destCreds) > 0 { - args = append(args, "--dest-creds", destCreds) - } else { - args = append(args, "--dest-no-creds") - } - - log.Ctx(ctx). - Trace(). - Str("app", app). - Strs("args", args). - Msg("execute command to copy image") - - output, cmdErr := exec.CommandContext(ctx, app, args...).CombinedOutput() - - // check if the command timed out during execution for proper logging - if err := ctx.Err(); err != nil { - return err - } - - // enrich error with output from the command which may contain the actual reason - if cmdErr != nil { - return fmt.Errorf("Command error, stderr: %s, stdout: %s", cmdErr.Error(), string(output)) - } - - return nil -} - -func (e *ECRClient) PullImage() error { - panic("implement me") -} - -func (e *ECRClient) PutImage() error { - panic("implement me") + return e.backend.Copy(ctx, srcRef, srcCredentials, destRef, dstCredentials) } func (e *ECRClient) ImageExists(ctx context.Context, imageRef ctypes.ImageReference) bool { + creds := backend.Credentials{ + Creds: e.Credentials(), + } + ref := imageRef.DockerReference().String() if _, found := e.cache.Get(ref); found { log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in cache") return true } - app := "skopeo" - args := []string{ - "inspect", - "--retry-times", "3", - "docker://" + ref, - "--creds", e.Credentials(), - } - - log.Ctx(ctx).Trace().Str("app", app).Strs("args", args).Msg("executing command to inspect image") - if err := exec.CommandContext(ctx, app, args...).Run(); err != nil { - log.Ctx(ctx).Trace().Str("ref", ref).Msg("not found in target repository") + exists, err := e.backend.Exists(ctx, imageRef, creds) + if err != nil { + log.Error().Err(err).Msg("unable to check existence of image") return false } - log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in target repository") - - e.cache.SetWithTTL(ref, "", 1, 24*time.Hour+time.Duration(rand.Intn(180))*time.Minute) - return true + return exists } func (e *ECRClient) Endpoint() string { @@ -312,6 +281,7 @@ func NewDummyECRClient(region string, targetAccount string, role string, options options: options, ecrDomain: fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", targetAccount, region), authToken: authToken, + backend: backend.NewSkopeo(), } } @@ -328,6 +298,7 @@ func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string ImageScanningConfiguration: config.ImageScanningConfiguration{ImageScanOnPush: true}, Tags: []config.Tag{{Key: "CreatedBy", Value: "k8s-image-swapper"}, {Key: "AnotherTag", Value: "another-tag"}}, }, + backend: backend.NewSkopeo(), } return client, nil diff --git a/pkg/registry/gar.go b/pkg/registry/gar.go index c129246b..da0a4464 100644 --- a/pkg/registry/gar.go +++ b/pkg/registry/gar.go @@ -13,7 +13,7 @@ import ( artifactregistry "cloud.google.com/go/artifactregistry/apiv1" "github.com/containers/image/v5/docker/reference" ctypes "github.com/containers/image/v5/types" - "github.com/dgraph-io/ristretto" + "github.com/estahn/k8s-image-swapper/pkg/backend" "github.com/estahn/k8s-image-swapper/pkg/config" "github.com/go-co-op/gocron" "google.golang.org/api/option" @@ -27,20 +27,12 @@ type GARAPI interface{} type GARClient struct { client GARAPI garDomain string - cache *ristretto.Cache scheduler *gocron.Scheduler authToken []byte + backend backend.Backend } -func NewGARClient(clientConfig config.GCP) (*GARClient, error) { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e7, // number of keys to track frequency of (10M). - MaxCost: 1 << 30, // maximum cost of cache (1GB). - BufferItems: 64, // number of keys per Get buffer. - }) - if err != nil { - panic(err) - } +func NewGARClient(clientConfig config.GCP, imageBackend backend.Backend) (*GARClient, error) { scheduler := gocron.NewScheduler(time.UTC) scheduler.StartAsync() @@ -48,8 +40,8 @@ func NewGARClient(clientConfig config.GCP) (*GARClient, error) { client := &GARClient{ client: nil, garDomain: clientConfig.GarDomain(), - cache: cache, scheduler: scheduler, + backend: imageBackend, } if err := client.scheduleTokenRenewal(); err != nil { @@ -69,93 +61,43 @@ func (e *GARClient) RepositoryExists() bool { } func (e *GARClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, srcCreds string, destRef ctypes.ImageReference, destCreds string) error { - src := srcRef.DockerReference().String() - dest := destRef.DockerReference().String() - - creds := []string{"--src-authfile", srcCreds} - - // use client credentials for any source GAR repositories - if strings.HasSuffix(reference.Domain(srcRef.DockerReference()), "-docker.pkg.dev") { - creds = []string{"--src-creds", e.Credentials()} - } - - app := "skopeo" - args := []string{ - "--override-os", "linux", - "copy", - "--multi-arch", "all", - "--retry-times", "3", - "docker://" + src, - "docker://" + dest, - } - - if len(creds[1]) > 0 { - args = append(args, creds...) - } else { - args = append(args, "--src-no-creds") + srcCredentials := backend.Credentials{ + AuthFile: srcCreds, } - - if len(destCreds) > 0 { - args = append(args, "--dest-creds", destCreds) - } else { - args = append(args, "--dest-no-creds") - } - - log.Ctx(ctx). - Trace(). - Str("app", app). - Strs("args", args). - Msg("execute command to copy image") - - output, cmdErr := exec.CommandContext(ctx, app, args...).CombinedOutput() - - // check if the command timed out during execution for proper logging - if err := ctx.Err(); err != nil { - return err + dstCredentials := backend.Credentials{ + Creds: destCreds, } - // enrich error with output from the command which may contain the actual reason - if cmdErr != nil { - return fmt.Errorf("Command error, stderr: %s, stdout: %s", cmdErr.Error(), string(output)) + // use client credentials for any source GAR repositories + if strings.HasSuffix(reference.Domain(srcRef.DockerReference()), "-docker.pkg.dev") { + srcCredentials = backend.Credentials{ + Creds: e.Credentials(), + } } - return nil -} - -func (e *GARClient) PullImage() error { - panic("implement me") -} - -func (e *GARClient) PutImage() error { - panic("implement me") + return e.backend.Copy(ctx, srcRef, srcCredentials, destRef, dstCredentials) } func (e *GARClient) ImageExists(ctx context.Context, imageRef ctypes.ImageReference) bool { - ref := imageRef.DockerReference().String() - if _, found := e.cache.Get(ref); found { - log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in cache") - return true + creds := backend.Credentials{ + Creds: e.Credentials(), } - app := "skopeo" - args := []string{ - "inspect", - "--retry-times", "3", - "docker://" + ref, - "--creds", e.Credentials(), - } - - log.Ctx(ctx).Trace().Str("app", app).Strs("args", args).Msg("executing command to inspect image") - if err := exec.CommandContext(ctx, app, args...).Run(); err != nil { - log.Trace().Str("ref", ref).Msg("not found in target repository") + exists, err := e.backend.Exists(ctx, imageRef, creds) + if err != nil { + log.Error().Err(err).Msg("unable to check existence of image") return false } +<<<<<<< HEAD log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in target repository") e.cache.SetWithTTL(ref, "", 1, 24*time.Hour+time.Duration(rand.Intn(180))*time.Minute) return true +======= + return exists +>>>>>>> 359ee16 (feat: Add native image handler backend) } func (e *GARClient) Endpoint() string { @@ -227,8 +169,8 @@ func NewMockGARClient(garClient GARAPI, garDomain string) (*GARClient, error) { client := &GARClient{ client: garClient, garDomain: garDomain, - cache: nil, scheduler: nil, + backend: backend.NewSkopeo(), authToken: []byte("oauth2accesstoken:mock-gar-client-fake-auth-token"), } From 9ddd05f20090482c458a7084c0ec615cf92903a5 Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Sat, 25 Mar 2023 10:57:45 +0100 Subject: [PATCH 2/9] feat: improve error handling --- pkg/backend/native.go | 58 +++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/pkg/backend/native.go b/pkg/backend/native.go index cdcc6125..10892d96 100644 --- a/pkg/backend/native.go +++ b/pkg/backend/native.go @@ -10,6 +10,7 @@ import ( "github.com/containers/image/v5/copy" "github.com/containers/image/v5/signature" ctypes "github.com/containers/image/v5/types" + "github.com/rs/zerolog/log" ) type Native struct { @@ -41,43 +42,58 @@ func (n *Native) newContext(creds Credentials) *ctypes.SystemContext { AuthFilePath: creds.AuthFile, DockerAuthConfig: dockerAuth, - // It actually defaults to the current runtime, ao we may not need to override it + // It actually defaults to the current runtime, so we may not need to override it // OSChoice: "linux", } } func (n *Native) Exists(ctx context.Context, imageRef ctypes.ImageReference, creds Credentials) (bool, error) { - srcImage, err := imageRef.NewImageSource(ctx, n.newContext(creds)) - if err != nil { - return false, err - } - defer srcImage.Close() - var rawManifest []byte + if err := retry.IfNecessary(ctx, func() error { + srcImage, err := imageRef.NewImageSource(ctx, n.newContext(creds)) + if err != nil { + log.Debug().Err(err).Msg("failed to read image source") + // There is no proper error type we can check, so check for existence of specific message :-( + // it will fail with something like: + // reading manifest in : name unknown: The repository with name '' does not exist in the registry with id '' + // reading manifest in : manifest unknown: Requested image not found + if strings.Contains(strings.ToLower(err.Error()), "name unknown:") { + return nil + } + if strings.Contains(strings.ToLower(err.Error()), "manifest unknown:") { + return nil + } + return err + } + defer srcImage.Close() + rawManifest, _, err = srcImage.GetManifest(ctx, nil) return err }, &n.retryOpts); err != nil { - // TODO: check if error is only client errors or also not found? - return false, fmt.Errorf("Error retrieving manifest for image: %w", err) + return false, fmt.Errorf("unable to retrieve manifest for image: %w", err) } exists := len(rawManifest) > 0 return exists, nil - } func (n *Native) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCreds Credentials, destRef ctypes.ImageReference, destCreds Credentials) error { policy, err := signature.DefaultPolicy(nil) if err != nil { - return fmt.Errorf("unable to get image copy policy: %q", err) + return fmt.Errorf("unable to get image copy policy: %w", err) } policyContext, err := signature.NewPolicyContext(policy) if err != nil { - return fmt.Errorf("unable to get image copy policy context: %q", err) + return fmt.Errorf("unable to get image copy policy context: %w", err) } - defer policyContext.Destroy() + + defer func() { + if err := policyContext.Destroy(); err != nil { + log.Err(err).Msg("failed to destroy policy context") + } + }() opts := ©.Options{ SourceCtx: n.newContext(srcCreds), @@ -85,12 +101,22 @@ func (n *Native) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCred ImageListSelection: copy.CopyAllImages, // multi-arch } - _, err = copy.Image(ctx, policyContext, destRef, srcRef, opts) - return retry.IfNecessary(ctx, func() error { + log.Debug(). + Str("dst", destRef.StringWithinTransport()). + Str("src", srcRef.StringWithinTransport()). + Msg("copy image started") + _, err := copy.Image(ctx, policyContext, destRef, srcRef, opts) + + log.Debug(). + Err(err). + Str("dst", destRef.StringWithinTransport()). + Str("src", srcRef.StringWithinTransport()). + Msg("copy image finished") + if err != nil { - return fmt.Errorf("failed to copy image: %q", err) + return fmt.Errorf("failed to copy image: %w", err) } return nil }, &n.retryOpts) From 54f2e61dad0c4015f1f1006d050f386b2f962cac Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Sat, 25 Mar 2023 11:02:31 +0100 Subject: [PATCH 3/9] fix(ci): add go build tags --- .github/workflows/test.yml | 3 +++ .goreleaser.yml | 1 + Makefile | 5 +++++ cmd/root.go | 2 -- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c8f61ba..4e96af4b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,9 @@ concurrency: group: ${{ github.ref }} cancel-in-progress: true +env: + GOFLAGS: -tags=containers_image_openpgp,exclude_graphdriver_btrfs,btrfs_noversion,exclude_graphdriver_devicemapper + jobs: lint: diff --git a/.goreleaser.yml b/.goreleaser.yml index f1d6038a..1c6c1573 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -17,6 +17,7 @@ builds: mod_timestamp: '{{ .CommitTimestamp }}' flags: - -trimpath + - -tags=containers_image_openpgp,exclude_graphdriver_btrfs,btrfs_noversion,exclude_graphdriver_devicemapper ldflags: - -s -w diff --git a/Makefile b/Makefile index 16680f92..44c89a33 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ TEST_OPTIONS?= export GO111MODULE := on export GOPROXY = https://proxy.golang.org,direct +export GOFLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,btrfs_noversion,exclude_graphdriver_devicemapper +export CGO_ENABLED ?=0 help: ## List targets & descriptions @cat Makefile* | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -30,3 +32,6 @@ lint: ## Run linters e2e: ## Run end-to-end tests go test -v -run TestHelmDeployment ./test + +build: ## Run go build + go build \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 49c86a8f..3a6e6ea0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -65,11 +65,9 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`, log.Trace().Interface("config", cfg).Msg("config") var imageBackend backend.Backend = backend.NewSkopeo() - if cfg.NativeBackend { log.Warn().Msg("experimental native image backend enabled") imageBackend = backend.NewNative() - } // Create registry clients for source registries From f4334cf50a6612f51f960b45946e633918394f6b Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Sun, 26 Mar 2023 14:06:40 +0200 Subject: [PATCH 4/9] feat: move cache layer into registry interface One central place to handle caching --- Makefile | 3 +- pkg/backend/cached.go | 44 --------------------------- pkg/backend/native.go | 3 -- pkg/backend/skopeo.go | 1 - pkg/registry/cache.go | 68 ++++++++++++++++++++++++++++++++++++++++++ pkg/registry/client.go | 15 ++++++---- pkg/registry/ecr.go | 53 ++++---------------------------- 7 files changed, 85 insertions(+), 102 deletions(-) delete mode 100644 pkg/backend/cached.go create mode 100644 pkg/registry/cache.go diff --git a/Makefile b/Makefile index 44c89a33..f6301d19 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ TEST_OPTIONS?= export GO111MODULE := on export GOPROXY = https://proxy.golang.org,direct export GOFLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,btrfs_noversion,exclude_graphdriver_devicemapper -export CGO_ENABLED ?=0 help: ## List targets & descriptions @cat Makefile* | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -34,4 +33,4 @@ e2e: ## Run end-to-end tests go test -v -run TestHelmDeployment ./test build: ## Run go build - go build \ No newline at end of file + go build diff --git a/pkg/backend/cached.go b/pkg/backend/cached.go deleted file mode 100644 index 7c19cb39..00000000 --- a/pkg/backend/cached.go +++ /dev/null @@ -1,44 +0,0 @@ -package backend - -import ( - "context" - - ctypes "github.com/containers/image/v5/types" - "github.com/dgraph-io/ristretto" - "github.com/rs/zerolog/log" -) - -// Cached backend adds a cache layer in front of a backend -type Cached struct { - Cache *ristretto.Cache - Backend Backend -} - -func NewCached(cache *ristretto.Cache, backend Backend) *Cached { - return &Cached{ - Backend: backend, - Cache: cache, - } -} - -func (c *Cached) Exists(ctx context.Context, imageRef ctypes.ImageReference, creds Credentials) (bool, error) { - ref := imageRef.DockerReference().String() - if _, found := c.Cache.Get(ref); found { - log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in cache") - return true, nil - } - - exists, err := c.Backend.Exists(ctx, imageRef, creds) - if err != nil { - return false, err - } - - if exists { - c.Cache.Set(ref, "", 1) - } - return exists, nil -} - -func (c *Cached) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCreds Credentials, destRef ctypes.ImageReference, destCreds Credentials) error { - return c.Backend.Copy(ctx, srcRef, srcCreds, destRef, destCreds) -} diff --git a/pkg/backend/native.go b/pkg/backend/native.go index 10892d96..34b2f09b 100644 --- a/pkg/backend/native.go +++ b/pkg/backend/native.go @@ -41,9 +41,6 @@ func (n *Native) newContext(creds Credentials) *ctypes.SystemContext { return &ctypes.SystemContext{ AuthFilePath: creds.AuthFile, DockerAuthConfig: dockerAuth, - - // It actually defaults to the current runtime, so we may not need to override it - // OSChoice: "linux", } } diff --git a/pkg/backend/skopeo.go b/pkg/backend/skopeo.go index 01d9a470..f92cbd91 100644 --- a/pkg/backend/skopeo.go +++ b/pkg/backend/skopeo.go @@ -65,7 +65,6 @@ func (s *Skopeo) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCred dest := destRef.DockerReference().String() app := "skopeo" args := []string{ - "--override-os", "linux", "copy", "--multi-arch", "all", "--retry-times", "3", diff --git a/pkg/registry/cache.go b/pkg/registry/cache.go new file mode 100644 index 00000000..dc1a31b0 --- /dev/null +++ b/pkg/registry/cache.go @@ -0,0 +1,68 @@ +package registry + +import ( + "context" + + ctypes "github.com/containers/image/v5/types" + "github.com/dgraph-io/ristretto" + "github.com/rs/zerolog/log" +) + +// Cached registry cache requests +type Cached struct { + Cache *ristretto.Cache + Registry Client +} + +func NewCachedClient(cache *ristretto.Cache, registry Client) (*Cached, error) { + return &Cached{ + Registry: registry, + Cache: cache, + }, nil +} + +func (c *Cached) CreateRepository(ctx context.Context, name string) error { + if _, found := c.Cache.Get(name); found { + log.Ctx(ctx).Trace().Str("name", name).Str("method", "CreateRepository").Msg("found in cache") + return nil + } + + err := c.Registry.CreateRepository(ctx, name) + + if err == nil { + c.Cache.Set(name, "", 1) + } + + return err +} + +func (c *Cached) ImageExists(ctx context.Context, imageRef ctypes.ImageReference) bool { + ref := imageRef.DockerReference().String() + if _, found := c.Cache.Get(ref); found { + log.Ctx(ctx).Trace().Str("ref", ref).Str("method", "ImageExists").Msg("found in cache") + return true + } + + exists := c.Registry.ImageExists(ctx, imageRef) + + if exists { + c.Cache.Set(ref, "", 1) + } + return exists +} + +func (c *Cached) CopyImage(ctx context.Context, src ctypes.ImageReference, srcCreds string, dest ctypes.ImageReference, destCreds string) error { + return c.Registry.CopyImage(ctx, src, srcCreds, dest, destCreds) +} + +func (c *Cached) Endpoint() string { + return c.Registry.Endpoint() +} + +func (c *Cached) Credentials() string { + return c.Registry.Credentials() +} + +func (c *Cached) IsOrigin(imageRef ctypes.ImageReference) bool { + return c.Registry.IsOrigin(imageRef) +} diff --git a/pkg/registry/client.go b/pkg/registry/client.go index bc4e3eca..105b1222 100644 --- a/pkg/registry/client.go +++ b/pkg/registry/client.go @@ -17,7 +17,6 @@ import ( // Client provides methods required to be implemented by the various target registry clients, e.g. ECR, Docker, Quay. type Client interface { CreateRepository(ctx context.Context, name string) error - RepositoryExists() bool CopyImage(ctx context.Context, src ctypes.ImageReference, srcCreds string, dest ctypes.ImageReference, destCreds string) error ImageExists(ctx context.Context, ref ctypes.ImageReference) bool @@ -48,6 +47,7 @@ func NewClient(r config.Registry, imageBackend backend.Backend) (Client, error) return nil, err } + // TODO: reduce cache size and/or make it configurable cache, err := ristretto.NewCache(&ristretto.Config{ NumCounters: 1e7, // number of keys to track frequency of (10M). MaxCost: 1 << 30, // maximum cost of cache (1GB). @@ -57,16 +57,21 @@ func NewClient(r config.Registry, imageBackend backend.Backend) (Client, error) return nil, err } - cachedBackend := backend.NewCached(cache, imageBackend) - + var registryClient Client switch registry { case types.RegistryAWS: - return NewECRClient(r.AWS, cachedBackend, cache) + if registryClient, err = NewECRClient(r.AWS, imageBackend); err != nil { + return nil, err + } case types.RegistryGCP: - return NewGARClient(r.GCP, cachedBackend) + if registryClient, err = NewGARClient(r.GCP, imageBackend); err != nil { + return nil, err + } default: return nil, fmt.Errorf(`registry of type "%s" is not supported`, r.Type) } + + return NewCachedClient(cache, registryClient) } func GenerateDockerConfig(c Client) ([]byte, error) { diff --git a/pkg/registry/ecr.go b/pkg/registry/ecr.go index f91f78b4..874870b6 100644 --- a/pkg/registry/ecr.go +++ b/pkg/registry/ecr.go @@ -4,14 +4,11 @@ import ( "context" "encoding/base64" "fmt" -<<<<<<< HEAD - "math/rand" -======= ->>>>>>> 359ee16 (feat: Add native image handler backend) "net/http" "time" "github.com/containers/image/v5/docker/reference" + "github.com/dgraph-io/ristretto" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -20,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go/service/ecr" "github.com/aws/aws-sdk-go/service/ecr/ecriface" ctypes "github.com/containers/image/v5/types" - "github.com/dgraph-io/ristretto" "github.com/estahn/k8s-image-swapper/pkg/backend" "github.com/estahn/k8s-image-swapper/pkg/config" "github.com/go-co-op/gocron" @@ -28,7 +24,6 @@ import ( ) type ECRClient struct { -<<<<<<< HEAD client ecriface.ECRAPI ecrDomain string authToken []byte @@ -36,21 +31,10 @@ type ECRClient struct { scheduler *gocron.Scheduler targetAccount string options config.ECROptions -======= - client ecriface.ECRAPI - ecrDomain string - authToken []byte - cache *ristretto.Cache - scheduler *gocron.Scheduler - targetAccount string - accessPolicy string - lifecyclePolicy string - tags []config.Tag - backend backend.Backend ->>>>>>> 359ee16 (feat: Add native image handler backend) + backend backend.Backend } -func NewECRClient(clientConfig config.AWS, imageBackend backend.Backend, cache *ristretto.Cache) (*ECRClient, error) { +func NewECRClient(clientConfig config.AWS, imageBackend backend.Backend) (*ECRClient, error) { ecrDomain := clientConfig.EcrDomain() var sess *session.Session @@ -85,24 +69,13 @@ func NewECRClient(clientConfig config.AWS, imageBackend backend.Backend, cache * scheduler.StartAsync() client := &ECRClient{ -<<<<<<< HEAD client: ecrClient, ecrDomain: ecrDomain, cache: cache, scheduler: scheduler, targetAccount: clientConfig.AccountID, options: clientConfig.ECROptions, -======= - client: ecrClient, - ecrDomain: ecrDomain, - cache: cache, - scheduler: scheduler, - targetAccount: clientConfig.AccountID, - accessPolicy: clientConfig.ECROptions.AccessPolicy, - lifecyclePolicy: clientConfig.ECROptions.LifecyclePolicy, - tags: clientConfig.ECROptions.Tags, - backend: imageBackend, ->>>>>>> 359ee16 (feat: Add native image handler backend) + backend: imageBackend, } if err := client.scheduleTokenRenewal(); err != nil { @@ -117,10 +90,6 @@ func (e *ECRClient) Credentials() string { } func (e *ECRClient) CreateRepository(ctx context.Context, name string) error { - if _, found := e.cache.Get(name); found { - return nil - } - log.Ctx(ctx).Debug().Str("repository", name).Msg("create repository") _, err := e.client.CreateRepositoryWithContext(ctx, &ecr.CreateRepositoryInput{ @@ -176,8 +145,6 @@ func (e *ECRClient) CreateRepository(ctx context.Context, name string) error { } } - e.cache.SetWithTTL(name, "", 1, time.Duration(24*time.Hour)) - return nil } @@ -212,19 +179,12 @@ func (e *ECRClient) ImageExists(ctx context.Context, imageRef ctypes.ImageRefere Creds: e.Credentials(), } - ref := imageRef.DockerReference().String() - if _, found := e.cache.Get(ref); found { - log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in cache") - return true - } - exists, err := e.backend.Exists(ctx, imageRef, creds) if err != nil { log.Error().Err(err).Msg("unable to check existence of image") return false } - return exists } @@ -281,7 +241,7 @@ func NewDummyECRClient(region string, targetAccount string, role string, options options: options, ecrDomain: fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", targetAccount, region), authToken: authToken, - backend: backend.NewSkopeo(), + backend: backend.NewSkopeo(), } } @@ -289,7 +249,6 @@ func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string client := &ECRClient{ client: ecrClient, ecrDomain: ecrDomain, - cache: nil, scheduler: nil, targetAccount: targetAccount, authToken: []byte("mock-ecr-client-fake-auth-token"), @@ -298,7 +257,7 @@ func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string ImageScanningConfiguration: config.ImageScanningConfiguration{ImageScanOnPush: true}, Tags: []config.Tag{{Key: "CreatedBy", Value: "k8s-image-swapper"}, {Key: "AnotherTag", Value: "another-tag"}}, }, - backend: backend.NewSkopeo(), + backend: backend.NewSkopeo(), } return client, nil From 020a9f011037cf199653ffc3155f0c03ab291489 Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Sun, 26 Mar 2023 14:27:31 +0200 Subject: [PATCH 5/9] fix(ci): add tags to pre-commit --- .github/workflows/pre-commit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a2b01f8a..584340e9 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -5,6 +5,9 @@ on: push: branches: [master] +env: + GOFLAGS: -tags=containers_image_openpgp,exclude_graphdriver_btrfs,btrfs_noversion,exclude_graphdriver_devicemapper + jobs: pre-commit: runs-on: ubuntu-latest From 407e84a39ee2bd9319aea876ee5c67d0a5691dd8 Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Thu, 6 Apr 2023 10:47:01 +0200 Subject: [PATCH 6/9] go mod tidy --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 4635c4a3..a660f408 100644 --- a/go.mod +++ b/go.mod @@ -160,6 +160,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect + github.com/opencontainers/runc v1.1.5 // indirect github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect From daa58e93982b63164f01a0233db6492845502ca4 Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Tue, 2 May 2023 11:11:40 +0200 Subject: [PATCH 7/9] fix: authentication --- pkg/backend/native.go | 4 ++-- pkg/backend/skopeo.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/backend/native.go b/pkg/backend/native.go index 34b2f09b..3cfead6d 100644 --- a/pkg/backend/native.go +++ b/pkg/backend/native.go @@ -27,8 +27,8 @@ func NewNative() *Native { } func (n *Native) newContext(creds Credentials) *ctypes.SystemContext { - // default is no creds - dockerAuth := &ctypes.DockerAuthConfig{} + // Needs to be nil to allow fallback to system given .docker/config.json + var dockerAuth *ctypes.DockerAuthConfig if creds.Creds != "" { username, password, _ := strings.Cut(creds.Creds, ":") diff --git a/pkg/backend/skopeo.go b/pkg/backend/skopeo.go index f92cbd91..c922a43b 100644 --- a/pkg/backend/skopeo.go +++ b/pkg/backend/skopeo.go @@ -73,7 +73,7 @@ func (s *Skopeo) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCred } args = append(args, s.credArgs(srcCreds, "src-")...) - args = append(args, s.credArgs(destCreds, "dst-")...) + args = append(args, s.credArgs(destCreds, "dest-")...) log.Ctx(ctx). Trace(). From a3ef67898439565450692a8e9d4ad28d409000fe Mon Sep 17 00:00:00 2001 From: Richard Hillmann Date: Tue, 2 May 2023 15:40:19 +0200 Subject: [PATCH 8/9] fix no-creds for skopeo --- pkg/backend/skopeo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/backend/skopeo.go b/pkg/backend/skopeo.go index c922a43b..a06a6938 100644 --- a/pkg/backend/skopeo.go +++ b/pkg/backend/skopeo.go @@ -32,7 +32,7 @@ func (s *Skopeo) credArgs(creds Credentials, prefix string) []string { } if len(args) == 0 { - args = append(args, fmt.Sprintf("--%sno-creds", prefix), creds.Creds) + args = append(args, fmt.Sprintf("--%sno-creds", prefix)) } return args From 0e73db76d74a375b4d1658fa46210c570c032cab Mon Sep 17 00:00:00 2001 From: Keir Mcluskey Date: Thu, 10 Oct 2024 20:20:22 +1100 Subject: [PATCH 9/9] Fix missed conflict --- pkg/registry/gar.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pkg/registry/gar.go b/pkg/registry/gar.go index da0a4464..64a57207 100644 --- a/pkg/registry/gar.go +++ b/pkg/registry/gar.go @@ -5,8 +5,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "math/rand" - "os/exec" "strings" "time" @@ -88,16 +86,8 @@ func (e *GARClient) ImageExists(ctx context.Context, imageRef ctypes.ImageRefere log.Error().Err(err).Msg("unable to check existence of image") return false } -<<<<<<< HEAD - log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in target repository") - - e.cache.SetWithTTL(ref, "", 1, 24*time.Hour+time.Duration(rand.Intn(180))*time.Minute) - - return true -======= return exists ->>>>>>> 359ee16 (feat: Add native image handler backend) } func (e *GARClient) Endpoint() string {