Add function for parsing DOI definition files (#172)

Add a Rego builtin called `attest.internals.parse_library_definition`
for parsing the DOI definition files in
https://github.com/docker-library/official-images/tree/master/library.
This will allow us to verify DOI provenance fields against these files
which are the source of truth for DOI images.

This function just defers to
https://github.com/docker-library/bashbrew/blob/master/manifest/rfc2822.go.

Author: jonnystoten

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/containerd/containerd/v2 v2.0.0-rc.4
 	github.com/containerd/platforms v0.2.1
 	github.com/distribution/reference v0.6.0
+	github.com/docker-library/bashbrew v0.1.12
 	github.com/go-openapi/runtime v0.28.0
 	github.com/go-openapi/strfmt v0.23.0
 	github.com/google/go-containerregistry v0.20.2
@@ -39,6 +40,7 @@ require (
 	cloud.google.com/go/iam v1.2.0 // indirect
 	cloud.google.com/go/kms v1.19.0 // indirect
 	cloud.google.com/go/longrunning v0.6.0 // indirect
+	github.com/Microsoft/hcsshim v0.12.6 // indirect
 	github.com/OneOfOne/xxhash v1.2.8 // indirect
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
@@ -62,6 +64,7 @@ require (
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.3.8 // indirect
+	github.com/containerd/containerd v1.7.21 // indirect
 	github.com/containerd/errdefs v0.1.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
@@ -177,4 +180,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.1 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
+	pault.ag/go/debian v0.12.0 // indirect
+	pault.ag/go/topsort v0.1.1 // indirect
 )

go.sum

@@ -54,10 +54,13 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0=
 github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/Microsoft/hcsshim v0.12.6 h1:qEnZjoHXv+4/s0LmKZWE0/AiZmMWEIkFfWBSf1a0wlU=
+github.com/Microsoft/hcsshim v0.12.6/go.mod h1:ZABCLVcvLMjIkzr9rUGcQ1QA0p0P3Ps+d3N1g2DsFfk=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
@@ -172,6 +175,8 @@ github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUo
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
 github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
 github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
+github.com/containerd/containerd v1.7.21 h1:USGXRK1eOC/SX0L195YgxTHb0a00anxajOzgfN0qrCA=
+github.com/containerd/containerd v1.7.21/go.mod h1:e3Jz1rYRUZ2Lt51YrH9Rz0zPyJBOlSvB3ghr2jbVD8g=
 github.com/containerd/containerd/v2 v2.0.0-rc.4 h1:Bvto4h5i2VZkQ+L5SrGupg5ilQ+zkVPILdjf9RWMego=
 github.com/containerd/containerd/v2 v2.0.0-rc.4/go.mod h1:p35nJi4Pl9ibzuoVOPc3MputVh6Gbp9xoDg9VHz6/YI=
 github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM=
@@ -208,6 +213,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker-library/bashbrew v0.1.12 h1:qykd2fxTMiudN/70XItEQqgk/7LeVoDiBTEnKTpkst8=
+github.com/docker-library/bashbrew v0.1.12/go.mod h1:6fyRRSm4vgBAgTw87EsfOT7wXKsc4JA9I5cdQJmwOm8=
 github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE=
 github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
@@ -401,6 +408,7 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -591,6 +599,7 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMc
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg=
 github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -632,6 +641,7 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
@@ -806,6 +816,11 @@ k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/A
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+pault.ag/go/debian v0.12.0 h1:b8ctSdBSGJ98NE1VLn06aSx70EUpczlP2qqSHEiYYJA=
+pault.ag/go/debian v0.12.0/go.mod h1:UbnMr3z/KZepjq7VzbYgBEfz8j4+Pyrm2L5X1fzhy/k=
+pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a/go.mod h1:INqx0ClF7kmPAMk2zVTX8DRnhZ/yaA/Mg52g8KFKE7k=
+pault.ag/go/topsort v0.1.1 h1:L0QnhUly6LmTv0e3DEzbN2q6/FGgAcQvaEw65S53Bg4=
+pault.ag/go/topsort v0.1.1/go.mod h1:r1kc/L0/FZ3HhjezBIPaNVhkqv8L0UJ9bxRuHRVZ0q4=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/release-utils v0.8.4 h1:4QVr3UgbyY/d9p74LBhg0njSVQofUsAZqYOzVZBhdBw=

policy/rego.go

@@ -1,12 +1,14 @@
 package policy
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 
+	"github.com/docker-library/bashbrew/manifest"
 	"github.com/docker/attest/attestation"
 	intoto "github.com/in-toto/in-toto-golang/in_toto"
 	"github.com/open-policy-agent/opa/ast"
@@ -149,6 +151,12 @@ var attestDecl = &ast.Builtin{
 	Nondeterministic: true,
 }
 
+var internalParseLibraryDefinitionDecl = &ast.Builtin{
+	Name:             "attest.internals.parse_library_definition",
+	Decl:             types.NewFunction(types.Args(types.S), dynamicObj),
+	Nondeterministic: false,
+}
+
 func wrapFunctionResult(value *ast.Term, err error) (*ast.Term, error) {
 	var terms [][2]*ast.Term
 	if err != nil {
@@ -174,28 +182,50 @@ func handleErrors2(f func(rCtx rego.BuiltinContext, a, b *ast.Term) (*ast.Term,
 
 func RegoFunctions(regoOpts *RegoFnOpts) []*tester.Builtin {
 	return []*tester.Builtin{
-		{
-			Decl: verifyDecl,
-			Func: rego.Function2(
-				&rego.Function{
-					Name:             verifyDecl.Name,
-					Decl:             verifyDecl.Decl,
-					Memoize:          true,
-					Nondeterministic: verifyDecl.Nondeterministic,
-				},
-				handleErrors2(regoOpts.verifyInTotoEnvelope)),
-		},
-		{
-			Decl: attestDecl,
-			Func: rego.Function1(
-				&rego.Function{
-					Name:             attestDecl.Name,
-					Decl:             attestDecl.Decl,
-					Memoize:          true,
-					Nondeterministic: attestDecl.Nondeterministic,
-				},
-				handleErrors1(regoOpts.fetchInTotoAttestations)),
-		},
+		builtin2(verifyDecl, regoOpts.verifyInTotoEnvelope),
+		builtin1(attestDecl, regoOpts.fetchInTotoAttestations),
+		builtin1(internalParseLibraryDefinitionDecl, regoOpts.internalParseLibraryDefinition),
+	}
+}
+
+func builtin1(decl *ast.Builtin, f rego.Builtin1) *tester.Builtin {
+	return &tester.Builtin{
+		Decl: decl,
+		Func: rego.Function1(
+			&rego.Function{
+				Name:             decl.Name,
+				Decl:             decl.Decl,
+				Memoize:          true,
+				Nondeterministic: decl.Nondeterministic,
+			},
+			handleErrors1(f)),
+	}
+}
+
+func builtin2(decl *ast.Builtin, f rego.Builtin2) *tester.Builtin {
+	return &tester.Builtin{
+		Decl: decl,
+		Func: rego.Function2(
+			&rego.Function{
+				Name:             decl.Name,
+				Decl:             decl.Decl,
+				Memoize:          true,
+				Nondeterministic: decl.Nondeterministic,
+			},
+			handleErrors2(f)),
+	}
+}
+
+type RegoFnOpts struct {
+	attestationResolver attestation.Resolver
+	attestationVerifier attestation.Verifier
+}
+
+// this is exported for testing here and in clients of the library.
+func NewRegoFunctionOptions(resolver attestation.Resolver, verifier attestation.Verifier) *RegoFnOpts {
+	return &RegoFnOpts{
+		attestationResolver: resolver,
+		attestationVerifier: verifier,
 	}
 }
 
@@ -229,19 +259,6 @@ func (regoOpts *RegoFnOpts) fetchInTotoAttestations(rCtx rego.BuiltinContext, pr
 	return set, nil
 }
 
-type RegoFnOpts struct {
-	attestationResolver attestation.Resolver
-	attestationVerifier attestation.Verifier
-}
-
-// this is exported for testing here and in clients of the library.
-func NewRegoFunctionOptions(resolver attestation.Resolver, verifier attestation.Verifier) *RegoFnOpts {
-	return &RegoFnOpts{
-		attestationResolver: resolver,
-		attestationVerifier: verifier,
-	}
-}
-
 // because we don't control the signature here (blame rego)
 // nolint:gocritic
 func (regoOpts *RegoFnOpts) verifyInTotoEnvelope(rCtx rego.BuiltinContext, envTerm, optsTerm *ast.Term) (*ast.Term, error) {
@@ -285,6 +302,26 @@ func (regoOpts *RegoFnOpts) verifyInTotoEnvelope(rCtx rego.BuiltinContext, envTe
 	return ast.NewTerm(value), nil
 }
 
+// because we don't control the signature here (blame rego)
+// nolint:gocritic
+func (regoOpts *RegoFnOpts) internalParseLibraryDefinition(_ rego.BuiltinContext, definitionTerm *ast.Term) (*ast.Term, error) {
+	definitionStr, ok := definitionTerm.Value.(ast.String)
+	if !ok {
+		return nil, fmt.Errorf("predicateTypeTerm is not a string")
+	}
+	definition := string(definitionStr)
+	defBuffer := bytes.NewBufferString(definition)
+	parsed, err := manifest.Parse2822(defBuffer)
+	if err != nil {
+		return nil, err
+	}
+	value, err := ast.InterfaceToValue(parsed)
+	if err != nil {
+		return nil, err
+	}
+	return ast.NewTerm(value), nil
+}
+
 func loadYAML(path string, bs []byte) (interface{}, error) {
 	var x interface{}
 	bs, err := yaml.YAMLToJSON(bs)

policy/rego_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestPolicy(t *testing.T) {
-	paths := []string{"testdata/policies/test"}
+	paths := []string{"testdata/policies/test/fetch"}
 	modules, store, err := tester.Load(paths, nil)
 	require.NoError(t, err)
 	resolver := &NullAttestationResolver{}
@@ -35,6 +35,30 @@ func TestPolicy(t *testing.T) {
 	assert.True(t, resolver.called)
 }
 
+func TestPolicyDefParse(t *testing.T) {
+	paths := []string{"testdata/policies/test/def_parse"}
+	modules, store, err := tester.Load(paths, nil)
+	require.NoError(t, err)
+	resolver := &NullAttestationResolver{}
+
+	opts := NewRegoFunctionOptions(resolver, nil)
+	ctx := context.Background()
+	ch, err := tester.NewRunner().
+		SetStore(store).
+		AddCustomBuiltins(RegoFunctions(opts)).
+		CapturePrintOutput(true).
+		RaiseBuiltinErrors(true).
+		EnableTracing(true).
+		SetModules(modules).
+		RunTests(ctx, nil)
+	require.NoError(t, err)
+	require.NoError(t, err)
+	results := buffer(ch)
+	t.Log(string(results[0].Output))
+	assert.Equalf(t, 1, len(results), "expected 1 results, got %d", len(results))
+	assert.Truef(t, results[0].Pass(), "expected result 1 to pass, got %v", results[0].Location)
+}
+
 func buffer[T any](ch chan T) []T {
 	var out []T
 	for v := range ch {

policy/testdata/policies/test/def_parse/def_parse_test.rego

@@ -0,0 +1,18 @@
+package def_parse_test
+
+import rego.v1
+
+test_parse_library_definition if {
+	def := `Maintainers: me <me@example.com> (@me)
+GitRepo: blah
+
+Tags: 1, 2, 3
+GitCommit: fa105cb3c26c8f0e87d7dbb1bf5293691ac2f688
+File: Dockerfile.foo`
+	result := attest.internals.parse_library_definition(def)
+	definition := result.value
+	definition.Entries[0].GitRepo == "blah"
+	definition.Entries[0].GitCommit == "fa105cb3c26c8f0e87d7dbb1bf5293691ac2f688"
+	definition.Entries[0].Tags == ["1", "2", "3"]
+	definition.Entries[0].File == "Dockerfile.foo"
+}

policy/testdata/policies/test/fetch.rego renamed to policy/testdata/policies/test/fetch/fetch.rego

@@ -0,0 +1,9 @@
+package attest_test
+
+import rego.v1
+
+import data.attest
+
+test_sucess if {
+	attest.success
+}