diff --git a/.travis.yml b/.travis.yml index 5b6a69a..874f8a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,20 +6,17 @@ services: - docker env: global: - - CGO_ENABLED=0 - FLYWAY_VERSION=9.4.0 - INPUT_BUILDARGS=FLYWAY_VERSION=$FLYWAY_VERSION before_install: # Requirement for 'test-local-deployment' - pip install --user awscli - export PATH=$PATH:$HOME/.local/bin -before_script: - - _script/start-pg gobuild_args: -a -tags netgo -ldflags '-w' go_import_path: github.com/adevinta/vulnerability-db script: - go install ./... - - go test -v -tags integration $(go list ./... | grep -v /vendor/) ./test + - _script/test after_success: - bash -c 'source <(curl -s https://raw.githubusercontent.com/adevinta/vulcan-cicd/master/docker.sh)' - cd local_deployment diff --git a/Dockerfile b/Dockerfile index 7e20617..032fffc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,10 @@ FROM golang:1.19.3-alpine3.15 as builder +# Required because the dependency +# https://github.com/confluentinc/confluent-kafka-go requires the gcc compiler. +RUN apk add gcc libc-dev + WORKDIR /app COPY go.mod . @@ -11,7 +15,9 @@ RUN go mod download COPY . . -RUN cd cmd/vulnerability-db-consumer/ && GOOS=linux GOARCH=amd64 go build . && cd - +# -tags musl argument is required for dependency github.com/confluentinc/confluent-kafka-go. +# see documentation: https://github.com/confluentinc/confluent-kafka-go#using-go-modules +RUN cd cmd/vulnerability-db-consumer/ && GOOS=linux GOARCH=amd64 go build -tags musl . && cd - FROM alpine:3.16.3 diff --git a/README.md b/README.md index 6dc7e40..6d4ab9b 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,12 @@ cd db && source flyway-migrate.sh && cd - vulnerability-db-consumer -c _resources/config/local.toml ``` +## How to generate AsyncAPI documentation +Generated [AsyncAPI documentation](https://www.asyncapi.com/) can be found in `./docs` directory. +``` +cd pkg/asyncapi/_gen && ./gen.sh && cd - +``` + ## How to run the Vulnerability DB in development mode You can test the Vulnerability DB Consumer locally in your machine. @@ -78,13 +84,19 @@ Those are the variables you have to use: |PG_PORT|Database port|5432| |PG_SSLMODE|One of these (disable,allow,prefer,require,verify-ca,verify-full)|disable| |PG_CA_B64|A base64 encoded CA certificate|| -|SQS_NUMBER_OF_PROCESSORS|Number of concurrent SQS processors|Default: 10| -|SQS_QUEUE_ARN|Checks queueu ARN|arn:aws:sqs:xxx:123456789012:yyy| -|SNS_TOPIC_ARN|ARN of topic to publish new vulnerabilities|arn:aws:sns:xxx:123456789012:yyy| |RESULTS_URL|External vulcan-results URL|https://results.vulcan.com| |RESULTS_INTERNAL_URL|Internal vulcan-results URL|http://vulcan-results| +|SQS_QUEUE_ARN|Checks queueu ARN|arn:aws:sqs:xxx:123456789012:yyy| +|SQS_NUMBER_OF_PROCESSORS|Number of concurrent SQS processors|Default: 10| |AWS_SQS_ENDPOINT|Endpoint for SQS creation queue (optional)|http://custom-aws-endpoint| +|SNS_ENABLED|Enables/Disables notifications sent to SNS|false| +|SNS_TOPIC_ARN|ARN of topic to publish new vulnerabilities|arn:aws:sns:xxx:123456789012:yyy| |AWS_SNS_ENDPOINT|Endpoint for SNS topic (optional)|http://custom-aws-endpoint| +|KAFKA_ENABLED|Enables/Disables notifications sent to Kafka|false| +|KAFKA_USER|Kafka user|| +|KAFKA_PASSWORD|Kafka password|| +|KAFKA_BROKER_URL|Kafka Broker URL|localhost:9092| +|KAFKA_TOPIC|Kafka topic|findings| ```bash docker build . -t vdb diff --git a/_resources/config/local.toml.example b/_resources/config/local.toml.example index c7b1b4f..28baad8 100644 --- a/_resources/config/local.toml.example +++ b/_resources/config/local.toml.example @@ -18,8 +18,15 @@ timeout = 30 queue_arn = "arn:aws:sqs:xxx:123456789012:yyy" [sns] +enabled = false topic_arn = "arn:aws:sns:xxx:123456789012:yyy" -enabled = true + +[kafka] +enabled = false +user = "user" +password = "password" +broker_url = "localhost:9092" +topic = "findings" [report] url_replace = "https://results.vulcan.example.com|http://localhost:8081" diff --git a/_script/start-pg b/_script/start-pg deleted file mode 100755 index 5c6e8cc..0000000 --- a/_script/start-pg +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Adevinta - -set -e - -docker run --name postgres -p 5432:5432 -e POSTGRES_USER=vulndb_test -e POSTGRES_PASSWORD=vulndb_test --rm -d postgres:13.3-alpine - -sleep 2 - -while ! docker exec -it postgres pg_isready; do echo "Waiting for postgres" && sleep 2; done; diff --git a/_script/test_local b/_script/test similarity index 100% rename from _script/test_local rename to _script/test diff --git a/cmd/vulnerability-db-consumer/config.go b/cmd/vulnerability-db-consumer/config.go index b9b0af5..6033077 100644 --- a/cmd/vulnerability-db-consumer/config.go +++ b/cmd/vulnerability-db-consumer/config.go @@ -19,6 +19,7 @@ type config struct { DB dbConfig SQS sqsConfig SNS snsConfig + Kafka kafkaConfig Report reportConfig Maintenance maintenanceConfig } @@ -38,19 +39,27 @@ type dbConfig struct { } type sqsConfig struct { - NProcessors uint8 `toml:"number_of_processors"` - WaitTime uint8 `toml:"wait_time"` - Timeout uint8 + NProcessors uint `toml:"number_of_processors"` + WaitTime uint `toml:"wait_time"` + Timeout uint QueueARN string `toml:"queue_arn"` Endpoint string `toml:"endpoint"` } type snsConfig struct { - TopicARN string `toml:"topic_arn"` Enabled bool + TopicARN string `toml:"topic_arn"` Endpoint string `toml:"endpoint"` } +type kafkaConfig struct { + Enabled bool `toml:"enabled"` + User string `toml:"user"` + Pass string `toml:"password"` + BrokerURL string `toml:"broker_url"` + Topic string `toml:"topic"` +} + type reportConfig struct { URLReplace string `toml:"url_replace"` } diff --git a/cmd/vulnerability-db-consumer/main.go b/cmd/vulnerability-db-consumer/main.go index e48c950..099a697 100644 --- a/cmd/vulnerability-db-consumer/main.go +++ b/cmd/vulnerability-db-consumer/main.go @@ -11,6 +11,7 @@ import ( "os" "sync" + "github.com/adevinta/vulnerability-db/pkg/asyncapi/kafka" "github.com/adevinta/vulnerability-db/pkg/maintenance" "github.com/adevinta/vulnerability-db/pkg/notify" "github.com/adevinta/vulnerability-db/pkg/processor" @@ -43,14 +44,9 @@ func main() { } // Build notifier. - snsConf := notify.SNSConfig{ - TopicArn: conf.SNS.TopicARN, - Enabled: conf.SNS.Enabled, - Endpoint: conf.SNS.Endpoint, - } - snsNotifier, err := notify.NewSNSNotifier(snsConf, logger) + notifier, err := buildNotifier(conf, logger) if err != nil { - log.Fatalf("Error creating notifier: %v", err) + log.Fatalf("Error building notifier: %v", err) } // Build processor. @@ -59,7 +55,7 @@ func main() { log.Fatalf("Error creating results client: %v", err) } - processor, err := processor.NewCheckProcessor(snsNotifier, db, resultsClient, conf.Report.URLReplace, conf.MaxEventAge, logger) + processor, err := processor.NewCheckProcessor(notifier, db, resultsClient, conf.Report.URLReplace, conf.MaxEventAge, logger) if err != nil { log.Fatalf("Error creating queue processor: %v", err) } @@ -97,6 +93,52 @@ func main() { wg.Wait() } +// buildNotifier builds the appropiate notifier given the defined configuration. +// TODO: Once the integrations dependent on the old notification format have been +// deprecated or updated to comply with the new format through Kafka topic channel +// we can get rid of SNS and multi implementations of notifier and just use Kafka. +func buildNotifier(conf *config, logger *log.Logger) (notify.Notifier, error) { + if !conf.SNS.Enabled && !conf.Kafka.Enabled { + logger.Info("using noop notifier") + return notify.NewNoopNotifier(), nil + } + if conf.SNS.Enabled && !conf.Kafka.Enabled { + logger.Info("using SNS notifier") + return buildSNSNotifier(conf, logger) + } + if !conf.SNS.Enabled && conf.Kafka.Enabled { + logger.Info("using Kafka notifier") + return buildKafkaNotifier(conf, logger) + } + // Multi Notifier + logger.Info("using multi notifier") + k, err := buildKafkaNotifier(conf, logger) + if err != nil { + return nil, err + } + s, err := buildSNSNotifier(conf, logger) + if err != nil { + return nil, err + } + return notify.NewMultiNotifier(k, s), nil +} + +func buildSNSNotifier(conf *config, logger *log.Logger) (*notify.SNSNotifier, error) { + return notify.NewSNSNotifier(notify.SNSConfig{ + TopicArn: conf.SNS.TopicARN, + Endpoint: conf.SNS.Endpoint, + }, logger) +} + +func buildKafkaNotifier(conf *config, logger *log.Logger) (*notify.KafkaNotifier, error) { + kafkaCli, err := kafka.NewClient(conf.Kafka.User, conf.Kafka.Pass, + conf.Kafka.BrokerURL, conf.Kafka.Topic) + if err != nil { + return nil, err + } + return notify.NewKafkaNotifier(kafkaCli, logger), nil +} + func setupLogger(cfg config) *log.Logger { var logger = log.New() diff --git a/config.toml b/config.toml index 5fbea1c..cd82f0a 100644 --- a/config.toml +++ b/config.toml @@ -17,15 +17,22 @@ name = "$PG_NAME" [sqs] number_of_processors = $SQS_NUMBER_OF_PROCESSORS wait_time = 20 -timeout = 30 +timeout = 60 queue_arn = "$SQS_QUEUE_ARN" endpoint = "$AWS_SQS_ENDPOINT" [sns] +enabled = $SNS_ENABLED topic_arn = "$SNS_TOPIC_ARN" -enabled = true endpoint = "$AWS_SNS_ENDPOINT" +[kafka] +enabled = $KAFKA_ENABLED +user = "$KAFKA_USER" +password = "$KAFKA_PASSWORD" +broker_url = "$KAFKA_BROKER_URL" +topic = "$KAFKA_TOPIC" + [report] url_replace = "$RESULTS_URL|$RESULTS_INTERNAL_URL" diff --git a/docs/asyncapi.yaml b/docs/asyncapi.yaml new file mode 100644 index 0000000..59079af --- /dev/null +++ b/docs/asyncapi.yaml @@ -0,0 +1,141 @@ +asyncapi: 2.4.0 +info: + title: Vulnerability DB + version: v0.0.1 +servers: + production: + url: broker.example.com + description: Dummy server + protocol: kafka +channels: + findings: + subscribe: + message: + $ref: '#/components/messages/FindingPayload' +components: + schemas: + FindingPayload: + properties: + affected_resource: + type: string + current_exposure: + type: integer + details: + type: string + id: + type: string + impact_details: + type: string + issue: + $ref: '#/components/schemas/StoreIssueLabels' + resources: + $ref: '#/components/schemas/StoreResources' + score: + type: number + source: + $ref: '#/components/schemas/StoreSource' + status: + type: string + target: + $ref: '#/components/schemas/StoreTargetTeams' + total_exposure: + type: integer + type: object + PqStringArray: + items: + type: string + type: + - array + - "null" + StoreIssueLabels: + properties: + cwe_id: + minimum: 0 + type: integer + description: + type: string + id: + type: string + labels: + items: + type: string + type: + - array + - "null" + recommendations: + $ref: '#/components/schemas/PqStringArray' + reference_links: + $ref: '#/components/schemas/PqStringArray' + summary: + type: string + type: object + StoreResourceGroup: + properties: + attributes: + items: + type: string + type: + - array + - "null" + name: + type: string + resources: + items: + additionalProperties: + type: string + type: object + type: + - array + - "null" + type: object + StoreResources: + items: + $ref: '#/components/schemas/StoreResourceGroup' + type: + - array + - "null" + StoreSource: + properties: + component: + type: string + id: + type: string + instance: + type: string + name: + type: string + options: + type: string + time: + format: date-time + type: string + type: object + StoreTargetTeams: + properties: + id: + type: string + identifier: + type: string + teams: + items: + type: string + type: + - array + - "null" + type: object + messages: + FindingPayload: + contentType: application/json + headers: + properties: + version: + description: schema version header + type: string + required: + - version + type: object + payload: + $ref: '#/components/schemas/FindingPayload' + summary: Events generated from Vulnerability DB findings state changes + name: Finding + title: Findings state diff --git a/go.mod b/go.mod index 1fd4c9e..9e709fe 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/BurntSushi/toml v0.4.1 github.com/adevinta/vulcan-report v1.0.0 github.com/aws/aws-sdk-go v1.44.98 - github.com/google/go-cmp v0.5.6 + github.com/confluentinc/confluent-kafka-go v1.9.2 + github.com/google/go-cmp v0.5.9 github.com/jmoiron/sqlx v1.3.4 github.com/lib/pq v1.10.3 github.com/sirupsen/logrus v1.8.1 @@ -14,6 +15,7 @@ require ( require ( github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/stretchr/testify v1.8.0 // indirect golang.org/x/sys v0.0.0-20220913175220-63ea55921009 // indirect - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 15d2dcd..10b1140 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,253 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/actgardner/gogen-avro/v10 v10.1.0/go.mod h1:o+ybmVjEa27AAr35FRqU98DJu1fXES56uXniYFv4yDA= +github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= +github.com/actgardner/gogen-avro/v9 v9.1.0/go.mod h1:nyTj6wPqDJoxM3qdnjcLv+EnMDSDFqE0qDpva2QRmKc= github.com/adevinta/vulcan-report v1.0.0 h1:44aICPZ+4svucgCSA5KmjlT3ZGzrvZXiSnkbnj6AC2k= github.com/adevinta/vulcan-report v1.0.0/go.mod h1:k34KaeoXc3H77WNMwI9F4F1G28hBjB95PeMUp9oHbEE= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go v1.44.98 h1:fX+NxebSdO/9T6DTNOLhpC+Vv6RNkKRfsMg0a7o/yBo= github.com/aws/aws-sdk-go v1.44.98/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/confluentinc/confluent-kafka-go v1.9.2 h1:gV/GxhMBUb03tFWkN+7kdhg+zf+QUM+wVkI9zwh770Q= +github.com/confluentinc/confluent-kafka-go v1.9.2/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8= +github.com/heetch/avro v0.3.1/go.mod h1:4xn38Oz/+hiEUTpbVfGVLfvOg0yKLlRP7Q9+gJJILgA= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= +github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= +github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= +github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220913175220-63ea55921009 h1:PuvuRMeLWqsf/ZdT1UUZz0syhioyv1mzuFZsXs4fvhw= golang.org/x/sys v0.0.0-20220913175220-63ea55921009/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/local.env.example b/local.env.example index 077f6b7..f09722f 100644 --- a/local.env.example +++ b/local.env.example @@ -11,9 +11,17 @@ PG_NAME=vulndb SQS_NUMBER_OF_PROCESSORS=1 SQS_QUEUE_ARN=arn:aws:sqs:xxx:123456789012:yyy -SNS_TOPIC_ARN=arn:aws:sns:xxx:123456789012:yyy AWS_SQS_ENDPOINT= + +SNS_ENABLED=false +SNS_TOPIC_ARN=arn:aws:sns:xxx:123456789012:yyy AWS_SNS_ENDPOINT= +KAFKA_ENABLED=false +KAFKA_USER= +KAFKA_PASSWORD= +KAFKA_BROKER_URL=localhost:29092 +KAFKA_TOPIC=findings + RESULTS_URL=http://vulcan-results RESULTS_INTERNAL_URL=http://vulcan-results diff --git a/local_deployment/docker-compose.yml b/local_deployment/docker-compose.yml index 6357388..1eb1124 100644 --- a/local_deployment/docker-compose.yml +++ b/local_deployment/docker-compose.yml @@ -19,6 +19,25 @@ services: - 4100:4100 volumes: - ./mocked_sns_sqs:/conf + + zookeeper: + image: confluentinc/cp-zookeeper:7.2.1 + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + kafka: + image: confluentinc/cp-kafka:7.2.1 + depends_on: + - zookeeper + ports: + - 29092:29092 + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 flyway: image: flyway/flyway:8.0.2-alpine diff --git a/local_deployment/local_deployment.toml b/local_deployment/local_deployment.toml index 6fc94d1..0332d1c 100644 --- a/local_deployment/local_deployment.toml +++ b/local_deployment/local_deployment.toml @@ -29,6 +29,13 @@ endpoint = "http://localhost:4100" [report] url_replace = "http://localhost:8080|http://localhost:8080" +[kafka] +enabled = false +user = "user" +password = "password" +broker_url = "localhost:9092" +topic = "findings" + [maintenance] # periodical tasks executed in background # - name: Custom task name diff --git a/pkg/asyncapi/_gen/gen.go b/pkg/asyncapi/_gen/gen.go new file mode 100644 index 0000000..7b35dbc --- /dev/null +++ b/pkg/asyncapi/_gen/gen.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + + "github.com/adevinta/vulnerability-db/pkg/notify" + + "github.com/swaggest/go-asyncapi/reflector/asyncapi-2.4.0" + "github.com/swaggest/go-asyncapi/spec-2.4.0" +) + +type FindingPayload struct { + notify.FindingNotification + Version string `header:"version" description:"schema version header" required:"true"` +} + +func main() { + asyncAPI := spec.AsyncAPI{} + asyncAPI.Info.Version = notify.Version + asyncAPI.Info.Title = "Vulnerability DB" + + asyncAPI.AddServer("production", spec.Server{ + URL: "broker.example.com", + Description: "Dummy server", + Protocol: "kafka", + }) + + reflector := asyncapi.Reflector{} + reflector.Schema = &asyncAPI + + mustNotFail(reflector.AddChannel(asyncapi.ChannelInfo{ + Name: "findings", + Subscribe: &asyncapi.MessageSample{ + MessageEntity: spec.MessageEntity{ + Name: "Finding", + Title: "Findings state", + Summary: "Events generated from Vulnerability DB findings state changes", + ContentType: "application/json", + }, + MessageSample: new(FindingPayload), + }, + })) + + yaml, err := reflector.Schema.MarshalYAML() + mustNotFail(err) + mustNotFail(os.WriteFile("../../../docs/asyncapi.yaml", yaml, 0o600)) +} + +func mustNotFail(err error) { + if err != nil { + panic(err) + } +} diff --git a/pkg/asyncapi/_gen/gen.sh b/pkg/asyncapi/_gen/gen.sh new file mode 100755 index 0000000..c4fa673 --- /dev/null +++ b/pkg/asyncapi/_gen/gen.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Copyright 2022 Adevinta + +set -eu + +go run gen.go diff --git a/pkg/asyncapi/_gen/go.mod b/pkg/asyncapi/_gen/go.mod new file mode 100644 index 0000000..fd49799 --- /dev/null +++ b/pkg/asyncapi/_gen/go.mod @@ -0,0 +1,22 @@ +module github.com/adevinta/vulnerability-db/pkg/asyncapi/_gen + +go 1.19 + +require ( + github.com/adevinta/vulnerability-db v1.1.5 + github.com/swaggest/go-asyncapi v0.8.0 +) + +require ( + github.com/aws/aws-sdk-go v1.44.98 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jmoiron/sqlx v1.3.4 // indirect + github.com/lib/pq v1.10.3 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/swaggest/jsonschema-go v0.3.39 // indirect + github.com/swaggest/refl v1.1.0 // indirect + golang.org/x/sys v0.0.0-20220913175220-63ea55921009 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +replace github.com/adevinta/vulnerability-db => ../../../ diff --git a/pkg/asyncapi/_gen/go.sum b/pkg/asyncapi/_gen/go.sum new file mode 100644 index 0000000..e3f706d --- /dev/null +++ b/pkg/asyncapi/_gen/go.sum @@ -0,0 +1,55 @@ +github.com/aws/aws-sdk-go v1.44.98 h1:fX+NxebSdO/9T6DTNOLhpC+Vv6RNkKRfsMg0a7o/yBo= +github.com/aws/aws-sdk-go v1.44.98/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/bool64/dev v0.2.20 h1:9eIRGdcg2kQW2NGza++QbOKidNNaK+KfWuUXcZFDejE= +github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= +github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/swaggest/assertjson v1.7.0 h1:SKw5Rn0LQs6UvmGrIdaKQbMR1R3ncXm5KNon+QJ7jtw= +github.com/swaggest/go-asyncapi v0.8.0 h1:aze7YL3o/4fkxx9ZL8ubKdyYCqFJy+9+JHpwLyF10H4= +github.com/swaggest/go-asyncapi v0.8.0/go.mod h1:s1urrZuYPcNPJrRUU/kF1eOnIBU02O4JfXCDS09wONI= +github.com/swaggest/jsonschema-go v0.3.39 h1:qX+a/LiK44xppnwUTZqJWNFcICxdWKeT27aFh4RvkEk= +github.com/swaggest/jsonschema-go v0.3.39/go.mod h1:ipIOmoFP64QyRUgyPyU/P9tayq2m2TlvUhyZHrhe3S4= +github.com/swaggest/refl v1.1.0 h1:a+9a75Kv6ciMozPjVbOfcVTEQe81t2R3emvaD9oGQGc= +github.com/swaggest/refl v1.1.0/go.mod h1:g3Qa6ki0A/L2yxiuUpT+cuBURuRaltF5SDQpg1kMZSY= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220913175220-63ea55921009 h1:PuvuRMeLWqsf/ZdT1UUZz0syhioyv1mzuFZsXs4fvhw= +golang.org/x/sys v0.0.0-20220913175220-63ea55921009/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/asyncapi/kafka/client.go b/pkg/asyncapi/kafka/client.go new file mode 100644 index 0000000..32b6d14 --- /dev/null +++ b/pkg/asyncapi/kafka/client.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 Adevinta +*/ + +package kafka + +import ( + "errors" + "fmt" + + "github.com/confluentinc/confluent-kafka-go/kafka" +) + +var ( + // ErrEmptyPayload is returned by the Push method of the [Client] when the + // given payload is empty. + ErrEmptyPayload = errors.New("payload can't be empty") +) + +const ( + kafkaSecurityProtocol = "sasl_ssl" + kafkaSaslMechanisms = "SCRAM-SHA-256" +) + +// Client implements an EventStreamClient using Kafka as the event stream +// system. +type Client struct { + producer *kafka.Producer + topic string +} + +// NewClient creates a new Kafka client connected to the a broker using the +// given credentials and setting the mapping between all the entities and their +// corresponding topics. +func NewClient(user string, password string, broker string, topic string) (*Client, error) { + config := kafka.ConfigMap{ + "bootstrap.servers": broker, + } + if password != "" { + config.SetKey("security.protocol", kafkaSecurityProtocol) + config.SetKey("sasl.mechanisms", kafkaSaslMechanisms) + config.SetKey("sasl.username", user) + config.SetKey("sasl.password", password) + } + p, err := kafka.NewProducer(&config) + if err != nil { + return &Client{}, err + } + return &Client{p, topic}, nil +} + +// Push sends the payload of an entity, with the specified id, to corresponding +// topic according to the specified entity, using the kafka broker the client +// is connected to. The method waits until kafka confirms the message has been +// stored in the topic. +func (c *Client) Push(id string, payload []byte, metadata map[string][]byte) error { + if len(payload) == 0 { + return ErrEmptyPayload + } + + delivered := make(chan kafka.Event) + defer close(delivered) + + var headers []kafka.Header + for k, v := range metadata { + headers = append(headers, kafka.Header{ + Key: k, + Value: v, + }) + } + msg := kafka.Message{ + TopicPartition: kafka.TopicPartition{ + Topic: &c.topic, + Partition: kafka.PartitionAny, + }, + Key: []byte(id), + Value: []byte(payload), + Headers: headers, + } + + err := c.producer.Produce(&msg, delivered) + if err != nil { + return fmt.Errorf("error producing message: %w", err) + } + e := <-delivered + m := e.(*kafka.Message) + if m.TopicPartition.Error != nil { + return fmt.Errorf("error delivering message: %w", m.TopicPartition.Error) + } + return nil +} diff --git a/pkg/notify/kafka.go b/pkg/notify/kafka.go new file mode 100644 index 0000000..b83442d --- /dev/null +++ b/pkg/notify/kafka.go @@ -0,0 +1,54 @@ +/* +Copyright 2022 Adevinta +*/ + +package notify + +import ( + "encoding/json" + + log "github.com/sirupsen/logrus" +) + +const ( + Version = "v0.0.1" // Defines the notification schema version +) + +// EventStreamClient represent a client of an event stream system, like Kafka +// or AWS FIFO SQS queues. +type EventStreamClient interface { + Push(id string, payload []byte, metadata map[string][]byte) error +} + +// KafkaNotifier represents a Notifier implementation to send notifications to +// a Kafka topic. +type KafkaNotifier struct { + c EventStreamClient + l *log.Logger +} + +// NewKafkaNotifier creates a new KafkaNotifier. +func NewKafkaNotifier(c EventStreamClient, l *log.Logger) *KafkaNotifier { + return &KafkaNotifier{ + c, + l, + } +} + +// PushFinding sends the given FindingNotification to the configured Kafka topic +// in the wrapped EventStreamClient. +func (k *KafkaNotifier) PushFinding(f FindingNotification) error { + k.l.WithFields(log.Fields{ + "notifier": "kafka", + "id": f.ID, + }).Debug("pushing finding notification") + + payload, err := json.Marshal(f) + if err != nil { + return err + } + meta := map[string][]byte{ + "version": []byte(Version), + } + return k.c.Push(f.ID, payload, meta) +} diff --git a/pkg/notify/kafka_test.go b/pkg/notify/kafka_test.go new file mode 100644 index 0000000..890e670 --- /dev/null +++ b/pkg/notify/kafka_test.go @@ -0,0 +1,242 @@ +package notify + +import ( + "errors" + "fmt" + "reflect" + "testing" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + + "github.com/adevinta/vulnerability-db/pkg/store" +) + +var ( + mockEventStreamClientErr = errors.New("mockEventStreamErr") +) + +type event struct { + id string + payload []byte + metadata map[string][]byte +} + +func (e event) String() string { + return fmt.Sprintf(` + { + "id": "%s", + "payload": "%s", + "metadata": "%v" + }`, e.id, string(e.payload), e.metadata) +} + +type mockEventStreamClient struct { + EventStreamClient + err error + e event +} + +func (m *mockEventStreamClient) Push(id string, payload []byte, metadata map[string][]byte) error { + if m.err != nil { + return m.err + } + m.e = event{ + id, + payload, + metadata, + } + return nil +} + +func (m *mockEventStreamClient) verify(want event) bool { + return reflect.DeepEqual(m.e, want) +} + +func TestKafka_PushFinding(t *testing.T) { + type fields struct { + logger *log.Logger + eventStreamCli *mockEventStreamClient + } + + logger := log.New() + + testCases := []struct { + name string + fields fields + f FindingNotification + want event + wantErr error + }{ + { + // Note: Define the whole FindingNotification struct and expected JSON payload + // so test fails if a modification is made to the notification struct fields or + // JSON export definition, so we are explicitly aware of a version change. + name: "Should send notification with version", + fields: fields{ + logger: logger, + eventStreamCli: &mockEventStreamClient{}, + }, + f: FindingNotification{ + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + ID: "FindingID-1", + AffectedResource: "AffectedResource-1", + Score: 9, + Status: "OPEN", + Details: "Details-1", + ImpactDetails: "ImpactDetails-1", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + ID: "IssueID-1", + Summary: "Summary-1", + CWEID: 1, + Description: "Description-1", + Recommendations: pq.StringArray{ + "Recommendation-1", + "Recommendation-2", + }, + ReferenceLinks: pq.StringArray{ + "ReferenceLink-1", + "ReferenceLink-2", + }, + }, + Labels: []string{ + "Label-1", + "Label-2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + ID: "TargetID-1", + Identifier: "Identifier-1", + }, + Teams: []string{ + "Team-1", + "Team-2", + }, + }, + Source: store.Source{ + ID: "SourceID-1", + Instance: "SourceInstance-1", + Options: "SourceOptions-1", + SourceFamily: store.SourceFamily{ + Name: "SourceName-1", + Component: "SourceComponent-1", + }, + }, + Resources: store.Resources{ + { + Name: "ResourceName-1", + Attributes: []string{ + "Attr-1", + "Attr-2", + }, + Resources: []map[string]string{ + { + "Attr-1": "1", + "Attr-2": "2", + }, + }, + }, + }, + TotalExposure: 10, + CurrentExposure: 5, + }, + Tag: "tag-1", + }, + want: event{ + id: "FindingID-1", + payload: []byte(removeSpaces(` + { + "id": "FindingID-1", + "affected_resource": "AffectedResource-1", + "score": 9, + "status": "OPEN", + "details": "Details-1", + "impact_details": "ImpactDetails-1", + "issue": { + "id": "IssueID-1", + "summary": "Summary-1", + "cwe_id": 1, + "description": "Description-1", + "recommendations": [ + "Recommendation-1", + "Recommendation-2" + ], + "reference_links": [ + "ReferenceLink-1", + "ReferenceLink-2" + ], + "labels": [ + "Label-1", + "Label-2" + ] + }, + "target": { + "id": "TargetID-1", + "identifier": "Identifier-1", + "teams": [ + "Team-1", + "Team-2" + ] + }, + "source": { + "id": "SourceID-1", + "instance": "SourceInstance-1", + "options": "SourceOptions-1", + "time": "0001-01-01T00:00:00Z", + "name": "SourceName-1", + "component": "SourceComponent-1" + }, + "resources": [ + { + "name": "ResourceName-1", + "attributes": [ + "Attr-1", + "Attr-2" + ], + "resources": [ + { + "Attr-1": "1", + "Attr-2": "2" + } + ] + } + ], + "total_exposure": 10, + "current_exposure": 5 + } + `)), + metadata: map[string][]byte{ + "version": []byte(Version), + }, + }, + }, + { + name: "Should propagate eventStreamClient error", + fields: fields{ + logger: logger, + eventStreamCli: &mockEventStreamClient{ + err: mockEventStreamClientErr, + }, + }, + wantErr: mockEventStreamClientErr, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kafka := NewKafkaNotifier(tc.fields.eventStreamCli, tc.fields.logger) + + err := kafka.PushFinding(tc.f) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("expected error: %v but got: %v", tc.wantErr, err) + } + if !tc.fields.eventStreamCli.verify(tc.want) { + t.Fatalf("error verifying finding notification event.\ngot: %v\nwant: %v", tc.fields.eventStreamCli.e, tc.want) + } + }) + } +} diff --git a/pkg/notify/multi.go b/pkg/notify/multi.go new file mode 100644 index 0000000..84a2bc9 --- /dev/null +++ b/pkg/notify/multi.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 Adevinta +*/ + +package notify + +// MultiNotifier represents a Notifier which delegates the +// notification delivery into multiple notifier implementations. +type MultiNotifier struct { + notifiers []Notifier +} + +// NewMultiNotifier creates a new MultiNotifier. +func NewMultiNotifier(notifiers ...Notifier) *MultiNotifier { + return &MultiNotifier{ + notifiers: notifiers, + } +} + +func (m *MultiNotifier) PushFinding(f FindingNotification) error { + // For every notifier wrapped by the multi implementation, push + // the given FindingNotification. Return an error on the first + // notifier that fails to deliver the notification, so possible + // repeated sending events might happen. This should be handled + // by the consumers complying with at-least-once semantics. + for iN := range m.notifiers { + if err := m.notifiers[iN].PushFinding(f); err != nil { + return err + } + } + return nil +} diff --git a/pkg/notify/multi_test.go b/pkg/notify/multi_test.go new file mode 100644 index 0000000..dc47411 --- /dev/null +++ b/pkg/notify/multi_test.go @@ -0,0 +1,82 @@ +package notify + +import ( + "errors" + "testing" +) + +var ( + mockNotifierErr = errors.New("mock notifier error") +) + +type mockNotifier struct { + err error + called bool +} + +func (m *mockNotifier) PushFinding(f FindingNotification) error { + m.called = true + return m.err +} + +func (m *mockNotifier) isCalled() bool { + return m.called +} + +func TestMulti_PushFinding(t *testing.T) { + testCases := []struct { + name string + notifiers []*mockNotifier + f FindingNotification + wantErr error + wantCalled int + }{ + { + name: "Should replay notification to multiple notifiers", + notifiers: []*mockNotifier{ + {}, + {}, + }, + f: FindingNotification{}, + wantCalled: 2, + }, + { + name: "Should propagate error", + notifiers: []*mockNotifier{ + {}, + { + err: mockNotifierErr, + }, + {}, + }, + f: FindingNotification{}, + wantErr: mockNotifierErr, + wantCalled: 2, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var nn []Notifier + for _, n := range tc.notifiers { + nn = append(nn, Notifier(n)) + } + multi := NewMultiNotifier(nn...) + + err := multi.PushFinding(tc.f) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("unexpected error, got: %v wanted: %v", err, tc.wantErr) + } + + var nCalled int + for _, n := range tc.notifiers { + if n.isCalled() { + nCalled++ + } + } + if nCalled != tc.wantCalled { + t.Fatalf("expected %d notifiers to be called, but got %d", tc.wantCalled, nCalled) + } + }) + } +} diff --git a/pkg/notify/noop.go b/pkg/notify/noop.go new file mode 100644 index 0000000..5e19df5 --- /dev/null +++ b/pkg/notify/noop.go @@ -0,0 +1,17 @@ +/* +Copyright 2022 Adevinta +*/ + +package notify + +// NoopNotifier represents a notifier which performs no action. +type NoopNotifier struct{} + +// NoopNotifier creates a new NoopNotifier. +func NewNoopNotifier() *NoopNotifier { + return &NoopNotifier{} +} + +func (n *NoopNotifier) PushFinding(f FindingNotification) error { + return nil // Do nothing +} diff --git a/pkg/notify/notifier.go b/pkg/notify/notifier.go index b60f826..8dc84fe 100644 --- a/pkg/notify/notifier.go +++ b/pkg/notify/notifier.go @@ -4,8 +4,25 @@ Copyright 2020 Adevinta package notify +import ( + "github.com/adevinta/vulnerability-db/pkg/store" +) + +// FindingNotification represents a notification associated with a finding state change. +type FindingNotification struct { + store.FindingExpanded + // TODO: Tag field is defined here in order for the FindingNotification struct to be + // backward compatible with the "old" vulnerability notifications format so integrations + // with external components that still use the old representation (e.g.: Hermes) are + // maintained and can be isolated in the notify pkg. Once these integrations are modified + // to use the new notification format we'll have to decide if we make this tag field + // serializable into JSON and keep using the tag field as identifier, or we migrate to use + // the current team identifier. + Tag string `json:"-"` +} + // Notifier is a connector that allows // to push messages to a notification service. type Notifier interface { - Push(mssg interface{}) error + PushFinding(f FindingNotification) error } diff --git a/pkg/notify/sns.go b/pkg/notify/sns.go index a0a484c..767890e 100644 --- a/pkg/notify/sns.go +++ b/pkg/notify/sns.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "strings" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -19,17 +20,43 @@ import ( // SNSConfig holds the required SNS config information. type SNSConfig struct { TopicArn string `mapstructure:"topic_arn"` - Enabled bool `mapstructure:"enabled"` Endpoint string `mapstructure:"endpoint"` } -// SNSNotifier sends push events to an SNS topic. +// SNSNotifier sends notifications to an SNS topic. type SNSNotifier struct { conf SNSConfig sns snsiface.SNSAPI logger *log.Logger } +// FindingNotification is the data that's notified when a new finding occurs. +// TODO: This struct has been moved from processor types so it's isolated in +// the SNS notifier implementation and it's easier to deprecate in the future +// in favor of Kafka implementation. This SNS implementation only exists in +// order to maintain compatibility with old integrations that expects this +// notification format. +type findingNotification struct { + TargetID string `json:"target_id"` + Target string `json:"target"` + IssueID string `json:"issue_id"` + FindingID string `json:"finding_id"` + CheckID string `json:"check_id"` + ChecktypeName string `json:"checktype_name"` + CheckTypeOptions string `json:"checktype_options"` + Tag string `json:"tag"` + Time time.Time `json:"time"` + Vulnerability vulnerability `json:"vulnerability"` +} + +type vulnerability struct { + ID string `json:"id"` + Summary string `json:"summary"` + Score float64 `json:"score"` + CWEID uint32 `json:"cwe_id"` + Description string `json:"description"` +} + // NewSNSNotifier creates a new SNSNotifier with the given configuration. func NewSNSNotifier(conf SNSConfig, logger *log.Logger) (*SNSNotifier, error) { sess, err := session.NewSession() @@ -55,14 +82,28 @@ func NewSNSNotifier(conf SNSConfig, logger *log.Logger) (*SNSNotifier, error) { return notifier, nil } -// Push pushes a notification to the configured sns topic. -func (n *SNSNotifier) Push(message interface{}) error { - if !n.conf.Enabled { +// PushFinding pushes a finding notification to the configured sns topic. +func (n *SNSNotifier) PushFinding(f FindingNotification) error { + // TODO: Due to the new events generation from VulnDB, now + // Finding Notifications received as input for SNS notifier + // might be for FIXED finding events, but these do not apply + // to former integrations tight to SNS, therefore we have to + // ignore them from this implementation until the dependent + // integrations are modified. + if f.Status != "OPEN" { + n.logger.WithFields(log.Fields{ + "notifier": "sns", + "id": f.ID, + }).Debug("ignoring FIXED finding notification") return nil } - n.logger.Info("Pushing notification to SNS") - content, err := json.Marshal(&message) + n.logger.WithFields(log.Fields{ + "notifier": "sns", + "id": f.ID, + }).Info("pushing finding notification") + + content, err := json.Marshal(toOldFmt(f)) if err != nil { return err } @@ -70,12 +111,11 @@ func (n *SNSNotifier) Push(message interface{}) error { Message: aws.String(string(content)), TopicArn: aws.String(n.conf.TopicArn), } + _, err = n.sns.Publish(input) if err != nil { return err } - n.logger.Info("Notification pushed to SNS successfully") - return nil } @@ -96,3 +136,26 @@ func parseSNSARN(snsARN string) snsData { endpoint: fmt.Sprintf("https://sns.%v.amazonaws.com/%v/%v", region, accountID, name), } } + +// toOldFmt translates a FindingNotification into the old +// findingNotification format. +func toOldFmt(f FindingNotification) findingNotification { + return findingNotification{ + TargetID: f.Target.ID, + Target: f.Target.Identifier, + IssueID: f.Issue.ID, + FindingID: f.ID, + CheckID: f.Source.Instance, + ChecktypeName: f.Source.Component, + CheckTypeOptions: f.Source.Options, + Tag: f.Tag, + Time: f.Source.Time, + Vulnerability: vulnerability{ + ID: f.Issue.ID, + Summary: f.Issue.Summary, + Score: f.Finding.Score, + CWEID: f.Issue.CWEID, + Description: f.Issue.Description, + }, + } +} diff --git a/pkg/notify/sns_test.go b/pkg/notify/sns_test.go index 4e1d97b..c8ac53a 100644 --- a/pkg/notify/sns_test.go +++ b/pkg/notify/sns_test.go @@ -6,14 +6,18 @@ package notify import ( "testing" + "unicode" "github.com/aws/aws-sdk-go/aws" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/lib/pq" "github.com/aws/aws-sdk-go/service/sns" "github.com/aws/aws-sdk-go/service/sns/snsiface" log "github.com/sirupsen/logrus" + + "github.com/adevinta/vulnerability-db/pkg/store" ) type snsMock struct { @@ -26,7 +30,7 @@ func (m *snsMock) Publish(s *sns.PublishInput) (*sns.PublishOutput, error) { return nil, nil } -func TestSNSNotifier_Push(t *testing.T) { +func TestSNS_PushFinding(t *testing.T) { type fields struct { conf SNSConfig sns *snsMock @@ -36,27 +40,133 @@ func TestSNSNotifier_Push(t *testing.T) { tests := []struct { name string fields fields - message map[string]interface{} + f FindingNotification want *sns.PublishInput wantErr bool }{ { - name: "PushesMsgsToTopic", + name: "Should translate notification to old format and send it", fields: fields{ sns: &snsMock{}, logger: log.New(), conf: SNSConfig{ TopicArn: "arn:aTopic", - Enabled: true, }, }, - message: map[string]interface{}{"a": "b"}, + f: FindingNotification{ + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + ID: "FindingID-1", + AffectedResource: "AffectedResource-1", + Score: 9, + Status: "OPEN", + Details: "Details-1", + ImpactDetails: "ImpactDetails-1", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + ID: "IssueID-1", + Summary: "Summary-1", + CWEID: 1, + Description: "Description-1", + Recommendations: pq.StringArray{ + "Recommendation-1", + "Recommendation-2", + }, + ReferenceLinks: pq.StringArray{ + "ReferenceLink-1", + "ReferenceLink-2", + }, + }, + Labels: []string{ + "Label-1", + "Label-2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + ID: "TargetID-1", + Identifier: "Identifier-1", + }, + Teams: []string{ + "Team-1", + "Team-2", + }, + }, + Source: store.Source{ + ID: "SourceID-1", + Instance: "SourceInstance-1", + Options: "SourceOptions-1", + SourceFamily: store.SourceFamily{ + Name: "SourceName-1", + Component: "SourceComponent-1", + }, + }, + Resources: store.Resources{ + { + Name: "ResourceName-1", + Attributes: []string{ + "Attr-1", + "Attr-2", + }, + Resources: []map[string]string{ + { + "Attr-1": "1", + "Attr-2": "2", + }, + }, + }, + }, + TotalExposure: 10, + CurrentExposure: 5, + }, + Tag: "tag-1", + }, want: &sns.PublishInput{ - Message: aws.String(`{"a":"b"}`), + Message: aws.String(removeSpaces(` + { + "target_id": "TargetID-1", + "target": "Identifier-1", + "issue_id": "IssueID-1", + "finding_id": "FindingID-1", + "check_id": "SourceInstance-1", + "checktype_name": "SourceComponent-1", + "checktype_options": "SourceOptions-1", + "tag": "tag-1", + "time": "0001-01-01T00:00:00Z", + "vulnerability": { + "id": "IssueID-1", + "summary": "Summary-1", + "score": 9, + "cwe_id": 1, + "description": "Description-1" + } + }`)), TopicArn: aws.String("arn:aTopic"), }, }, + { + name: "Should ignore FIXED finding notification", + fields: fields{ + sns: &snsMock{}, + logger: log.New(), + }, + f: FindingNotification{ + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + ID: "FindingID-2", + AffectedResource: "AffectedResource-2", + Score: 7, + Status: "FIXED", + Details: "Details-2", + ImpactDetails: "ImpactDetails-2", + }, + }, + }, + want: nil, + }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &SNSNotifier{ @@ -64,13 +174,23 @@ func TestSNSNotifier_Push(t *testing.T) { sns: tt.fields.sns, logger: tt.fields.logger, } - if err := s.Push(tt.message); (err != nil) != tt.wantErr { - t.Errorf("SNSNotifier.Push() error = %v, wantErr %v", err, tt.wantErr) + if err := s.PushFinding(tt.f); (err != nil) != tt.wantErr { + t.Errorf("got error: %v but wanted: %v", err, tt.wantErr) } diff := cmp.Diff(tt.want, tt.fields.sns.notification, cmpopts.IgnoreUnexported(sns.PublishInput{})) if diff != "" { - t.Errorf("want!= got. Diffs:%s", diff) + t.Errorf("SNS payload does not match with expected one. Diff:\n%s", diff) } }) } } + +func removeSpaces(s string) string { + rr := make([]rune, 0, len(s)) + for _, r := range s { + if !unicode.IsSpace(r) { + rr = append(rr, r) + } + } + return string(rr) +} diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 8550276..94ca7bb 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -146,7 +146,7 @@ func (p *CheckProcessor) ProcessMessage(m string) error { source := p.sourceFromCheck(*check, target.ID, checkTime) l.Debugf("Processing source with %d source findings: %#v", len(sourceFindings), source) - source, err = p.store.ProcessSourceExecution(source, sourceFindings) + _, findingsState, err := p.store.ProcessSourceExecution(source, sourceFindings) if err != nil { l.Errorf("Error while processing source: %#v", err) if !store.IsDuplicateErr(err) { @@ -154,8 +154,8 @@ func (p *CheckProcessor) ProcessMessage(m string) error { } } - l.Debug("Sending open findings") - err = p.notifyOpenFindings(source, check.Target, check.Tag) + l.Debug("Notifying findings") + err = p.notifyFindings(findingsState, check.Tag) if err != nil { return err } @@ -273,50 +273,33 @@ func (p *CheckProcessor) processVulns(vulns []report.Vulnerability, targetIdenti return created, nil } -func (p *CheckProcessor) notifyOpenFindings(source store.Source, target, tag string) error { - // Notify the findings found by this source that are still open. Note that - // we assume that a finding can be notified multiple times when, for - // instance, if checks for the same target are processed not in the same - // order that they were executed. - sourceF, err := p.store.GetOpenSourceFindings(source.ID) +func (p *CheckProcessor) notifyFindings(findingsState []store.FindingState, tag string) error { + if len(findingsState) == 0 { + return nil + } + + // Retrieve expanded findings for findings which state + // has been updated due to current source execution + var fIDs []string + for _, fs := range findingsState { + fIDs = append(fIDs, fs.ID) + } + findings, err := p.store.GetFindingsExpanded(fIDs) if err != nil { return err } p.logger.WithFields(log.Fields{ - "check_id": source.Instance, - "target": target, - "tag": tag, - }).Debugf("Sending %d finding events", len(sourceF)) - - for _, sf := range sourceF { - issue, err := p.store.FindIssueByID(sf.IssueID) - if err != nil { - return err - } - f, err := p.store.FindFinding(store.Finding{TargetID: source.Target, IssueID: issue.ID, AffectedResource: sf.AffectedResource}) - if err != nil { - return err - } - err = p.notifier.Push( - FindingNotification{ - TargetID: source.Target, - Target: target, - IssueID: issue.ID, - FindingID: f.ID, - CheckID: source.Instance, - ChecktypeName: source.Component, - CheckTypeOptions: source.Options, - Tag: tag, - Time: source.Time, - Vulnerability: vulnerability{ - ID: issue.ID, - Summary: issue.Summary, - Score: float32(sf.Score), - CWEID: issue.CWEID, - Description: issue.Description, - }}, - ) + "target": findings[0].Target.Identifier, + "tag": tag, + }).Debugf("sending %d finding notifications", len(findings)) + + // Notify finding state update + for _, f := range findings { + err = p.notifier.PushFinding(notify.FindingNotification{ + FindingExpanded: f, + Tag: tag, + }) if err != nil { return err } diff --git a/pkg/processor/processor_test.go b/pkg/processor/processor_test.go index a6a5df0..eeb652c 100644 --- a/pkg/processor/processor_test.go +++ b/pkg/processor/processor_test.go @@ -24,37 +24,44 @@ const ( pqNotFoundErr = "sql: no rows in result set" ) +var ( + mockNotifierErr = errors.New("mockNotifierErr") +) + type mockNotifier struct { calls uint8 } -func (n *mockNotifier) Push(mssg interface{}) error { +func (n *mockNotifier) PushFinding(f notify.FindingNotification) error { n.calls++ return nil } type inMemMockNotifier struct { - sent []FindingNotification + sent []notify.FindingNotification + err error } -func (n *inMemMockNotifier) Push(mssg interface{}) error { - notif, _ := mssg.(FindingNotification) - n.sent = append(n.sent, notif) +func (n *inMemMockNotifier) PushFinding(f notify.FindingNotification) error { + if n.err != nil { + return n.err + } + n.sent = append(n.sent, f) return nil } type mockStore struct { store.VulnStore - wantFindFindingErr bool // Specifies if FindFinding must return error. - wantNewFinding bool // Specifies if FindFinding must return NotFoundErr. - wantNewCreateIssueIfNotExistErr bool // Specifies if CreateIssueIfNotExists must return an error. - returnFindingStatus string // Specifies FindFinding returned finding's status. - returnCreateEventStatus string // Specifies the finding status after calling CreateFindingEvent. - returnLastFindingEvent *store.FindingEvent // Specifies the finding event to return for GetLastFindingEvent. - returnSourceIssues []*store.Issue // Specifies the list of issues returned for GetIssuesBySource. - returnOpenSourceFindings []store.SourceFinding // Specifies the list of Finding statuses returned by the returnFindingStatuses, - createIssueIfNotExistCalls uint8 // RecalculateFindingStatusResult return status. + wantFindFindingErr bool // Specifies if FindFinding must return error. + wantNewFinding bool // Specifies if FindFinding must return NotFoundErr. + wantNewCreateIssueIfNotExistErr bool // Specifies if CreateIssueIfNotExists must return an error. + returnFindingStatus string // Specifies FindFinding returned finding's status. + returnCreateEventStatus string // Specifies the finding status after calling CreateFindingEvent. + returnLastFindingEvent *store.FindingEvent // Specifies the finding event to return for GetLastFindingEvent. + returnSourceIssues []*store.Issue // Specifies the list of issues returned for GetIssuesBySource. + returnFindingsExpanded []store.FindingExpanded // Specifies the list of Expanded Findings returned by the GetFindingsExpanded. + createIssueIfNotExistCalls uint8 // RecalculateFindingStatusResult return status. getOpenIssuesForSource func(id string) ([]string, error) findIssueByID func(id string) (*store.Issue, error) } @@ -112,8 +119,8 @@ func (s *mockStore) FindIssueByID(id string) (*store.Issue, error) { return s.findIssueByID(id) } -func (s *mockStore) GetOpenSourceFindings(id string) ([]store.SourceFinding, error) { - return s.returnOpenSourceFindings, nil +func (s *mockStore) GetFindingsExpanded(id []string) ([]store.FindingExpanded, error) { + return s.returnFindingsExpanded, nil } type mockResClient struct { @@ -161,7 +168,7 @@ func TestProcess(t *testing.T) { }, input: input{ vulns: []report.Vulnerability{ - report.Vulnerability{ + { Summary: "Vulnerability 1", }, }}, @@ -179,10 +186,10 @@ func TestProcess(t *testing.T) { expectedCreateIssueIfNotExist: 2, input: input{ vulns: []report.Vulnerability{ - report.Vulnerability{ + { Summary: "Vulnerability 1", }, - report.Vulnerability{ + { Summary: "Vulnerability 2", }, }}, @@ -211,115 +218,92 @@ func TestProcess(t *testing.T) { } func TestNotifyOpenFindings(t *testing.T) { + type input struct { + findingsState []store.FindingState + tag string + } + logger := log.New() - tests := []struct { - name string - fields fields - source store.Source - expectedNotifications []FindingNotification - expectedError string + testCases := []struct { + name string + fields fields + input input + expectedNotifs []notify.FindingNotification + expectedErr error }{ { - name: "Notifies2Vulns", - source: store.Source{ - SourceFamily: store.SourceFamily{ - Component: "c", - Name: "n", - Target: "t", + name: "Should send notificatons adding check tag", + fields: fields{ + logger: logger, + notifier: &inMemMockNotifier{}, + store: &mockStore{ + returnFindingsExpanded: []store.FindingExpanded{ + {Finding: store.Finding{ID: "1"}}, + {Finding: store.Finding{ID: "2"}}, + }, }, - ID: "i", - Instance: "ins", - Options: "o", }, - fields: fields{ - notifier: &inMemMockNotifier{ - sent: make([]FindingNotification, 2), + input: input{ + findingsState: []store.FindingState{ + {ID: "1"}, + {ID: "2"}, }, - store: &mockStore{ - getOpenIssuesForSource: func(id string) ([]string, error) { - return []string{"id1", "id2"}, nil + tag: "test-tag", + }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ID: "1"}, }, - - findIssueByID: func(id string) (*store.Issue, error) { - issues := map[string]store.Issue{ - "id1": store.Issue{ - ID: "id1", - Summary: "Summary one", - CWEID: 1, - Description: "description", - }, - "id2": store.Issue{ - ID: "id2", - Summary: "Summary two", - CWEID: 2, - Description: "description 2", - }, - } - i, ok := issues[id] - if !ok { - // Todo export error not found constant in order - // to not having to mock it like this. - return nil, errors.New("sql: no rows in result set") - } - return &i, nil + Tag: "test-tag", + }, + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ID: "2"}, }, + Tag: "test-tag", }, - resClient: mockResClient{}, - logger: logger, }, - expectedNotifications: []FindingNotification{ - FindingNotification{ - TargetID: "t", - Target: "target", - IssueID: "id1", - CheckID: "ins", - ChecktypeName: "c", - CheckTypeOptions: "o", - Tag: "tag", - Vulnerability: vulnerability{ - ID: "id1", - Summary: "Summary one", - Score: 6.0, - CWEID: 1, - Description: "description", - }, + }, + { + name: "Should propagate error from notifier", + fields: fields{ + logger: logger, + notifier: &inMemMockNotifier{ + err: mockNotifierErr, }, - FindingNotification{ - TargetID: "t", - Target: "target", - IssueID: "id1", - CheckID: "ins", - ChecktypeName: "c", - CheckTypeOptions: "o", - Tag: "tag", - Vulnerability: vulnerability{ - ID: "id2", - Summary: "Summary two", - Score: 7.0, - CWEID: 2, - Description: "description 2", + store: &mockStore{ + returnFindingsExpanded: []store.FindingExpanded{ + {Finding: store.Finding{ID: "1"}}, }, }, }, + input: input{ + findingsState: []store.FindingState{ + {ID: "1"}, + }, + tag: "test-tag", + }, + expectedErr: mockNotifierErr, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { processor := &CheckProcessor{ - notifier: tt.fields.notifier, - store: tt.fields.store, - resultsClient: tt.fields.resClient, - logger: tt.fields.logger, + notifier: tc.fields.notifier, + store: tc.fields.store, + resultsClient: tc.fields.resClient, + logger: tc.fields.logger, } - err := processor.notifyOpenFindings(tt.source, "target", "tag") - if errToStr(err) != tt.expectedError { - t.Fatalf("expected error different from got err, got: %v, expected %s", err, tt.expectedError) + err := processor.notifyFindings(tc.input.findingsState, tc.input.tag) + if !errors.Is(err, tc.expectedErr) { + t.Fatalf("expected error: %v but got: %v", tc.expectedErr, err) } - got := tt.fields.notifier.(*inMemMockNotifier).sent - if reflect.DeepEqual(tt.expectedNotifications, got) { - t.Errorf("got != want, got %+v, want %+v", got, tt.expectedNotifications) + got := tc.fields.notifier.(*inMemMockNotifier).sent + if !reflect.DeepEqual(tc.expectedNotifs, got) { + t.Fatalf("expected:\n%v\nbut got:\n%v", got, tc.expectedNotifs) } }) } diff --git a/pkg/processor/types.go b/pkg/processor/types.go index 2e23ae1..195a4f9 100644 --- a/pkg/processor/types.go +++ b/pkg/processor/types.go @@ -4,8 +4,6 @@ Copyright 2020 Adevinta package processor -import "time" - // Notification is the message received in the queue from the Vulcan Core SNS topic. type Notification struct { Type string `json:"Type"` @@ -41,25 +39,3 @@ type CheckMessage struct { QueueName string `json:"queue_name"` Tag string `json:"tag"` } - -// FindingNotification is the data that's notified when a new finding occurs. -type FindingNotification struct { - TargetID string `json:"target_id"` - Target string `json:"target"` - IssueID string `json:"issue_id"` - FindingID string `json:"finding_id"` - CheckID string `json:"check_id"` - ChecktypeName string `json:"checktype_name"` - CheckTypeOptions string `json:"checktype_options"` - Tag string `json:"tag"` - Time time.Time `json:"time"` - Vulnerability vulnerability `json:"vulnerability"` -} - -type vulnerability struct { - ID string `json:"id"` - Summary string `json:"summary"` - Score float32 `json:"score"` - CWEID uint32 `json:"cwe_id"` - Description string `json:"description"` -} diff --git a/pkg/queue/sqs.go b/pkg/queue/sqs.go index abeb3eb..52fa3ab 100644 --- a/pkg/queue/sqs.go +++ b/pkg/queue/sqs.go @@ -53,7 +53,7 @@ type awsData struct { } // NewSQSConsumerGroup creates a new SQSConsumerGroup. -func NewSQSConsumerGroup(nConsumers uint8, config SQSConfig, processor Processor, logger *log.Logger) (*SQSConsumerGroup, error) { +func NewSQSConsumerGroup(nConsumers uint, config SQSConfig, processor Processor, logger *log.Logger) (*SQSConsumerGroup, error) { var consumerGroup SQSConsumerGroup awsSess, err := session.NewSession() @@ -64,7 +64,7 @@ func NewSQSConsumerGroup(nConsumers uint8, config SQSConfig, processor Processor awsData := parseQueueARN(config.QueueArn) var consumers []*SQSConsumer - for i := uint8(0); i < nConsumers; i++ { + for i := uint(0); i < nConsumers; i++ { c, err := newSQSConsumer(awsSess, awsData, config, processor, logger) if err != nil { return nil, err @@ -158,13 +158,14 @@ func (c *SQSConsumer) readAndProcess(ctx context.Context) error { } // If message is valid, process it - c.logger.Debugf("Processing SQS message: %s", *mssg.Body) + c.logger.Infof("Processing SQS message: %s", *mssg.Body) if err = c.processor.ProcessMessage(*mssg.Body); err != nil { c.logger.Errorf("Error processing SQS message: %s", err.Error()) continue } // Delete it + c.logger.Debugf("Deleting SQS message: %s", *mssg.Body) if err = c.deleteMessage(mssg); err != nil { c.logger.Errorf("Error deleting processed message: %s", err.Error()) } diff --git a/pkg/store/findings.go b/pkg/store/findings.go index 50ee17e..c4873fb 100644 --- a/pkg/store/findings.go +++ b/pkg/store/findings.go @@ -6,40 +6,52 @@ package store import ( "context" + "database/sql" + "encoding/json" + "errors" "fmt" "sort" "strings" "time" "github.com/jmoiron/sqlx" + "github.com/lib/pq" log "github.com/sirupsen/logrus" ) const ( - FindingStatusOpen = "OPEN" - FindingStatusFixed = "FIXED" - FindingStatusNew = "NEW" - FindingStatusInvalidated = "INVALIDATED" - FindingDefaultFingerprint = "NOT_PROVIDED" + FindingStatusOpen = "OPEN" + FindingStatusFixed = "FIXED" + FindingStatusExpired = "EXPIRED" + FindingStatusFalsePositive = "FALSE_POSITIVE" + FindingStatusNew = "NEW" + FindingStatusInvalidated = "INVALIDATED" + FindingDefaultFingerprint = "NOT_PROVIDED" + + dateTimeFmt = "2006-01-02 15:04:05" +) + +var ( + // ErrParsingFinding indicates an error when parsing finding data retrieved from database. + ErrParsingFinding = errors.New("error parsing finding data") ) // Finding represents the finding of a // vulnerability in a target. type Finding struct { - ID string - IssueID string `db:"issue_id"` - TargetID string `db:"target_id"` - AffectedResource string `db:"affected_resource"` - AffectedResourceString string `db:"affected_resource_string"` - Fingerprint string `db:"fingerprint"` - Score float64 - Status string - Details string - ImpactDetails string `db:"impact_details"` + ID string `json:"id" db:"id"` + IssueID string `json:"-" db:"issue_id"` + TargetID string `json:"-" db:"target_id"` + AffectedResource string `json:"affected_resource" db:"affected_resource"` + AffectedResourceString string `json:"-" db:"affected_resource_string"` + Fingerprint string `json:"-" db:"fingerprint"` + Score float64 `json:"score" db:"score"` + Status string `json:"status" db:"status"` + Details string `json:"details" db:"details"` + ImpactDetails string `json:"impact_details" db:"impact_details"` // Resources contains the vulnerability resources tables mashalled into a // json. - Resources *[]byte - Exposure uint32 + Resources *[]byte `json:"-"` } // FindingEvent is an event related to a finding @@ -68,7 +80,30 @@ type FindingExposure struct { TTR *int `db:"fixed_at"` } -type findingState struct { +// FindingExpanded represents a finding expanding the associated target, issue +// and source data. +type FindingExpanded struct { + Finding + Issue IssueLabels `json:"issue"` + Target TargetTeams `json:"target"` + Source Source `json:"source"` + Resources Resources `json:"resources"` + TotalExposure int64 `json:"total_exposure"` + CurrentExposure int64 `json:"current_exposure,omitempty"` +} + +// Resources defines the structure of a the resources of a finding. +type Resources []ResourceGroup + +// ResourceGroup reprents a resource in a finding. +type ResourceGroup struct { + Name string `json:"name"` + Attributes []string `json:"attributes"` + Resources []map[string]string `json:"resources"` +} + +// FindingState represents a state update for a finding. +type FindingState struct { ID string Exposure []FindingExposure Status string @@ -196,6 +231,51 @@ func (db *psqlxStore) GetLastFindingEvent(findingID string) (*FindingEvent, erro return &event, nil } +func (db *psqlxStore) GetFindingsExpanded(ids []string) ([]FindingExpanded, error) { + query := `SELECT f.id AS finding_id, s.id AS source_id, + f.issue_id, f.target_id, f.affected_resource, f.affected_resource_string, f.status, f.details, f.impact_details, f.resources, f.score, + s.name, s.component, s.instance, s.options, s.time, + i.*, t.*, + (SELECT ARRAY_AGG(tt.team_id) FROM target_teams tt WHERE t.id = tt.target_id) teams, + (SELECT ARRAY_AGG(fexp.found_at) FROM finding_exposures fexp WHERE fexp.finding_id = f.id) as found_at, + (SELECT ARRAY_AGG(fexp.fixed_at) FROM finding_exposures fexp WHERE fexp.finding_id = f.id) as fixed_at, + (SELECT ARRAY_AGG(fexp.expired_at) FROM finding_exposures fexp WHERE fexp.finding_id = f.id) as expired_at, + (SELECT ARRAY_AGG(il.label) FROM issue_labels il WHERE il.issue_id = f.issue_id) labels + FROM findings f + INNER JOIN issues i ON f.issue_id = i.id + INNER JOIN targets t ON f.target_id = t.id + INNER JOIN finding_events fe ON fe.finding_id = f.id + INNER JOIN sources s ON fe.source_id = s.id + WHERE f.id = ANY ($1) + AND fe.id IN ( + SELECT id FROM finding_events + WHERE finding_id = f.id ORDER BY time DESC LIMIT 1 + )` + rows, err := db.DB.Queryx(query, pq.Array(ids)) + if err == sql.ErrNoRows { + return []FindingExpanded{}, nil + } + if err != nil { + return nil, err + } + + var findings []FindingExpanded + for rows.Next() { + row := make(map[string]any) + err = rows.MapScan(row) + if err != nil { + return nil, err + } + f, err := buildFindingExpanded(row) + if err != nil { + return nil, err + } + findings = append(findings, f) + } + + return findings, nil +} + // CreateFindingEvent creates a new finding event . Returns finding after event // insert. func (db *psqlxStore) CreateFindingEvent(eventTime time.Time, findingID, sourceID string, score float64, fingerprint, affectedResourceString string) (*Finding, error) { @@ -328,8 +408,7 @@ func (db *psqlxStore) RecalculateFindingsStatus(s SourceFamily) error { return tx.Commit() } -// createFindingEvent creates a finding event for the given finding. It creates the -// finding if it does not exist. +// createFindingEvent creates a finding and/or the associated finding event if they do not exist. func createFindingEvent(ctx context.Context, tx *sqlx.Tx, f Finding, s Source, score float32, resources *[]byte, details string, fingerprint, affectedResourceString string) (*FindingEvent, error) { // Ensure finding for the finding event exists. q := ` @@ -352,15 +431,30 @@ func createFindingEvent(ctx context.Context, tx *sqlx.Tx, f Finding, s Source, s return nil, err } - // Create the finding event. + // Create the finding event if not exists. var findingEvent FindingEvent - r := tx.QueryRowxContext(ctx, `INSERT INTO finding_events (finding_id, source_id, time, score, resources, details, fingerprint, affected_resource_string) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *`, f.ID, s.ID, s.Time, score, sanitizeJSONB(f.Resources), f.Details, f.Fingerprint, affectedResourceString) + r := tx.QueryRowxContext(ctx, ` + WITH ins AS ( + INSERT INTO finding_events + (finding_id, source_id, time, score, resources, details, fingerprint, affected_resource_string) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT ON CONSTRAINT unique_finding_events DO UPDATE + SET finding_id = NULL -- never executed + WHERE FALSE + RETURNING * + ) + SELECT * + FROM ins + UNION ALL + SELECT * + FROM finding_events + WHERE finding_id = $1 AND source_id = $2 and time = $3`, + f.ID, s.ID, s.Time, score, sanitizeJSONB(f.Resources), f.Details, f.Fingerprint, affectedResourceString) err = r.StructScan(&findingEvent) return &findingEvent, err } -func findingsStates(ctx context.Context, tx *sqlx.Tx, log *log.Logger, s SourceFamily) ([]findingState, error) { +func findingsStates(ctx context.Context, tx *sqlx.Tx, log *log.Logger, s SourceFamily) ([]FindingState, error) { findingsQ := ` WITH relevant_sources AS( SELECT s.* @@ -416,7 +510,7 @@ func findingsStates(ctx context.Context, tx *sqlx.Tx, log *log.Logger, s SourceF return states, nil } -func buildFindingStates(sourceF []sourceFindings) []findingState { +func buildFindingStates(sourceF []sourceFindings) []FindingState { if len(sourceF) == 0 { return nil } @@ -516,7 +610,7 @@ func buildFindingStates(sourceF []sourceFindings) []findingState { } // Build the findings state from their timeline. - var findingStates = make([]findingState, 0) + var findingStates = make([]FindingState, 0) notFoundF := func(e timeEvent) bool { return !e.found } foundF := func(e timeEvent) bool { return e.found } for id, f := range findings { @@ -597,7 +691,7 @@ func buildFindingStates(sourceF []sourceFindings) []findingState { } score, details, resources, fingerprint, affectedResourceString := tl.LastFoundAttributes() - state := findingState{ + state := FindingState{ Exposure: exposures, ID: id, Score: score, @@ -681,7 +775,7 @@ func buildFindingTimeline(f findingData) timeline { return tl } -func replaceFindingsState(ctx context.Context, tx *sqlx.Tx, l *log.Logger, states []findingState) error { +func replaceFindingsState(ctx context.Context, tx *sqlx.Tx, l *log.Logger, states []FindingState) error { qd := "DELETE FROM finding_exposures where finding_id = $1" di := "INSERT INTO finding_exposures(finding_id,found_at,fixed_at,ttr) VALUES ($1, $2, $3, $4)" qF := `UPDATE findings SET @@ -726,3 +820,352 @@ func sanitizeJSONB(jsonb *[]byte) *[]byte { sanitized := []byte(strings.Replace(string(*jsonb), "\\u0000", "", -1)) return &sanitized } + +// buildFindingExpanded builds a finding notification. +// Note: This function and associated ones are copied and adapted from vulnerability-db-api. +func buildFindingExpanded(row map[string]interface{}) (FindingExpanded, error) { + var f FindingExpanded + atTime := time.Now() + + findingID, ok := row["finding_id"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.ID = findingID + + issueID, ok := row["issue_id"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Issue.ID = issueID + + summary, ok := row["summary"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Issue.Summary = summary + + CWEID, ok := row["cwe_id"].(int64) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Issue.CWEID = uint32(CWEID) + + description, ok := row["description"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Issue.Description = description + + recommendations, err := parseStringArrayNotNil(row["recommendations"]) + if err != nil { + return FindingExpanded{}, ErrParsingFinding + } + f.Issue.Recommendations = recommendations + + referenceLinks, err := parseStringArrayNotNil(row["reference_links"]) + if err != nil { + return FindingExpanded{}, ErrParsingFinding + } + f.Issue.ReferenceLinks = referenceLinks + + labels, ok := row["labels"] + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + if labels != nil { + labels, ok := row["labels"].([]uint8) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Issue.Labels = parseStringArray(labels) + } + + targetID, ok := row["target_id"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Target.ID = targetID + + affectedResource, ok := row["affected_resource"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.AffectedResource = affectedResource + + affectedResourceString, ok := row["affected_resource_string"] + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + if affectedResourceString != nil { + affectedResourceString, ok := affectedResourceString.(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + // NOTE: The problem with the affected resource is that in some cases + // it might not be human-readable, and that was the reason to introduce + // the affected resource string field. + // Given that clients of the vulnerability-db-api are using the + // affected resource field just as information to be shown to users, + // and therefore there are no specific API queries using that field as + // a parameter, the API will return the human-readable version of the + // affected resource field when existing. + // The reason is to avoid propagating the new field to the clients + // (e.g. vulcan-api or vulcan-ui) if not required. + if affectedResourceString != "" { + f.AffectedResource = affectedResourceString + } + } + + identifier, ok := row["identifier"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Target.Identifier = identifier + + teams, ok := row["teams"] + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + if teams != nil { + teams, ok := row["teams"].([]uint8) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Target.Teams = parseStringArray(teams) + } + + sourceID, ok := row["source_id"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Source.ID = sourceID + + name, ok := row["name"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Source.Name = name + + component, ok := row["component"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Source.Component = component + + instance, ok := row["instance"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Source.Instance = instance + + options, ok := row["options"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Source.Options = options + + time, ok := row["time"].(time.Time) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Source.Time = time // Serialized to ISO8601 compliant format 2006-01-02T15:04:05Z + + details, ok := row["details"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Details = details + + impactDetails, ok := row["impact_details"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.ImpactDetails = impactDetails + + status, ok := row["status"].(string) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Status = status + + score, ok := row["score"].(float64) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + f.Score = score + + var resourcesContent []byte + if row["resources"] != nil { + var ok bool + resourcesContent, ok = row["resources"].([]byte) + if !ok { + return FindingExpanded{}, errors.New("unexpected resources type") + } + } + resources, err := parseFindingResources(resourcesContent) + if err != nil { + return FindingExpanded{}, err + } + f.Resources = resources + + found, ok := row["found_at"].([]uint8) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + foundAt := parseStringArray(found) + + fixed, ok := row["fixed_at"].([]uint8) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + fixedAt := parseStringArray(fixed) + + expired, ok := row["expired_at"].([]uint8) + if !ok { + return FindingExpanded{}, ErrParsingFinding + } + expiredAt := parseStringArray(expired) + + // Don't calculate finding exposure if the finding was marked as a FALSE POSITIVE. + // A finding marked as false positive must not impact metrics. + if f.Status != FindingStatusFalsePositive { + err = fillFindingExposures(&f, foundAt, fixedAt, expiredAt, atTime) + if err != nil { + return FindingExpanded{}, err + } + } + + return f, nil +} + +// fillFindingExposures calculates and modifies finding exposures for input finding +// at the time specified by atTime parameter. +func fillFindingExposures(f *FindingExpanded, foundAt, fixedAt, expiredAt []string, atTime time.Time) error { + if f == nil { + return errors.New("the finding to fill the exposure can not be nil") + } + if len(foundAt) != len(fixedAt) { + return errors.New("finding exposures time bounds expected to have the same len") + } + + var ( + total float64 + current float64 + ) + + for i, found := range foundAt { + // Parse found time. + foundT, err := parseDBTime(found) + if err != nil { + return err + } + if foundT.After(atTime) { + // If found time is after reference time + // do not process this entry. + continue + } + + // Parse fixed time. + var fixedT time.Time + if fixedAt[i] != "NULL" { + fixedT, err = parseDBTime(fixedAt[i]) + if err != nil { + return err + } + } + + // Parse expired time. + var expiredT time.Time + if expiredAt[i] != "NULL" { + expiredT, err = parseDBTime(expiredAt[i]) + if err != nil { + return err + } + } + + // If finding is expired and time reference is + // posterior to expiration date, set current + // exposure to -1 and do not add to total exposure. + if !expiredT.IsZero() && atTime.After(expiredT) { + current = -1 + continue + } + + // If finding is expired and time reference is before + // expiration date, or finding is open, or finding is + // fixed but was fixed after time reference, then only + // count exposure up to time reference. + if (!expiredT.IsZero() && atTime.Before(expiredT)) || + (expiredT.IsZero() && (fixedT.IsZero() || fixedT.After(atTime))) { + fixedT = atTime + current = fixedT.Sub(foundT).Hours() + } + + // Add to total exposure. + total = total + fixedT.Sub(foundT).Hours() + } + + f.TotalExposure = int64(total) + if current > 0 { + f.CurrentExposure = int64(current) + } + return nil +} + +// parseDBTime parses a datetime string retrieved from DB. +func parseDBTime(dbTime string) (time.Time, error) { + return time.Parse(dateTimeFmt, strings.Replace(dbTime, "\"", "", -1)) +} + +func parseStringArrayNotNil(row interface{}) ([]string, error) { + if row == nil { + return nil, nil + } + safe, ok := row.([]uint8) + if !ok { + return nil, ErrParsingFinding + } + return parseStringArray(safe), nil +} + +// parseStringArray parses a string array retrieved from database +// stripping the initial and trailing brackets and splitting elements +// separated by comma. +func parseStringArray(array []uint8) []string { + // Void array case. + if len(array) <= 2 { + return []string{} + } + + // Remove JSON brackets first. + auxString := string(array[1 : len(array)-1]) + + var splitter string + if len(auxString) > 2 && auxString[0] == '"' && + !strings.Contains(auxString, "NULL") { + // Strings are separated by "\",\"" + // so remove first and last quote and + // split by it. + auxString = auxString[1 : len(auxString)-1] + splitter = "\",\"" + } else { + // Strings are separated by ',' with + // no quotes between them, or it's a string + // like: ""2019-01-01",NULL". + splitter = "," + } + return strings.Split(auxString, splitter) +} + +func parseFindingResources(content []byte) (Resources, error) { + var resources = make([]ResourceGroup, 0) + if content == nil { + return resources, nil + } + err := json.Unmarshal(content, &resources) + if err != nil { + return nil, err + } + return resources, nil +} diff --git a/pkg/store/findings_test.go b/pkg/store/findings_test.go index 9738090..563dc60 100644 --- a/pkg/store/findings_test.go +++ b/pkg/store/findings_test.go @@ -24,8 +24,8 @@ var ( if _, ok := a.(FindingExposure); ok { return compareFindingExposures(a, b) } - e1 := a.(findingState) - e2 := b.(findingState) + e1 := a.(FindingState) + e2 := b.(FindingState) c1 := strings.Compare(e1.ID, e2.ID) if c1 != 0 { return c1 < 0 @@ -48,7 +48,7 @@ func Test_buildFindingStates(t *testing.T) { tests := []struct { name string sourceF []sourceFindings - wantFindingStates []findingState + wantFindingStates []FindingState }{ { name: "buildFindingStatesOldModelHappyPathNewOpen", @@ -61,7 +61,7 @@ func Test_buildFindingStates(t *testing.T) { Fingerprint: strToPtr(FindingDefaultFingerprint), }, }, - wantFindingStates: []findingState{ + wantFindingStates: []FindingState{ { ID: "finding1", Status: "OPEN", @@ -89,7 +89,7 @@ func Test_buildFindingStates(t *testing.T) { Fingerprint: nil, }, }, - wantFindingStates: []findingState{ + wantFindingStates: []FindingState{ { ID: "finding1", Status: "FIXED", @@ -123,7 +123,7 @@ func Test_buildFindingStates(t *testing.T) { Fingerprint: nil, }, }, - wantFindingStates: []findingState{ + wantFindingStates: []FindingState{ { ID: "finding1", Status: "FIXED", @@ -158,7 +158,7 @@ func Test_buildFindingStates(t *testing.T) { Fingerprint: strToPtr("000001"), }, }, - wantFindingStates: []findingState{ + wantFindingStates: []FindingState{ { ID: "findingOldModel", Status: "INVALIDATED", @@ -198,7 +198,7 @@ func Test_buildFindingStates(t *testing.T) { AffectedResourceString: strToPtr("affected_resource_string"), }, }, - wantFindingStates: []findingState{ + wantFindingStates: []FindingState{ { ID: "finding1", Status: "OPEN", @@ -229,7 +229,7 @@ func Test_buildFindingStates(t *testing.T) { AffectedResourceString: strToPtr("affected_resource_string_2"), }, }, - wantFindingStates: []findingState{ + wantFindingStates: []FindingState{ { ID: "finding1", Status: "OPEN", @@ -259,7 +259,7 @@ func Test_buildFindingStates(t *testing.T) { Fingerprint: strToPtr("fingerprint"), }, }, - wantFindingStates: []findingState{ + wantFindingStates: []FindingState{ { ID: "finding1", Status: "OPEN", @@ -287,42 +287,40 @@ func Test_buildFindingExposures(t *testing.T) { name string sourceF []sourceFindings wantIDs []string - wantFindingStates []findingState + wantFindingStates []FindingState }{ { name: "BuildsProperFindingExposuresWithMultiplePeriods", sourceF: []sourceFindings{ - sourceFindings{ + { FindingID: strToPtr("finding1"), SourceID: "source1", SourceTime: mustParseTime("2019-04-10 20:40:56"), Score: intToFloatPtr(1), Fingerprint: strToPtr(FindingDefaultFingerprint), }, - sourceFindings{ + { FindingID: strToPtr("finding1"), SourceID: "source2", SourceTime: mustParseTime("2019-04-15 22:58:21"), Score: intToFloatPtr(2), Fingerprint: strToPtr(FindingDefaultFingerprint), }, - sourceFindings{ + { FindingID: strToPtr("finding1"), SourceID: "source3", SourceTime: mustParseTime("2019-05-08 23:46:36"), Score: intToFloatPtr(3), Fingerprint: strToPtr(FindingDefaultFingerprint), }, - - sourceFindings{ + { FindingID: strToPtr("finding2"), SourceID: "source4", SourceTime: mustParseTime("2019-04-20 00:00:00"), Score: intToFloatPtr(1), Fingerprint: strToPtr(FindingDefaultFingerprint), }, - - sourceFindings{ + { FindingID: strToPtr("finding3"), SourceID: "source5", SourceTime: mustParseTime("2019-06-08 20:47:08"), @@ -331,20 +329,20 @@ func Test_buildFindingExposures(t *testing.T) { }, }, - wantFindingStates: []findingState{ - findingState{ + wantFindingStates: []FindingState{ + { ID: "finding1", Status: "FIXED", Score: 3, Fingerprint: FindingDefaultFingerprint, Exposure: []FindingExposure{ - FindingExposure{ + { FindingID: "finding1", FoundAT: mustParseTime("2019-04-10 20:40:56"), FixedAT: timeToPtr(mustParseTime("2019-04-20 00:00:00")), TTR: intToPtr(219), }, - FindingExposure{ + { FindingID: "finding1", FoundAT: mustParseTime("2019-05-08 23:46:36"), FixedAT: timeToPtr(mustParseTime("2019-06-08 20:47:08")), @@ -352,26 +350,26 @@ func Test_buildFindingExposures(t *testing.T) { }, }, }, - findingState{ + { ID: "finding2", Status: "FIXED", Score: 1, Fingerprint: FindingDefaultFingerprint, Exposure: []FindingExposure{ - FindingExposure{ + { FindingID: "finding2", FoundAT: mustParseTime("2019-04-20 00:00:00"), FixedAT: timeToPtr(mustParseTime("2019-05-08 23:46:36")), TTR: intToPtr(455), }, }}, - findingState{ + { ID: "finding3", Status: "OPEN", Score: 1, Fingerprint: FindingDefaultFingerprint, Exposure: []FindingExposure{ - FindingExposure{ + { FindingID: "finding3", FoundAT: mustParseTime("2019-06-08 20:47:08"), }, @@ -383,29 +381,28 @@ func Test_buildFindingExposures(t *testing.T) { { name: "BuildsProperFindingExposuresProperScoreAnsStatus", sourceF: []sourceFindings{ - sourceFindings{ + { FindingID: strToPtr("finding1"), SourceID: "source1", SourceTime: mustParseTime("2019-04-10 20:40:56"), Score: intToFloatPtr(2), Fingerprint: strToPtr(FindingDefaultFingerprint), }, - sourceFindings{ + { FindingID: strToPtr("finding1"), SourceID: "source2", SourceTime: mustParseTime("2019-04-15 22:58:21"), Score: intToFloatPtr(2), Fingerprint: strToPtr(FindingDefaultFingerprint), }, - sourceFindings{ + { FindingID: strToPtr("finding1"), SourceID: "source3", SourceTime: mustParseTime("2019-05-08 23:46:36"), Score: intToFloatPtr(3), Fingerprint: strToPtr(FindingDefaultFingerprint), }, - - sourceFindings{ + { FindingID: strToPtr("finding1"), SourceID: "source5", SourceTime: mustParseTime("2019-06-08 20:47:08"), @@ -414,14 +411,14 @@ func Test_buildFindingExposures(t *testing.T) { }, }, - wantFindingStates: []findingState{ - findingState{ + wantFindingStates: []FindingState{ + { ID: "finding1", Status: "OPEN", Score: 1, Fingerprint: FindingDefaultFingerprint, Exposure: []FindingExposure{ - FindingExposure{ + { FindingID: "finding1", FoundAT: mustParseTime("2019-04-10 20:40:56"), }, diff --git a/pkg/store/issues.go b/pkg/store/issues.go index 2d90a6a..b10f6b5 100644 --- a/pkg/store/issues.go +++ b/pkg/store/issues.go @@ -13,12 +13,18 @@ import ( // Issue represents a security vulnerability. type Issue struct { - ID string - Summary string - CWEID uint32 `db:"cwe_id"` - Description string - Recommendations pq.StringArray - ReferenceLinks pq.StringArray `db:"reference_links"` + ID string `json:"id" db:"id"` + Summary string `json:"summary" db:"summary"` + CWEID uint32 `json:"cwe_id" db:"cwe_id"` + Description string `json:"description" db:"description"` + Recommendations pq.StringArray `json:"recommendations" db:"recommendations"` + ReferenceLinks pq.StringArray `json:"reference_links" db:"reference_links"` +} + +// IssueLabels represents an issue along with its associated labels. +type IssueLabels struct { + Issue + Labels []string `json:"labels"` } func (db *psqlxStore) CreateIssueIfNotExists(i Issue) (*Issue, error) { diff --git a/pkg/store/sources.go b/pkg/store/sources.go index 7083cb6..9f5416b 100644 --- a/pkg/store/sources.go +++ b/pkg/store/sources.go @@ -18,19 +18,19 @@ import ( // Source represents a source // which reports vulnerabilities. type Source struct { - ID string - Instance string - Options string - Time time.Time + ID string `json:"id" db:"id"` + Instance string `json:"instance" db:"instance"` + Options string `json:"options" db:"options"` + Time time.Time `json:"time" db:"time"` SourceFamily } // SourceFamily represents the set of sources with same // name, component and target. type SourceFamily struct { - Name string - Component string - Target string `db:"target_id"` + Name string `json:"name" db:"name"` + Component string `json:"component" db:"component"` + Target string `json:"-" db:"target_id"` } // SourceFamilies represents a slice of SourceFamily elements. @@ -76,7 +76,7 @@ func (db *psqlxStore) CreateSourceIfNotExists(s Source) (*Source, error) { if err != nil { return nil, err } - source, err := createSourceIfNotExists(context.Background(), tx, s) + source, err := createSource(context.Background(), tx, s) if err != nil { return nil, err } @@ -87,37 +87,22 @@ func (db *psqlxStore) CreateSourceIfNotExists(s Source) (*Source, error) { return source, nil } -// GetOpenSourceFindings returns the tuple issue_id,score for the findings that -// have been found by a given the source. -func (db *psqlxStore) GetOpenSourceFindings(id string) ([]SourceFinding, error) { - openFindingsForSource := ` - SELECT distinct f.issue_id as issue_id, fe.score as score, f.affected_resource - FROM findings f JOIN finding_events fe on fe.finding_id=f.id - JOIN sources s on s.id = fe.source_id - WHERE f.status='OPEN' AND fe.source_id = $1` - sf := []SourceFinding{} - err := db.DB.Select(&sf, openFindingsForSource, id) - if err != nil && !IsNotFoundErr(err) { - return nil, err - } - return sf, nil -} - // ProcessSourceExecution updates the store from a source and the issues it has found. -func (db *psqlxStore) ProcessSourceExecution(s Source, sourceFindings []SourceFinding) (Source, error) { +// Returns the newly created source and the state of the findings affected by its execution. +func (db *psqlxStore) ProcessSourceExecution(s Source, sourceFindings []SourceFinding) (Source, []FindingState, error) { ctx := context.Background() tx, err := db.DB.BeginTxx(ctx, nil) if err != nil { - return Source{}, err + return Source{}, nil, err } screated, err := createSource(ctx, tx, s) if err != nil { tx.Rollback() - return Source{}, err + return Source{}, nil, err } if screated == nil { - return Source{}, errors.New("unexpected nil pointer after source creation") + return Source{}, nil, errors.New("unexpected nil pointer after source creation") } // Before updating the findings we have to lock the tx to prevent race @@ -126,18 +111,18 @@ func (db *psqlxStore) ProcessSourceExecution(s Source, sourceFindings []SourceFi // have an effect on the current source being processed and its findings. relatedFamilies, err := db.relatedFamilies(s.SourceFamily) if err != nil { - return Source{}, err + return Source{}, nil, err } err = db.lockTxBySources(tx, relatedFamilies) if err != nil { - return Source{}, err + return Source{}, nil, err } for _, sf := range sourceFindings { err = createSourceIssue(ctx, tx, sf.IssueID, screated.ID) if err != nil { tx.Rollback() // nolint - return Source{}, err + return Source{}, nil, err } f := Finding{ IssueID: sf.IssueID, @@ -153,28 +138,28 @@ func (db *psqlxStore) ProcessSourceExecution(s Source, sourceFindings []SourceFi fe, err := createFindingEvent(ctx, tx, f, *screated, sf.Score, sf.Resources, sf.Details, sf.Fingerprint, sf.AffectedResourceString) if err != nil { tx.Rollback() // nolint - return Source{}, err + return Source{}, nil, err } err = updateLastSource(ctx, tx, fe.FindingID, screated.ID, screated.Time) if err != nil { tx.Rollback() // nolint - return Source{}, err + return Source{}, nil, err } } findingStates, err := findingsStates(ctx, tx, db.logger, screated.SourceFamily) if err != nil { tx.Rollback() - return Source{}, err + return Source{}, nil, err } err = replaceFindingsState(ctx, tx, db.logger, findingStates) if err != nil { tx.Rollback() - return Source{}, err + return Source{}, nil, err } - return *screated, tx.Commit() + return *screated, findingStates, tx.Commit() } func (db *psqlxStore) CreateSource(s Source) (*Source, error) { @@ -271,8 +256,23 @@ func (db *psqlxStore) lockTxBySources(tx *sqlx.Tx, sff SourceFamilies) error { } func createSource(ctx context.Context, tx *sqlx.Tx, s Source) (*Source, error) { - r := tx.QueryRowxContext(ctx, `INSERT INTO sources (name, component, instance, options, time, target_id) - VALUES ($1, $2, $3, $4, $5, $6) RETURNING *`, s.Name, s.Component, s.Instance, s.Options, s.Time, s.Target) + r := tx.QueryRowxContext(ctx, ` + WITH ins AS ( + INSERT INTO sources + (name, component, instance, options, time, target_id) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT ON CONSTRAINT sources_name_component_instance_options_key DO UPDATE + SET id = NULL -- never executed + WHERE FALSE + RETURNING * + ) + SELECT * + FROM ins + UNION ALL + SELECT * + FROM sources + WHERE name = $1 AND component = $2 AND instance = $3 AND options = $4`, + s.Name, s.Component, s.Instance, s.Options, s.Time, s.Target) err := r.StructScan(&s) if err != nil { return nil, err @@ -280,33 +280,6 @@ func createSource(ctx context.Context, tx *sqlx.Tx, s Source) (*Source, error) { return &s, nil } -func createSourceIfNotExists(ctx context.Context, tx *sqlx.Tx, s Source) (*Source, error) { - var err error - q := ` - WITH q as ( - SELECT * FROM sources - WHERE name=$1 AND component=$2 AND instance=$3 AND options=$4 - ), - c AS ( - INSERT INTO sources (name, component, instance, options, time, target_id) - SELECT $1, $2, $3, $4, $5, $6 - WHERE NOT EXISTS (SELECT 1 FROM q) - RETURNING * - ) - SELECT * FROM c - UNION ALL - SELECT * FROM q` - - r := tx.QueryRowContext(ctx, q, - s.Name, s.Component, s.Instance, s.Options, s.Time, s.Target) - var created Source - - err = r.Scan(&created.ID, &created.Name, &created.Component, &created.Instance, - &created.Options, &created.Time, &created.Target) - - return &created, err -} - // updateLastSource inserts or updates information about last source for specified finding. // Last sources table stores, for each finding, the ID of the last source that detected it. // This is precalculated so it improves performance for other queries like list findings. diff --git a/pkg/store/store.go b/pkg/store/store.go index 0b88b2e..9f44714 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -4,7 +4,9 @@ Copyright 2020 Adevinta package store -import "time" +import ( + "time" +) // VulnStore specifies the methods available for // the vulnerability database adapter. @@ -18,7 +20,7 @@ type VulnStore interface { CreateSourceIfNotExists(s Source) (*Source, error) FindSource(s Source) (*Source, error) SourceFamilies() (SourceFamilies, error) - ProcessSourceExecution(s Source, finding []SourceFinding) (Source, error) + ProcessSourceExecution(s Source, finding []SourceFinding) (Source, []FindingState, error) // Issues CreateIssue(i Issue) (*Issue, error) @@ -32,8 +34,8 @@ type VulnStore interface { FindFinding(f Finding) (*Finding, error) GetLastFindingEvent(findingID string) (*FindingEvent, error) CreateFindingEvent(eventTime time.Time, findingID, sourceID string, score float64, fingerprint, affectedResourceString string) (*Finding, error) - GetOpenSourceFindings(id string) ([]SourceFinding, error) RecalculateFindingsStatus(s SourceFamily) error FindIssueByID(id string) (*Issue, error) ExpireFindings(source string, ttl int) (int64, error) + GetFindingsExpanded(ids []string) ([]FindingExpanded, error) } diff --git a/pkg/store/targets.go b/pkg/store/targets.go index 6411af6..c0f26e9 100644 --- a/pkg/store/targets.go +++ b/pkg/store/targets.go @@ -9,8 +9,14 @@ import "context" // Target represents the target // scope for a check execution. type Target struct { - ID string - Identifier string + ID string `json:"id"` + Identifier string `json:"identifier"` +} + +// TargetTeams represents a target along with its associated teams. +type TargetTeams struct { + Target + Teams []string `json:"teams"` } func (db *psqlxStore) CreateTarget(t Target) (*Target, error) { diff --git a/test/docker-compose.yml b/test/docker-compose.yml index cd7d35c..5400999 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -3,6 +3,7 @@ version: '2' services: + # PostgreSQL vulndb_test: container_name: vulndb_test image: postgres:11-alpine @@ -17,3 +18,22 @@ services: - POSTGRES_USER=vulndb_test - POSTGRES_PASSWORD=vulndb_test - POSTGRES_DB=vulndb_test + # Kafka + zookeeper: + image: confluentinc/cp-zookeeper:7.2.1 + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + kafka: + image: confluentinc/cp-kafka:7.2.1 + depends_on: + - zookeeper + ports: + - 29092:29092 + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 diff --git a/test/processor_integration_test.go b/test/processor_integration_test.go index a50f3f1..bd1a7fb 100644 --- a/test/processor_integration_test.go +++ b/test/processor_integration_test.go @@ -15,35 +15,46 @@ import ( "testing" "time" - "github.com/adevinta/vulnerability-db/pkg/processor" - "github.com/adevinta/vulnerability-db/pkg/store" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus/hooks/test" + + "github.com/adevinta/vulnerability-db/pkg/notify" + "github.com/adevinta/vulnerability-db/pkg/processor" + "github.com/adevinta/vulnerability-db/pkg/store" + "github.com/adevinta/vulnerability-db/test/utils" ) const ( // Default maxEventAge (days). maxEventAge = 0 - // DB tables. - fEventsTable = "finding_events" - fExposuresTable = "finding_exposures" - findingsTable = "findings" - issuesTable = "issues" - sourcesTable = "sources" - targetsTable = "targets" - // timeFmt. timeFmt = "2006-01-02 15:04:05" ) +var ( + testTopic = fmt.Sprintf("test_findings_%d", time.Now().Unix()) + + cmpFindingsExpOpts = cmp.Options{cmpopts.IgnoreFields(notify.FindingNotification{}, + "Finding.ID", "Finding.IssueID", "Finding.TargetID", "Finding.ImpactDetails", + "TotalExposure", "CurrentExposure", "Resources", + "Issue.ID", "Issue.Recommendations", "Issue.ReferenceLinks", "Issue.Labels", + "Target.ID", "Target.Teams", + "Source.ID", "Source.Time", + )} +) + // Mock notifier. type mockNotifier struct { - pushEvents uint32 + pushEvents int + events []notify.FindingNotification } -func (n *mockNotifier) Push(mssg interface{}) error { +func (n *mockNotifier) PushFinding(f notify.FindingNotification) error { n.pushEvents++ + n.events = append(n.events, f) return nil } @@ -79,9 +90,10 @@ func TestProcessor(t *testing.T) { // regarding precision comparison using reflect.DeepEqual. // E.g.: 8.9 is stored and retrieved as 8.899996. testCases := []struct { - name string - checkData string - expected []expectedData + name string + checkData string + expected []expectedData + expectedNotifs []notify.FindingNotification }{ { name: "Happy path", @@ -97,7 +109,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Managed AWS databases using CA about to expire", "description": "Mock description", @@ -105,13 +117,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.adevinta.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-mock-check", @@ -120,7 +132,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(8.0), @@ -131,7 +143,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2020-01-21 16:03:25", "score": float64(8.0), @@ -141,7 +153,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-21 16:03:25", "fixed_at": nil, @@ -149,6 +161,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:rds:eu-west-1:123456789012:db:myRDS", + Score: 8.0, + Status: "OPEN", + Details: "Managed AWS details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Managed AWS databases using CA about to expire", + CWEID: 216, + Description: "Mock description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.adevinta.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000001", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-mock-check", + }, + }, + }, + }, + }, }, { name: "Should fix initial finding", @@ -164,7 +208,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000001", "target_id": "a0000000-0000-0000-0000-000000000001", @@ -174,7 +218,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 00:00:00", "fixed_at": "2020-01-01 10:00:00", @@ -182,6 +226,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.example.com", + Score: 7.0, + Status: "FIXED", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial issue", + CWEID: 1, + Description: "Initial issue description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000000", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-check", + }, + }, + }, + }, + }, }, { name: "Should create new finding event for initial finding", @@ -197,7 +273,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000001", "target_id": "a0000000-0000-0000-0000-000000000001", @@ -210,7 +286,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 00:00:00", "fixed_at": nil, @@ -218,7 +294,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(7.0), "time": "2020-01-02 10:00:00", @@ -228,6 +304,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.example.com", + Score: 7.0, + Status: "OPEN", + Details: "Managed AWS details - modified", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial issue", + CWEID: 1, + Description: "Initial issue description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000003", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-check", + }, + }, + }, + }, + }, }, { name: "Should reopen initial fixed finding", @@ -243,20 +351,20 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000002", "target_id": "a0000000-0000-0000-0000-000000000002", "status": "OPEN", "score": float64(9.0), "details": "Initial fixed issue details", - "affected_resource": "www.initial.fixed.example.com", + "affected_resource": "arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", "fingerprint": "NOT_PROVIDED", "resources": []byte(`[{"name": "resource name", "resources": [{"CVE": "CVE-8888-9999"}], "attributes": ["CVE"]}]`), }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-03 01:00:00", "fixed_at": nil, @@ -264,7 +372,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-03 01:00:00", @@ -274,6 +382,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", + Score: 9.0, + Status: "OPEN", + Details: "Initial fixed issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial fixed issue", + CWEID: 2, + Description: "Initial fixed issue description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.fixed.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000004", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-fixed-check", + }, + }, + }, + }, + }, }, { name: "Should reopen initial fixed finding via 'cross checktype'", @@ -289,21 +429,20 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ - "issue_id": "c0000000-0000-0000-0000-000000000002", - "target_id": "a0000000-0000-0000-0000-000000000002", - "status": "OPEN", - "score": float64(6.0), - // Score is overwritten by latest check. TODO: Rethink about this situation. + "issue_id": "c0000000-0000-0000-0000-000000000002", + "target_id": "a0000000-0000-0000-0000-000000000002", + "status": "OPEN", + "score": float64(6.0), "details": "Initial fixed issue details - modified", - "affected_resource": "www.initial.fixed.example.com", + "affected_resource": "arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", "fingerprint": "NOT_PROVIDED", "resources": []byte(`[{"name": "resource name", "resources": [{"CVE": "CVE-9999-9999"}], "attributes": ["CVE"]}]`), }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-03 01:00:00", "fixed_at": nil, @@ -311,7 +450,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(6.0), "time": "2020-01-03 01:00:00", @@ -321,6 +460,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", + Score: 6.0, + Status: "OPEN", + Details: "Initial fixed issue details - modified", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial fixed issue", + CWEID: 2, + Description: "Initial fixed issue description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.fixed.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000005", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-cross-checktype-check", + }, + }, + }, + }, + }, }, { name: "Should open multiple new findings", @@ -336,7 +507,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "New issue one", "description": "New issue one description", @@ -344,13 +515,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.new.example.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-new-check", @@ -359,7 +530,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(5.0), @@ -370,7 +541,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(5.0), "time": "2020-01-04 01:00:00", @@ -380,7 +551,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-04 01:00:00", "fixed_at": nil, @@ -388,7 +559,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "New issue two", "description": "New issue two description", @@ -396,7 +567,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ // issue_id will be overwritten in // related data map when parsing this @@ -412,7 +583,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(6.0), "time": "2020-01-04 01:00:00", @@ -422,7 +593,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-04 01:00:00", "fixed_at": nil, @@ -430,6 +601,68 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.new.example.com", + Score: 5.0, + Status: "OPEN", + Details: "New issue one details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "New issue one", + CWEID: 3, + Description: "New issue one description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.new.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000006", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-new-check", + }, + }, + }, + }, + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.new.example.com", + Score: 6.0, + Status: "OPEN", + Details: "New issue two details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "New issue two", + CWEID: 4, + Description: "New issue two description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.new.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000006", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-new-check", + }, + }, + }, + }, + }, }, { name: "Should reopen initially EXPIRED finding", @@ -445,7 +678,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Initial expired issue", "description": "Initial expired issue description", @@ -453,13 +686,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.initial.expired.example.com", }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(9.0), @@ -470,7 +703,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": nil, @@ -479,12 +712,44 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.expired.example.com", + Score: 9.0, + Status: "OPEN", + Details: "Initial expired issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial expired issue", + CWEID: 2, + Description: "Initial expired issue description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.expired.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000007", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-expired-check", + }, + }, + }, + }, + }, }, { name: "Should fix initially EXPIRED finding", checkData: ` { - "id":"00000000-0000-0000-0000-000000000007", + "id":"00000000-0000-0000-0000-000000000008", "checktype_name":"vulcan-initial-expired-check", "status":"FINISHED", "target":"www.initial.expired.example.com", @@ -494,7 +759,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Initial expired issue", "description": "Initial expired issue description", @@ -502,13 +767,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.initial.expired.example.com", }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "FIXED", "score": float64(9.0), @@ -517,7 +782,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-03-01 01:00:00", @@ -526,6 +791,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.expired.example.com", + Score: 9.0, + Status: "FIXED", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial expired issue", + CWEID: 2, + Description: "Initial expired issue description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.expired.example.com", + }, + }, + Source: store.Source{ + Instance: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-expired-check", + }, + }, + }, + }, + }, }, { name: "Should not reopen finding marked as false positive", @@ -541,7 +838,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000004", "target_id": "a0000000-0000-0000-0000-000000000004", @@ -554,7 +851,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": nil, @@ -562,7 +859,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-02 01:00:00", @@ -572,7 +869,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -580,6 +877,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.false.positive.example.com", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "Initial false positive issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue", + CWEID: 2, + Description: "Initial false positive description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-FFFFFFFFFFFF", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-check", + }, + }, + }, + }, + }, }, { name: "Should not fix a finding marked as false positive", @@ -595,7 +924,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000004", "target_id": "a0000000-0000-0000-0000-000000000004", @@ -606,7 +935,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-01-03 01:00:00", @@ -614,7 +943,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -622,6 +951,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.false.positive.example.com", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue", + CWEID: 2, + Description: "Initial false positive description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.example.com", + }, + }, + Source: store.Source{ + Instance: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeee4", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-check", + }, + }, + }, + }, + }, }, { name: "Should not reopen finding marked as false positive - previously FIXED", @@ -637,7 +998,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000004", "target_id": "a0000000-0000-0000-0000-000000000005", @@ -650,14 +1011,14 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-03 01:00:00", "ttr": nil, }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-01-01 02:00:00", @@ -665,7 +1026,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-03 01:00:00", @@ -675,7 +1036,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -683,6 +1044,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.false.positive.2.example.com", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "Initial false positive issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue", + CWEID: 2, + Description: "Initial false positive description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.2.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-FFFFFFFFFFF2", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-check", + }, + }, + }, + }, + }, }, { name: "Should not FIX finding marked as false positive - previously FIXED", @@ -698,7 +1091,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000004", "target_id": "a0000000-0000-0000-0000-000000000005", @@ -709,7 +1102,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-01-01 02:00:00", @@ -717,7 +1110,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -725,6 +1118,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.false.positive.2.example.com", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue", + CWEID: 2, + Description: "Initial false positive description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.2.example.com", + }, + }, + Source: store.Source{ + Instance: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeee5", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-check", + }, + }, + }, + }, + }, }, { name: "Should reopen FALSE POSITIVE finding due to fingerprint variation", @@ -740,7 +1165,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Initial false positive issue", "description": "Initial false positive description", @@ -748,13 +1173,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.initial.false.positive.example.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-initial-false-positive-check", @@ -763,7 +1188,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(9.0), @@ -772,7 +1197,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2020-01-01 01:00:00", "score": float64(9.0), @@ -780,7 +1205,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2021-01-21 16:03:25", "score": float64(9.0), @@ -788,7 +1213,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": nil, @@ -796,6 +1221,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.false.positive.example.com", + Score: 9.0, + Status: "OPEN", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue", + CWEID: 2, + Description: "Initial false positive description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.example.com", + }, + }, + Source: store.Source{ + Instance: "aaaaaaaa-0000-0000-0000-000000000000", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-check", + }, + }, + }, + }, + }, }, { name: "Should invalidate v1 model finding and open a new v2 model finding", @@ -812,7 +1269,7 @@ func TestProcessor(t *testing.T) { expected: []expectedData{ // Invalidated v1 finding expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Exposed SSH Ports", "description": "An SSH port is accessible from the public internet", @@ -820,13 +1277,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.model.test.example.com", }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "INVALIDATED", "score": float64(7.0), @@ -837,7 +1294,7 @@ func TestProcessor(t *testing.T) { }, // New v2 finding expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-model-check", @@ -846,7 +1303,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(7.0), @@ -856,7 +1313,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2021-01-21 16:03:25", "score": float64(7.0), @@ -865,7 +1322,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2021-01-21 16:03:25", "fixed_at": nil, @@ -873,6 +1330,68 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.model.test.example.com", + Score: 7.0, + Status: "INVALIDATED", + Details: "Exposed SSH Port in 22", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Exposed SSH Ports", + CWEID: 1, + Description: "An SSH port is accessible from the public internet", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.model.test.example.com", + }, + }, + Source: store.Source{ + Instance: "aaaaaaaa-0000-0000-0000-000000000001", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-model-check", + }, + }, + }, + }, + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "TCP22", + Score: 7.0, + Status: "OPEN", + Details: "Exposed SSH Port in 22", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Exposed SSH Ports", + CWEID: 1, + Description: "An SSH port is accessible from the public internet", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.model.test.example.com", + }, + }, + Source: store.Source{ + Instance: "aaaaaaaa-0000-0000-0000-000000000002", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-model-check", + }, + }, + }, + }, + }, }, { name: "Should create new v2 model finding event for old v1 model finding continuing its lifecycle due to target and affected resource match", @@ -888,7 +1407,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Exposed SSH Ports", "description": "An SSH port is accessible from the public internet", @@ -896,13 +1415,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.model.test.example.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-model-check", @@ -911,7 +1430,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(7.0), @@ -921,7 +1440,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2021-01-22 16:03:25", "score": float64(7.0), @@ -930,7 +1449,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2021-01-20 16:03:25", "fixed_at": nil, @@ -938,6 +1457,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.model.test.example.com", + Score: 7.0, + Status: "OPEN", + Details: "Exposed SSH Port in 22", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Exposed SSH Ports", + CWEID: 1, + Description: "An SSH port is accessible from the public internet", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.model.test.example.com", + }, + }, + Source: store.Source{ + Instance: "aaaaaaaa-0000-0000-0000-000000000003", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-model-check", + }, + }, + }, + }, + }, }, { name: "Should fix finding with continued lifecycle from v1 model to v2 model instead of invalidating it", @@ -953,7 +1504,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Certificate Host Mismatch", "description": "Certificate Host Mismatch Mock Desc", @@ -961,13 +1512,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.model.test.bis.example.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-model-bis-check", @@ -976,7 +1527,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "FIXED", "score": float64(7.0), @@ -986,7 +1537,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2021-02-20 16:03:25", "fixed_at": "2021-02-23 16:03:25", @@ -994,6 +1545,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.model.test.bis.example.com", + Score: 7.0, + Status: "FIXED", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Certificate Host Mismatch", + CWEID: 1, + Description: "Certificate Host Mismatch Mock Desc", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.model.test.bis.example.com", + }, + }, + Source: store.Source{ + Instance: "aaaaaaaa-0000-0000-0000-000000000022", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-model-bis-check", + }, + }, + }, + }, + }, }, { name: "Happy path v2", @@ -1009,7 +1592,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Managed AWS databases using CA about to expire v2", "description": "Mock description v2", @@ -1017,13 +1600,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.v2.adevinta.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-mock-check-v2", @@ -1032,7 +1615,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(8.0), @@ -1044,7 +1627,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2020-01-21 16:03:25", "score": float64(8.0), @@ -1053,7 +1636,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-21 16:03:25", "fixed_at": nil, @@ -1061,6 +1644,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "MyRDS", + Score: 8.0, + Status: "OPEN", + Details: "Managed AWS details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Managed AWS databases using CA about to expire v2", + CWEID: 216, + Description: "Mock description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.v2.adevinta.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000001", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-mock-check-v2", + }, + }, + }, + }, + }, }, { name: "Should fix initial v2 finding", @@ -1076,7 +1691,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000008", "target_id": "a0000000-0000-0000-0000-000000000008", @@ -1086,7 +1701,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 00:00:00", "fixed_at": "2020-01-01 10:00:00", @@ -1094,6 +1709,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:vpc/vpc-11223344556677889", + Score: 7.0, + Status: "FIXED", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial issue v2", + CWEID: 1, + Description: "Initial issue v2 description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000000", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-v2-check", + }, + }, + }, + }, + }, }, { name: "Should create new finding event for initial v2 finding", @@ -1109,7 +1756,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000008", "target_id": "a0000000-0000-0000-0000-000000000008", @@ -1123,7 +1770,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 00:00:00", "fixed_at": nil, @@ -1131,7 +1778,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(7.0), "time": "2020-01-02 10:00:00", @@ -1141,6 +1788,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:vpc/vpc-11223344556677889", + Score: 7.0, + Status: "OPEN", + Details: "Managed AWS details - modified", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial issue v2", + CWEID: 1, + Description: "Initial issue v2 description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000003", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-v2-check", + }, + }, + }, + }, + }, }, { name: "Should reopen initial fixed v2 finding", @@ -1156,7 +1835,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000009", "target_id": "a0000000-0000-0000-0000-000000000009", @@ -1169,7 +1848,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-03 01:00:00", "fixed_at": nil, @@ -1177,7 +1856,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-03 01:00:00", @@ -1187,6 +1866,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", + Score: 9.0, + Status: "OPEN", + Details: "Initial fixed issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial fixed issue v2", + CWEID: 2, + Description: "Initial fixed issue description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.fixed.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000004", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-fixed-v2-check", + }, + }, + }, + }, + }, }, { name: "Should reopen initial v2 fixed finding via 'cross checktype'", @@ -1202,13 +1913,12 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ - "issue_id": "c0000000-0000-0000-0000-000000000009", - "target_id": "a0000000-0000-0000-0000-000000000009", - "status": "OPEN", - "score": float64(6.0), - // Score is overwritten by latest check. TODO: Rethink about this situation. + "issue_id": "c0000000-0000-0000-0000-000000000009", + "target_id": "a0000000-0000-0000-0000-000000000009", + "status": "OPEN", + "score": float64(6.0), "details": "Initial fixed issue details - modified", "affected_resource": "arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", "affected_resource_string": "Updated resource", @@ -1217,7 +1927,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-03 01:00:00", "fixed_at": nil, @@ -1225,7 +1935,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(6.0), "time": "2020-01-03 01:00:00", @@ -1236,6 +1946,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "Updated resource", + Score: 6.0, + Status: "OPEN", + Details: "Initial fixed issue details - modified", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial fixed issue v2", + CWEID: 2, + Description: "Initial fixed issue description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.fixed.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000005", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-cross-checktype-v2-check", + }, + }, + }, + }, + }, }, { name: "Should open multiple new v2 findings", @@ -1251,7 +1993,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "New issue one v2", "description": "New issue one description v2", @@ -1259,13 +2001,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.new.v2.example.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-new-v2-check", @@ -1274,7 +2016,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(5.0), @@ -1285,7 +2027,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(5.0), "time": "2020-01-04 01:00:00", @@ -1295,7 +2037,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-04 01:00:00", "fixed_at": nil, @@ -1303,7 +2045,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "New issue two v2", "description": "New issue two description v2", @@ -1311,7 +2053,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ // issue_id will be overwritten in // related data map when parsing this @@ -1327,7 +2069,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(6.0), "time": "2020-01-04 01:00:00", @@ -1337,7 +2079,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-04 01:00:00", "fixed_at": nil, @@ -1345,6 +2087,68 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-1", + Score: 5.0, + Status: "OPEN", + Details: "New issue one details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "New issue one v2", + CWEID: 3, + Description: "New issue one description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.new.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000006", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-new-v2-check", + }, + }, + }, + }, + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-2", + Score: 6.0, + Status: "OPEN", + Details: "New issue two details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "New issue two v2", + CWEID: 4, + Description: "New issue two description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.new.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000006", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-new-v2-check", + }, + }, + }, + }, + }, }, { name: "Should reopen initially EXPIRED v2 finding", @@ -1360,7 +2164,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Initial expired issue v2", "description": "Initial expired issue description v2", @@ -1368,13 +2172,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.initial.expired.v2.example.com", }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(9.0), @@ -1385,7 +2189,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": nil, @@ -1394,6 +2198,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-1111", + Score: 9.0, + Status: "OPEN", + Details: "Initial expired issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial expired issue v2", + CWEID: 2, + Description: "Initial expired issue description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.expired.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000007", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-expired-v2-check", + }, + }, + }, + }, + }, }, { name: "Should fix initially EXPIRED v2 finding", @@ -1409,7 +2245,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Initial expired issue v2", "description": "Initial expired issue description v2", @@ -1417,13 +2253,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.initial.expired.v2.example.com", }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "affected_resource": "arn:aws:ec2:region:777788889999:sg/sg-1111", "fingerprint": "v2ExpiredFP", @@ -1432,7 +2268,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-03-01 01:00:00", @@ -1441,6 +2277,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-1111", + Score: 9.0, + Status: "FIXED", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial expired issue v2", + CWEID: 2, + Description: "Initial expired issue description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.expired.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeeb", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-expired-v2-check", + }, + }, + }, + }, + }, }, { name: "Should fix first initial double v2 finding and keep the second one open", @@ -1456,7 +2324,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000013", "target_id": "a0000000-0000-0000-0000-000000000013", @@ -1466,7 +2334,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 00:00:00", "fixed_at": "2020-01-01 10:00:00", @@ -1474,7 +2342,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000013", "target_id": "a0000000-0000-0000-0000-000000000013", @@ -1484,7 +2352,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(7.0), "time": "2020-01-01 10:00:00", @@ -1492,7 +2360,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": nil, @@ -1500,6 +2368,68 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:000011112222:vpc/vpc-99887766554433221", + Score: 7.0, + Status: "FIXED", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial double issue v2", + CWEID: 1, + Description: "Initial double issue v2 description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.double.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000014", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-double-v2-check", + }, + }, + }, + }, + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:333355557777:vpc/vpc-22553311007788996", + Score: 7.0, + Status: "OPEN", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial double issue v2", + CWEID: 1, + Description: "Initial double issue v2 description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.double.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-000000000009", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-double-v2-check", + }, + }, + }, + }, + }, }, { name: "Should not reopen v2 finding marked as false positive", @@ -1515,7 +2445,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000011", "target_id": "a0000000-0000-0000-0000-000000000011", @@ -1528,7 +2458,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": nil, @@ -1536,7 +2466,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-02 01:00:00", @@ -1546,7 +2476,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -1554,6 +2484,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-FFFF", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "Initial false positive issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue v2", + CWEID: 2, + Description: "Initial false positive description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-FFFFFFFFFFFF", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-v2-check", + }, + }, + }, + }, + }, }, { name: "Should not fix v2 finding marked as false positive", @@ -1569,7 +2531,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000011", "target_id": "a0000000-0000-0000-0000-000000000011", @@ -1580,7 +2542,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-01-03 01:00:00", @@ -1588,7 +2550,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -1596,6 +2558,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-FFFF", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue v2", + CWEID: 2, + Description: "Initial false positive description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeee11", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-v2-check", + }, + }, + }, + }, + }, }, { name: "Should not reopen v2 finding marked as false positive - previously FIXED", @@ -1611,7 +2605,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000011", "target_id": "a0000000-0000-0000-0000-000000000012", @@ -1623,14 +2617,14 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-03 01:00:00", "ttr": nil, }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-01-01 02:00:00", @@ -1638,7 +2632,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-03 01:00:00", @@ -1648,7 +2642,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -1656,6 +2650,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-FFF2", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "Initial false positive issue details", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue v2", + CWEID: 2, + Description: "Initial false positive description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.2.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-FFFFFFFFFFF2", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-v2-check", + }, + }, + }, + }, + }, }, { name: "Should not FIX v2 finding marked as false positive - previously FIXED", @@ -1671,7 +2697,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000011", "target_id": "a0000000-0000-0000-0000-000000000012", @@ -1682,7 +2708,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": "2020-01-01 02:00:00", @@ -1690,7 +2716,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "score": float64(9.0), "time": "2020-01-01 01:00:00", @@ -1698,6 +2724,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-FFF2", + Score: 9.0, + Status: "FALSE_POSITIVE", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue v2", + CWEID: 2, + Description: "Initial false positive description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.2.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeb5", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-v2-check", + }, + }, + }, + }, + }, }, { name: "Should reopen FALSE POSITIVE v2 finding due to fingerprint variation", @@ -1713,7 +2771,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Initial false positive issue v2", "description": "Initial false positive description v2", @@ -1721,13 +2779,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "www.initial.false.positive.v2.example.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-initial-false-positive-v2-check", @@ -1736,7 +2794,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(9.0), @@ -1745,7 +2803,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2020-01-01 01:00:00", "score": float64(9.0), @@ -1753,7 +2811,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2021-01-21 16:03:25", "score": float64(9.0), @@ -1761,7 +2819,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2020-01-01 01:00:00", "fixed_at": nil, @@ -1769,6 +2827,38 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "arn:aws:ec2:region:777788889999:sg/sg-FFFF", + Score: 9.0, + Status: "OPEN", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial false positive issue v2", + CWEID: 2, + Description: "Initial false positive description v2", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.false.positive.v2.example.com", + }, + }, + Source: store.Source{ + Instance: "bbbbbbbb-0000-0000-0000-FFFFFFFFFFF4", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-false-positive-v2-check", + }, + }, + }, + }, + }, }, { name: "Should handle NULL unicode character", @@ -1784,7 +2874,7 @@ func TestProcessor(t *testing.T) { }`, expected: []expectedData{ expectedData{ - table: issuesTable, + table: utils.IssuesTable, data: map[string]interface{}{ "summary": "Unicode character vulnerability", "description": "Unicode character vulnerability description", @@ -1792,13 +2882,13 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: targetsTable, + table: utils.TargetsTable, data: map[string]interface{}{ "identifier": "unicode.example.com", }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-unicode-check", @@ -1807,7 +2897,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "status": "OPEN", "score": float64(6.0), @@ -1818,7 +2908,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "time": "2022-01-01 16:03:25", "score": float64(6.0), @@ -1828,7 +2918,7 @@ func TestProcessor(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "found_at": "2022-01-01 16:03:25", "fixed_at": nil, @@ -1836,20 +2926,134 @@ func TestProcessor(t *testing.T) { }, }, }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "unicode.example.com/encoding", + Score: 6.0, + Status: "OPEN", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Unicode character vulnerability", + CWEID: 2, + Description: "Unicode character vulnerability description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "unicode.example.com", + }, + }, + Source: store.Source{ + Instance: "uuuuuuuu-0000-0000-0000-000000000001", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-unicode-check", + }, + }, + }, + }, + }, + }, + { + name: "Should verify checks processing idempotency", + checkData: ` + { + "id":"00000000-0000-0000-0000-000000000000", + "checktype_name":"vulcan-initial-check", + "status":"FINISHED", + "target":"www.initial.example.com", + "options":"{}", + "report":"https://dummy.com/v1/reports/00000000-0000-0000-0000-000000000000.json", + "tag":"adrn:adevinta:team:idempotency" + }`, + expected: []expectedData{ + expectedData{ + table: utils.FindingsTable, + data: map[string]interface{}{ + "issue_id": "c0000000-0000-0000-0000-000000000001", + "target_id": "a0000000-0000-0000-0000-000000000001", + "status": "OPEN", + "score": float64(7.0), + "details": "", + "affected_resource": "www.initial.example.com", + "fingerprint": "NOT_PROVIDED", + "resources": nil, + }, + }, + expectedData{ + table: utils.FExposuresTable, + data: map[string]interface{}{ + "found_at": "2020-01-01 00:00:00", + "fixed_at": nil, + "ttr": nil, + }, + }, + expectedData{ + table: utils.FEventsTable, + data: map[string]interface{}{ + "score": float64(7.0), + "time": "2020-01-01 00:00:00", + "details": "", + "fingerprint": "NOT_PROVIDED", + "resources": nil, + }, + }, + }, + expectedNotifs: []notify.FindingNotification{ + { + FindingExpanded: store.FindingExpanded{ + Finding: store.Finding{ + AffectedResource: "www.initial.example.com", + Score: 7.0, + Status: "OPEN", + Details: "", + }, + Issue: store.IssueLabels{ + Issue: store.Issue{ + Summary: "Initial issue", + CWEID: 1, + Description: "Initial issue description", + }, + }, + Target: store.TargetTeams{ + Target: store.Target{ + Identifier: "www.initial.example.com", + }, + }, + Source: store.Source{ + Instance: "00000000-0000-0000-0000-000000000000", + Options: "{}", + SourceFamily: store.SourceFamily{ + Name: "vulcan", + Component: "vulcan-initial-check", + }, + }, + }, + }, + }, }, } - testDB, err := db() + err := utils.CreateTopics([]string{testTopic}) if err != nil { - t.Fatalf("Error connecting to test DB: %v", err) + t.Fatalf("error creating test topic: %v", err) + } + testDB, err := utils.DB() + if err != nil { + t.Fatalf("error connecting to test DB: %v", err) } - checksProcessor := setupProcessor(t) + checksProcessor := setupProcessor(t, true) for _, tc := range testCases { - err := resetDB() + err := utils.ResetDB() if err != nil { - t.Fatalf("Error resetting DB: %v", err) + t.Fatalf("error resetting DB: %v", err) } t.Run(tc.name, func(t *testing.T) { @@ -1857,19 +3061,27 @@ func TestProcessor(t *testing.T) { // SNS notification. checkMssg, err := serializeCheckMssg(tc.checkData) if err != nil { - t.Fatalf("Error serializing check mssg: %v", err) + t.Fatalf("error serializing check mssg: %v", err) } // Process check message. err = checksProcessor.ProcessMessage(checkMssg) if err != nil { - t.Fatalf("Error processing mssg: %v", err) + t.Fatalf("error processing mssg: %v", err) } // Check expected data. err = checkData(t, tc.expected, testDB) if err != nil { - t.Fatalf("Error checking expected data: %v", err) + t.Fatalf("error checking expected data: %v", err) + } + findingsData, err := utils.ReadAllFindingsTopic(testTopic) + if err != nil { + t.Fatalf("error reading finding notifications from topic: %v", err) + } + err = checkNotifications(t, tc.expectedNotifs, findingsData) + if err != nil { + t.Fatalf("error checking notifications data: %v", err) } }) } @@ -1889,7 +3101,6 @@ func TestProcessor(t *testing.T) { // becouse these are the fields that can particularly be messed up // when handling concurrency wrongly. func TestProcessorConcurrency(t *testing.T) { - // checks is a slice of unordered check messages. // That means that their position in the slice does not represent // the time of execution specified in the reports related to them. @@ -1954,7 +3165,7 @@ func TestProcessorConcurrency(t *testing.T) { expected := []expectedData{ expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-initial-check", @@ -1963,7 +3174,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-initial-check", @@ -1972,7 +3183,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-initial-check", @@ -1981,7 +3192,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-initial-check", @@ -1990,7 +3201,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: sourcesTable, + table: utils.SourcesTable, data: map[string]interface{}{ "name": "vulcan", "component": "vulcan-initial-check", @@ -1999,7 +3210,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: findingsTable, + table: utils.FindingsTable, data: map[string]interface{}{ "issue_id": "c0000000-0000-0000-0000-000000000001", "target_id": "a0000000-0000-0000-0000-000000000001", @@ -2009,7 +3220,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "finding_id": "f0000000-0000-0000-0000-000000000001", "score": float64(7.0), @@ -2017,7 +3228,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "finding_id": "f0000000-0000-0000-0000-000000000001", "score": float64(7.0), @@ -2025,7 +3236,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: fEventsTable, + table: utils.FEventsTable, data: map[string]interface{}{ "finding_id": "f0000000-0000-0000-0000-000000000001", "score": float64(7.0), @@ -2033,7 +3244,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "finding_id": "f0000000-0000-0000-0000-000000000001", "found_at": "2020-01-01 00:00:00", @@ -2042,7 +3253,7 @@ func TestProcessorConcurrency(t *testing.T) { }, }, expectedData{ - table: fExposuresTable, + table: utils.FExposuresTable, data: map[string]interface{}{ "finding_id": "f0000000-0000-0000-0000-000000000001", "found_at": "2020-01-04 01:00:00", @@ -2052,18 +3263,18 @@ func TestProcessorConcurrency(t *testing.T) { }, } - testDB, err := db() + testDB, err := utils.DB() if err != nil { - t.Fatalf("Error connecting to test DB: %v", err) + t.Fatalf("error connecting to test DB: %v", err) } - err = resetDB() + err = utils.ResetDB() if err != nil { - t.Fatalf("Error resetting DB: %v", err) + t.Fatalf("error resetting DB: %v", err) } processorsPool := make([]*processor.CheckProcessor, len(checks)) for i := 0; i < len(checks); i++ { - processorsPool[i] = setupProcessor(t) + processorsPool[i] = setupProcessor(t, false) } var wg sync.WaitGroup @@ -2091,29 +3302,38 @@ func TestProcessorConcurrency(t *testing.T) { err = checkData(t, expected, testDB) if err != nil { - t.Fatalf("Error checking expected data: %v", err) + t.Fatalf("error checking expected data: %v", err) } } -func setupProcessor(t *testing.T) *processor.CheckProcessor { +func setupProcessor(t *testing.T, enableNotifications bool) *processor.CheckProcessor { t.Helper() - // Build logger. + // Logger logger, _ := test.NewNullLogger() - // Build notifier. - notifier := &mockNotifier{} - // Build results client. - resultsClient := &mockResultsClient{} - // Build DB. - db, err := store.NewDB(dbConnStr(), logger) + // DB + db, err := store.NewDB(utils.DBConnStr(), logger) if err != nil { - t.Fatalf("Error connecting to DB: %v", err) + t.Fatalf("error connecting to DB: %v", err) + } + // Notifier + var notifier notify.Notifier + if enableNotifications { + kafkaCli, err := utils.NewKafka(testTopic) + if err != nil { + t.Fatalf("error building Kafka client: %v", err) + } + notifier = notify.NewKafkaNotifier(kafkaCli, logger) + } else { + notifier = &mockNotifier{} } + // Results + resultsClient := &mockResultsClient{} - // Build processor. + // Build processor processor, err := processor.NewCheckProcessor(notifier, db, resultsClient, "", maxEventAge, logger) if err != nil { - t.Fatalf("Error building processor: %v", err) + t.Fatalf("error building processor: %v", err) } return processor } @@ -2146,13 +3366,13 @@ func checkData(t *testing.T, expected []expectedData, db *sqlx.DB) error { // expected data always has preference. mergeMaps(relatedData, ed.data, true) - data, err := fetchDBData(ed.table, relatedData, db) + data, err := utils.FetchDBData(ed.table, relatedData, db) if err != nil { return err } if !isSameData(ed.data, data) { - return fmt.Errorf("Expected (table: %v):\n%v\nBut got:\n%v", ed.table, ed.data, data) + return fmt.Errorf("expected (table: %v):\n%v\nBut got:\n%v", ed.table, ed.data, data) } // merge current related data @@ -2198,6 +3418,21 @@ func isSameData(eData, data map[string]interface{}) bool { return true } +func checkNotifications(t *testing.T, expected []notify.FindingNotification, got []utils.FindingsTopicData) error { + t.Helper() + + if len(expected) != len(got) { + return fmt.Errorf("mismatched number of notifications. got: %d but expected: %d", + len(got), len(expected)) + } + for i := range got { + if diff := cmp.Diff(got[i].Payload, expected[i], cmpFindingsExpOpts); diff != "" { + return fmt.Errorf("mismatched notifications. diff: %s", diff) + } + } + return nil +} + // serializeCheckMssg encapsulates the check data into a // mock SNS notification struct and serializes it into a string. func serializeCheckMssg(check string) (string, error) { diff --git a/test/reports_mock.go b/test/reports_mock.go index 6cfb0d7..da843c0 100644 --- a/test/reports_mock.go +++ b/test/reports_mock.go @@ -147,7 +147,7 @@ var ( "cwe_id": 2, "description":"Initial fixed issue description", "references":[], - "affected_resource":"www.initial.fixed.example.com", + "affected_resource":"arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", "resources":[{"name":"resource name", "rows": [{"CVE":"CVE-8888-9999"}], "header": ["CVE"]}] } ], @@ -165,7 +165,7 @@ var ( "cwe_id": 2, "description":"Initial fixed issue description", "references":[], - "affected_resource":"www.initial.fixed.example.com", + "affected_resource":"arn:aws:ec2:region:777788889999:sg/sg-11223344556677889", "resources":[{"name":"resource name", "rows": [{"CVE":"CVE-9999-9999"}], "header": ["CVE"]}] } ], @@ -596,6 +596,27 @@ var ( "start_time":"2022-01-01 16:03:13", "end_time":"2022-01-01 16:03:25" }`, + // Report that contains the same data as the "initial OPEN finding" present + // in the V2.0_initial_test_data.sql in order to test check processing idempotency + `{ + "check_id":"00000000-0000-0000-0000-000000000000", + "vulnerabilities": + [ + { + "summary":"Initial issue", + "score":7.0, + "details":"", + "cwe_id": 1, + "description":"Initial issue description", + "references":[], + "affected_resource":"www.initial.example.com", + "fingerprint":"NOT_PROVIDED", + "resources":[] + } + ], + "start_time":"2020-01-01 00:00:00", + "end_time":"2020-01-01 00:00:00" + }`, // Processor Concurrency: // Report that adds new finding event to initial finding. `{ diff --git a/test/utils.go b/test/utils/db.go similarity index 84% rename from test/utils.go rename to test/utils/db.go index 50bf5cf..2493d3d 100644 --- a/test/utils.go +++ b/test/utils/db.go @@ -5,7 +5,7 @@ Copyright 2020 Adevinta */ -package test +package utils import ( "errors" @@ -41,6 +41,14 @@ const ( flywayClean = "clean" flywayMigrate = "migrate" + // DB tables. + FEventsTable = "finding_events" + FExposuresTable = "finding_exposures" + FindingsTable = "findings" + IssuesTable = "issues" + SourcesTable = "sources" + TargetsTable = "targets" + // Query statements. sourcesStmt = "SELECT id as source_id, * FROM sources WHERE instance = :instance" targetsStmt = "SELECT id as target_id, * FROM targets WHERE identifier = :identifier" @@ -56,21 +64,21 @@ func init() { } } -// dbConnStr returns the connection +// DBConnStr returns the connection // str for postgres test db. -func dbConnStr() string { +func DBConnStr() string { return fmt.Sprintf(dbConnStrFmt, dbHost, dbPort, dbUser, dbPass, dbName, dbSSLMode) } -// db returns a new sqlx DB. -func db() (*sqlx.DB, error) { - return sqlx.Connect("postgres", dbConnStr()) +// DB returns a new sqlx DB. +func DB() (*sqlx.DB, error) { + return sqlx.Connect("postgres", DBConnStr()) } -// resetDB cleans the current vulndb database +// ResetDB cleans the current vulndb database // in the test postgres container and loads the // initial schema again. -func resetDB() error { +func ResetDB() error { return runFlywayCmd(dbDirPath, flywayClean, flywayMigrate) } @@ -115,25 +123,25 @@ func runFlywayCmd(dbDirPath string, flywayCommand ...string) error { return nil } -// fetchData retrieves data from test db. +// FetchData retrieves data from test db. // Because DB IDs are autogenerated, we can not query by them, // so we have to retrieve data based on other attributes. // E.g.: Issue{summary, description}, target{identifier}, etc. -func fetchDBData(table string, args map[string]interface{}, db *sqlx.DB) (map[string]interface{}, error) { +func FetchDBData(table string, args map[string]interface{}, db *sqlx.DB) (map[string]interface{}, error) { var stmt string switch table { - case sourcesTable: + case SourcesTable: stmt = sourcesStmt - case targetsTable: + case TargetsTable: stmt = targetsStmt - case issuesTable: + case IssuesTable: stmt = issuesStmt - case findingsTable: + case FindingsTable: stmt = findingsStmt - case fEventsTable: + case FEventsTable: stmt = fEventsStmt - case fExposuresTable: + case FExposuresTable: stmt = fExposuresStmt } diff --git a/test/utils/kafka.go b/test/utils/kafka.go new file mode 100644 index 0000000..c056250 --- /dev/null +++ b/test/utils/kafka.go @@ -0,0 +1,144 @@ +package utils + +import ( + "context" + "encoding/json" + "fmt" + "time" + + asyncapi "github.com/adevinta/vulnerability-db/pkg/asyncapi/kafka" + "github.com/adevinta/vulnerability-db/pkg/notify" + "github.com/confluentinc/confluent-kafka-go/kafka" +) + +// KafkaTestBroker contains the address of the local broker used for tests. +const KafkaTestBroker = "localhost:29092" + +func NewKafka(topic string) (*asyncapi.Client, error) { + return asyncapi.NewClient("", "", KafkaTestBroker, topic) +} + +type topicsOpResult []kafka.TopicResult + +func (t topicsOpResult) Error() kafka.ErrorCode { + for _, res := range t { + if res.Error.Code() != kafka.ErrNoError { + return res.Error.Code() + } + } + return kafka.ErrNoError +} + +func CreateTopics(names []string) error { + config := kafka.ConfigMap{ + "bootstrap.servers": KafkaTestBroker, + } + client, err := kafka.NewAdminClient(&config) + if err != nil { + return err + } + + waitDuration := time.Duration(time.Second * 60) + opTimeout := kafka.SetAdminOperationTimeout(waitDuration) + + results, err := client.DeleteTopics(context.Background(), names, opTimeout) + if err != nil { + return err + } + tResults := topicsOpResult(results) + if tResults.Error() != kafka.ErrNoError && tResults.Error() != kafka.ErrUnknownTopicOrPart { + return fmt.Errorf("error deleting topic %s", tResults.Error()) + } + + var topics []kafka.TopicSpecification + for _, name := range names { + topic := kafka.TopicSpecification{ + Topic: name, + NumPartitions: 1, + } + topics = append(topics, topic) + } + for { + results, err = client.CreateTopics(context.Background(), topics, opTimeout) + if err != nil { + return err + } + tResults = topicsOpResult(results) + if tResults.Error() == kafka.ErrNoError { + break + } + if tResults.Error() == kafka.ErrTopicAlreadyExists { + continue + } + return fmt.Errorf("error creating topics: %s", tResults.Error()) + } + return nil +} + +type FindingsTopicData struct { + Payload notify.FindingNotification + Headers map[string][]byte +} + +func ReadAllFindingsTopic(topic string) ([]FindingsTopicData, error) { + broker := KafkaTestBroker + config := kafka.ConfigMap{ + "go.events.channel.enable": true, + "bootstrap.servers": broker, + "group.id": "test_" + topic, + "enable.partition.eof": true, + "auto.offset.reset": "earliest", + "enable.auto.commit": false, + } + c, err := kafka.NewConsumer(&config) + if err != nil { + return nil, err + } + defer c.Close() + if err = c.Subscribe(topic, nil); err != nil { + return nil, err + } + + var topicFindingsData []FindingsTopicData +Loop: + for ev := range c.Events() { + switch e := ev.(type) { + case *kafka.Message: + data := e.Value + finding := notify.FindingNotification{} + // The data will be empty in case the event is a tombstone. + if len(data) > 0 { + err = json.Unmarshal(data, &finding) + if err != nil { + return nil, err + } + } + headers := map[string][]byte{} + for _, v := range e.Headers { + headers[v.Key] = v.Value + } + topicData := FindingsTopicData{ + Payload: finding, + Headers: headers, + } + topicFindingsData = append(topicFindingsData, topicData) + _, err := c.CommitOffsets([]kafka.TopicPartition{ + { + Topic: e.TopicPartition.Topic, + Partition: e.TopicPartition.Partition, + Offset: e.TopicPartition.Offset + 1, + }, + }) + if err != nil { + return nil, err + } + case kafka.Error: + return nil, e + case kafka.PartitionEOF: + break Loop + default: + return nil, fmt.Errorf("received unexpected message %v", e) + } + } + return topicFindingsData, nil +}