diff --git a/README.md b/README.md index 8933764..e670247 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,9 @@ You can then configure it using the following directives: | `firetail_api_token` | `http` | Your API token from the FireTail platform | `PS-02-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` | | `firetail_url` | `http` | The URL of the API endpoint the FireTail NGINX module will send logs to. | `https://api.logging.eu-west-1.prod.firetail.app/logs/bulk` | | `firetail_enable` | `location` | Use this in every location block for which you want FireTail to be enabled. | This directive takes no arguments. | -| `firetail_allow_undefined_routes` | `http` | If set to `1`, `t`, `T`, `TRUE`, `true`, or `True`, requests to routes not defined in your OpenAPI specification will not be blocked. | `1`, `t`, `T`, `TRUE`, `true`, `True`, `0`, `f`, `F`, `FALSE`, `false`, `False` | +| `firetail_allow_undefined_routes` | `http` | If set to `on`, requests to routes not defined in your OpenAPI specification will not be blocked. | `on`, `off` (default) | +| `firetail_redact_request_bodies` | `http` | If set to `on`, request bodies will be redacted in your logs. | `on`, `off` (default) | +| `firetail_redact_response_bodies` | `http` | If set to `on`, response bodies will be redacted in your logs. | `on`, `off` (default) | See [dev/nginx.conf](./dev/nginx.conf) for an example of these in use. diff --git a/dev/nginx.conf b/dev/nginx.conf index f8d3b5b..f260885 100644 --- a/dev/nginx.conf +++ b/dev/nginx.conf @@ -11,7 +11,9 @@ http { # You should use lua-nginx-module to pull the API token in from an environment variable here firetail_api_token "YOUR-API-TOKEN"; firetail_url "https://api.logging.eu-west-1.prod.firetail.app/logs/bulk"; - firetail_allow_undefined_routes "true"; + firetail_allow_undefined_routes on; + firetail_redact_request_bodies off; + firetail_redact_response_bodies off; server { listen 80; diff --git a/src/nginx_module/access_phase_handler.c b/src/nginx_module/access_phase_handler.c index 35cb27f..56a990e 100644 --- a/src/nginx_module/access_phase_handler.c +++ b/src/nginx_module/access_phase_handler.c @@ -15,8 +15,8 @@ struct ValidateRequestBody_return { int r0; char *r1; }; -typedef struct ValidateRequestBody_return (*ValidateRequestBody)(void *, int, void *, int, void *, int, void *, int, - void *, int); +typedef struct ValidateRequestBody_return (*ValidateRequestBody)(int, void *, int, void *, int, void *, int, void *, + int); typedef struct { ngx_int_t status; @@ -150,9 +150,9 @@ static ngx_int_t FiretailClientBodyHandlerInternal(ngx_http_request_t *request) ngx_log_debug(NGX_LOG_DEBUG, request->connection->log, 0, "Validating request body..."); struct ValidateRequestBody_return validation_result = request_body_validator( - main_config->FiretailAllowUndefinedRoutes.data, main_config->FiretailAllowUndefinedRoutes.len, ctx->request_body, - ctx->request_body_size, request->unparsed_uri.data, request->unparsed_uri.len, request->method_name.data, - request->method_name.len, (char *)ctx->request_headers_json, ctx->request_headers_json_size); + (int)main_config->FiretailAllowUndefinedRoutes, ctx->request_body, ctx->request_body_size, + request->unparsed_uri.data, request->unparsed_uri.len, request->method_name.data, request->method_name.len, + (char *)ctx->request_headers_json, ctx->request_headers_json_size); ngx_log_debug(NGX_LOG_DEBUG, request->connection->log, 0, "Validation request result: %d", validation_result.r0); ngx_log_debug(NGX_LOG_DEBUG, request->connection->log, 0, "Validating request body: %s", validation_result.r1); diff --git a/src/nginx_module/filter_response_body.c b/src/nginx_module/filter_response_body.c index 4ee93d8..0118c4d 100644 --- a/src/nginx_module/filter_response_body.c +++ b/src/nginx_module/filter_response_body.c @@ -10,9 +10,9 @@ struct ValidateResponseBody_return { int r0; char *r1; }; -typedef struct ValidateResponseBody_return (*ValidateResponseBody)(char *, int, char *, int, char *, int, char *, int, - char *, int, void *, int, char *, int, void *, int, - int, void *, int); +typedef struct ValidateResponseBody_return (*ValidateResponseBody)(char *, int, char *, int, int, int, int, char *, int, + char *, int, char *, int, void *, int, char *, int, + int, char *, int); static ngx_buf_t *FiretailResponseBodyFilterBuffer(ngx_http_request_t *request, u_char *response); static ngx_int_t FiretailResponseBodyFilterFinalise(ngx_http_request_t *request, FiretailFilterContext *ctx, @@ -106,14 +106,27 @@ ngx_int_t FiretailResponseBodyFilter(ngx_http_request_t *request, ngx_chain_t *c ngx_log_debug(NGX_LOG_DEBUG, request->connection->log, 0, "Validating response body..."); FiretailConfig *main_config = ngx_http_get_module_main_conf(request, ngx_firetail_module); - validation_result = response_body_validator( - (char *)main_config->FiretailUrl.data, main_config->FiretailUrl.len, (char *)main_config->FiretailApiToken.data, - main_config->FiretailApiToken.len, (char *)main_config->FiretailAllowUndefinedRoutes.data, - (int)main_config->FiretailAllowUndefinedRoutes.len, (char *)ctx->request_body, (int)ctx->request_body_size, - (char *)ctx->request_headers_json, (int)ctx->request_headers_json_size, ctx->response_body, - ctx->response_body_size, response_headers_json_string, strlen(response_headers_json_string), - request->unparsed_uri.data, request->unparsed_uri.len, ctx->status_code, request->method_name.data, - request->method_name.len); + validation_result = response_body_validator((char *)main_config->FiretailUrl.data, // urlCharPtr + main_config->FiretailUrl.len, // urlLength + (char *)main_config->FiretailApiToken.data, // tokenCharPtr + main_config->FiretailApiToken.len, // tokenLength + (int)main_config->FiretailAllowUndefinedRoutes, // allowUndefinedRoutes + (int)main_config->FiretailRedactRequestBodies, // redactRequestBodies + (int)main_config->FiretailRedactResponseBodies, // redactResponseBodies + (char *)ctx->request_body, // reqBodyCharPtr + (int)ctx->request_body_size, // reqBodyLength + (char *)ctx->request_headers_json, // reqHeadersJsonCharPtr + (int)ctx->request_headers_json_size, // reqHeadersJsonLength + (char *)ctx->response_body, // resBodyCharPtr + ctx->response_body_size, // resBodyLength + response_headers_json_string, // resHeadersJsonCharPtr + strlen(response_headers_json_string), // resHeadersJsonLength + (char *)request->unparsed_uri.data, // pathCharPtr + request->unparsed_uri.len, // pathLength + ctx->status_code, // statusCode + (char *)request->method_name.data, // methodCharPtr + request->method_name.len // methodLength + ); ngx_log_debug(NGX_LOG_DEBUG, request->connection->log, 0, "Validation response result: %d", validation_result.r0); ngx_log_debug(NGX_LOG_DEBUG, request->connection->log, 0, "Validating response body: %s", validation_result.r1); diff --git a/src/nginx_module/firetail_config.h b/src/nginx_module/firetail_config.h index c6ed9fe..bbf6fbc 100644 --- a/src/nginx_module/firetail_config.h +++ b/src/nginx_module/firetail_config.h @@ -3,6 +3,8 @@ typedef struct { ngx_str_t FiretailApiToken; // TODO: this should probably be a *ngx_str_t ngx_str_t FiretailUrl; - ngx_str_t FiretailAllowUndefinedRoutes; + ngx_flag_t FiretailAllowUndefinedRoutes; ngx_int_t FiretailEnabled; + ngx_flag_t FiretailRedactRequestBodies; + ngx_flag_t FiretailRedactResponseBodies; } FiretailConfig; diff --git a/src/nginx_module/firetail_context.c b/src/nginx_module/firetail_context.c index 65e6d36..bffd65c 100644 --- a/src/nginx_module/firetail_context.c +++ b/src/nginx_module/firetail_context.c @@ -36,12 +36,10 @@ void *CreateFiretailConfig(ngx_conf_t *configuration_object) { if (firetail_config == NULL) { return NULL; } - - ngx_str_t firetail_api_token = ngx_string(""); - ngx_str_t firetail_url = ngx_string(""); - firetail_config->FiretailApiToken = firetail_api_token; - firetail_config->FiretailUrl = firetail_url; firetail_config->FiretailEnabled = 0; + firetail_config->FiretailAllowUndefinedRoutes = NGX_CONF_UNSET; + firetail_config->FiretailRedactRequestBodies = NGX_CONF_UNSET; + firetail_config->FiretailRedactResponseBodies = NGX_CONF_UNSET; return firetail_config; } diff --git a/src/nginx_module/firetail_directives.c b/src/nginx_module/firetail_directives.c index 77e709e..33a9198 100644 --- a/src/nginx_module/firetail_directives.c +++ b/src/nginx_module/firetail_directives.c @@ -1,48 +1,5 @@ #include -char *FiretailApiTokenDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, - void *http_main_config) { - // TODO: validate the args given to the directive - - // Find the firetail_api_key_field given the config pointer & offset in cmd - char *firetail_config = http_main_config; - ngx_str_t *firetail_api_key_field = (ngx_str_t *)(firetail_config + command_definition->offset); - - // Get the string value from the configuraion object - ngx_str_t *value = configuration_object->args->elts; - *firetail_api_key_field = value[1]; - - return NGX_CONF_OK; -} - -char *FiretailUrlDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, - void *http_main_config) { - // TODO: validate the args given to the directive - - // Find the firetail_api_key_field given the config pointer & offset in cmd - char *firetail_config = http_main_config; - ngx_str_t *firetail_url_field = (ngx_str_t *)(firetail_config + command_definition->offset); - - // Get the string value from the configuraion object - ngx_str_t *value = configuration_object->args->elts; - *firetail_url_field = value[1]; - - return NGX_CONF_OK; -} - -char *FiretailAllowUndefinedRoutesDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, - void *http_main_config) { - // Find the firetail_api_key_field given the config pointer & offset in cmd - char *firetail_config = http_main_config; - ngx_str_t *firetail_allow_undefined_routes_field = (ngx_str_t *)(firetail_config + command_definition->offset); - - // Get the string value from the configuraion object - ngx_str_t *value = configuration_object->args->elts; - *firetail_allow_undefined_routes_field = value[1]; - - return NGX_CONF_OK; -} - char *FiretailEnableDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, void *http_main_config) { // Find the firetail_enable_field given the config pointer & offset in cmd diff --git a/src/nginx_module/firetail_directives.h b/src/nginx_module/firetail_directives.h index c79472d..54709f1 100644 --- a/src/nginx_module/firetail_directives.h +++ b/src/nginx_module/firetail_directives.h @@ -2,38 +2,45 @@ #include #include "firetail_config.h" -char *FiretailApiTokenDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, - void *http_main_config); -char *FiretailUrlDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, - void *http_main_config); -char *FiretailAllowUndefinedRoutesDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, - void *http_main_config); char *FiretailEnableDirectiveCallback(ngx_conf_t *configuration_object, ngx_command_t *command_definition, void *http_main_config); -ngx_command_t kFiretailCommands[5] = { +ngx_command_t kFiretailCommands[7] = { {// Name of the directive ngx_string("firetail_api_token"), // Valid in the main config and takes one arg NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, // A callback function to be called when the directive is found in the // configuration - FiretailApiTokenDirectiveCallback, NGX_HTTP_MAIN_CONF_OFFSET, offsetof(FiretailConfig, FiretailApiToken), NULL}, + ngx_conf_set_str_slot, NGX_HTTP_MAIN_CONF_OFFSET, offsetof(FiretailConfig, FiretailApiToken), NULL}, {// Name of the directive ngx_string("firetail_url"), // Valid in the main config and takes one arg NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, // A callback function to be called when the directive is found in the // configuration - FiretailUrlDirectiveCallback, NGX_HTTP_MAIN_CONF_OFFSET, offsetof(FiretailConfig, FiretailUrl), NULL}, + ngx_conf_set_str_slot, NGX_HTTP_MAIN_CONF_OFFSET, offsetof(FiretailConfig, FiretailUrl), NULL}, {// Name of the directive ngx_string("firetail_allow_undefined_routes"), // Valid in the main config and takes one arg NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, // A callback function to be called when the directive is found in the // configuration - FiretailAllowUndefinedRoutesDirectiveCallback, NGX_HTTP_MAIN_CONF_OFFSET, - offsetof(FiretailConfig, FiretailAllowUndefinedRoutes), NULL}, + ngx_conf_set_flag_slot, NGX_HTTP_MAIN_CONF_OFFSET, offsetof(FiretailConfig, FiretailAllowUndefinedRoutes), NULL}, + {// Name of the directive + ngx_string("firetail_redact_request_bodies"), + // Valid in the main config and takes one arg + NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, + // A callback function to be called when the directive is found in the + // configuration + ngx_conf_set_flag_slot, NGX_HTTP_MAIN_CONF_OFFSET, offsetof(FiretailConfig, FiretailRedactRequestBodies), NULL}, + {// Name of the directive + ngx_string("firetail_redact_response_bodies"), + // Valid in the main config and takes one arg + NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, + // A callback function to be called when the directive is found in the + // configuration + ngx_conf_set_flag_slot, NGX_HTTP_MAIN_CONF_OFFSET, offsetof(FiretailConfig, FiretailRedactResponseBodies), NULL}, {// Name of the directive ngx_string("firetail_enable"), // Valid in location configs and takes no args diff --git a/src/validator/go.mod b/src/validator/go.mod index 6b800bc..34ef349 100644 --- a/src/validator/go.mod +++ b/src/validator/go.mod @@ -2,7 +2,7 @@ module firetail-validator go 1.20 -require github.com/FireTail-io/firetail-go-lib v0.0.0 +require github.com/FireTail-io/firetail-go-lib v0.3.0 require ( github.com/getkin/kin-openapi v0.110.0 // indirect @@ -15,5 +15,3 @@ require ( github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -replace github.com/FireTail-io/firetail-go-lib => github.com/FireTail-io/firetail-go-lib v0.2.3 diff --git a/src/validator/go.sum b/src/validator/go.sum index 77fb7eb..02a634e 100644 --- a/src/validator/go.sum +++ b/src/validator/go.sum @@ -2,6 +2,8 @@ github.com/FireTail-io/firetail-go-lib v0.2.1 h1:Jf3C1mAtCHAHHqSeuaMx6sSRmUS7Ojg github.com/FireTail-io/firetail-go-lib v0.2.1/go.mod h1:PH4aGBwry6z/3vzXEdcMaxK22E3xqPq2+w2y3FzETj4= github.com/FireTail-io/firetail-go-lib v0.2.3 h1:MDWnEpa/cyqc5sfhyHLjSXz8C+510nmHvpVxiOXnFAg= github.com/FireTail-io/firetail-go-lib v0.2.3/go.mod h1:PH4aGBwry6z/3vzXEdcMaxK22E3xqPq2+w2y3FzETj4= +github.com/FireTail-io/firetail-go-lib v0.3.0 h1:P8qc6hV2qLffQ0peX9oqdQyhswFnSw4xgcK8h3NBvIo= +github.com/FireTail-io/firetail-go-lib v0.3.0/go.mod h1:PH4aGBwry6z/3vzXEdcMaxK22E3xqPq2+w2y3FzETj4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/src/validator/main.go b/src/validator/main.go index 90cdb43..4513d84 100644 --- a/src/validator/main.go +++ b/src/validator/main.go @@ -16,7 +16,6 @@ import ( ) import ( "net/http" - "strconv" ) var firetailRequestMiddleware func(next http.Handler) http.Handler @@ -24,21 +23,19 @@ var firetailResponseMiddleware func(next http.Handler) http.Handler //export ValidateRequestBody func ValidateRequestBody( - allowUndefinedRoutes unsafe.Pointer, allowUndefinedRoutesLength C.int, - bodyCharPtr unsafe.Pointer, bodyLength C.int, - pathCharPtr unsafe.Pointer, pathLength C.int, - methodCharPtr unsafe.Pointer, methodLength C.int, - headersCharPtr unsafe.Pointer, headersLength C.int, + allowUndefinedRoutes C.int, + bodyCharPtr unsafe.Pointer, + bodyLength C.int, + pathCharPtr unsafe.Pointer, + pathLength C.int, + methodCharPtr unsafe.Pointer, + methodLength C.int, + headersCharPtr unsafe.Pointer, + headersLength C.int, ) (C.int, *C.char) { // Create the middleware if it hasn't already been done if firetailRequestMiddleware == nil { - allowUndefinedRoutesBool, err := strconv.ParseBool( - string(C.GoBytes(allowUndefinedRoutes, allowUndefinedRoutesLength)), - ) - if err != nil { - log.Println("Failed to initialise Firetail middleware, err:", err.Error()) - } - + var err error firetailRequestMiddleware, err = firetail.GetMiddleware(&firetail.Options{ OpenapiSpecPath: "/etc/nginx/appspec.yml", LogsApiToken: "", @@ -46,11 +43,10 @@ func ValidateRequestBody( DebugErrs: true, EnableRequestValidation: true, EnableResponseValidation: false, - AllowUndefinedRoutes: allowUndefinedRoutesBool, + AllowUndefinedRoutes: allowUndefinedRoutes == 1, }) if err != nil { log.Println("Failed to initialise Firetail middleware, err:", err.Error()) - // return 1 is error by convention return 1, nil } } @@ -91,39 +87,42 @@ func ValidateRequestBody( middlewareResponseBodyBytes, err := io.ReadAll(localResponseWriter.Body) responseCString := C.CString(string(middlewareResponseBodyBytes)) if err != nil { - return 1, responseCString // return 1 is error by convention + return 1, responseCString } // If the body differs after being passed through the middleware then we'll just infer it doesn't match the spec if string(middlewareResponseBodyBytes) != string(placeholderResponse) { - return 1, responseCString // return 1 is error by convention + return 1, responseCString } - return 0, responseCString // return 0 is success by convention + return 0, responseCString } //export ValidateResponseBody func ValidateResponseBody( urlCharPtr unsafe.Pointer, urlLength C.int, - tokenCharPtr unsafe.Pointer, tokenLength C.int, - allowUndefinedRoutes unsafe.Pointer, allowUndefinedRoutesLength C.int, - reqBodyCharPtr unsafe.Pointer, reqBodyLength C.int, - reqHeadersJsonCharPtr unsafe.Pointer, reqHeadersJsonLength C.int, - resBodyCharPtr unsafe.Pointer, resBodyLength C.int, - resHeadersJsonCharPtr unsafe.Pointer, resHeadersJsonLength C.int, - pathCharPtr unsafe.Pointer, pathLength C.int, + tokenCharPtr unsafe.Pointer, + tokenLength C.int, + allowUndefinedRoutes C.int, + redactRequestBodies C.int, + redactResponseBodies C.int, + reqBodyCharPtr unsafe.Pointer, + reqBodyLength C.int, + reqHeadersJsonCharPtr unsafe.Pointer, + reqHeadersJsonLength C.int, + resBodyCharPtr unsafe.Pointer, + resBodyLength C.int, + resHeadersJsonCharPtr unsafe.Pointer, + resHeadersJsonLength C.int, + pathCharPtr unsafe.Pointer, + pathLength C.int, statusCode C.int, - methodCharPtr unsafe.Pointer, methodLength C.int, + methodCharPtr unsafe.Pointer, + methodLength C.int, ) (C.int, *C.char) { if firetailResponseMiddleware == nil { - allowUndefinedRoutesBool, err := strconv.ParseBool( - string(C.GoBytes(allowUndefinedRoutes, allowUndefinedRoutesLength)), - ) - if err != nil { - log.Println("Failed to initialise Firetail middleware, err:", err.Error()) - } - + var err error firetailResponseMiddleware, err = firetail.GetMiddleware(&firetail.Options{ OpenapiSpecPath: "/etc/nginx/appspec.yml", LogsApiToken: strings.TrimSpace(string(C.GoBytes(tokenCharPtr, tokenLength))), @@ -131,11 +130,13 @@ func ValidateResponseBody( DebugErrs: true, EnableRequestValidation: false, EnableResponseValidation: true, - AllowUndefinedRoutes: allowUndefinedRoutesBool, + AllowUndefinedRoutes: allowUndefinedRoutes == 1, + RedactJSONRequestBodies: redactRequestBodies == 1, + RedactJSONResponseBodies: redactResponseBodies == 1, }) if err != nil { log.Println("Failed to initialise Firetail middleware, err:", err.Error()) - return 0, nil + return 1, nil } } @@ -192,15 +193,15 @@ func ValidateResponseBody( middlewareResponseBodyBytes, err := io.ReadAll(localResponseWriter.Body) responseCString := C.CString(string(middlewareResponseBodyBytes)) if err != nil { - return 1, responseCString // return 1 is error by convention + return 1, responseCString } // If the body differs after being passed through the middleware then we'll just infer it doesn't match the spec if string(middlewareResponseBodyBytes) != string(resBodySlice) || localResponseWriter.Code != int(statusCode) { - return 1, responseCString // return 1 is error by convention + return 1, responseCString } - return 0, C.CString(string(resBodySlice)) // return 0 is success by convention + return 0, C.CString(string(resBodySlice)) } func main() {}