From 475d17692b53f5617229d3e340ccda46a2866ae6 Mon Sep 17 00:00:00 2001 From: artem-v Date: Sun, 14 Sep 2025 00:53:29 +0300 Subject: [PATCH 1/3] WIP ... --- .../services/gateway/http/HttpGateway.java | 20 +++++ .../gateway/http/HttpGatewayAcceptor.java | 42 ++-------- .../gateway/files/FileUploadTest.java | 78 ++++++++++++++++--- 3 files changed, 96 insertions(+), 44 deletions(-) diff --git a/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGateway.java b/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGateway.java index 880ec1c54..8da665f15 100644 --- a/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGateway.java +++ b/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGateway.java @@ -24,6 +24,7 @@ import java.util.function.Function; import reactor.netty.DisposableServer; import reactor.netty.http.server.HttpServer; +import reactor.netty.http.server.HttpServerFormDecoderProvider; import reactor.netty.resources.LoopResources; public class HttpGateway implements Gateway { @@ -38,6 +39,7 @@ public class HttpGateway implements Gateway { private final HttpGatewayAuthenticator authenticator; private final boolean corsEnabled; private final CorsConfigBuilder corsConfigBuilder; + private final Consumer formDecoderBuilder; private DisposableServer server; private LoopResources loopResources; @@ -50,6 +52,7 @@ private HttpGateway(Builder builder) { this.authenticator = builder.authenticator; this.corsEnabled = builder.corsEnabled; this.corsConfigBuilder = builder.corsConfigBuilder; + this.formDecoderBuilder = builder.formDecoderBuilder; } public static Builder builder() { @@ -80,6 +83,12 @@ public Gateway start(ServiceCall call, ServiceRegistry serviceRegistry) { .handle( new HttpGatewayAcceptor( callFactory.apply(call), serviceRegistry, errorMapper, authenticator)) + .httpFormDecoder( + builder -> { + if (formDecoderBuilder != null) { + formDecoderBuilder.accept(builder); + } + }) .bind() .toFuture() .get(); @@ -129,6 +138,8 @@ public static class Builder { .allowedRequestHeaders("*") .exposeHeaders("*") .maxAge(3600); + private Consumer formDecoderBuilder = + builder -> builder.maxSize(100 * 1024 * 1024); private Builder() {} @@ -196,6 +207,15 @@ public Builder corsConfigBuilder(Consumer consumer) { return this; } + public Consumer formDecoderBuilder() { + return formDecoderBuilder; + } + + public Builder formDecoderBuilder(Consumer consumer) { + this.formDecoderBuilder = consumer; + return this; + } + public HttpGateway build() { return new HttpGateway(this); } diff --git a/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java b/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java index ffcc40275..a002d4541 100644 --- a/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java +++ b/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java @@ -13,7 +13,6 @@ import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.ByteBufOutputStream; import io.netty.buffer.Unpooled; -import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.multipart.FileUpload; @@ -52,7 +51,6 @@ public class HttpGatewayAcceptor private static final String ERROR_NAMESPACE = "io.scalecube.services.error"; private static final long MAX_SERVICE_MESSAGE_SIZE = 1024 * 1024; - private static final long MAX_FILE_UPLOAD_SIZE = 100 * 1024 * 1024; private final ServiceCall serviceCall; private final ServiceRegistry serviceRegistry; @@ -104,21 +102,20 @@ private Mono handleFileUploadRequest( Map principal, HttpServerRequest httpRequest, HttpServerResponse httpResponse) { - return Mono.fromRunnable(() -> validateFileUploadRequest(httpRequest)) - .thenMany(httpRequest.receiveForm()) - .doOnNext(HttpGatewayAcceptor::doPostFileUploadCheck) + return httpRequest + .receiveForm() + .cast(FileUpload.class) .flatMap( - httpData -> + fileUpload -> serviceCall .requestBidirectional( - createFileFlux(httpData) + createFileFlux(fileUpload) .map( data -> toMessage( httpRequest, builder -> { - final var filename = - ((FileUpload) httpData).getFilename(); + final var filename = fileUpload.getFilename(); builder .headers(principal) .header(HEADER_UPLOAD_FILENAME, filename) @@ -132,7 +129,7 @@ private Mono handleFileUploadRequest( : response.hasData() // check data ? ok(httpResponse, response) : noContent(httpResponse)) - .doFinally(signalType -> httpData.delete())) + .doFinally(signalType -> fileUpload.delete())) .then() .onErrorResume(ex -> error(httpResponse, errorMapper.toMessage(ERROR_NAMESPACE, ex))); } @@ -321,29 +318,4 @@ private static Flux createFileFlux(HttpData httpData) { throw new RuntimeException(e); } } - - private static void validateFileUploadRequest(HttpServerRequest httpRequest) { - final var contentLengthHeader = - httpRequest.requestHeaders().get(HttpHeaderNames.CONTENT_LENGTH); - final var contentLength = - contentLengthHeader != null ? Long.parseLong(contentLengthHeader) : -1L; - final var limit = MAX_FILE_UPLOAD_SIZE; - if (contentLength > limit) { - throw new BadRequestException( - "File upload is too large, size: " + contentLength + ", limit: " + limit); - } - } - - private static void doPostFileUploadCheck(HttpData httpData) { - if (!(httpData instanceof FileUpload)) { - throw new BadRequestException("File upload is missing or invalid"); - } - final long fileSize = httpData.length(); - final var limit = MAX_FILE_UPLOAD_SIZE; - if (fileSize > limit) { - httpData.delete(); - throw new BadRequestException( - "File upload is too large, size: " + fileSize + ", limit: " + limit); - } - } } diff --git a/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java b/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java index 79e97da73..1e3c7e5e9 100644 --- a/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java +++ b/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.function.Function; +import java.util.stream.Stream; import okhttp3.MediaType; import okhttp3.MultipartBody; import okhttp3.OkHttpClient; @@ -31,11 +33,15 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import reactor.core.publisher.Mono; public class FileUploadTest { + private static final int MAX_SIZE = 15 * 1024 * 1024; + private static final int SIZE_OVER_LIMIT = MAX_SIZE + 1024 * 1024; + private static Microservices gateway; private static Microservices microservices; private static Address httpAddress; @@ -56,7 +62,12 @@ static void beforeAll() { .options(opts -> opts.metadata(serviceEndpoint))) .transport( () -> new RSocketServiceTransport().credentialsSupplier(credentialsSupplier)) - .gateway(() -> HttpGateway.builder().id("HTTP").build())); + .gateway( + () -> + HttpGateway.builder() + .id("HTTP") + .formDecoderBuilder(builder -> builder.maxSize(MAX_SIZE)) + .build())); microservices = Microservices.start( @@ -85,9 +96,9 @@ static void afterAll() { } } - @ParameterizedTest(name = "Upload file of size {0} bytes") - @ValueSource(longs = {0, 64, 512, 1024, 1024 * 1024, 10 * 1024 * 1024}) - public void uploadSuccessfully(long fileSize) throws Exception { + @ParameterizedTest(name = "Upload successfully, size: {0} bytes") + @ValueSource(longs = {0, 512, 1024, 1024 * 1024, 10 * 1024 * 1024}) + void uploadSuccessfully(long fileSize) throws Exception { final var client = new OkHttpClient(); final var file = generateFile(Files.createTempFile("export_report_", null), fileSize); @@ -113,10 +124,11 @@ public void uploadSuccessfully(long fileSize) throws Exception { } } - @Test - public void uploadFailed() throws Exception { + @ParameterizedTest + @MethodSource("uploadFailedSource") + void uploadFailed(UploadFailedArgs args) throws Exception { final var client = new OkHttpClient(); - final var file = generateFile(Files.createTempFile("export_report_", null), 1024); + final var file = generateFile(Files.createTempFile("export_report_", null), args.fileSize()); final var body = new MultipartBody.Builder() @@ -135,8 +147,44 @@ public void uploadFailed() throws Exception { assertEquals( HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), response.code(), "response.code"); assertEquals( - "{\"errorCode\":500,\"errorMessage\":\"Upload report failed: %s\"}" - .formatted(file.getName()), + "{\"errorCode\":%s,\"errorMessage\":\"%s\"}" + .formatted(args.errorCode(), args.errorMessageFunc().apply(file)), + response.body().string(), + "response.body"); + } + } + + @Test + void onlySingleFileAllowed() throws Exception { + final var client = new OkHttpClient(); + + final var fileSize = 1024 * 1024; + final var file1 = generateFile(Files.createTempFile("export_report_1_", null), fileSize); + final var file2 = generateFile(Files.createTempFile("export_report_2_", null), fileSize); + + final var body = + new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart( + "file", + file1.getName(), + RequestBody.create(file1, MediaType.get("application/text"))) + .addFormDataPart( + "file", + file2.getName(), + RequestBody.create(file2, MediaType.get("application/text"))) + .build(); + + final var request = + new Request.Builder() + .url("http://localhost:" + httpAddress.port() + "/v1/api/uploadReportError") + .post(body) + .build(); + + try (Response response = client.newCall(request).execute()) { + assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.code(), "response.code"); + assertEquals( + "{\"errorCode\":400,\"errorMessage\":\"Too many file-upload parts\"}", response.body().string(), "response.body"); } @@ -152,4 +200,16 @@ private static void assertFilesEqual(Path expected, Path actual) { throw new RuntimeException(e); } } + + private static Stream uploadFailedSource() { + return Stream.of( + new UploadFailedArgs(1024 * 1024, 500, file -> "Upload report failed: " + file.getName()), + new UploadFailedArgs( + SIZE_OVER_LIMIT, + 500, + file -> "java.io.IOException: Size exceed allowed maximum capacity")); + } + + private record UploadFailedArgs( + long fileSize, int errorCode, Function errorMessageFunc) {} } From 843cf64b7dbe1e98d3d91d27667f787dd0d0fbb9 Mon Sep 17 00:00:00 2001 From: artem-v Date: Sun, 14 Sep 2025 12:30:17 +0300 Subject: [PATCH 2/3] WIP --- .../gateway/http/HttpGatewayAcceptor.java | 75 +++++++++++++------ .../gateway/files/FileUploadTest.java | 16 ++-- 2 files changed, 63 insertions(+), 28 deletions(-) diff --git a/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java b/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java index a002d4541..d243cc8b9 100644 --- a/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java +++ b/services-gateway/src/main/java/io/scalecube/services/gateway/http/HttpGatewayAcceptor.java @@ -104,32 +104,57 @@ private Mono handleFileUploadRequest( HttpServerResponse httpResponse) { return httpRequest .receiveForm() - .cast(FileUpload.class) + .map( + data -> { + if (!(data instanceof FileUpload file)) { + throw new BadRequestException( + "Non file-upload part is not allowed, name=" + data.getName()); + } + return file.retain(); + }) + .collectList() .flatMap( - fileUpload -> - serviceCall - .requestBidirectional( - createFileFlux(fileUpload) - .map( - data -> - toMessage( - httpRequest, - builder -> { - final var filename = fileUpload.getFilename(); + files -> { + if (files.size() != 1) { + return Mono.error( + new BadRequestException( + "Exactly one file-upload part is expected (received: " + + files.size() + + ")")); + } + + final var fileUpload = files.get(0); + final var filename = fileUpload.getFilename(); + + return serviceCall + .requestBidirectional( + createFileFlux(fileUpload) + .map( + data -> + toMessage( + httpRequest, + builder -> builder .headers(principal) .header(HEADER_UPLOAD_FILENAME, filename) - .data(data); - }))) - .last() - .flatMap( - response -> - response.isError() // check error - ? error(httpResponse, response) - : response.hasData() // check data - ? ok(httpResponse, response) - : noContent(httpResponse)) - .doFinally(signalType -> fileUpload.delete())) + .data(data)))) + .last() + .flatMap( + response -> + response.isError() // check error + ? error(httpResponse, response) + : response.hasData() // check data + ? ok(httpResponse, response) + : noContent(httpResponse)) + .doFinally( + signalType -> { + try { + fileUpload.delete(); + } finally { + safestRelease(fileUpload); + } + }); + }) .then() .onErrorResume(ex -> error(httpResponse, errorMapper.toMessage(ERROR_NAMESPACE, ex))); } @@ -147,7 +172,11 @@ private Mono handleServiceRequest( final var limit = MAX_SERVICE_MESSAGE_SIZE; if (readableBytes >= limit) { throw new BadRequestException( - "Service message is too large, size: " + readableBytes + ", limit: " + limit); + "Service message is too large (size: " + + readableBytes + + ", limit: " + + limit + + ")"); } return reduce.writeBytes(byteBuf); }) diff --git a/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java b/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java index 1e3c7e5e9..67657a740 100644 --- a/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java +++ b/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java @@ -39,8 +39,8 @@ public class FileUploadTest { - private static final int MAX_SIZE = 15 * 1024 * 1024; - private static final int SIZE_OVER_LIMIT = MAX_SIZE + 1024 * 1024; + private static final int MAX_SIZE = 10 * 1024 * 1024; + private static final int SIZE_OVER_LIMIT = MAX_SIZE << 1; private static Microservices gateway; private static Microservices microservices; @@ -184,7 +184,10 @@ void onlySingleFileAllowed() throws Exception { try (Response response = client.newCall(request).execute()) { assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.code(), "response.code"); assertEquals( - "{\"errorCode\":400,\"errorMessage\":\"Too many file-upload parts\"}", + "{" + + "\"errorCode\":400," + + "\"errorMessage\":\"Exactly one file-upload part is expected (received: 2)\"" + + "}", response.body().string(), "response.body"); } @@ -203,10 +206,13 @@ private static void assertFilesEqual(Path expected, Path actual) { private static Stream uploadFailedSource() { return Stream.of( - new UploadFailedArgs(1024 * 1024, 500, file -> "Upload report failed: " + file.getName()), + new UploadFailedArgs( + 1024 * 1024, + HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), + file -> "Upload report failed: " + file.getName()), new UploadFailedArgs( SIZE_OVER_LIMIT, - 500, + HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), file -> "java.io.IOException: Size exceed allowed maximum capacity")); } From 55468a5cfa9a33f174f92cf9ee20c7437aed5385 Mon Sep 17 00:00:00 2001 From: artem-v Date: Sun, 14 Sep 2025 13:02:08 +0300 Subject: [PATCH 3/3] WIP --- .../io/scalecube/services/gateway/files/FileUploadTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java b/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java index 67657a740..3c920ae29 100644 --- a/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java +++ b/services-gateway/src/test/java/io/scalecube/services/gateway/files/FileUploadTest.java @@ -39,7 +39,7 @@ public class FileUploadTest { - private static final int MAX_SIZE = 10 * 1024 * 1024; + private static final int MAX_SIZE = 15 * 1024 * 1024; private static final int SIZE_OVER_LIMIT = MAX_SIZE << 1; private static Microservices gateway;