fix: Compress optimize tours request payload (#194)

- Add compression middleware to the backend
- Change optimize tours endpoint to accept either a JSON body or a gzip-compressed JSON file
- Compress optimize tours request bodies client-side in the frontend
- Upgrade backend dependencies
- Upgrade container image to Node 22
- Upgrade Terraform providers
- Increase Cloud Run memory limit

Author: cmorabito-woolpert

.github/workflows/actions.yaml

@@ -32,10 +32,10 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Use Node.js v16 LTS
+    - name: Use Node.js v22 LTS
       uses: actions/setup-node@v3
       with:
-        node-version: '16'
+        node-version: '22'
         cache: 'npm'
         cache-dependency-path: application/backend/package-lock.json
 
@@ -56,10 +56,10 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Use Node.js v16 LTS
+    - name: Use Node.js v22 LTS
       uses: actions/setup-node@v3
       with:
-        node-version: '16'
+        node-version: '22'
         cache: 'npm'
         cache-dependency-path: application/frontend/package-lock.json
 

application/.nvmrc

@@ -1 +1 @@
-16
+22

application/Dockerfile

@@ -1,5 +1,5 @@
 # build frontend
-FROM node:16-alpine as build
+FROM node:22-alpine as build
 
 WORKDIR /usr/src/app
 
@@ -17,7 +17,7 @@ ARG configuration=production
 RUN npm run build -- --output-path=./dist/out --configuration $configuration
 
 # build backend
-FROM node:16-alpine
+FROM node:22-alpine
 
 WORKDIR /usr/src/app
 

application/backend/.eslintrc.json

@@ -17,10 +17,9 @@ limitations under the License.
 import { promises as fs } from "fs";
 
 import bodyParser from "body-parser";
+import compression from "compression";
 import cors from "cors";
 import express, { Response, Request } from "express";
-import expressPinoLogger from "express-pino-logger";
-import multer from "multer";
 
 import { router as apiRoutes } from "./routes/api";
 import { router as optimizationRoutes } from "./routes/optimization";
@@ -30,15 +29,16 @@ import { log } from "./logging";
 export const app = express();
 
 // logging
-app.use(
-  expressPinoLogger({
-    logger: log,
-  })
-);
+app.use(log);
 
 // headers
 app.disable("x-powered-by");
 
+// compression
+app.use(
+  compression()
+);
+
 // cors
 app.use(
   cors({
@@ -48,22 +48,15 @@ app.use(
   })
 );
 
-// multi-part form data
-app.use(
-  multer({
-    storage: multer.memoryStorage(),
-  }).single("file")
-);
-
 // body parser
-app.use(bodyParser.json({ limit: "50mb" }));
-app.use(bodyParser.urlencoded({ limit: "50mb", extended: true }));
+app.use(bodyParser.json({ limit: "1gb" }));
+app.use(bodyParser.urlencoded({ limit: "1gb", extended: true }));
 
 /**
  * Readiness/liveness probe
  */
 app.get("/healthz", async (req: Request, res: Response) => {
-  log.debug("Health check");
+  log.logger.debug("Health check");
   res.status(200).send("OK");
 });
 
@@ -76,13 +69,13 @@ app.get("/config.json", async (req: Request, res: Response) => {
   // get config.json from angular app when proxied
   if (process.env.FRONTEND_PROXY) {
     try {
-      const axios = (await import("axios")) as any;
+      const axios = (await import("axios") as any);
       const response = await axios.get(
         process.env.FRONTEND_PROXY + "config.json"
       );
       config = response.data;
     } catch (err) {
-      log.error(err);
+      log.logger.error(err);
     }
   }
 
@@ -92,7 +85,7 @@ app.get("/config.json", async (req: Request, res: Response) => {
       const data = await fs.readFile("public/config.json");
       config = JSON.parse(data.toString());
     } catch (err) {
-      log.error(err);
+      log.logger.error(err);
       return res.sendStatus(404);
     }
   }
@@ -122,7 +115,7 @@ app.get("/config.json", async (req: Request, res: Response) => {
 
     res.status(200).send(config);
   } catch (err) {
-    log.error(err);
+    log.logger.error(err);
     return res.sendStatus(500);
   }
 });

application/backend/app.ts

@@ -0,0 +1,21 @@
+import eslint from '@eslint/js';
+import tseslint from 'typescript-eslint';
+
+export default tseslint.config(
+  [
+    {
+      ignores: [
+        "node_modules/**",
+        "lib/**",
+        "**/*.js"
+      ],
+    },
+    eslint.configs.recommended,
+    tseslint.configs.recommended,
+  ],
+  {
+    rules: {
+      "@typescript-eslint/no-explicit-any": "warn"
+    }
+  }
+);

application/backend/eslint.config.mjs

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { pino, Logger } from "pino";
+import { pinoHttp, HttpLogger } from "pino-http";
 
 // map pino level to google cloud logging severity
 const levelToGoogleSeverity: { [level: string]: string } = {
@@ -55,9 +55,9 @@ if (!process.env.LOG_FORMAT || process.env.LOG_FORMAT === "google") {
 
         return object;
       },
-      log: (o: any) => {
+      log: (o: Record<string, unknown>) => {
         const { err, ...reshaped } = o;
-        if (err) {
+        if (err instanceof Error) {
           reshaped.stack_trace = err.stack;
         }
         return { serviceContext, ...reshaped };
@@ -82,6 +82,6 @@ else if (process.env.LOG_FORMAT === "pretty") {
   );
 }
 
-const p = pino(options);
+const p = pinoHttp(options);
 
-export const log: Logger = p;
+export const log: HttpLogger = p;

application/backend/logging.ts

@@ -1,4 +1,4 @@
-openapi: 3.0.3
+openapi: 3.1.1
 info:
   title: Fleet Routing App - Backend
   version: 4.0.0
@@ -9,6 +9,10 @@ paths:
         Static web content.
         Paths not matched by other API routes are directed to static file serving middleware
         to deliver the frontend application.
+      responses:
+        200:
+          description:
+            OK
 
   /healthz:
     get:
@@ -29,11 +33,19 @@ paths:
         description:
           A [`google.cloud.optimization.v1.IOptimizeToursRequest`](https://cloud.google.com/optimization/docs/reference/rpc/google.cloud.optimization.v1#optimizetoursrequest),
           with the `parent` property omitted (to be added by the backend).
+
         required: true
         content:
           application/json:
             schema:
-              type: array
+              type: object
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                file:
+                  format: binary
+                  description: A gzip-compressed JSON file containing the IOptimizeToursRequest data
       responses:
         200:
           description:
@@ -90,22 +102,21 @@ paths:
               schema:
                 type: array
                 items:
-                  schema:
-                    type: object
-                    properties:
-                      name:
-                        type: string
-                      pageToken:
-                        type: object
-                        properties:
-                          prefix:
-                            type: string
-                          maxResults:
-                            type: integer
-                          autoPaginate:
-                            type: boolean
-                          pageToken:
-                            type: string
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    pageToken:
+                      type: object
+                      properties:
+                        prefix:
+                          type: string
+                        maxResults:
+                          type: integer
+                        autoPaginate:
+                          type: boolean
+                        pageToken:
+                          type: string
                       required: ['name']
               example:
                 - name: "2020-10-23/0-vehicles-10-shipments-dayton.request.json"
@@ -259,22 +270,21 @@ paths:
               schema:
                 type: array
                 items:
-                  schema:
-                    type: object
-                    properties:
-                      name:
-                        type: string
-                      pageToken:
-                        type: object
-                        properties:
-                          prefix:
-                            type: string
-                          maxResults:
-                            type: integer
-                          autoPaginate:
-                            type: boolean
-                          pageToken:
-                            type: string
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    pageToken:
+                      type: object
+                      properties:
+                        prefix:
+                          type: string
+                        maxResults:
+                          type: integer
+                        autoPaginate:
+                          type: boolean
+                        pageToken:
+                          type: string
               example:
                 - name: "2020-10-23/0-vehicles-10-shipments-dayton.request.json"
                   pageToken: