Use new zip.js version to allow zip64 uploads (#6939)

daniel-wer · fm3 · philippotto · web-flow · commit 27dcf807daa2 · 2023-03-27T14:10:08.000+02:00
* Reject dataset uploads if organization storage quota is exceeded

* warn user in upload-dataset-view when storage is exceeded and disable upload button

* changelog

* Use new zip.js version to allow zip64 uploads, for example

* update changelog

* also notify airbrake for zip errors

* Remove jszip dependency and rewrite code to use zip.js instead

---------

Co-authored-by: Florian M &lt;florian@scm.io&gt;
Co-authored-by: Philipp Otto &lt;philippotto@users.noreply.github.com&gt;
Co-authored-by: Philipp Otto &lt;philipp.4096@gmail.com&gt;
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added list of all respective team members to the administration page for teams. [#6915](https://github.com/scalableminds/webknossos/pull/6915)
 - Added email notifications for WK worker jobs. [#6918](https://github.com/scalableminds/webknossos/pull/6918)
 - Added support for viewing sharded neuroglancer precomputed datasets. [#6920](https://github.com/scalableminds/webknossos/pull/6920)
+- Added support for uploading zip64 files. [#6939](https://github.com/scalableminds/webknossos/pull/6939)
 
 ### Changed
 - Interpolation during rendering is now more performance intensive, since the rendering approach was changed. Therefore, interpolation is disabled by default. On the flip side, the rendered quality is often higher than it used to be. [#6748](https://github.com/scalableminds/webknossos/pull/6748)
diff --git a/frontend/javascripts/admin/dataset/dataset_upload_view.tsx b/frontend/javascripts/admin/dataset/dataset_upload_view.tsx
@@ -32,8 +32,7 @@ import Toast from "libs/toast";
 import * as Utils from "libs/utils";
 import messages from "messages";
 import { trackAction } from "oxalis/model/helpers/analytics";
-// @ts-expect-error ts-migrate(2306) FIXME: File ... Remove this comment to see the full error message
-import { createReader, BlobReader, ZipReader, Entry } from "zip-js-webpack";
+import { BlobReader, ZipReader, Entry } from "@zip.js/zip.js";
 import {
   CardContainer,
   DatasetNameFormItem,
@@ -473,7 +472,7 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
     );
   };
 
-  validateFiles = (files: FileWithPath[]) => {
+  validateFiles = async (files: FileWithPath[]) => {
     if (files.length === 0) {
       return;
     }
@@ -490,41 +489,40 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
       });
 
       if (fileExtension === "zip") {
-        createReader(
-          new BlobReader(file),
-          (reader: ZipReader) => {
-            reader.getEntries((entries: Array<Entry>) => {
-              const wkwFile = entries.find((entry: Entry) =>
-                Utils.isFileExtensionEqualTo(entry.filename, "wkw"),
-              );
-              const needsConversion = wkwFile == null;
-              this.handleNeedsConversionInfo(needsConversion);
-
-              const nmlFile = entries.find((entry: Entry) =>
-                Utils.isFileExtensionEqualTo(entry.filename, "nml"),
-              );
-              if (nmlFile) {
-                Modal.error({
-                  content: messages["dataset.upload_zip_with_nml"],
-                });
-              }
-            });
-          },
-          () => {
+        try {
+          const reader = new ZipReader(new BlobReader(file));
+          const entries = await reader.getEntries();
+          await reader.close();
+          const wkwFile = entries.find((entry: Entry) =>
+            Utils.isFileExtensionEqualTo(entry.filename, "wkw"),
+          );
+          const needsConversion = wkwFile == null;
+          this.handleNeedsConversionInfo(needsConversion);
+
+          const nmlFile = entries.find((entry: Entry) =>
+            Utils.isFileExtensionEqualTo(entry.filename, "nml"),
+          );
+          if (nmlFile) {
             Modal.error({
-              content: messages["dataset.upload_invalid_zip"],
+              content: messages["dataset.upload_zip_with_nml"],
             });
-            const form = this.formRef.current;
-
-            if (!form) {
-              return;
-            }
+          }
+        } catch (e) {
+          console.error(e);
+          ErrorHandling.notify(e as Error);
+          Modal.error({
+            content: messages["dataset.upload_invalid_zip"],
+          });
+          const form = this.formRef.current;
+
+          if (!form) {
+            return;
+          }
 
-            form.setFieldsValue({
-              zipFile: [],
-            });
-          },
-        );
+          form.setFieldsValue({
+            zipFile: [],
+          });
+        }
         // We return here since not more than 1 zip archive is supported anyway.
         return;
       } else if (fileExtension === "wkw") {
diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/skeleton_tab_view.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/skeleton_tab_view.tsx
@@ -21,7 +21,7 @@ import {
 import { batchActions } from "redux-batched-actions";
 import { connect } from "react-redux";
 import { saveAs } from "file-saver";
-import JSZip from "jszip";
+import { BlobReader, BlobWriter, ZipReader, Entry } from "@zip.js/zip.js";
 import * as React from "react";
 import _ from "lodash";
 import memoizeOne from "memoize-one";
@@ -208,22 +208,29 @@ export async function importTracingFiles(files: Array<File>, createGroupForEachF
 
     const tryParsingFileAsZip = async (file: File) => {
       try {
-        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'Promise<ArrayBuffer>' is not ass... Remove this comment to see the full error message
-        const zipFile = await JSZip().loadAsync(readFileAsArrayBuffer(file));
-        const nmlFileName = Object.keys(zipFile.files).find((key) =>
-          Utils.isFileExtensionEqualTo(key, "nml"),
+        const reader = new ZipReader(new BlobReader(file));
+        const entries = await reader.getEntries();
+        const nmlFileEntry = entries.find((entry: Entry) =>
+          Utils.isFileExtensionEqualTo(entry.filename, "nml"),
         );
-        // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
-        const nmlFile = await zipFile.file(nmlFileName).async("blob");
+
+        if (nmlFileEntry == null) {
+          await reader.close();
+          throw Error("Zip file doesn't contain an NML file.");
+        }
+
+        const nmlBlob = await nmlFileEntry.getData(new BlobWriter());
+        const nmlFile = new File([nmlBlob], nmlFileEntry.filename);
+
         const nmlImportActions = await tryParsingFileAsNml(nmlFile);
-        const dataFileName = Object.keys(zipFile.files).find((key) =>
-          Utils.isFileExtensionEqualTo(key, "zip"),
+
+        const dataFileEntry = entries.find((entry: Entry) =>
+          Utils.isFileExtensionEqualTo(entry.filename, "zip"),
         );
 
-        if (dataFileName) {
-          // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
-          const dataBlob = await zipFile.file(dataFileName).async("blob");
-          const dataFile = new File([dataBlob], dataFileName);
+        if (dataFileEntry) {
+          const dataBlob = await dataFileEntry.getData(new BlobWriter());
+          const dataFile = new File([dataBlob], dataFileEntry.filename);
           await Model.ensureSavedState();
           const storeState = Store.getState();
           const { tracing, dataset } = storeState;
@@ -263,6 +270,7 @@ export async function importTracingFiles(files: Array<File>, createGroupForEachF
           }
         }
 
+        await reader.close();
         return nmlImportActions;
       } catch (error) {
         // @ts-ignore
diff --git a/package.json b/package.json
@@ -151,6 +151,7 @@
     "@types/pngjs": "^6.0.1",
     "@types/three": "^0.142.0",
     "@use-it/interval": "^1.0.0",
+    "@zip.js/zip.js": "^2.6.81",
     "ansi-to-react": "^6.1.6",
     "antd": "^4.24.8",
     "backbone-events-standalone": "^0.2.7",
@@ -182,7 +183,6 @@
     "javascript-natural-sort": "^0.7.1",
     "js-priority-queue": "^0.1.5",
     "jsonschema": "^1.2.4",
-    "jszip": "^3.7.0",
     "lodash": "^4.17.21",
     "lz-string": "^1.4.4",
     "lz4-wasm": "^0.9.2",
@@ -229,8 +229,7 @@
     "typed-redux-saga": "^1.4.0",
     "url": "^0.11.0",
     "url-join": "^4.0.0",
-    "worker-loader": "^3.0.8",
-    "zip-js-webpack": "^1.0.0"
+    "worker-loader": "^3.0.8"
   },
   "resolutions": {
     "**/mini-store": "^1.1.0"
diff --git a/yarn.lock b/yarn.lock
@@ -2311,6 +2311,11 @@
   resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
   integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==
 
+"@zip.js/zip.js@^2.6.81":
+  version "2.6.81"
+  resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.6.81.tgz#c3c9618a8e02f3a24d359a0a14d46985fea971f5"
+  integrity sha512-VXrwa5fthYq74sIZsHarCFVSwnKdispTd/WQBgcNEuB9X0N3L5s8odRCjx9Zw6XsvpG5krqB4ZN4X0lLMyjgDA==
+
 JSONStream@^1.0.3, JSONStream@^1.0.4:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0"
@@ -7168,11 +7173,6 @@ image-size@~0.5.0:
   resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c"
   integrity sha1-Cd/Uq50g4p6xw+gLiZA3jfnjy5w=
 
-immediate@~3.0.5:
-  version "3.0.6"
-  resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
-  integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
-
 immutability-helper@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/immutability-helper/-/immutability-helper-3.1.1.tgz#2b86b2286ed3b1241c9e23b7b21e0444f52f77b7"
@@ -8161,16 +8161,6 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
-jszip@^3.7.0:
-  version "3.7.0"
-  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.7.0.tgz#9b8b995a4e7c9024653ce743e902076a82fdf4e6"
-  integrity sha512-Y2OlFIzrDOPWUnpU0LORIcDn2xN7rC9yKffFM/7pGhQuhO+SUhfm2trkJ/S5amjFvem0Y+1EALz/MEPkvHXVNw==
-  dependencies:
-    lie "~3.3.0"
-    pako "~1.0.2"
-    readable-stream "~2.3.6"
-    set-immediate-shim "~1.0.1"
-
 just-extend@^4.0.2:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744"
@@ -8313,13 +8303,6 @@ levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
-lie@~3.3.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
-  integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==
-  dependencies:
-    immediate "~3.0.5"
-
 lines-and-columns@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@@ -10027,7 +10010,7 @@ packet-reader@1.0.0:
   resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74"
   integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==
 
-pako@^1.0.5, pako@~1.0.2:
+pako@^1.0.5:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
   integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
@@ -12507,11 +12490,6 @@ set-blocking@^2.0.0:
   resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
   integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
 
-set-immediate-shim@~1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61"
-  integrity sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=
-
 set-value@^2.0.0, set-value@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b"
@@ -14672,11 +14650,6 @@ yocto-queue@^0.1.0:
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
 
-zip-js-webpack@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/zip-js-webpack/-/zip-js-webpack-1.0.0.tgz#553dd15c76a9b2cfb4de97f338f4eb0fe0e2a897"
-  integrity sha512-epPHhnoh3nxrpEzM07yLZZtKANIubMeWUH2bslUIPMGo/vBDKJhdYqoupsfI6uT24i8tR9i7+y/ajC3Sa4R21A==
-
 zustand@^3.7.2:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/zustand/-/zustand-3.7.2.tgz#7b44c4f4a5bfd7a8296a3957b13e1c346f42514d"
