Encourage coarser mags when annotating a high number of voxels

conf/application.conf

@@ -170,6 +170,7 @@ features {
   publicDemoDatasetUrl = "https://webknossos.org/datasets/scalable_minds/l4dense_motta_et_al_demo"
   exportTiffMaxVolumeMVx = 1024
   exportTiffMaxEdgeLengthVx = 8192
+  bucketSaveWarningThreshold = 1000
   defaultToLegacyBindings = false
   editableMappingsEnabled = true
   # The only valid item value is currently "ConnectomeView":

docs/volume_annotation/import_export.md

@@ -16,4 +16,19 @@ After finishing the annotation of a volume layer with a fallback layer, the comb
 Icon to open the materialize volume annotation dialog
 ///
 
-This button opens up a dialog that starts a long-running job which will materialize the volume annotation.
+This button opens up a dialog that starts a long-running job which will materialize the volume annotation.
+
+## Restricting magnifications
+
+WEBKNOSSOS allows data annotation in different magnifications.
+Restricting the available magnifications can greatly improve the performance when annotating large structures, such as nuclei, since the volume data does not need to be stored in all quality levels. 
+How to read: Mag 1 is the most detailed, 4-4-2 is downsampled by factor 4 in x and y, and by factor 2 in z.
+When annotating large structures, consider restricting the available magnifications in this specific annotation or in this volume annotation layer.
+
+### Restricting magnifications for a whole annotation
+
+Go to the dataset dashboard. Click on the three dots next to `New Annotation` for the dataset you want to annotate. Use the slider to restrict the volume magnifications, e.g. by omitting the finest magnification, which is usually 1-1-1, or the two finest magnifications.
+
+### Restricting magnifications for volume annotation layers
+
+Within an existing annotation, go to the `Layers` tab on the left. Click `Add Volume Annotation Layer` at the bottom of the list of all layers in this annotation. Restrict the magnifications as explained above.

frontend/javascripts/libs/toast.tsx

@@ -17,7 +17,9 @@ export type ToastConfig = {
   sticky?: boolean;
   timeout?: number;
   key?: string;
+  customFooter?: React.ReactNode;
   onClose?: () => void;
+  className?: string;
 };
 
 export type NotificationAPI = ReturnType<typeof notification.useNotification>[0];
@@ -148,8 +150,9 @@ const Toast = {
       duration: useManualTimeout || sticky ? 0 : timeOutInSeconds,
       message: toastMessage,
       style: {},
-      className: "",
+      className: config.className || "",
       onClose,
+      btn: config.customFooter,
     };
 
     if (type === "error") {

frontend/javascripts/test/backend-snapshot-tests/__snapshots__/misc.e2e.ts.snap

@@ -61,6 +61,7 @@ exports[`Misc APIs (E2E)  > getFeatureToggles() 1`] = `
 {
   "alignmentCostPerGVx": 0.5,
   "allowDeleteDatasets": true,
+  "bucketSaveWarningThreshold": 1000,
   "costPerCreditInDollar": 5.75,
   "costPerCreditInEuro": 5,
   "defaultToLegacyBindings": false,

frontend/javascripts/test/helpers/apiHelpers.ts

@@ -52,6 +52,7 @@ import {
 } from "test/fixtures/hybridtracing_server_objects";
 import type { ElementClass, ServerTracing } from "types/api_types";
 import { getConstructorForElementClass } from "viewer/model/helpers/typed_buffer";
+import { __setFeatures } from "features";
 
 const TOKEN = "secure-token";
 const ANNOTATION_TYPE = "annotationTypeValue";
@@ -377,6 +378,8 @@ export async function setupWebknossosForTesting(
     segmentMeshController: { meshesGroupsPerSegmentId: {} },
   });
 
+  __setFeatures({});
+
   try {
     await Model.fetch(
       null, // no compound annotation

frontend/javascripts/test/model/binary/layers/wkstore_adapter.spec.ts

@@ -312,7 +312,7 @@ describe("wkstore_adapter", () => {
 
     // @ts-ignore pushTransaction is a private method
     return pushQueue.pushTransaction(batch).then(() => {
-      expect(Store.dispatch).toHaveBeenCalledTimes(1);
+      expect(Store.dispatch).toHaveBeenCalledTimes(2);
       expect(Store.dispatch).toHaveBeenCalledWith(expectedSaveQueueItems);
     });
   });

frontend/javascripts/types/api_types.ts

@@ -760,6 +760,7 @@ export type APIFeatureToggles = {
   readonly publicDemoDatasetUrl: string;
   readonly exportTiffMaxVolumeMVx: number;
   readonly exportTiffMaxEdgeLengthVx: number;
+  readonly bucketSaveWarningThreshold: number;
   readonly defaultToLegacyBindings: boolean;
   readonly editableMappingsEnabled?: boolean;
   readonly optInTabs?: Array<string>;

frontend/javascripts/viewer/model/actions/annotation_actions.ts

@@ -77,6 +77,9 @@ export type AddPrecomputedMeshAction = ReturnType<typeof addPrecomputedMeshActio
 export type SetOthersMayEditForAnnotationAction = ReturnType<
   typeof setOthersMayEditForAnnotationAction
 >;
+export type ShowManyBucketUpdatesWarningAction = ReturnType<
+  typeof showManyBucketUpdatesWarningAction
+>;
 
 export type AnnotationActionTypes =
   | InitializeAnnotationAction
@@ -373,6 +376,11 @@ export const setOthersMayEditForAnnotationAction = (othersMayEdit: boolean) =>
     othersMayEdit,
   }) as const;
 
+export const showManyBucketUpdatesWarningAction = () =>
+  ({
+    type: "SHOW_MANY_BUCKET_UPDATES_WARNING",
+  }) as const;
+
 export const dispatchMaybeFetchMeshFilesAsync = async (
   dispatch: Dispatch<any>,
   segmentationLayer: APIDataLayer | null | undefined,

frontend/javascripts/viewer/model/actions/save_actions.ts

@@ -14,6 +14,7 @@ export type PushSaveQueueTransaction = {
   items: UpdateAction[];
   transactionId: string;
 };
+export type NotifyAboutUpdatedBucketsAction = ReturnType<typeof notifyAboutUpdatedBucketsAction>;
 type SaveNowAction = ReturnType<typeof saveNowAction>;
 export type ShiftSaveQueueAction = ReturnType<typeof shiftSaveQueueAction>;
 type DiscardSaveQueuesAction = ReturnType<typeof discardSaveQueuesAction>;
@@ -31,6 +32,7 @@ export type SaveAction =
   | PushSaveQueueTransaction
   | SaveNowAction
   | ShiftSaveQueueAction
+  | NotifyAboutUpdatedBucketsAction
   | DiscardSaveQueuesAction
   | SetSaveBusyAction
   | SetLastSaveTimestampAction
@@ -61,6 +63,9 @@ export const pushSaveQueueTransactionIsolated = (
     transactionId: getUid(),
   }) as const;
 
+export const notifyAboutUpdatedBucketsAction = (count: number) =>
+  ({ type: "NOTIFY_ABOUT_UPDATED_BUCKETS", count }) as const;
+
 export const saveNowAction = () =>
   ({
     type: "SAVE_NOW",

frontend/javascripts/viewer/model/bucket_data_handling/pushqueue.ts

@@ -6,7 +6,7 @@ import type DataCube from "viewer/model/bucket_data_handling/data_cube";
 import { createCompressedUpdateBucketActions } from "viewer/model/bucket_data_handling/wkstore_adapter";
 import Store from "viewer/store";
 import { escalateErrorAction } from "../actions/actions";
-import { pushSaveQueueTransaction } from "../actions/save_actions";
+import { notifyAboutUpdatedBucketsAction, pushSaveQueueTransaction } from "../actions/save_actions";
 import type { UpdateActionWithoutIsolationRequirement } from "../sagas/volume/update_actions";
 
 // Only process the PushQueue after there was no user interaction (or bucket modification due to
@@ -155,7 +155,8 @@ class PushQueue {
         createCompressedUpdateBucketActions(batch),
       );
       Store.dispatch(pushSaveQueueTransaction(items));
-
+      Store.dispatch(notifyAboutUpdatedBucketsAction(items.length));
+      console.log("notify about ", items.length, " items");
       this.compressingBucketCount -= batch.length;
     } catch (error) {
       // See other usage of escalateErrorAction for a detailed explanation.

frontend/javascripts/viewer/model/sagas/saving/save_saga.ts → frontend/javascripts/viewer/model/sagas/saving/save_saga.tsx

@@ -1,19 +1,24 @@
 import { getUpdateActionLog } from "admin/rest_api";
+import features from "features";
 import ErrorHandling from "libs/error_handling";
 import Toast from "libs/toast";
 import { sleep } from "libs/utils";
 import _ from "lodash";
 import { call, fork, put, takeEvery } from "typed-redux-saga";
 import type { APIUpdateActionBatch } from "types/api_types";
 import { getLayerByName, getMappingInfo } from "viewer/model/accessors/dataset_accessor";
-import { setVersionNumberAction } from "viewer/model/actions/save_actions";
+import { showManyBucketUpdatesWarningAction } from "viewer/model/actions/annotation_actions";
+import {
+  type NotifyAboutUpdatedBucketsAction,
+  setVersionNumberAction,
+} from "viewer/model/actions/save_actions";
 import { applySkeletonUpdateActionsFromServerAction } from "viewer/model/actions/skeletontracing_actions";
 import { applyVolumeUpdateActionsFromServerAction } from "viewer/model/actions/volumetracing_actions";
 import { globalPositionToBucketPositionWithMag } from "viewer/model/helpers/position_converter";
 import type { Saga } from "viewer/model/sagas/effect-generators";
 import { select } from "viewer/model/sagas/effect-generators";
 import { ensureWkReady } from "viewer/model/sagas/ready_sagas";
-import { Model } from "viewer/singletons";
+import { Model, Store } from "viewer/singletons";
 import type { SkeletonTracing, VolumeTracing } from "viewer/store";
 import { takeEveryWithBatchActionSupport } from "../saga_helpers";
 import { updateLocalHdf5Mapping } from "../volume/mapping_saga";
@@ -31,11 +36,49 @@ export function* setupSavingToServer(): Saga<void> {
   yield* takeEvery("INITIALIZE_ANNOTATION_WITH_TRACINGS", setupSavingForAnnotation);
   yield* takeEveryWithBatchActionSupport("INITIALIZE_SKELETONTRACING", setupSavingForTracingType);
   yield* takeEveryWithBatchActionSupport("INITIALIZE_VOLUMETRACING", setupSavingForTracingType);
+  yield* takeEvery("WK_READY", watchForNumberOfBucketsInSaveQueue);
 }
 
 const VERSION_POLL_INTERVAL_COLLAB = 10 * 1000;
 const VERSION_POLL_INTERVAL_READ_ONLY = 60 * 1000;
 const VERSION_POLL_INTERVAL_SINGLE_EDITOR = 30 * 1000;
+const CHECK_NUMBER_OF_BUCKETS_IN_SAVE_QUEUE_INTERVAL = 10 * 1000;
+const CHECK_NUMBER_OF_BUCKETS_SLIDING_WINDOW = 120 * 1000;
+
+function* watchForNumberOfBucketsInSaveQueue(): Saga<void> {
+  const bucketSaveWarningThreshold = features().bucketSaveWarningThreshold;
+  let bucketsForCurrentInterval = 0;
+  let currentBuckets: Array<number> = [];
+  const bucketCountArrayLength = Math.floor(
+    CHECK_NUMBER_OF_BUCKETS_SLIDING_WINDOW / CHECK_NUMBER_OF_BUCKETS_IN_SAVE_QUEUE_INTERVAL,
+  );
+  yield* call(
+    setInterval,
+    () => {
+      const sumOfBuckets = _.sum(currentBuckets);
+      console.log(
+        "buckets in last interval: ",
+        bucketsForCurrentInterval,
+        "currentBucketsArray: ",
+        currentBuckets,
+        "sumOfBuckets: ",
+        sumOfBuckets,
+      );
+      if (sumOfBuckets > bucketSaveWarningThreshold) {
+        Store.dispatch(showManyBucketUpdatesWarningAction());
+      }
+      currentBuckets.push(bucketsForCurrentInterval);
+      if (currentBuckets.length > bucketCountArrayLength) {
+        currentBuckets.shift();
+      }
+      bucketsForCurrentInterval = 0;
+    },
+    CHECK_NUMBER_OF_BUCKETS_IN_SAVE_QUEUE_INTERVAL,
+  );
+  yield* takeEvery("NOTIFY_ABOUT_UPDATED_BUCKETS", (action: NotifyAboutUpdatedBucketsAction) => {
+    bucketsForCurrentInterval += action.count;
+  });
+}
 
 function* watchForSaveConflicts(): Saga<void> {
   function* checkForNewVersion(): Saga<boolean> {

frontend/javascripts/viewer/view/components/many_bucket_updates_warning.tsx

@@ -0,0 +1,99 @@
+import { Button, Checkbox, type CheckboxChangeEvent, Space } from "antd";
+import { useInterval } from "libs/react_helpers";
+import Toast from "libs/toast";
+import UserLocalStorage from "libs/user_local_storage";
+import { useCallback, useEffect, useRef } from "react";
+import { useReduxActionListener } from "viewer/model/helpers/listener_helpers";
+
+const TOO_MANY_BUCKETS_TOAST_KEY = "manyBucketUpdatesWarningToast";
+const WARNING_SUPPRESSION_USER_STORAGE_KEY = "suppressBucketWarning";
+
+export function ManyBucketUpdatesWarning(): React.ReactNode {
+  const notificationAPIRef = useRef(Toast.notificationAPI);
+  const neverShowAgainRef = useRef(false);
+  const dontShowAgainInThisSessionRef = useRef(false);
+
+  useEffect(() => {
+    if (Toast.notificationAPI != null) {
+      notificationAPIRef.current = Toast.notificationAPI;
+    }
+  }, []);
+  useInterval(() => {
+    UserLocalStorage.setItem("suppressBucketWarning", "false");
+    console.log("resetting suppressBucketWarning to false every 120s for dev purposes");
+  }, 120 * 1000); //TODO_C dev
+
+  const onClose = useCallback(() => {
+    notificationAPIRef.current?.destroy(TOO_MANY_BUCKETS_TOAST_KEY);
+    UserLocalStorage.setItem("suppressBucketWarning", neverShowAgainRef.current.toString());
+  }, []);
+  const handleCheckboxChange = (event: CheckboxChangeEvent) => {
+    neverShowAgainRef.current = event.target.checked;
+  };
+
+  const warningMessage =
+    "You are annotating a large area with fine magnifications. This can significantly slow down WEBKNOSSOS. Consider creating an annotation or annotation layer with restricted magnifications.";
+  const linkToDocs =
+    "https://docs.webknossos.org/volume_annotation/import_export.html#restricting-magnifications";
+  const neverShowAgainCheckbox = (
+    <Checkbox onChange={handleCheckboxChange} style={{ marginTop: "8px", marginBottom: "5px" }}>
+      Never show this again
+    </Checkbox>
+  );
+  const closeButton = (
+    <Button
+      onClick={() => {
+        onClose();
+      }}
+    >
+      Close
+    </Button>
+  );
+  const linkToDocsButton = (
+    <Button href={linkToDocs} target="_blank" rel="noopener noreferrer" type="primary">
+      Learn how
+    </Button>
+  );
+  const footer = (
+    <div>
+      <Space>
+        {linkToDocsButton}
+        {closeButton}
+      </Space>
+    </div>
+  );
+
+  const showWarningToast = () => {
+    const suppressManyBucketUpdatesWarning = UserLocalStorage.getItem(
+      WARNING_SUPPRESSION_USER_STORAGE_KEY,
+    );
+    if (notificationAPIRef.current == null) {
+      return null;
+    }
+    if (
+      suppressManyBucketUpdatesWarning !== "true" &&
+      dontShowAgainInThisSessionRef.current !== true
+    ) {
+      console.warn(warningMessage + " For more info, visit: " + linkToDocs);
+      Toast.warning(
+        <>
+          {warningMessage}
+          <br />
+          {neverShowAgainCheckbox}
+        </>,
+        {
+          customFooter: footer,
+          key: TOO_MANY_BUCKETS_TOAST_KEY,
+          sticky: true,
+          onClose,
+          className: "many-bucket-updates-warning",
+        },
+      );
+      dontShowAgainInThisSessionRef.current = true;
+    } else {
+      console.log("suppressing warning toast"); //TODO_C dev
+    }
+  };
+  useReduxActionListener("SHOW_MANY_BUCKET_UPDATES_WARNING", () => showWarningToast());
+  return null;
+}

frontend/javascripts/viewer/view/layouting/tracing_layout_view.tsx

@@ -50,6 +50,7 @@ import WelcomeToast from "viewer/view/novel_user_experiences/welcome_toast";
 import { importTracingFiles } from "viewer/view/right-border-tabs/trees_tab/skeleton_tab_view";
 import TracingView from "viewer/view/tracing_view";
 import VersionView from "viewer/view/version_view";
+import { ManyBucketUpdatesWarning } from "../components/many_bucket_updates_warning";
 import TabTitle from "../components/tab_title_component";
 import { determineLayout } from "./default_layout_configs";
 import FlexLayoutWrapper from "./flex_layout_wrapper";
@@ -364,6 +365,7 @@ class TracingLayoutView extends React.PureComponent<PropsWithRouter, State> {
                       display: "flex",
                     }}
                   >
+                    <ManyBucketUpdatesWarning />
                     <ActionBarView
                       layoutProps={{
                         storedLayoutNamesForView: currentLayoutNames,

frontend/stylesheets/main.less

@@ -716,4 +716,10 @@ button.narrow {
   >div:last-of-type {
     margin-bottom: 15px;
   }
+}
+
+.many-bucket-updates-warning{
+  .ant-notification-notice-btn{
+    margin-top: 0 !important;
+  }
 }