feat: retry acquiring lockfile, change timeouts (#289)

Author: AlCalzone

README.md

@@ -82,13 +82,20 @@ The following options exist:
 
 To create a compressed copy of the database in `/path/to/file.dump`, use the `dump()` method. If any data is written to the db during the dump, it is appended to the dump but most likely compressed.
 
-### Changing where the lockfile is created
+### Lockfile-related options
 
-Normally, the lockfile to avoid concurrent access to the DB file is created right next to the DB file. You can change this, e.g. to put the lockfile into a `tmpfs`:
+A lockfile is used to avoid concurrent access to the DB file. Multiple options exist to control where this lockfile is created and how it is accessed:
 ```ts
-const db = new DB("/path/to/file", { lockfileDirectory: "/var/tmp" });
+const db = new DB("/path/to/file", { lockfile: { /* lockfile options */ } });
 ```
-If the directory does not exist, it will be created when opening the DB.
+
+| Option | Default | Description |
+|-----------------|---------|-------------|
+| `directory` | - | Change where the lockfile is created, e.g. to put the lockfile into a `tmpfs`. By default the lockfile is created in the same directory as the DB file. If the directory does not exist, it will be created when opening the DB. |
+| `staleMs` | `10000` | Duration after which the lock is considered stale. Minimum: `5000` |
+| `updateMs` | `staleMs/2` | The interval in which the lockfile's `mtime` will be updated. Range: `1000 ... staleMs/2` |
+| `retries` | `0` | How often to retry acquiring a lock before giving up. The retries progressively wait longer with an exponential backoff strategy. |
+
 
 ### Copying and compressing the database
 

src/lib/db.test.ts

@@ -132,6 +132,76 @@ describe("lib/db", () => {
 				).toThrowError("maxBufferedCommands");
 			});
 		});
+
+		describe("validates lockfile options", () => {
+			it("staleMs < 5000", () => {
+				expect(
+					() =>
+						new JsonlDB("foo", {
+							lockfile: {
+								staleMs: 2500,
+							},
+						}),
+				).toThrowError("staleMs");
+			});
+
+			it("updateMs < 1000", () => {
+				expect(
+					() =>
+						new JsonlDB("foo", {
+							lockfile: {
+								updateMs: 999,
+							},
+						}),
+				).toThrowError("updateMs");
+			});
+
+			it("updateMs > staleMs/2", () => {
+				expect(
+					() =>
+						new JsonlDB("foo", {
+							lockfile: {
+								staleMs: 5000,
+								updateMs: 10001,
+							},
+						}),
+				).toThrowError("updateMs");
+			});
+
+			it("retries < 0", () => {
+				expect(
+					() =>
+						new JsonlDB("foo", {
+							lockfile: {
+								retries: -1,
+							},
+						}),
+				).toThrowError("retries");
+			});
+
+			it("retries > 10", () => {
+				expect(
+					() =>
+						new JsonlDB("foo", {
+							lockfile: {
+								retries: 11,
+							},
+						}),
+				).toThrowError("retries");
+			});
+
+			it("lockfileDirectory and lockfile.directory both present", () => {
+				expect(
+					() =>
+						new JsonlDB("foo", {
+							lockfile: {
+								directory: "/lock/dir",
+							},
+							lockfileDirectory: "/lock/dir2",
+						}),
+				).toThrowError("lockfileDirectory");
+			});
+		});
 	});
 
 	describe("open()", () => {

src/lib/db.ts

@@ -68,7 +68,28 @@ export interface JsonlDBOptions<V> {
 		maxBufferedCommands?: number;
 	};
 
+	/** Configure settings related to the lockfile */
+	lockfile?: Partial<{
+		/**
+		 * Override in which directory the lockfile is created.
+		 * Defaults to the directory in which the DB file is located.
+		 */
+		directory?: string;
+
+		/** Duration after which the lock is considered stale. Minimum: 5000, Default: 10000 */
+		staleMs?: number;
+		/** The interval in which the lockfile's `mtime` will be updated. Range: [1000...staleMs/2]. Default: staleMs/2  */
+		updateMs?: number;
+		/**
+		 * How often to retry acquiring a lock before giving up. The retries progressively wait longer with an exponential backoff strategy.
+		 * Range: [0...10]. Default: 0
+		 */
+		retries?: number;
+	}>;
+
 	/**
+	 * @deprecated Use lockfile.directory instead.
+	 *
 	 * Override in which directory the lockfile is created.
 	 * Defaults to the directory in which the DB file is located.
 	 */
@@ -150,8 +171,10 @@ export class JsonlDB<V extends unknown = unknown> {
 		this.filename = filename;
 		this.dumpFilename = this.filename + ".dump";
 		this.backupFilename = this.filename + ".bak";
-		this.lockfileName = options.lockfileDirectory
-			? path.join(options.lockfileDirectory, path.basename(this.filename))
+		const lockfileDirectory =
+			options.lockfile?.directory ?? options.lockfileDirectory;
+		this.lockfileName = lockfileDirectory
+			? path.join(lockfileDirectory, path.basename(this.filename))
 			: this.filename;
 
 		this.options = options;
@@ -198,6 +221,37 @@ export class JsonlDB<V extends unknown = unknown> {
 				throw new Error("maxBufferedCommands must be >= 0");
 			}
 		}
+		if (options.lockfile) {
+			const {
+				directory,
+				retries,
+				staleMs = 10000,
+				updateMs = staleMs / 2,
+			} = options.lockfile;
+			if (staleMs < 5000) {
+				throw new Error("staleMs must be >= 5000");
+			}
+			if (updateMs < 1000) {
+				throw new Error("updateMs must be >= 1000");
+			}
+			if (updateMs > staleMs / 2) {
+				throw new Error(`updateMs must be <= ${staleMs / 2}`);
+			}
+			if (retries != undefined && retries < 0) {
+				throw new Error("retries must be >= 0");
+			}
+			if (retries != undefined && retries > 10) {
+				throw new Error("retries must be <= 10");
+			}
+			if (
+				options.lockfileDirectory != undefined &&
+				directory != undefined
+			) {
+				throw new Error(
+					"lockfileDirectory and lockfile.directory must not both be specified",
+				);
+			}
+		}
 	}
 
 	public readonly filename: string;
@@ -252,6 +306,14 @@ export class JsonlDB<V extends unknown = unknown> {
 		// Open the file for appending and reading
 		await fs.ensureDir(path.dirname(this.filename));
 
+		let retryOptions: lockfile.LockOptions["retries"];
+		if (this.options.lockfile?.retries) {
+			retryOptions = {
+				retries: this.options.lockfile.retries,
+				factor: 1.25,
+			};
+		}
+
 		try {
 			await fs.ensureDir(path.dirname(this.lockfileName));
 			await lockfile.lock(this.lockfileName, {
@@ -262,7 +324,10 @@ export class JsonlDB<V extends unknown = unknown> {
 					// Avoid timeouts during testing
 					process.env.NODE_ENV === "test"
 						? 100000
-						: /* istanbul ignore next - this is impossible to test */ undefined,
+						: /* istanbul ignore next - this is impossible to test */ this
+								.options.lockfile?.staleMs,
+				update: this.options.lockfile?.updateMs,
+				retries: retryOptions,
 
 				onCompromised: /* istanbul ignore next */ () => {
 					// do nothing