Skip to content

Commit 28494fd

Browse files
MiccWanMiccWan
committed
upload Pathways data in workshop session
1 parent d743a2a commit 28494fd

File tree

6 files changed

+156
-47
lines changed

6 files changed

+156
-47
lines changed

‎dbux-code/src/preActivate.js‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { workspace, commands, window } from 'vscode';
22
import { newLogger } from '@dbux/common/src/log/logger';
3+
import EmptyObject from '@dbux/common/src/util/EmptyObject';
34
import { initMemento, get as mementoGet, set as mementoSet } from './memento';
45
import { initInstallId } from './installId';
56
import { initLogging } from './logging';
@@ -70,10 +71,9 @@ export default async function preActivate(context) {
7071
commands.executeCommand('setContext', 'dbux.context.researchEnabled', !!process.env.RESEARCH_ENABLED);
7172

7273
// the following should ensures `doActivate` will be called at least once
73-
// autoStart = (process.env.NODE_ENV === 'development') ||
74-
// workspace.getConfiguration('dbux').get('autoStart');
75-
// TODO-M: recover this
76-
autoStart = workspace.getConfiguration('dbux').get('autoStart');
74+
autoStart = (process.env.NODE_ENV === 'development') ||
75+
workspace.getConfiguration('dbux').get('autoStart');
76+
// autoStart = workspace.getConfiguration('dbux').get('autoStart');
7777
if (autoStart) {
7878
await ensureActivate(context);
7979
}
@@ -122,7 +122,7 @@ function initPreActivateCommand(context) {
122122
await ensureActivate(context);
123123
}
124124
else {
125-
await showInformationMessage(`Workshop code "${code}" is invalid.`, { modal: true });
125+
await showInformationMessage(`Workshop code "${code}" is invalid.`, EmptyObject, { modal: true });
126126
}
127127
});
128128
}

‎dbux-code/src/workshop/PathwaysDataBuffer.js‎

Lines changed: 41 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,16 @@ import SafetyStorage from './SafetyStorage';
77
// eslint-disable-next-line no-unused-vars
88
const { log, debug, warn, error: logError } = newLogger('PathwaysDataBuffer');
99

10-
const DefaultBufferSize = 50;
10+
// const Verbose = true;
11+
const Verbose = false;
12+
13+
const DefaultBufferSize = 10;
1114
const DefaultBufferFlushTime = 2 * 60 * 1000; // 2 minutes
15+
// const DefaultBufferFlushTime = 1 * 1000; // 1 second
1216

17+
/**
18+
* @template T
19+
*/
1320
export default class PathwaysDataBuffer {
1421
constructor(sessionId, collectionName) {
1522
const storageKeyName = `dbux.pathwaysDataBuffer.${collectionName}`;
@@ -26,11 +33,14 @@ export default class PathwaysDataBuffer {
2633
return this.buffer.get() || [];
2734
}
2835

36+
/**
37+
* @param {T[]} entries
38+
*/
2939
async add(entries) {
3040
await this.buffer.acquireLock();
3141

3242
try {
33-
let buffer = this.safeGetBuffer();
43+
const buffer = this.safeGetBuffer();
3444
buffer.push(...entries);
3545
await this.buffer.set(buffer);
3646
}
@@ -41,40 +51,47 @@ export default class PathwaysDataBuffer {
4151

4252
async maybeFlush() {
4353
if (!this._flushing && (this.safeGetBuffer().length >= DefaultBufferSize || Date.now() - this._previousFlushTime >= DefaultBufferFlushTime)) {
44-
await this.forceFlush();
54+
await this.flush();
4555
}
4656
}
4757

48-
async forceFlush() {
49-
this._flushing = true;
50-
await this._flush();
51-
}
52-
53-
async _flush() {
58+
async flush() {
59+
Verbose && log(`Flushing collection ${this.collectionName}`);
5460
this._flushing = true;
55-
await this.buffer.acquireLock();
5661

57-
let buffer;
5862
try {
59-
buffer = this.safeGetBuffer();
60-
await this.buffer.set([]);
61-
}
62-
finally {
63-
this.buffer.releaseLock();
64-
}
63+
await this.buffer.acquireLock();
6564

66-
try {
67-
await this.addDoc(buffer);
65+
let buffer;
66+
try {
67+
buffer = this.safeGetBuffer();
68+
await this.buffer.set([]);
69+
}
70+
finally {
71+
this.buffer.releaseLock();
72+
}
73+
74+
try {
75+
await this.addDoc(buffer);
76+
}
77+
catch (err) {
78+
throw new NestedError(`Failed when flushing`, err);
79+
}
80+
81+
this._previousFlushTime = Date.now();
6882
}
69-
catch (err) {
83+
finally {
7084
this._flushing = false;
71-
throw new NestedError(`Failed when flushing`, err);
7285
}
73-
74-
this._previousFlushTime = Date.now();
7586
}
7687

7788
async addDoc(entries) {
78-
return await uploadPathways(this.sessionId, this.collectionName, entries);
89+
if (entries.length) {
90+
Verbose && log(`Uploading ${entries.length} "${this.collectionName}" of session "${this.sessionId}"`);
91+
return await uploadPathways(this.sessionId, this.collectionName, entries);
92+
}
93+
else {
94+
return null;
95+
}
7996
}
8097
}

‎dbux-code/src/workshop/PathwaysDataContainer.js‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import { newLogger } from '@dbux/common/src/log/logger';
2+
import PathwaysDataBuffer from './PathwaysDataBuffer';
3+
4+
// eslint-disable-next-line no-unused-vars
5+
const { log, debug, warn, error: logError } = newLogger('PathwaysDataContainer');
6+
7+
// const Verbose = true;
8+
const Verbose = false;
9+
10+
export class PathwaysDataContainer {
11+
/**
12+
* @type {Object<string, PathwaysDataBuffer>}
13+
*/
14+
buffers;
15+
16+
constructor() {
17+
this.prevListener = null;
18+
this.reset();
19+
}
20+
21+
reset(sessionId = null) {
22+
Verbose && log(`Reset with sessionId ${sessionId}`);
23+
this.buffers = {};
24+
if (sessionId) {
25+
this.buffers = {
26+
applications: new PathwaysDataBuffer(sessionId, 'applications'),
27+
testRuns: new PathwaysDataBuffer(sessionId, 'testRuns'),
28+
steps: new PathwaysDataBuffer(sessionId, 'steps'),
29+
actionGroups: new PathwaysDataBuffer(sessionId, 'actionGroups'),
30+
userActions: new PathwaysDataBuffer(sessionId, 'userActions'),
31+
};
32+
}
33+
}
34+
35+
onSessionChanged = async (session) => {
36+
if (this.sessionId !== session?.sessionId) {
37+
this.sessionId = session?.sessionId;
38+
39+
// stop listening on previous events
40+
this.prevListener?.();
41+
42+
// flush all current data
43+
const flushOldPromise = this.flushAll();
44+
45+
// make new buffers
46+
this.reset(this.sessionId);
47+
48+
// listen on new events
49+
this.prevListener = session?.pdp.onAnyData((allData) => {
50+
const serializedData = session?.pdp.serializeJson(Object.entries(allData));
51+
this.addAllData(serializedData);
52+
});
53+
54+
await flushOldPromise;
55+
}
56+
}
57+
58+
addAllData = ({ collections }) => {
59+
for (const collectionName of Object.keys(collections)) {
60+
this.buffers[collectionName].add(collections[collectionName]);
61+
}
62+
}
63+
64+
async maybeFlushAll() {
65+
return await Promise.all(Object.values(this.buffers).map((buffer) => buffer.maybeFlush()));
66+
}
67+
68+
async flushAll() {
69+
return await Promise.all(Object.values(this.buffers).map((buffer) => buffer.flush()));
70+
}
71+
}
72+
73+
/**
74+
* @type {PathwaysDataContainer}
75+
*/
76+
let container;
77+
78+
export function initPathwaysDataContainer() {
79+
container = new PathwaysDataContainer();
80+
81+
return container;
82+
}

‎dbux-code/src/workshop/SafetyStorage.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { newLogger } from '@dbux/common/src/log/logger';
55
import { get, set } from '../memento';
66

77
// eslint-disable-next-line no-unused-vars
8-
const { log, debug, warn, error: logError } = newLogger('SavetyStorage');
8+
const { log, debug, warn, error: logError } = newLogger('SafetyStorage');
99

1010
function getLockfileName(name) {
1111
const lockfilePath = path.join(os.tmpdir(), `dbux-lockfile.${name}`);

‎dbux-code/src/workshop/index.js‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
/** @typedef {import('@dbux/projects/src/ProjectsManager').default} ProjectsManager */
22

3+
import NestedError from '@dbux/common/src/NestedError';
4+
import { newLogger } from '@dbux/common/src/log/logger';
5+
import sleep from '@dbux/common/src/util/sleep';
6+
import { initPathwaysDataContainer } from './PathwaysDataContainer';
7+
8+
// eslint-disable-next-line no-unused-vars
9+
const { log, debug, warn, error: logError } = newLogger('WorkshopSession');
10+
11+
// const Verbose = true;
12+
const Verbose = false;
13+
314
const ValidCodes = new Set([]);
415
let currentCode = '';
516

@@ -38,24 +49,23 @@ export function setupWorkshopSession(code) {
3849
*/
3950
export async function initWorkshopSession(projectsManager) {
4051
if (isWorkshopSessionActive()) {
41-
addHook();
42-
projectsManager.onPracticeSessionStateChanged(addHook);
52+
const pathwaysDataContainer = initPathwaysDataContainer();
53+
projectsManager.onPracticeSessionStateChanged(pathwaysDataContainer.onSessionChanged);
54+
scheduleUpload(pathwaysDataContainer);
4355
}
4456
}
4557

46-
let sessionId, prevListener;
58+
const UploadLoopInterval = 2 * 60 * 1000;
59+
// const UploadLoopInterval = 10 * 1000;
4760

48-
function addHook(session) {
49-
if (sessionId !== session?.sessionId) {
50-
// stop listening on previous events
51-
prevListener?.();
52-
53-
prevListener = session?.pdp.onAnyData(addData);
61+
async function scheduleUpload(pathwaysDataContainer) {
62+
while (isWorkshopSessionActive()) {
63+
await sleep(UploadLoopInterval);
64+
try {
65+
await pathwaysDataContainer.maybeFlushAll();
66+
}
67+
catch (err) {
68+
throw new NestedError(`Failed in PathwaysDataContainer upload loop`, err);
69+
}
5470
}
5571
}
56-
57-
function addData(allData) {
58-
for (const collectionName of Object.keys(allData)) {
59-
// TODO-M: push data into buffer
60-
}
61-
}

‎dbux-projects/src/userEvents/index.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import { newLogger } from '@dbux/common/src/log/logger';
88
// eslint-disable-next-line no-unused-vars
99
const { log, debug, warn, error: logError } = newLogger('UserEvents');
1010

11-
const Verbose = true;
12-
// const Verbose = false;
11+
// const Verbose = true;
12+
const Verbose = false;
1313

1414
// ###########################################################################
1515
// event registry

0 commit comments

Comments
 (0)