Skip to content

Commit abd8d3c

Browse files
VafilorVafilor
authored
Merge pull request #933 from Vafilor/feat/filesyncer.update.migrations
feat: added migrations to update filesyncer to version 1.0.0
2 parents 5b69793 + 6f8e3f5 commit abd8d3c

File tree

6 files changed

+451
-0
lines changed

6 files changed

+451
-0
lines changed

‎db/go/20210719190719_update_filesyncer.go

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package migration
2+
3+
import (
4+
"database/sql"
5+
"github.com/pressly/goose"
6+
"path/filepath"
7+
)
8+
9+
func initialize20210719190719() {
10+
if _, ok := initializedMigrations[20210719190719]; !ok {
11+
goose.AddMigration(Up20210719190719, Down20210719190719)
12+
initializedMigrations[20210719190719] = true
13+
}
14+
}
15+
16+
// Up20210719190719 updates the workspace templates to use new v1.0.0 of filesyncer
17+
func Up20210719190719(tx *sql.Tx) error {
18+
// This code is executed when the migration is applied.
19+
if err := updateWorkspaceTemplateManifest(
20+
filepath.Join("workspaces", "cvat", "20210719190719.yaml"),
21+
cvatTemplateName); err != nil {
22+
return err
23+
}
24+
25+
if err := updateWorkspaceTemplateManifest(
26+
filepath.Join("workspaces", "jupyterlab", "20210719190719.yaml"),
27+
jupyterLabTemplateName); err != nil {
28+
return err
29+
}
30+
31+
if err := updateWorkspaceTemplateManifest(
32+
filepath.Join("workspaces", "vnc", "20210719190719.yaml"),
33+
deepLearningDesktopTemplateName); err != nil {
34+
return err
35+
}
36+
37+
return updateWorkspaceTemplateManifest(
38+
filepath.Join("workspaces", "vscode", "20210719190719.yaml"),
39+
vscodeWorkspaceTemplateName)
40+
}
41+
42+
// Down20210719190719 rolls back the change to update filesyncer
43+
func Down20210719190719(tx *sql.Tx) error {
44+
// This code is executed when the migration is rolled back.
45+
if err := updateWorkspaceTemplateManifest(
46+
filepath.Join("workspaces", "cvat", "20210323175655.yaml"),
47+
cvatTemplateName); err != nil {
48+
return err
49+
}
50+
51+
if err := updateWorkspaceTemplateManifest(
52+
filepath.Join("workspaces", "jupyterlab", "20210323175655.yaml"),
53+
jupyterLabTemplateName); err != nil {
54+
return err
55+
}
56+
57+
if err := updateWorkspaceTemplateManifest(
58+
filepath.Join("workspaces", "vnc", "20210414165510.yaml"),
59+
deepLearningDesktopTemplateName); err != nil {
60+
return err
61+
}
62+
63+
return updateWorkspaceTemplateManifest(
64+
filepath.Join("workspaces", "vscode", "20210323175655.yaml"),
65+
vscodeWorkspaceTemplateName)
66+
}

‎db/go/db.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ func Initialize() {
9595
initialize20210329171739()
9696
initialize20210329194731()
9797
initialize20210414165510()
98+
initialize20210719190719()
9899

99100
if err := client.DB.Close(); err != nil {
100101
log.Printf("[error] closing db %v", err)

‎db/yaml/workspaces/cvat/20210719190719.yaml

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
containers:
2+
- name: cvat-db
3+
image: postgres:10-alpine
4+
env:
5+
- name: POSTGRES_USER
6+
value: root
7+
- name: POSTGRES_DB
8+
value: cvat
9+
- name: POSTGRES_HOST_AUTH_METHOD
10+
value: trust
11+
- name: PGDATA
12+
value: /var/lib/psql/data
13+
ports:
14+
- containerPort: 5432
15+
name: tcp
16+
volumeMounts:
17+
- name: db
18+
mountPath: /var/lib/psql
19+
- name: cvat-redis
20+
image: redis:4.0-alpine
21+
ports:
22+
- containerPort: 6379
23+
name: tcp
24+
- name: cvat
25+
image: onepanel/cvat:v0.19.0_cvat.1.0.0
26+
env:
27+
- name: DJANGO_MODWSGI_EXTRA_ARGS
28+
value: ""
29+
- name: ALLOWED_HOSTS
30+
value: '*'
31+
- name: CVAT_REDIS_HOST
32+
value: localhost
33+
- name: CVAT_POSTGRES_HOST
34+
value: localhost
35+
- name: CVAT_SHARE_URL
36+
value: /cvat/data
37+
- name: CVAT_SHARE_DIR
38+
value: /share
39+
- name: CVAT_DATA_DIR
40+
value: /cvat/data
41+
- name: CVAT_MEDIA_DATA_DIR
42+
value: /cvat/data/data
43+
- name: CVAT_KEYS_DIR
44+
value: /cvat/data/keys
45+
- name: CVAT_MODELS_DIR
46+
value: /cvat/data/models
47+
- name: CVAT_LOGS_DIR
48+
value: /cvat/logs
49+
- name: CVAT_ANNOTATIONS_OBJECT_STORAGE_PREFIX
50+
value: 'artifacts/$(ONEPANEL_RESOURCE_NAMESPACE)/annotations/'
51+
- name: CVAT_ONEPANEL_WORKFLOWS_LABEL
52+
value: 'key=used-by,value=cvat'
53+
- name: NVIDIA_VISIBLE_DEVICES
54+
value: all
55+
- name: NVIDIA_DRIVER_CAPABILITIES
56+
value: compute,utility
57+
- name: NVIDIA_REQUIRE_CUDA
58+
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
59+
- name: ONEPANEL_MAIN_CONTAINER
60+
value: 'true'
61+
ports:
62+
- containerPort: 8080
63+
name: http
64+
volumeMounts:
65+
- name: cvat-data
66+
mountPath: /cvat
67+
- name: share
68+
mountPath: /share
69+
- name: sys-namespace-config
70+
mountPath: /etc/onepanel
71+
readOnly: true
72+
- name: cvat-ui
73+
image: onepanel/cvat-ui:v0.19.0_cvat.1.0.0
74+
ports:
75+
- containerPort: 80
76+
name: http
77+
- name: sys-filesyncer
78+
image: onepanel/filesyncer:v1.0.0
79+
imagePullPolicy: Always
80+
args:
81+
- server
82+
- -server-prefix=/sys/filesyncer
83+
volumeMounts:
84+
- name: share
85+
mountPath: /share
86+
- name: sys-namespace-config
87+
mountPath: /etc/onepanel
88+
readOnly: true
89+
ports:
90+
- name: cvat-ui
91+
port: 80
92+
protocol: TCP
93+
targetPort: 80
94+
- name: cvat
95+
port: 8080
96+
protocol: TCP
97+
targetPort: 8080
98+
- name: fs
99+
port: 8888
100+
protocol: TCP
101+
targetPort: 8888
102+
routes:
103+
- match:
104+
- uri:
105+
prefix: /sys/filesyncer
106+
route:
107+
- destination:
108+
port:
109+
number: 8888
110+
- match:
111+
- uri:
112+
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
113+
- queryParams:
114+
id:
115+
regex: \d+.*
116+
route:
117+
- destination:
118+
port:
119+
number: 8080
120+
- match:
121+
- uri:
122+
prefix: /
123+
route:
124+
- destination:
125+
port:
126+
number: 80
127+
volumeClaimTemplates:
128+
- metadata:
129+
name: db
130+
spec:
131+
accessModes: ["ReadWriteOnce"]
132+
resources:
133+
requests:
134+
storage: 20Gi

‎db/yaml/workspaces/jupyterlab/20210719190719.yaml

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
containers:
2+
- name: jupyterlab
3+
image: onepanel/dl:v0.20.0
4+
command: ["/bin/bash", "-c", "pip install onepanel-sdk && start.sh LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64 jupyter lab --LabApp.token='' --LabApp.allow_remote_access=True --LabApp.allow_origin=\"*\" --LabApp.disable_check_xsrf=True --LabApp.trust_xheaders=True --LabApp.base_url=/ --LabApp.tornado_settings='{\"headers\":{\"Content-Security-Policy\":\"frame-ancestors * 'self'\"}}' --notebook-dir='/data' --allow-root"]
5+
workingDir: /data
6+
env:
7+
- name: tornado
8+
value: "'{'headers':{'Content-Security-Policy':\"frame-ancestors\ *\ 'self'\"}}'"
9+
- name: TENSORBOARD_PROXY_URL
10+
value: '//$(ONEPANEL_RESOURCE_UID)--$(ONEPANEL_RESOURCE_NAMESPACE).$(ONEPANEL_DOMAIN)/tensorboard'
11+
- name: ONEPANEL_MAIN_CONTAINER
12+
value: 'true'
13+
ports:
14+
- containerPort: 8888
15+
name: jupyterlab
16+
- containerPort: 6006
17+
name: tensorboard
18+
- containerPort: 8080
19+
name: nni
20+
volumeMounts:
21+
- name: data
22+
mountPath: /data
23+
lifecycle:
24+
postStart:
25+
exec:
26+
command:
27+
- /bin/sh
28+
- -c
29+
- >
30+
condayml="/data/.environment.yml";
31+
jupytertxt="/data/.jupexported.txt";
32+
if [ -f "$condayml" ]; then conda env update -f $condayml; fi;
33+
if [ -f "$jupytertxt" ]; then cat $jupytertxt | xargs -n 1 jupyter labextension install --no-build && jupyter lab build --minimize=False; fi;
34+
preStop:
35+
exec:
36+
command:
37+
- /bin/sh
38+
- -c
39+
- >
40+
conda env export > /data/.environment.yml -n base;
41+
jupyter labextension list 1>/dev/null 2> /data/.jup.txt;
42+
cat /data/.jup.txt | sed -n '2,$p' | awk 'sub(/v/,"@", $2){print $1$2}' > /data/.jupexported.txt;
43+
- name: sys-filesyncer
44+
image: onepanel/filesyncer:v1.0.0
45+
imagePullPolicy: Always
46+
args:
47+
- server
48+
- -host=localhost:8889
49+
- -server-prefix=/sys/filesyncer
50+
volumeMounts:
51+
- name: data
52+
mountPath: /data
53+
- name: sys-namespace-config
54+
mountPath: /etc/onepanel
55+
readOnly: true
56+
ports:
57+
- name: jupyterlab
58+
port: 80
59+
protocol: TCP
60+
targetPort: 8888
61+
- name: tensorboard
62+
port: 6006
63+
protocol: TCP
64+
targetPort: 6006
65+
- name: nni
66+
port: 8080
67+
protocol: TCP
68+
targetPort: 8080
69+
- name: fs
70+
port: 8889
71+
protocol: TCP
72+
targetPort: 8889
73+
routes:
74+
- match:
75+
- uri:
76+
prefix: /sys/filesyncer
77+
route:
78+
- destination:
79+
port:
80+
number: 8889
81+
- match:
82+
- uri:
83+
prefix: /tensorboard
84+
route:
85+
- destination:
86+
port:
87+
number: 6006
88+
- match:
89+
- uri:
90+
prefix: /nni
91+
route:
92+
- destination:
93+
port:
94+
number: 8080
95+
- match:
96+
- uri:
97+
prefix: /
98+
route:
99+
- destination:
100+
port:
101+
number: 80

‎db/yaml/workspaces/vnc/20210719190719.yaml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
arguments:
2+
parameters:
3+
# parameter screen-resolution allows users to select screen resolution
4+
- name: screen-resolution
5+
value: 1680x1050
6+
type: select.select
7+
displayName: Screen Resolution
8+
options:
9+
- name: 1280x1024
10+
value: 1280x1024
11+
- name: 1680x1050
12+
value: 1680x1050
13+
- name: 2880x1800
14+
value: 2880x1800
15+
containers:
16+
- name: ubuntu
17+
image: onepanel/vnc:dl-vnc
18+
env:
19+
- name: VNC_PASSWORDLESS
20+
value: true
21+
- name: VNC_RESOLUTION
22+
value: '{{workflow.parameters.screen-resolution}}'
23+
ports:
24+
- containerPort: 6901
25+
name: vnc
26+
volumeMounts:
27+
- name: data
28+
mountPath: /data
29+
- name: sys-filesyncer
30+
image: onepanel/filesyncer:v1.0.0
31+
imagePullPolicy: Always
32+
args:
33+
- server
34+
- -host=localhost:8889
35+
- -server-prefix=/sys/filesyncer
36+
volumeMounts:
37+
- name: data
38+
mountPath: /data
39+
- name: sys-namespace-config
40+
mountPath: /etc/onepanel
41+
readOnly: true
42+
ports:
43+
- name: vnc
44+
port: 80
45+
protocol: TCP
46+
targetPort: 6901
47+
- name: fs
48+
port: 8889
49+
protocol: TCP
50+
targetPort: 8889
51+
routes:
52+
- match:
53+
- uri:
54+
prefix: /sys/filesyncer
55+
route:
56+
- destination:
57+
port:
58+
number: 8889
59+
- match:
60+
- uri:
61+
prefix: /
62+
route:
63+
- destination:
64+
port:
65+
number: 80
66+
# DAG Workflow to be executed once a Workspace action completes (optional)
67+
#postExecutionWorkflow:
68+
# entrypoint: main
69+
# templates:
70+
# - name: main
71+
# dag:
72+
# tasks:
73+
# - name: slack-notify
74+
# template: slack-notify
75+
# - name: slack-notify
76+
# container:
77+
# image: technosophos/slack-notify
78+
# args:
79+
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
80+
# command:
81+
# - sh

0 commit comments

Comments
 (0)