Skip to content

add fake run #733

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 6, 2020
Merged

add fake run #733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a7ee71f
add fake run
ShenYuhan Aug 3, 2020
10621bc
add display_name docs
ShenYuhan Aug 3, 2020
2ad5e0b
fix review
ShenYuhan Aug 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/overview.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ class LogWriter(logdir=None,
| flush_secs | int | 日志记录消息队列的最大缓存时间，达到此时间则立即写入到日志文件 |
| filename_suffix | string | 为默认的日志文件名添加后缀 |
| write_to_disk | boolean | 是否写入到磁盘 |
| display_name | string | 实际展示在面板中的`runs`名称，可隐藏路径信息 |

#### 示例

Expand Down
21 changes: 21 additions & 0 deletions visualdl/component/base_component.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,27 @@ def scalar(tag, value, step, walltime=None):
])


def meta_data(tag='meta_data_tag', display_name="", step=0, walltime=None):
"""Package data to one meta_data.

Meta data is info for one record file, include `display_name` etc.

Args:
tag (string): Data identifier
display_name (string): Replace
step (int): Step of scalar
walltime (int): Wall time of scalar

Return:
Package with format of record_pb2.Record
"""
meta = Record.MetaData(display_name=display_name)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime,
meta_data=meta)
])


def imgarray2bytes(np_array):
"""Convert image ndarray to bytes.

Expand Down
5 changes: 5 additions & 0 deletions visualdl/proto/record.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ message Record {
repeated double recall = 6;
}

message MetaData {
string display_name = 1;
}

message Value {
int64 id = 1;
string tag = 2;
Expand All @@ -54,6 +58,7 @@ message Record {
Embeddings embeddings = 7;
Histogram histogram = 8;
PRCurve pr_curve = 9;
MetaData meta_data = 10;
}
}

Expand Down
60 changes: 55 additions & 5 deletions visualdl/proto/record_pb2.py
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 12 additions & 0 deletions visualdl/reader/reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ def __init__(self, logdir):
self.readers = {}
self.walks = None
self._tags = {}
self.name2tags = {}
self.tags2name = {}

self.file_readers = {}
self._environments = components
Expand All @@ -64,6 +66,7 @@ def __init__(self, logdir):

self._model = ""


@property
def model(self):
return self._model
Expand Down Expand Up @@ -110,12 +113,21 @@ def parse_from_bin(self, record_bin):
component = "histogram"
elif "pr_curve" == value_type:
component = "pr_curve"
elif "meta_data" == value_type:
self.update_meta_data(record)
component = "meta_data"
else:
raise TypeError("Invalid value type `%s`." % value_type)
self._tags[path] = component

return self._tags[path], self.reader.dir, tag, value

def update_meta_data(self, record):
meta = record.values[0].meta_data
if meta.display_name:
self.name2tags[meta.display_name] = self.reader.dir
self.tags2name[self.reader.dir] = meta.display_name

def get_all_walk(self):
self.walks = {}
for dir in self.dir:
Expand Down
7 changes: 5 additions & 2 deletions visualdl/server/data_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
"histogram": 100,
"embeddings": 50000,
"audio": 10,
"pr_curve": 300
"pr_curve": 300,
"meta_data": 100
}


Expand Down Expand Up @@ -277,7 +278,9 @@ def __init__(self):
"audio":
Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["audio"]),
"pr_curve":
Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["pr_curve"])
Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["pr_curve"]),
"meta_data":
Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["meta_data"])
}
self._mutex = threading.Lock()

Expand Down
27 changes: 25 additions & 2 deletions visualdl/server/lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,13 @@ def get_components(log_reader):


def get_runs(log_reader):
return log_reader.runs()
runs = []
for item in log_reader.runs():
if item in log_reader.tags2name:
runs.append(log_reader.tags2name[item])
else:
runs.append(item)
return runs


def get_tags(log_reader):
Expand All @@ -47,14 +53,23 @@ def get_logs(log_reader, component):
tags[run].append(tag)
else:
tags[run] = [tag]
return tags
fake_tags = {}
for key, value in tags.items():

if key in log_reader.tags2name:
fake_tags[log_reader.tags2name[key]] = value
else:
fake_tags[key] = value

return fake_tags


def get_scalar_tags(log_reader):
return get_logs(log_reader, "scalar")


def get_scalar(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("scalar").get_items(
run, decode_tag(tag))
Expand All @@ -67,6 +82,7 @@ def get_image_tags(log_reader):


def get_image_tag_steps(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("image").get_items(
run, decode_tag(tag))
Expand All @@ -78,6 +94,7 @@ def get_image_tag_steps(log_reader, run, tag):


def get_individual_image(log_reader, run, tag, step_index):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("image").get_items(
run, decode_tag(tag))
Expand All @@ -89,6 +106,7 @@ def get_audio_tags(log_reader):


def get_audio_tag_steps(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("audio").get_items(
run, decode_tag(tag))
Expand All @@ -100,6 +118,7 @@ def get_audio_tag_steps(log_reader, run, tag):


def get_individual_audio(log_reader, run, tag, step_index):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("audio").get_items(
run, decode_tag(tag))
Expand All @@ -120,6 +139,7 @@ def get_pr_curve_tags(log_reader):


def get_pr_curve(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("pr_curve").get_items(
run, decode_tag(tag))
Expand All @@ -141,6 +161,7 @@ def get_pr_curve(log_reader, run, tag):


def get_pr_curve_step(log_reader, run, tag=None):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
tag = get_pr_curve_tags(log_reader)[run][0] if tag is None else tag
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("pr_curve").get_items(
Expand All @@ -150,6 +171,7 @@ def get_pr_curve_step(log_reader, run, tag=None):


def get_embeddings(log_reader, run, tag, reduction, dimension=2):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))
Expand All @@ -173,6 +195,7 @@ def get_embeddings(log_reader, run, tag, reduction, dimension=2):


def get_histogram(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("histogram").get_items(
run, decode_tag(tag))
Expand Down
Loading