Merge pull request #80 from PhilipVinc/threadsafe

Upgrade to ProtoBuf 0.9 and make TensorBoardLogger threadsafe

Author: PhilipVinc

.gitignore

@@ -3,3 +3,6 @@
 events.out.tfevents*
 docs/build
 test/test_logs
+
+gen/proto
+gen/protojl

Project.toml

@@ -1,7 +1,7 @@
 name = "TensorBoardLogger"
 uuid = "899adc3e-224a-11e9-021f-63837185c80f"
 authors = ["Filippo Vicentini <filippovicentini@gmail.com>"]
-version = "0.1.12"
+version = "0.1.13"
 
 [deps]
 CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
@@ -18,7 +18,7 @@ ColorTypes = "0.7, 0.8, 0.9, 0.10"
 FileIO = "1"
 FixedPointNumbers = "0.6, 0.7, 0.8"
 ImageCore = "0.7, 0.8"
-ProtoBuf = "0.7, 0.8"
+ProtoBuf = "0.10"
 Requires = "0.5, 1"
 StatsBase = "0.27, 0.28, 0.29, 0.30, 0.31, 0.32, 0.33"
 julia = "0.7, 1"

compat/Project.toml

@@ -1,7 +1,7 @@
 name = "TensorBoardLogger"
 uuid = "899adc3e-224a-11e9-021f-63837185c80f"
 authors = ["Filippo Vicentini <filippovicentini@gmail.com>"]
-version = "0.1.12"
+version = "0.1.13"
 
 [deps]
 CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
@@ -18,7 +18,7 @@ ColorTypes = "0.7, 0.8, 0.9, 0.10"
 FileIO = "1"
 FixedPointNumbers = "0.6, 0.7, 0.8"
 ImageCore = "0.7, 0.8"
-ProtoBuf = "0.7, 0.8"
+ProtoBuf = "0.10"
 Requires = "0.5, 1"
 StatsBase = "0.27, 0.28, 0.29, 0.30, 0.31, 0.32, 0.33"
 julia = "0.7, 1"

gen/Project.toml

@@ -0,0 +1,8 @@
+[deps]
+FilePaths = "8fc22ac5-c921-52a6-82fd-178b2807b824"
+FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
+Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
+ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429"
+
+[comapt]
+ProtoBuf = "0.9.1"

gen/README

@@ -0,0 +1,7 @@
+README.md
+
+Download proto files from tensorboard and tensorflow git master using download_proto_source bash script
+
+Commpile them using compile_proto
+
+then move gen/protojl/tensorboard to src/ and add submodules to protojl/tensorboard.jl

gen/compile_proto.jl

@@ -0,0 +1,83 @@
+## Note: this file is supposed to be run from the TensorBoardLogger/gen folder as
+## $ cd .../TensorBoardLogger/gen
+## $ julia --project=. compile_proto.jl
+
+# The proto folder is generated by the following script
+# and then manually cleaning the output from useless stuff
+#
+# mkdir -p tmp/proto && cd tmp
+# git clone --recurse-submodules git@github.com:tensorflow/tensorboard.git
+# git clone --recurse-submodules git@github.com:tensorflow/tensorflow.git
+# rsync -zarvm --include="*/" --include="*.proto" --exclude="*" tensorboard proto
+# rsync -zarvm --include="*/" --include="*.proto" --exclude="*" tensorflow proto
+
+using ProtoBuf
+using Glob 
+using FilePaths; using FilePathsBase: /
+
+pbpath =dirname(dirname(PosixPath(pathof(ProtoBuf))))/p"gen"
+
+## Setup input and output paths
+cur_path = cwd()
+TBL_root = dirname(cur_path)
+
+src_dir = cur_path/"proto"
+out_dir   = cur_path/"protojl"
+
+## Clean the output directory
+rm(out_dir, force=true, recursive=true)
+
+## First module
+function process_module(cur_module::AbstractString; base_module::AbstractString=cur_module, input_path=cur_module)
+    # Include search paths
+    includes   = [src_dir, src_dir/base_module]
+
+    # Output folder
+    module_out_dir    = out_dir/cur_module
+
+    # Input files
+    infiles = glob("*.proto", src_dir/input_path)
+
+    mkpath(module_out_dir)
+    includes_str=["--proto_path=$path" for path=includes]
+    run(ProtoBuf.protoc(`$includes_str --julia_out=$module_out_dir $infiles`))
+
+    nothing
+end
+
+#process_module("tensorflow", input_path="tensorflow/core/protobuf")
+
+process_module("tensorboard", input_path="tensorboard/compat/proto")
+
+#plugins = ["audio", "custom_scalar", "hparams", "histogram", "image", "scalar", "text"]
+plugins = ["custom_scalar", "hparams", "text"]
+for plugin in plugins
+    process_module("tensorboard/plugins/$plugin", base_module="tensorboard")
+end
+
+
+## this fails but would be better
+#cur_module  = "tensorboard"
+#base_module = cur_module
+#
+## Include search paths
+#includes   = [src_dir, src_dir/base_module]
+#
+## Output folder
+#module_out_dir    = out_dir/("$cur_module"*"2")
+#
+## Input files
+#infiles = glob("*.proto", src_dir/cur_module/"compat/proto")
+#
+#for plugin in plugins
+#    plugin_proto_files = glob("*.proto", src_dir/cur_module/"plugins/$plugin")
+#    append!(infiles, plugin_proto_files)
+#end
+#
+#mkpath(module_out_dir)
+#includes_str=["--proto_path=$path" for path=includes]
+#run(ProtoBuf.protoc(`$includes_str --julia_out=$module_out_dir $infiles`))
+
+
+# Finally move the output directory to the src folder
+mv(out_dir, TBL_root/"src"/"protojl")

gen/download_proto_source

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -o xtrace
+
+# The proto folder is generated by the following script
+# and then manually cleaning the output from useless stuff
+
+echo "Cleaning up tmp and output directory"
+rm -rf tmp
+rm -rf proto
+
+echo "Creating tmp directory"
+mkdir -p tmp/proto
+
+echo "Entering tmp..."
+cd tmp
+
+echo "Cloning all submodules of git@github.com:tensorflow/tensorboard.git"
+git clone --recurse-submodules git@github.com:tensorflow/tensorboard.git
+
+echo "Cloning all submodules of git@github.com:tensorflow/tensorflow.git"
+git clone --recurse-submodules git@github.com:tensorflow/tensorflow.git
+
+echo "Copy from tmp/tensorboard to tmp/proto only proto files, keeping folder structure"
+rsync -zarvm --include="*/" --include="*.proto" --exclude="*" tensorboard/ proto
+
+echo "Copy from tmp/tensorflow to tmp/proto only proto files, keeping folder structure"
+rsync -zarvm --include="*/" --include="*.proto" --exclude="*" tensorflow/ proto
+
+# Tensorboard has a file named struct.proto which conflicts with google's stdlib struct.proto,
+# so we rename it to struct_tb.proto to avoid the conflict
+mv proto/tensorboard/compat/proto/struct.proto proto/tensorboard/compat/proto/struct_tb.proto
+
+# find and replace usages "compat/proto/struct.proto" with "compat/proto/struct.proto"
+if [[ $(uname) == "Darwin" ]]; then
+	echo "Workaround for sed on macos."
+	LC_ALL=C
+fi
+grep -rl "compat/proto/struct.proto" . | xargs sed -i '' -e 's+compat/proto/struct\.proto+compat/proto/struct_tb\.proto+g'
+
+cd ..
+mv tmp/proto proto
+rm -rf tmp