Merge pull request #44 from JuliaReach/schillic/refactor

Refactoring and remove type restriction of `DenseLayerOp`

Author: schillic

src/Architecture/Architecture.jl

@@ -7,16 +7,16 @@ module Architecture
 
 using Requires
 
-export AbstractNeuralNetwork, AbstractLayerOp,
-       FeedforwardNetwork, DenseLayerOp,
+export AbstractNeuralNetwork, FeedforwardNetwork,
+       AbstractLayerOp, DenseLayerOp,
        layers, dim_in, dim_out,
        ActivationFunction, Id, ReLU, Sigmoid, Tanh, LeakyReLU
 
-include("AbstractNeuralNetwork.jl")
-include("AbstractLayerOp.jl")
 include("ActivationFunction.jl")
-include("DenseLayerOp.jl")
-include("FeedforwardNetwork.jl")
+include("LayerOps/AbstractLayerOp.jl")
+include("LayerOps/DenseLayerOp.jl")
+include("NeuralNetworks/AbstractNeuralNetwork.jl")
+include("NeuralNetworks/FeedforwardNetwork.jl")
 
 include("init.jl")
 

src/Architecture/AbstractLayerOp.jl renamed to src/Architecture/LayerOps/AbstractLayerOp.jl

@@ -1,5 +1,5 @@
 """
-    DenseLayerOp{F, M<:AbstractMatrix, B} <: AbstractLayerOp
+    DenseLayerOp{F, M, B} <: AbstractLayerOp
 
 A dense layer operation is an affine map followed by an activation function.
 
@@ -13,23 +13,23 @@ A dense layer operation is an affine map followed by an activation function.
 
 Conversion from a `Flux.Dense` is supported.
 """
-struct DenseLayerOp{F,M<:AbstractMatrix,B} <: AbstractLayerOp
-    weights::M
+struct DenseLayerOp{F,W,B} <: AbstractLayerOp
+    weights::W
     bias::B
     activation::F
 
-    function DenseLayerOp(weights::M, bias::B, activation::F;
-                          validate=Val(true)) where {F,M<:AbstractMatrix,B}
-        if validate isa Val{true} && !_isconsistent(weights, bias)
+    function DenseLayerOp(weights::W, bias::B, activation::F;
+                          validate=Val(true)) where {F,W,B}
+        if validate isa Val{true} && !_isconsistent_DenseLayerOp(weights, bias)
             throw(ArgumentError("inconsistent dimensions of weights " *
                                 "($(size(weights, 1))) and bias ($(length(bias)))"))
         end
 
-        return new{F,M,B}(weights, bias, activation)
+        return new{F,W,B}(weights, bias, activation)
     end
 end
 
-function _isconsistent(weights, bias)
+function _isconsistent_DenseLayerOp(weights, bias)
     return size(weights, 1) == length(bias)
 end
 
@@ -73,7 +73,7 @@ dim_in(L::DenseLayerOp) = size(L.weights, 2)
 
 dim_out(L::DenseLayerOp) = length(L.bias)
 
-function load_Flux_convert_layer()
+function load_Flux_convert_Dense_layer()
     return quote
         function Base.convert(::Type{DenseLayerOp}, layer::Flux.Dense)
             act = get(activations_Flux, layer.σ, nothing)

src/Architecture/DenseLayerOp.jl renamed to src/Architecture/LayerOps/DenseLayerOp.jl

@@ -2,7 +2,7 @@
 function __init__()
     @require Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" begin
         eval(load_Flux_activations())
-        eval(load_Flux_convert_layer())
+        eval(load_Flux_convert_Dense_layer())
         eval(load_Flux_convert_network())
     end
 end

src/Architecture/AbstractNeuralNetwork.jl renamed to src/Architecture/NeuralNetworks/AbstractNeuralNetwork.jl

@@ -109,7 +109,7 @@ function read_ONNX(filename::String; input_dimension=nothing)
                 @assert args[2]._op.id == idx - 1
                 act = args[1]
             else
-                @assert false "cannot parse activation $op"
+                throw(ArgumentError("cannot parse activation $op"))  # COV_EXCL_LINE
             end
             a = available_activations[string(act)]
             idx += 1

src/Architecture/FeedforwardNetwork.jl renamed to src/Architecture/NeuralNetworks/FeedforwardNetwork.jl

@@ -3,3 +3,7 @@ io = IOBuffer()
 for act in (Id(), ReLU(), Sigmoid(), Tanh(), LeakyReLU(0.1))
     println(io, act)
 end
+
+# leaky ReLU on a vector
+act = LeakyReLU(0.01)
+@test act([-1.0, 0, 1, -100]) == [-0.01, 0, 1, -1]

src/Architecture/init.jl

@@ -79,7 +79,3 @@ for act in subtypes(ActivationFunction)
     end
     test_layer(DenseLayerOp(W, b, act_inst))
 end
-
-# leaky ReLU on a vector
-act = LeakyReLU(0.01)
-@test act([-1.0, 0, 1, -100]) == [-0.01, 0, 1, -1]

src/FileFormats/ONNX.jl

@@ -1,37 +1,37 @@
 import Flux
 
-l1 = Flux.Dense(1, 2, Flux.relu)
-l1.weight .= 1, 2
-l1.bias .= 3, 4
+L1 = Flux.Dense(1, 2, Flux.relu)
+L1.weight .= 1, 2
+L1.bias .= 3, 4
 
-l2 = Flux.Dense(2, 3, Flux.sigmoid)
-l2.weight .= [1 2; 3 4; 5 6]
+L2 = Flux.Dense(2, 3, Flux.sigmoid)
+L2.weight .= [1 2; 3 4; 5 6]
 
-l3 = Flux.Dense(3, 1)
-l3.weight .= [1 2 3;]
+L3 = Flux.Dense(3, 1)
+L3.weight .= [1 2 3;]
 
-l_unsupported = Flux.Dense(1 => 1, Flux.trelu)
+L_unsupported = Flux.Dense(1 => 1, Flux.trelu)
 
-c = Flux.Chain(l1, l2, l3)
+c = Flux.Chain(L1, L2, L3)
 
 activations = [ReLU(), Sigmoid(), Id()]
 
 # `==` is not defined for Flux types
-function compare_Flux_layer(l1, l2)
-    return l1.weight == l2.weight && l1.bias == l2.bias && l1.σ == l2.σ
+function compare_Flux_layer(L1, L2)
+    return L1.weight == L2.weight && L1.bias == L2.bias && L1.σ == L2.σ
 end
 
 # layer conversion
-for (i, l) in enumerate(c.layers)
-    op = convert(DenseLayerOp, l)
-    @test op.weights == l.weight
-    @test op.bias == l.bias
+for (i, L) in enumerate(c.layers)
+    op = convert(DenseLayerOp, L)
+    @test op.weights == L.weight
+    @test op.bias == L.bias
     @test op.activation == activations[i]
 
-    l_back = convert(Flux.Dense, op)
-    @test compare_Flux_layer(l, l_back)
+    L_back = convert(Flux.Dense, op)
+    @test compare_Flux_layer(L, L_back)
 end
-@test_throws ArgumentError convert(DenseLayerOp, l_unsupported)
+@test_throws ArgumentError convert(DenseLayerOp, L_unsupported)
 
 # network conversion
 net = convert(FeedforwardNetwork, c)