Skip to content

Commit 3c4933f

Browse files
schillicschillic
committed
add ConvolutionalLayerOp
1 parent c558c44 commit 3c4933f

File tree

8 files changed

+278
-1
lines changed

8 files changed

+278
-1
lines changed

‎Project.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ uuid = "02ac4b2c-022a-44aa-84a5-ea45a5754bcc"
33
version = "0.2.2"
44

55
[deps]
6+
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
67
ReachabilityBase = "379f33d0-9447-4353-bd03-d664070e549f"
78
Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
89
Requires = "ae029012-a4dd-5104-9daa-d747884805df"
910
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
1011

1112
[compat]
13+
LinearAlgebra = "<0.0.1, 1.6"
1214
ReachabilityBase = "0.1.1 - 0.2"
1315
Reexport = "0.2, 1"
1416
Requires = "0.5, 1"

‎docs/src/lib/Architecture.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ AbstractPoolingLayerOp
6363

6464
```@docs
6565
DenseLayerOp
66+
ConvolutionalLayerOp
6667
FlattenLayerOp
6768
MaxPoolingLayerOp
6869
MeanPoolingLayerOp

‎src/Architecture/Architecture.jl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,19 @@ Module containing data structures to represent controllers.
66
module Architecture
77

88
using Requires
9+
using LinearAlgebra: dot
910
using Statistics: mean
1011

1112
export AbstractNeuralNetwork, FeedforwardNetwork,
12-
AbstractLayerOp, DenseLayerOp, FlattenLayerOp,
13+
AbstractLayerOp, DenseLayerOp, ConvolutionalLayerOp, FlattenLayerOp,
1314
AbstractPoolingLayerOp, MaxPoolingLayerOp, MeanPoolingLayerOp,
1415
layers, dim_in, dim_out,
1516
ActivationFunction, Id, ReLU, Sigmoid, Tanh, LeakyReLU
1617

1718
include("ActivationFunction.jl")
1819
include("LayerOps/AbstractLayerOp.jl")
1920
include("LayerOps/DenseLayerOp.jl")
21+
include("LayerOps/ConvolutionalLayerOp.jl")
2022
include("LayerOps/FlattenLayerOp.jl")
2123
include("LayerOps/PoolingLayerOp.jl")
2224
include("NeuralNetworks/AbstractNeuralNetwork.jl")

‎src/Architecture/LayerOps/ConvolutionalLayerOp.jl

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
"""
2+
ConvolutionalLayerOp{F, M, B} <: AbstractLayerOp
3+
4+
A convolutional layer operation is a series of filters, each of which computes a
5+
small affine map followed by an activation function.
6+
7+
### Fields
8+
9+
- `weights` -- vector with one weight matrix for each filter
10+
- `bias` -- vector with one bias value for each filter
11+
- `activation` -- activation function
12+
13+
### Notes
14+
15+
Conversion from a `Flux.Conv` is supported.
16+
"""
17+
struct ConvolutionalLayerOp{F,W,B} <: AbstractLayerOp
18+
weights::W
19+
bias::B
20+
activation::F
21+
22+
function ConvolutionalLayerOp(weights::W, bias::B, activation::F;
23+
validate=Val(true)) where {F,W,B}
24+
if validate isa Val{true} && !_isconsistent_ConvolutionalLayerOp(weights, bias)
25+
throw(ArgumentError("inconsistent filter dimensions: weights " *
26+
"($(length(weights))) and biases ($(length(bias)))"))
27+
end
28+
29+
return new{F,W,B}(weights, bias, activation)
30+
end
31+
end
32+
33+
function _isconsistent_ConvolutionalLayerOp(weights, bias)
34+
if length(weights) != length(bias)
35+
return false
36+
elseif length(bias) == 0
37+
return false
38+
end
39+
@inbounds begin
40+
s = size(first(weights))
41+
if length(s) != 3 || s[1] == 0 || s[2] == 0 || s[3] == 0
42+
return false
43+
end
44+
for e in weights
45+
if size(e) != s
46+
return false
47+
end
48+
end
49+
end
50+
return true
51+
end
52+
53+
n_filters(L::ConvolutionalLayerOp) = length(L.bias)
54+
55+
kernel(L::ConvolutionalLayerOp) = @inbounds size(first(L.weights))
56+
57+
# application to a tensor
58+
function (L::ConvolutionalLayerOp)(T)
59+
s = size(T)
60+
if length(s) != 3
61+
throw(ArgumentError("a convolutional layer requires at least two dimensions, but got $s"))
62+
end
63+
p, q, r = kernel(L)
64+
@inbounds begin
65+
if p > s[1] || q > s[2] || r != s[3]
66+
throw(ArgumentError("convolution with kernel size $(kernel(L)) " *
67+
"does not apply to a tensor of dimension $s"))
68+
end
69+
d1 = s[1] - p + 1
70+
d2 = s[2] - q + 1
71+
end
72+
t = n_filters(L)
73+
s = (d1, d2, t)
74+
O = similar(T, s)
75+
@inbounds for f in 1:t
76+
W = L.weights[f]
77+
b = L.bias[f]
78+
for k in 1:r
79+
for j in 1:d2
80+
for i in 1:d1
81+
T′ = view(T, i:(i + p - 1), j:(j + q - 1), k)
82+
O[i, j, f] = L.activation(dot(W, T′) + b)
83+
end
84+
end
85+
end
86+
end
87+
return O
88+
end
89+
90+
function Base.:(==)(L1::ConvolutionalLayerOp, L2::ConvolutionalLayerOp)
91+
return L1.weights == L2.weights &&
92+
L1.bias == L2.bias &&
93+
L1.activation == L2.activation
94+
end
95+
96+
function Base.:isapprox(L1::ConvolutionalLayerOp, L2::ConvolutionalLayerOp; atol::Real=0,
97+
rtol=nothing)
98+
if isnothing(rtol)
99+
if iszero(atol)
100+
N = @inbounds promote_type(eltype(first(L1.weights)), eltype(first(L2.weights)),
101+
eltype(L1.bias), eltype(L2.bias))
102+
rtol = Base.rtoldefault(N)
103+
else
104+
rtol = zero(atol)
105+
end
106+
end
107+
return isapprox(L1.weights, L2.weights; atol=atol, rtol=rtol) &&
108+
isapprox(L1.bias, L2.bias; atol=atol, rtol=rtol) &&
109+
L1.activation == L2.activation
110+
end
111+
112+
function Base.show(io::IO, L::ConvolutionalLayerOp)
113+
str = "$(string(ConvolutionalLayerOp)) of $(n_filters(L)) filters with " *
114+
"kernel size $(kernel(L)) and $(typeof(L.activation)) activation"
115+
return print(io, str)
116+
end
117+
118+
function load_Flux_convert_Conv_layer()
119+
return quote
120+
function Base.convert(::Type{ConvolutionalLayerOp}, layer::Flux.Conv)
121+
if !all(isone, layer.stride)
122+
throw(ArgumentError("stride $(layer.stride) != 1 is not supported")) # COV_EXCL_LINE
123+
end
124+
if !all(iszero, layer.pad)
125+
throw(ArgumentError("pad $(layer.pad) != 0 is not supported")) # COV_EXCL_LINE
126+
end
127+
if !all(isone, layer.dilation)
128+
throw(ArgumentError("dilation $(layer.dilation) != 1 is not supported")) # COV_EXCL_LINE
129+
end
130+
if !all(isone, layer.groups)
131+
throw(ArgumentError("groups $(layer.groups) != 1 is not supported")) # COV_EXCL_LINE
132+
end
133+
act = get(activations_Flux, layer.σ, nothing)
134+
if isnothing(act)
135+
throw(ArgumentError("unsupported activation function $(layer.σ)")) # COV_EXCL_LINE
136+
end
137+
# Flux stores a 4D matrix instead of a vector of 3D matrices
138+
weights = @inbounds [layer.weight[:, :, :, i] for i in 1:size(layer.weight, 4)]
139+
return ConvolutionalLayerOp(weights, layer.bias, act)
140+
end
141+
142+
function Base.convert(::Type{Flux.Conv}, layer::ConvolutionalLayerOp)
143+
act = get(activations_Flux, layer.activation, nothing)
144+
if isnothing(act)
145+
throw(ArgumentError("unsupported activation function $(layer.activation)")) # COV_EXCL_LINE
146+
end
147+
# Flux stores a 4D matrix instead of a vector of 3D matrices
148+
weights = cat(layer.weights...; dims=4)
149+
return Flux.Conv(weights, layer.bias, act)
150+
end
151+
end
152+
end

‎src/Architecture/init.jl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ function __init__()
33
@require Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" begin
44
eval(load_Flux_activations())
55
eval(load_Flux_convert_Dense_layer())
6+
eval(load_Flux_convert_Conv_layer())
67
eval(load_Flux_convert_network())
78
end
89
end

‎test/Architecture/ConvolutionalLayerOp.jl

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
using ControllerFormats.Architecture: kernel, n_filters
2+
using ReachabilityBase.Subtypes: subtypes
3+
4+
# 4x4x1 input tensor
5+
T441 = reshape([0 4 2 1; -1 0 1 -2; 3 1 2 0; 0 1 4 1], (4, 4, 1))
6+
O_Id = reshape([2 7 -2; -1 4 0; 6 9 1], (3, 3, 1))
7+
# 2x2x3 input tensor
8+
T223 = reshape(1:12, (2, 2, 3))
9+
10+
W1 = reshape([1 0; -1 2], (2, 2, 1))
11+
b1 = 1
12+
W2 = W1
13+
b2 = 2
14+
# 2x2 kernel and 1 filter
15+
Ws = [W1]
16+
bs = [b1]
17+
18+
# invalid weight/bias combination
19+
@test_throws ArgumentError ConvolutionalLayerOp(Ws, [1, 0], Id())
20+
@test_throws ArgumentError ConvolutionalLayerOp([], [], Id())
21+
@test_throws ArgumentError ConvolutionalLayerOp([W1, hcat(1)], [1, 0], Id())
22+
@test_throws ArgumentError ConvolutionalLayerOp([[1 0; -1 2]], [1], Id())
23+
24+
# one filter
25+
L = ConvolutionalLayerOp(Ws, bs, ReLU())
26+
# two filters
27+
L2 = ConvolutionalLayerOp([W1, W2], [b1, b2], ReLU())
28+
29+
# printing
30+
io = IOBuffer()
31+
println(io, L)
32+
33+
# output for tensors
34+
@test L(T441) == reshape([2 7 0; 0 4 0; 6 9 1], (3, 3, 1))
35+
@test L2(T441) == cat([2 7 0; 0 4 0; 6 9 1], [3 8 0; 0 5 1; 7 10 2]; dims=(3))
36+
@test_throws ArgumentError L(T223)
37+
@test_throws ArgumentError L(reshape(1:4.0, (2, 2)))
38+
39+
# equality
40+
@test L == ConvolutionalLayerOp(Ws, bs, ReLU())
41+
@test L != ConvolutionalLayerOp([W1 .+ 1], bs, ReLU()) &&
42+
L != ConvolutionalLayerOp(Ws, [b1 .+ 1], ReLU()) &&
43+
L != ConvolutionalLayerOp(Ws, bs, Id())
44+
45+
# approximate equality
46+
@test L ≈ ConvolutionalLayerOp(Ws, bs, ReLU())
47+
@test L ≈ ConvolutionalLayerOp([W1 .+ 1e-10], bs, ReLU()) &&
48+
L ≈ ConvolutionalLayerOp(Ws, [b1 .+ 1e-10], ReLU()) &&
49+
!≈(L, ConvolutionalLayerOp([W1 .+ 1e-10], bs, ReLU()); rtol=1e-12) &&
50+
!≈(L, ConvolutionalLayerOp(Ws, [b1 .+ 1e-10], ReLU()); rtol=1e-12) &&
51+
≈(L, ConvolutionalLayerOp([W1 .+ 1e-1], bs, ReLU()); atol=1) &&
52+
≈(L, ConvolutionalLayerOp(Ws, [b1 .+ 1e-1], ReLU()); atol=1) &&
53+
!(L ≈ ConvolutionalLayerOp([W1 .+ 1], bs, ReLU())) &&
54+
!(L ≈ ConvolutionalLayerOp(Ws, [b1 .+ 1], ReLU())) &&
55+
!(L ≈ ConvolutionalLayerOp(Ws, bs, Id()))
56+
57+
# kernel size and number of filters
58+
@test kernel(L) == kernel(L2) == (2, 2, 1)
59+
@test n_filters(L) == 1 && n_filters(L2) == 2
60+
61+
# test methods for all activations
62+
function test_layer(L::ConvolutionalLayerOp{Id})
63+
@test L(T441) == O_Id
64+
end
65+
66+
function test_layer(L::ConvolutionalLayerOp{ReLU})
67+
@test L(T441) == reshape([2 7 0; 0 4 0; 6 9 1], (3, 3, 1))
68+
end
69+
70+
function test_layer(L::ConvolutionalLayerOp{Sigmoid})
71+
@test L(float(T441)) ≈ Sigmoid().(O_Id) atol = 1e-3
72+
end
73+
74+
function test_layer(L::ConvolutionalLayerOp{Tanh})
75+
@test L(float(T441)) ≈ Tanh().(O_Id) atol = 1e-3
76+
end
77+
78+
function test_layer(L::ConvolutionalLayerOp{<:LeakyReLU})
79+
@test L(T441) == O_Id
80+
end
81+
82+
function test_layer(L::ConvolutionalLayerOp)
83+
return error("untested activation function: ", typeof(L.activation))
84+
end
85+
86+
# run test with all activations
87+
for act in subtypes(ActivationFunction)
88+
if act == TestActivation
89+
continue
90+
elseif act == LeakyReLU
91+
act_inst = LeakyReLU(1)
92+
else
93+
act_inst = act()
94+
end
95+
test_layer(ConvolutionalLayerOp(Ws, bs, act_inst))
96+
end

‎test/Architecture/Flux.jl

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
import Flux
22

3+
################
4+
# Dense layers #
5+
################
6+
37
L1 = Flux.Dense(1, 2, Flux.relu)
48
L1.weight .= 1, 2
59
L1.bias .= 3, 4
@@ -48,3 +52,19 @@ W = hcat([1 0.5; -0.5 0.5; -1 -0.5])
4852
b = [1.0, 0, -2]
4953
L = DenseLayerOp(W, b, TestActivation())
5054
@test_throws ArgumentError convert(Flux.Dense, L)
55+
56+
########################
57+
# Convolutional layers #
58+
########################
59+
60+
LC = Flux.Conv((2, 2), 1 => 1, Flux.relu)
61+
LC.weight .= reshape([1 0; -1 2], (2, 2, 1, 1))
62+
LC.bias .= 1
63+
64+
# layer conversion
65+
op = convert(ConvolutionalLayerOp, LC)
66+
@test op.weights[1] == LC.weight[:, :, :]
67+
@test op.bias == LC.bias
68+
@test op.activation == ReLU()
69+
L_back = convert(Flux.Conv, op)
70+
@test compare_Flux_layer(LC, L_back)

‎test/runtests.jl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ struct TestActivation <: ActivationFunction end
1616
@testset "DenseLayerOp" begin
1717
include("Architecture/DenseLayerOp.jl")
1818
end
19+
@testset "ConvolutionalLayerOp" begin
20+
include("Architecture/ConvolutionalLayerOp.jl")
21+
end
1922
@testset "FlattenLayerOp" begin
2023
include("Architecture/FlattenLayerOp.jl")
2124
end

0 commit comments

Comments
 (0)