Skip to content

Commit 098968a

Browse files
kmesh-botkmesh-bot
authored
Merge pull request #1496 from zrggw/ipsec_issue
Resolving communication issue between pods after enabling IPsec
2 parents c700423 + bf64ca4 commit 098968a

File tree

6 files changed

+149
-21
lines changed

6 files changed

+149
-21
lines changed

‎bpf/kmesh/general/include/tc.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@ struct tc_info {
2424
};
2525
};
2626

27-
#define PARSER_FAILED 1
28-
#define PARSER_SUCC 0
27+
#define PARSER_FAILED 1
28+
#define PARSER_SUCC 0
29+
#define IPSEC_DECRYPTED_MARK 0x00d0
2930

3031
static inline bool is_ipv4(struct tc_info *info)
3132
{

‎bpf/kmesh/general/tc_mark_decrypt.c‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,34 @@ int tc_mark_decrypt(struct __sk_buff *ctx)
1212
{
1313
struct nodeinfo *nodeinfo;
1414
struct tc_info info = {0};
15+
__u8 protocol = 0;
16+
bool decrypted = false;
17+
__u32 mark = 0;
1518

1619
if (parser_tc_info(ctx, &info)) {
1720
return TC_ACT_OK;
1821
}
19-
nodeinfo = check_remote_manage_by_kmesh(ctx, &info, info.iph->saddr, info.ip6h->saddr.s6_addr32);
20-
if (!nodeinfo) {
22+
if (is_ipv4(&info)) {
23+
protocol = info.iph->protocol;
24+
} else if (is_ipv6(&info)) {
25+
protocol = info.ip6h->nexthdr;
26+
} else {
2127
return TC_ACT_OK;
2228
}
23-
// 0x00d0 mean need decryption in ipsec
24-
ctx->mark = 0x00d0;
29+
30+
if (protocol == IPPROTO_ESP) {
31+
return TC_ACT_OK;
32+
}
33+
34+
mark = ctx->mark;
35+
decrypted = (mark == IPSEC_DECRYPTED_MARK); // IPSEC_DECRYPTED_MARK is same with xfmr state output-mark, which means
36+
// packet was decrypted and then back to ingress
37+
38+
if (decrypted) {
39+
return TC_ACT_OK;
40+
}
41+
42+
ctx->mark = 0;
2543
return TC_ACT_OK;
2644
}
2745

‎pkg/constants/constants.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,11 @@ const (
3131
ENABLED = uint32(1)
3232
DISABLED = uint32(0)
3333

34-
TC_MARK_DECRYPT = "tc_mark_decrypt"
35-
TC_MARK_ENCRYPT = "tc_mark_encrypt"
34+
TC_MARK_DECRYPT = "tc_mark_decrypt"
35+
TC_MARK_ENCRYPT = "tc_mark_encrypt"
36+
XfrmDecryptedMark = 0x00d0
37+
XfrmEncryptMark = 0x00e0
38+
XfrmMarkMask = 0xffffffff
3639

3740
TC_ATTACH = 0
3841
TC_DETACH = 1

‎pkg/controller/encryption/ipsec/ipsec_handler.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,10 @@ func (is *IpSecHandler) createStateRule(src net.IP, dst net.IP, key []byte, ipse
280280
Key: key,
281281
ICVLen: ipsecKey.Length,
282282
},
283+
OutputMark: &netlink.XfrmMark{
284+
Value: constants.XfrmDecryptedMark,
285+
Mask: constants.XfrmMarkMask,
286+
},
283287
}
284288
err := netlink.XfrmStateAdd(state)
285289
if err != nil && !os.IsExist(err) {
@@ -302,12 +306,12 @@ func (is *IpSecHandler) createPolicyRule(srcCIDR, dstCIDR *net.IPNet, src, dst n
302306
},
303307
},
304308
Mark: &netlink.XfrmMark{
305-
Mask: 0xffffffff,
309+
Mask: constants.XfrmMarkMask,
306310
},
307311
}
308312
if out {
309313
// ingress
310-
mark := uint32(0xd0)
314+
mark := uint32(constants.XfrmDecryptedMark)
311315
policy.Mark.Value = mark
312316

313317
policy.Dir = netlink.XFRM_DIR_IN
@@ -321,7 +325,7 @@ func (is *IpSecHandler) createPolicyRule(srcCIDR, dstCIDR *net.IPNet, src, dst n
321325
}
322326
} else {
323327
// egress, update SPI
324-
mark := uint32(0xe0)
328+
mark := uint32(constants.XfrmEncryptMark)
325329

326330
policy.Mark.Value = uint32(mark)
327331
policy.Tmpls[0].Spi = int(spi)

‎test/bpf_ut/bpftest/general_test.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,23 @@ func testGeneralTC(t *testing.T) {
4646
objFilename: "tc_mark_decrypt_test.o",
4747
uts: []unitTest_BPF_PROG_TEST_RUN{
4848
{
49-
name: "tc_mark_decrypt",
49+
name: "tc_mark_decrypt_case1",
50+
setupInUserSpace: func(t *testing.T, coll *ebpf.Collection) {
51+
setBpfConfig(t, coll, &factory.GlobalBpfConfig{
52+
BpfLogLevel: constants.BPF_LOG_DEBUG,
53+
})
54+
},
55+
},
56+
{
57+
name: "tc_mark_decrypt_case2",
58+
setupInUserSpace: func(t *testing.T, coll *ebpf.Collection) {
59+
setBpfConfig(t, coll, &factory.GlobalBpfConfig{
60+
BpfLogLevel: constants.BPF_LOG_DEBUG,
61+
})
62+
},
63+
},
64+
{
65+
name: "tc_mark_decrypt_case3",
5066
setupInUserSpace: func(t *testing.T, coll *ebpf.Collection) {
5167
setBpfConfig(t, coll, &factory.GlobalBpfConfig{
5268
BpfLogLevel: constants.BPF_LOG_DEBUG,

‎test/bpf_ut/tc_mark_decrypt_test.c‎

Lines changed: 95 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,34 +52,120 @@ const struct tcphdr l4 = {
5252
};
5353
const char body[20] = "Should not change!!";
5454

55-
PKTGEN("tc", "tc_mark_decrypt")
55+
// Test Case 1: TCP protocol, mark 0x00a0, expect mark to be 0x0
56+
PKTGEN("tc", "tc_mark_decrypt_case1")
5657
int test1_pktgen(struct __sk_buff *ctx)
5758
{
5859
return build_tc_packet(ctx, &l2, &l3, &l4, body, (uint)sizeof(body));
5960
}
6061

61-
JUMP("tc", "tc_mark_decrypt")
62+
JUMP("tc", "tc_mark_decrypt_case1")
6263
int test1_jump(struct __sk_buff *ctx)
6364
{
64-
// build context
65-
struct lpm_key key = {0};
66-
key.trie_key.prefixlen = 32;
67-
key.ip.ip4 = SRC_IP;
68-
__u32 value = 1;
69-
bpf_map_update_elem(&map_of_nodeinfo, &key, &value, BPF_ANY);
65+
// Set initial mark to 0x00a0
66+
ctx->mark = 0x00a0;
7067

7168
bpf_tail_call(ctx, &entry_call_map, 0);
7269
return TEST_ERROR;
7370
}
7471

75-
CHECK("tc", "tc_mark_decrypt")
72+
CHECK("tc", "tc_mark_decrypt_case1")
7673
int test1_check(struct __sk_buff *ctx)
7774
{
7875
const __u32 expected_status_code = TC_ACT_OK;
7976

8077
test_init();
8178

8279
check_tc_packet(ctx, &expected_status_code, &l2, &l3, &l4, body, (uint)sizeof(body));
80+
if (ctx->mark != 0x0)
81+
test_fatal("ctx->mark mismatch, expected 0x0, got %u", ctx->mark);
82+
83+
test_finish();
84+
}
85+
86+
// Test Case 2: TCP protocol, mark 0x00d0, expect mark to remain 0x00d0
87+
PKTGEN("tc", "tc_mark_decrypt_case2")
88+
int test2_pktgen(struct __sk_buff *ctx)
89+
{
90+
return build_tc_packet(ctx, &l2, &l3, &l4, body, (uint)sizeof(body));
91+
}
92+
93+
JUMP("tc", "tc_mark_decrypt_case2")
94+
int test2_jump(struct __sk_buff *ctx)
95+
{
96+
// Set initial mark to 0x00d0 (already decrypted)
97+
ctx->mark = 0x00d0;
98+
99+
bpf_tail_call(ctx, &entry_call_map, 0);
100+
return TEST_ERROR;
101+
}
102+
103+
CHECK("tc", "tc_mark_decrypt_case2")
104+
int test2_check(struct __sk_buff *ctx)
105+
{
106+
const __u32 expected_status_code = TC_ACT_OK;
107+
108+
test_init();
109+
110+
check_tc_packet(ctx, &expected_status_code, &l2, &l3, &l4, body, (uint)sizeof(body));
111+
if (ctx->mark != 0x00d0)
112+
test_fatal("ctx->mark mismatch, expected 0x00d0, got %u", ctx->mark);
113+
114+
test_finish();
115+
}
116+
117+
// Test Case 3: ESP protocol, mark 0x00d0, expect mark to remain 0x00d0
118+
PKTGEN("tc", "tc_mark_decrypt_case3")
119+
int test3_pktgen(struct __sk_buff *ctx)
120+
{
121+
// Use ESP protocol header for this test
122+
struct iphdr l3_esp = {
123+
.version = 4,
124+
.ihl = 5,
125+
.tot_len = 40,
126+
.id = 0x5438,
127+
.frag_off = bpf_htons(IP_DF),
128+
.ttl = 64,
129+
.protocol = IPPROTO_ESP,
130+
.saddr = SRC_IP,
131+
.daddr = DEST_IP,
132+
};
133+
134+
return build_tc_packet(ctx, &l2, &l3_esp, &l4, body, (uint)sizeof(body));
135+
}
136+
137+
JUMP("tc", "tc_mark_decrypt_case3")
138+
int test3_jump(struct __sk_buff *ctx)
139+
{
140+
// Set initial mark to 0x00d0
141+
ctx->mark = 0x00d0;
142+
143+
// build context - no need for nodeinfo map for ESP packets
144+
bpf_tail_call(ctx, &entry_call_map, 0);
145+
return TEST_ERROR;
146+
}
147+
148+
CHECK("tc", "tc_mark_decrypt_case3")
149+
int test3_check(struct __sk_buff *ctx)
150+
{
151+
const __u32 expected_status_code = TC_ACT_OK;
152+
153+
test_init();
154+
155+
// Use ESP protocol header for verification
156+
struct iphdr l3_esp = {
157+
.version = 4,
158+
.ihl = 5,
159+
.tot_len = 40,
160+
.id = 0x5438,
161+
.frag_off = bpf_htons(IP_DF),
162+
.ttl = 64,
163+
.protocol = IPPROTO_ESP,
164+
.saddr = SRC_IP,
165+
.daddr = DEST_IP,
166+
};
167+
168+
check_tc_packet(ctx, &expected_status_code, &l2, &l3_esp, &l4, body, (uint)sizeof(body));
83169
if (ctx->mark != 0x00d0)
84170
test_fatal("ctx->mark mismatch, expected 0x00d0, got %u", ctx->mark);
85171

0 commit comments

Comments
 (0)