Skip to content

Commit 971c3cb

Browse files
Apoorv Raj (apooraj)GitHub Enterprise
authored andcommitted
Merge pull request #275 from network-intelligence/dev
Dev to trunk merge
2 parents eb04c7b + f044f3d commit 971c3cb

27 files changed

+3660
-283
lines changed

VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.5.29
1+
2.5.30

doc/CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
# CHANGELOG for Mercury
22

3+
## Version 2.5.30
4+
* Dramatic improvements to TCP reassembly for output, performance and TCP segments handling.
5+
* Improved error handling for malformed UTF-8 strings encountered in protocol fields.
6+
* Support to parse and output an Encrypted Client Hello extension features.
7+
* Concise Data Definition Language (CDDL) definitions for Network Protocol Fingerprinting.
8+
* Concise Binary Object Representation (CBOR) encoding and decoding for fingerprints and Fingerprint and Destination Context.
9+
* Support for reading classifier feature weights from resource file, whenever available.
10+
311
## Version 2.5.29
412
* Support for "dual DB" resource archives, as described in
513
[doc/resources.md](../doc/resources.md).

doc/npf.cddl

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
; Concise Data Definition Language (CDDL) definitions for Network
2+
; Protocol Fingerprinting
3+
;
4+
5+
; An npf fingerprint is a map from unsigned integers (the
6+
; fingerprint_type enumeration) to a protocol-specific fingerprint
7+
; object
8+
;
9+
fingerprint = {
10+
? tls: tls_fp
11+
? http: http_fp
12+
}
13+
14+
; The fingerprint_type enumeration identifies a protocol types
15+
; associated with a fingerprint
16+
;
17+
fingerprint_type = &(
18+
; unknown: 0,
19+
tls: 1,
20+
; tls_server: 2,
21+
http: 3,
22+
; http_server: 4,
23+
; ssh: 5,
24+
; ssh_kex: 6,
25+
; tcp: 7,
26+
; dhcp: 8,
27+
; smtp_server: 9,
28+
; dtls: 10,
29+
; dtls_server: 11,
30+
; quic: 12,
31+
; tcp_server: 13,
32+
; openvpn: 14,
33+
; tofsee: 15,
34+
)
35+
36+
; A tls_fp is a map from a version number (unsigned integer) to a
37+
; specific version of a tls fingerprint
38+
;
39+
tls_fp = {
40+
? 1: tls_fp_1
41+
}
42+
43+
; tls_fp_1 is a record-style array of data elements that represents a
44+
; tls fingerprint, which corresponds to the tls/1 fingerprint:
45+
;
46+
; "tls/1" (TLS_Version) (TLS_Ciphersuite) [ TLS_Extension* ]
47+
;
48+
;
49+
tls_fp_1 = [
50+
version: bstr .size 2,
51+
ciphersuites: bstr,
52+
extensions: [ ; sorted into ascending lexicographic order
53+
* bstr
54+
]
55+
? rank: uint ; rank of sort-permutation
56+
]
57+
58+
; An http_fp is a map from a version number (unsigned integer) to a
59+
; specific version of an http fingerprint
60+
;
61+
http_fp = {
62+
? 0: http_fp_0
63+
}
64+
65+
; http_fp_0 is a record-style array of data elements that represents
66+
; an http fingerprint, which corresponds to the http/ fingerprint:
67+
;
68+
; "http/" (method) (version) ((selected-header)*)
69+
;
70+
http_fp_0 = [
71+
method: bstr,
72+
version: bstr,
73+
selected-headers: [
74+
* bstr
75+
]
76+
]
77+
78+
;
79+
; IP address and flow key definitions
80+
;
81+
ipv4_address = bstr .size 4
82+
ipv6_address = bstr .size 16
83+
84+
ipv4_flow_key = [
85+
src_ip: ipv4_address,
86+
dst_ip: ipv4_address,
87+
src_port: uint,
88+
dst_port: uint,
89+
protocol: uint
90+
]
91+
92+
ipv6_flow_key = [
93+
src_ip: ipv4_address,
94+
dst_ip: ipv4_address,
95+
src_port: uint,
96+
dst_port: uint,
97+
protocol: uint
98+
]
99+

src/Makefile.in

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,9 @@ archive_reader: archive_reader.cc libmerc/archive.h
197197
string: string.cc stringalgs.h options.h
198198
$(CXX) $(CFLAGS) string.cc -o string
199199

200+
cbor: cbor.cpp libmerc/cbor.hpp libmerc/fdc.hpp libmerc/static_dict.hpp libmerc/file_datum.hpp options.h
201+
$(CXX) $(CFLAGS) cbor.cpp -o cbor
202+
200203
decode: decode.cc
201204
$(CXX) $(CFLAGS) decode.cc -o decode
202205

@@ -245,7 +248,7 @@ intercept.so: intercept.cc libmerc.a
245248

246249
.PHONY: clean
247250
clean: libmerc-clean
248-
rm -rf mercury libmerc_test libmerc_util intercept_server tls_scanner cert_analyze os_identifier archive_reader batch_gcd string decode pcap pcap_filter format intercept.so gmon.out *.o *.json.gz
251+
rm -rf mercury libmerc_test libmerc_util intercept_server tls_scanner cert_analyze os_identifier archive_reader batch_gcd string cbor decode pcap pcap_filter format intercept.so gmon.out *.o *.json.gz
249252
for file in Makefile.in README.md configure.ac; do if [ -e "$$file~" ]; then rm -f "$$file~" ; fi; done
250253
for file in mercury.c libmerc_test.c tls_scanner.cc cert_analyze.cc $(MERC) $(MERC_H); do if [ -e "$$file~" ]; then rm -f "$$file~" ; fi; done
251254

src/cbor.cpp

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// cbor.cpp
2+
3+
#include <cstdio>
4+
#include <vector>
5+
#include <cctype>
6+
#include <iostream>
7+
#include <fstream>
8+
9+
#include "options.h"
10+
#include "libmerc/cbor.hpp"
11+
#include "libmerc/fdc.hpp"
12+
#include "libmerc/file_datum.hpp"
13+
14+
using namespace mercury_option;
15+
16+
int main(int argc, char *argv[]) {
17+
18+
// run unit tests (when NDEBUG is not defined)
19+
//
20+
assert(static_dictionary<0>::unit_test() == true);
21+
assert(cbor::unit_test() == true);
22+
assert(cbor_fingerprint::unit_test() == true);
23+
assert(fdc::unit_test() == true);
24+
25+
const char *summary = "usage: %s [OPTIONS]\n";
26+
option_processor opt({
27+
{ argument::none, "--decode-cbor", "decode input as generic CBOR" },
28+
{ argument::none, "--decode-fdc", "decode input as FDC" },
29+
{ argument::none, "--encode-fingerprint", "encode fingerprint string as CBOR" },
30+
{ argument::required, "--input-file", "read data from file <filename>" },
31+
{ argument::none, "--verbose-tests", "run unit tests in verbose mode" },
32+
{ argument::none, "--help", "print out help message" },
33+
});
34+
35+
if (!opt.process_argv(argc, argv)) {
36+
opt.usage(stderr, argv[0], summary);
37+
return EXIT_FAILURE;
38+
}
39+
40+
auto [ input_file_is_set, input_file ] = opt.get_value("--input-file");
41+
bool decode_cbor = opt.is_set("--decode-cbor");
42+
bool decode_fdc = opt.is_set("--decode-fdc");
43+
bool encode_fp = opt.is_set("--encode-fingerprint");
44+
bool verbose_tests = opt.is_set("--verbose-tests");
45+
bool help_needed = opt.is_set("--help");
46+
if (help_needed) {
47+
opt.usage(stdout, argv[0], summary);
48+
return 0;
49+
}
50+
51+
if (verbose_tests) {
52+
bool sd_result = static_dictionary<0>::unit_test(stdout);
53+
bool cbor_result = cbor::unit_test(stdout);
54+
bool cbor_fingerprint_result = cbor_fingerprint::unit_test(stdout);
55+
bool fdc_result = fdc::unit_test(stdout);
56+
57+
printf("static_dictionary::unit_test: %s\n", sd_result ? "passed" : "failed");
58+
printf("cbor::unit_test: %s\n", cbor_result ? "passed" : "failed");
59+
printf("cbor_fingerprint::unit_test: %s\n", cbor_fingerprint_result ? "passed" : "failed");
60+
printf("fdc::unit_test: %s\n", fdc_result ? "passed" : "failed");
61+
62+
if (sd_result and cbor_result and cbor_fingerprint_result and fdc_result) {
63+
printf("all unit tests passed\n");
64+
return EXIT_SUCCESS;
65+
}
66+
fprintf(stderr, "error: one or more unit tests failed\n");
67+
return EXIT_FAILURE;
68+
}
69+
70+
if (input_file_is_set) {
71+
72+
if (encode_fp) {
73+
fprintf(stderr, "error: option --encode-fingerprint is incompatible with --input-file\n");
74+
return EXIT_FAILURE;
75+
}
76+
77+
file_datum input{input_file.c_str()};
78+
if (decode_cbor) {
79+
bool decode_ok = cbor::decode_fprint(input, stdout);
80+
if (!decode_ok) {
81+
fprintf(stderr, "error: could not decode complete input file\n");
82+
return EXIT_FAILURE;
83+
}
84+
}
85+
if (decode_fdc) {
86+
static const size_t MAX_DST_ADDR_LEN = 48;
87+
static const size_t MAX_SNI_LEN = 257;
88+
static const size_t MAX_USER_AGENT_LEN = 512;
89+
static const size_t MAX_FP_STR_LEN = 4096;
90+
char fp_str[MAX_FP_STR_LEN];
91+
char dst_ip_str[MAX_DST_ADDR_LEN];
92+
char sn_str[MAX_SNI_LEN];
93+
char ua_str[MAX_USER_AGENT_LEN];
94+
uint16_t dst_port;
95+
96+
bool ok = fdc::decode(input,
97+
writeable{(uint8_t*)fp_str, MAX_FP_STR_LEN},
98+
writeable{(uint8_t*)sn_str, MAX_SNI_LEN},
99+
writeable{(uint8_t*)dst_ip_str, MAX_DST_ADDR_LEN},
100+
dst_port,
101+
writeable{(uint8_t*)ua_str, MAX_USER_AGENT_LEN});
102+
if (ok) {
103+
fprintf(stdout, "{\"fdc\":{");
104+
fprintf(stdout, "\"fingerprint\": \"%s\",", fp_str);
105+
fprintf(stdout, "\"sni\": \"%s\",", sn_str);
106+
fprintf(stdout, "\"dst_ip_str\": \"%s\",", dst_ip_str);
107+
fprintf(stdout, "\"dst_port\": %u,", dst_port);
108+
fprintf(stdout, "\"user-agent\": \"%s\"", ua_str);
109+
fprintf(stdout, "}}\n");
110+
} else {
111+
fprintf(stderr, "error: could not decode FDC\n");
112+
}
113+
}
114+
115+
} else if (encode_fp) {
116+
117+
std::ios::sync_with_stdio(false); // for performance
118+
std::string line;
119+
while (std::getline(std::cin, line)) {
120+
if (line.length() == 0) {
121+
continue; // ignore empty line
122+
}
123+
124+
// verify that we can represent this fingerprint in CBOR
125+
//
126+
if (!cbor_fingerprint::test_fingerprint(line.c_str())){
127+
fprintf(stderr, "error: could not encode/decode fingerprint %s\n", line.c_str());
128+
return EXIT_FAILURE;
129+
}
130+
131+
// convert fingerprint to CBOR
132+
//
133+
datum fp_data{(uint8_t *)line.c_str(), (uint8_t *)line.c_str() + line.length()};
134+
dynamic_buffer data_buf{4096};
135+
cbor_fingerprint::encode_cbor_fingerprint(fp_data, data_buf);
136+
137+
// print out hex representation
138+
//
139+
data_buf.contents().fprint_hex(stdout); fputc('\n', stdout);
140+
141+
// print out human-readable CBOR
142+
//
143+
cbor::decode_fprint(data_buf.contents(), stdout);
144+
}
145+
return EXIT_SUCCESS;
146+
147+
} else {
148+
fprintf(stderr, "error: no input file specified\n");
149+
return EXIT_FAILURE;
150+
}
151+
152+
return EXIT_SUCCESS;
153+
}

0 commit comments

Comments
 (0)