Merge pull request #283 from network-intelligence/dev

Merging dev into trunk

Author: davidmcgrew

VERSION

@@ -1 +1 @@
-2.5.30
+2.5.31

doc/CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG for Mercury
 
+## Version 2.5.31
+* Mercury now outputs `tls.client.certs` and `tls.undetermined.certs`
+  as well as `tls.server.certs`, for TLS version 1.2 and earlier.
+  Client and server certificate chains are distinguished by the
+  handshake type of the key exchange data that follows the
+  `Certificate` data.  If no key exchange data is present, then the
+  certificate is reported as `tls.undetermined.certs`.
+* Timestamp counter reading support for ARM added in
+  [tsc_clock.hpp](src/libmerc/tsc_clock.hpp).
+* If a timestamp of `0` is passed to `libmerc`, a timestamp is
+  computed in order to improve the reassembly of TCP messages, as needed.
+
 ## Version 2.5.30
 * Dramatic improvements to TCP reassembly for output, performance and TCP segments handling.
 * Improved error handling for malformed UTF-8 strings encountered in protocol fields.

src/libmerc/bench.h

@@ -5,46 +5,13 @@
 #ifndef BENCH_H
 #define BENCH_H
 
-// The cycle_counter class will compile anywhere, but will only work
-// correctly on platforms that provide a function to read the
-// timestamp counter.  The following preprocessor conditionals
-// identify the appropriate function, if one is present, and set
-// benchmark_is_valid to true or false.   That value can be accessed
-// through the constexpr static boolean benchmark::is_valid.
-//
-#ifdef HAVE_X86INTRIN_H
-   #include <x86intrin.h>
-   #define read_timestamp_counter() __rdtsc()
-   #define benchmark_is_valid true
-#else
-   #define read_timestamp_counter() 0
-   #define benchmark_is_valid false
-#endif
-//
-// TODO: add the corresponding ARM equivalent function
-
 #include <cmath>  // for sqrt()
+#include "tsc_clock.hpp"
 
 namespace benchmark {
 
-    static constexpr bool is_valid = benchmark_is_valid;
-
-    // An object of class cycle_counter counts the number of clock
-    // cycles between its construction and the invocation of the
-    // delta() function
-    //
-    class cycle_counter {
-        uint64_t value;
-
-    public:
-
-        cycle_counter() : value{read_timestamp_counter()} { }
-
-        uint64_t delta() const { return read_timestamp_counter() - value; }
-
-    };
+    static bool is_valid = tsc_clock::is_valid();
 
-    // An object of class count_and_mean maintains a count and mean of
     // all of the observed numbers.  Each observation is reported with
     // the member function +=, e.g. 's += x' observes x.
     //

src/libmerc/libmerc.cc

@@ -14,6 +14,7 @@
 #include "pkt_proc.h"
 #include "config_generator.h"
 #include "global_config.h"
+#include "tsc_clock.hpp"
 
 #ifndef  MERCURY_SEMANTIC_VERSION
 #warning MERCURY_SEMANTIC_VERSION is not defined
@@ -89,7 +90,9 @@ mercury_context mercury_init(const struct libmerc_config *vars, int verbosity) {
     // taking place
     //
     assert(printf_err(log_info, "libmerc is running assert() tests\n") != 0);
-
+    // Calculate the cpu ticks per sec during initialization time
+    // to avoid delay during packet processing path
+    tsc_clock::init();
     try {
         m = new mercury{vars, verbosity};
         return m;

src/libmerc/pkt_proc.cc

@@ -53,6 +53,7 @@
 #include "openvpn.h"
 #include "mysql.hpp"
 #include "geneve.hpp"
+#include "tsc_clock.hpp"
 
 // double malware_prob_threshold = -1.0; // TODO: document hidden option
 
@@ -217,9 +218,11 @@ void stateful_pkt_proc::set_tcp_protocol(protocol &x,
             break;
         }
     case tcp_msg_type_tls_server_hello:
-    case tcp_msg_type_tls_certificate:
         x.emplace<tls_server_hello_and_certificate>(pkt, tcp_pkt);
         break;
+    case tcp_msg_type_tls_certificate:
+        x.emplace<tls_certificate>(pkt, tcp_pkt);
+        break;
     case tcp_msg_type_ssh:
         x.emplace<ssh_init_packet>(pkt);
         break;
@@ -508,6 +511,11 @@ size_t stateful_pkt_proc::ip_write_json(void *buffer,
         }
     }
 
+    if (ts->tv_sec == 0) {
+        tsc_clock time_now;
+        ts->tv_sec = time_now.time_in_seconds();
+    }
+
     // process transport/application protocols
     //
     protocol x;
@@ -785,6 +793,12 @@ bool stateful_pkt_proc::analyze_ip_packet(const uint8_t *packet,
     if (reassembler) {
         reassembler->dump_pkt = false;
     }
+
+    if (ts->tv_sec == 0) {
+        tsc_clock time_now;
+        ts->tv_sec = time_now.time_in_seconds();
+    }
+
     if (transport_proto == ip::protocol::tcp) {
         tcp_packet tcp_pkt{pkt, &ip_pkt};
         if (!tcp_pkt.is_valid()) {

src/libmerc/pkt_proc_util.h

@@ -38,6 +38,7 @@ struct http_request;                      // start of tcp protocols
 struct http_response;
 struct tls_client_hello;
 class tls_server_hello_and_certificate;
+class tls_certificate;
 struct ssh_init_packet;
 struct ssh_kex_init;
 class smtp_client;
@@ -72,6 +73,7 @@ using protocol = std::variant<std::monostate,
                               http_response,
                               tls_client_hello,
                               tls_server_hello_and_certificate,
+                              tls_certificate,
                               ssh_init_packet,
                               ssh_kex_init,
                               smtp_client,
@@ -240,6 +242,10 @@ struct write_metadata {
         r.write_json(record, metadata_output_, certs_json_output_);
     }
 
+    void operator()(tls_certificate &r) {
+        r.write_json(record, metadata_output_, certs_json_output_);
+    }
+
     void operator()(std::monostate &) { }
 
 };

src/libmerc/tls.h

@@ -203,8 +203,10 @@ enum class handshake_type : uint8_t {
     end_of_early_data = 5,
     encrypted_extensions = 8,
     certificate = 11,
+    server_key_exchange = 12,
     certificate_request = 13,
     certificate_verify = 15,
+    client_key_exchange = 16,
     finished = 20,
     key_update = 24,
     message_hash = 254
@@ -570,6 +572,107 @@ class tls_server_hello_and_certificate : public base_protocol {
 
 };
 
+
+//   ClientKeyExchange, following RFC 5246 Section 7.4.7
+//
+//   struct {
+//       select (KeyExchangeAlgorithm) {
+//           case rsa:
+//               EncryptedPreMasterSecret;
+//           case dhe_dss:
+//           case dhe_rsa:
+//           case dh_dss:
+//           case dh_rsa:
+//           case dh_anon:
+//               ClientDiffieHellmanPublic;
+//       } exchange_keys;
+//   } ClientKeyExchange;
+
+// enum { dhe_dss, dhe_rsa, dh_anon, rsa, dh_dss, dh_rsa
+//     /* may be extended, e.g., for ECDH -- see [TLSECC] */
+// } KeyExchangeAlgorithm;
+
+
+enum role {
+    client,
+    server,
+    undetected
+};
+
+class tls_certificate : public base_protocol {
+    struct tls_server_certificate certificate;
+    role entity;
+
+public:
+
+    tls_certificate(struct datum &pkt, struct tcp_packet *tcp_pkt) : certificate{}, entity{undetected} {
+        parse(pkt, tcp_pkt);
+    }
+
+    void parse(struct datum &pkt, struct tcp_packet *tcp_pkt) {
+
+        // parse certificate
+        //
+        struct tls_record rec{pkt};
+        struct tls_handshake handshake{rec.fragment};
+        if (handshake.msg_type == handshake_type::certificate) {
+            certificate.parse(handshake.body);
+
+            if (rec.fragment.is_not_empty()) {
+                tls_handshake handshake{rec.fragment};
+                if (handshake.msg_type == handshake_type::client_key_exchange) {
+                    entity = client;
+                } else if (handshake.msg_type == handshake_type::server_key_exchange) {
+                    entity = server;
+                }
+            } else if (pkt.is_not_empty()) {
+                tls_record rec2{pkt};
+                tls_handshake handshake{rec2.fragment};
+                if (handshake.msg_type == handshake_type::client_key_exchange) {
+                    entity = client;
+                } else if (handshake.msg_type == handshake_type::server_key_exchange) {
+                    entity = server;
+                }
+            }
+
+        }
+        if (tcp_pkt && certificate.additional_bytes_needed) {
+            tcp_pkt->reassembly_needed(certificate.additional_bytes_needed);
+        }
+    }
+
+    bool is_not_empty() {
+        return certificate.is_not_empty();
+    }
+
+    void write_json(struct json_object &record, bool metadata_output, bool certs_json_output) {
+        (void)metadata_output;
+
+        bool have_certificate = certificate.is_not_empty();
+        if (have_certificate) {
+
+            // output certificate
+            //
+            const char *role = "undetermined";
+            if (entity == client) {
+                role = "client";
+            } else if (entity == server) {
+                role = "server";
+            }
+            struct json_object tls{record, "tls"};
+            json_object client_or_server{tls, role};
+            struct json_array certs{client_or_server, "certs"};
+            certificate.write_json(certs, certs_json_output);
+            certs.close();
+            client_or_server.close();
+            tls.close();
+
+        }
+    }
+
+};
+
+
 static uint16_t degrease_uint16(uint16_t x) {
     switch(x) {
     case 0x0a0a: