Merge pull request #309 from network-intelligence/dev

Merging Dev into Trunk

Author: davidmcgrew

Makefile

@@ -192,6 +192,7 @@ else
 	cd resources && $(MAKE) distclean
 	rm -rf autom4te.cache config.log config.status Makefile_helper.mk
 	rm -f lib/*.so
+	-git clean -xf
 endif
 
 .PHONY: package-deb

README.md

@@ -104,13 +104,14 @@ GENERAL OPTIONS
    --config c                            # read configuration from file c
    [-a or --analysis]                    # analyze fingerprints
    --resources=f                         # use resource file f
+   --format=f                            # report fingerprints with formats(s) f
    --stats=f                             # write stats to file f
    --stats-time=T                        # write stats every T seconds
    --stats-limit=L                       # limit stats to L entries
    [-s or --select] filter               # select traffic by filter (see --help)
    --nonselected-tcp-data                # tcp data for nonselected traffic
    --nonselected-udp-data                # udp data for nonselected traffic
-   --tcp-reassembly                      # reassemble tcp data segments
+   --reassembly                          # reassemble protocol messages over multiple transport segments
    [-l or --limit] l                     # rotate output file after l records
    --output-time=T                       # rotate output file after T seconds
    --dns-json                            # output DNS as JSON, not base64
@@ -147,27 +148,43 @@ DETAILS
 
    "[-s or --select] f" selects packets according to the metadata filter f, which
    is a comma-separated list of the following strings:
+      arp               ARP message
+      bittorrent        Bittorrent Handshake Message, LSD message, DHT message
+      cdp               CDP message
       dhcp              DHCP discover message
+      dnp3              DNP3 industrial control message
       dns               DNS messages
       dtls              DTLS clientHello, serverHello, and certificates
+      gre               GRE message
       http              HTTP request and response
       http.request      HTTP request
       http.response     HTTP response
+      icmp              ICMP message
       iec               IEC 60870-5-104
+      lldp              LLDP message
       mdns              multicast DNS
+      mysql             MySQL Client/Server Protocol
       nbns              NetBIOS Name Service
+      nbds              NetBIOS Datagram Service
+      nbss              NetBIOS Session Service
       openvpn_tcp       OpenVPN over TCP
+      ospf              OSPF message
       quic              QUIC handshake
+      sctp              SCTP message
       ssh               SSH handshake and KEX
       smb               SMB v1 and v2
+      smtp              SMTP client and server messages
       stun              STUN messages
       ssdp              SSDP (UPnP)
+      socks             SOCKS4,SOCKS5 messages
       tcp               TCP headers
       tcp.message       TCP initial message
+      tcp.syn_ack       TCP syn ack message
       tls               TLS clientHello, serverHello, and certificates
       tls.client_hello  TLS clientHello
       tls.server_hello  TLS serverHello
       tls.certificates  TLS serverCertificates
+      tofsee            Tofsee malware communication
       wireguard         WG handshake initiation message
       all               all of the above
       <no option>       all of the above
@@ -185,10 +202,10 @@ DETAILS
    --select filter affects the UDP data written by this option; use
    '--select=none' to obtain the UDP data for each flow.
 
-   --tcp-reassembly enables the tcp reassembly
-   This option allows mercury to keep track of tcp segment state and 
-   and reassemble these segments based on the application in tcp payload
-
+   --reassembly enables reassembly
+   This option allows mercury to keep track of tcp or udp segment state and
+   and reassemble these segments based on the application in payload
+ 
    "[-u or --user] u" sets the UID and GID to those of user u, so that
    output file(s) are owned by this user.  If this option is not set, then
    the UID is set to SUDO_UID, so that privileges are dropped to those of
@@ -206,6 +223,15 @@ DETAILS
    object in the JSON records.   This option only works with the option
    [-f or --fingerprint].
 
+   "--format=f" reports fingerprints with formats(s) f, where f is either a
+   fingerprint protocol and format like "tls/1", or is a comma separated
+   list of below fingerprint protocol and format strings.
+       tls
+       tls/1
+       tls/2
+       quic
+       quic/1
+
    "[-l or --limit] l" rotates output files so that each file has at most
    l records or packets; filenames include a sequence number, date and time.
 
@@ -224,7 +250,6 @@ DETAILS
    --license and --version write their information to stdout, then halt.
 
    [-h or --help] writes this extended help message to stdout.
-
 ```
 
 ### SYSTEM

VERSION

@@ -1 +1 @@
-2.5.31
+2.6.1

doc/CHANGELOG.md

@@ -1,5 +1,70 @@
 # CHANGELOG for Mercury
 
+## VERSION 2.6.1
+* Improved STUN implementation: added test cases, fixed fingerprint
+  feature nits, renamed variables for consistency with the RFCs, and
+  simplified the message_type check.
+
+## VERSION 2.6.0
+* Added reassembly for QUIC initial messages, to ensure metadata and
+  fingerprint capture even for very long messages (e.g. due to
+  quantum-safe cryptography or encrypted client hellos).
+* Added the `--reassembly` keyword, which applies to both TCP and
+  QUIC, and retired the `--tcp-reassembly` option.
+* Improved support for the STUN protocol.
+    * Added support for "classic" (RFC3489) STUN.  In classic STUN,
+      `stun.magic_cookie` field is `false` and the
+      `stun.transaction_id` field is 16 bytes long.  In modern STUN,
+      the former field is `true` and the latter is 12 bytes long.
+    * Added a STUN fingerprint that uses data features selected by
+      automated feature-mining fingerprint.
+    * The STUN "usage" (STUN/TURN/ICE/etc.) is reported in the new
+      `stun.usage` field.
+    * Details are now reported for the STUN
+      attributes `BANDWIDTH`, `SOURCE-ADDRESS`, `CHANGED-ADDRESS`,
+      `RESPONSE-ADDRESS`, and `REFLECTED-FROM`.
+    * The `stun.message_type` field has been renamed to `stun.class`.
+* Added new output fields `dns.id`, `ip.version`, `ip.id`, `ip.ttl`
+  that are present when the `--metadata` option is used.
+* Added a `--raw-features=<protocols>` command line option that
+      specifies which protocols should have a raw feature vector
+      output.  Currently supported options include `bittorrent`,
+      `smb`, `ssdp`, `stun`, `tls`, `all`, and `none`.
+* Refactored HTTP header processing, enabling
+    * Non-standard delimeters are accepted, and their value is reported.
+    * HTTP 0.9 is accepted.
+    * With the `--metadata` option, all of the headers are output
+      using an object to represent each key/value pair.  This
+      simplifies the processing of header keys that appear more than
+      once in an HTTP request or response.
+* Added the first 512 bytes of the HTTP Body to `http` JSON records
+* Added BSD Loopback support for GENEVE.
+* Improved `tofsee` message format checking and `--stats` reporting.
+* Fixed the UTF-8 fuzz test seed file locations.
+* Changes to fingerprint and destination statistics output (`--stats`)
+    * Removed source IP address (`src_ip`) anonymization in stats file output.
+    * When processing a PCAP file, stats now uses lossless (blocking)
+      processing of events.  This enables the stats test to be
+      deterministic, avoiding occasional spurious failures during `make test`.
+    * Increased stats message queue size from 256 to 512.
+    * Stats aggregator now uses adaptive sleep time.
+    * Makefile addition: pre-clean the `test/` directory before
+      running the stats test.  This prevents leftover files in the
+      test directory from throwing off the counts of mercury JSON
+      output vs. mercury stats output.
+* The `--select=all` configuration option now actually selects all
+  protocols, including layer 2 protocols like ARP.
+* Added the `event_start` timestamp field to layer 2 JSON records.
+* Improved signal handling for code safety.
+* TLS ALPN GREASE is now normalized to `0x0a0a (\n\n)`.
+* In `libmerc`, truncated fingerprints now have a `fingerprint_status`
+  set to `fingerprint_status_unlabeled`.
+* Added informational messages to `libmerc`: resource file load time,
+  total number of loaded fingerprints, and the end time of a telemetry
+  stats dump.
+* Moved `hasher` to `crypto_engine.h`, so that it is more accesible.
+
+
 ## Version 2.5.31
 * Mercury now outputs `tls.client.certs` and `tls.undetermined.certs`
   as well as `tls.server.certs`, for TLS version 1.2 and earlier.

doc/npf.md

@@ -2,8 +2,6 @@
 
 **Draft**
 
-December 6, 2022
-
 David McGrew (Editor)
 
 
@@ -381,6 +379,33 @@ An example of a OPENVPN fingerprint is
 openvpn/(06)(03)(0400)(14)(0301)(c014c00ac022c0210039003800880087c00fc00500350084c012c008c01cc01b00160013c00dc003000ac013c009c01fc01e00330032009a009900450044c00ec004002f00960041c011c007c00cc002000500040015001200090014001100080006000300ff)((000b000403000102)(000a00340032000e000d0019000b000c00180009000a00160017000800060007001400150004000500120013000100020003000f00100011)(0023)(000f000101))
 ```
 
+## STUN
+
+The STUN fingerprint is formed from the messages of the "classic STUN" (RFC 3489) or modern STUN (RFC 8489) protocol.  The fingerprint format is
+```
+    "stun/1/" (class)(method)(magic)((attribute)*)
+```
+where the elements are defined as
+- `class` (string, one byte) is the (RFC 8489) class field
+- `method` (string, two bytes) is the (RFC 8489) method field
+- `magic` (string, one byte) is 01 if the Magic Cookie is present, and 00 otherwise.   The former indicates modern STUN, the latter classic STUN.
+- `attribute` (string, variable length) contains the type field, and optionally the length and data fields, of an attribute, in the order that they appear in the message.  The attribute types included in this field, and the extent of the data included, are as per the following table:
+```
+   USERNAME                  type
+   MESSAGE_INTEGRITY         type
+   XOR_MAPPED_ADDRESS        type
+   0x8007                    type
+   MS_VERSION                type
+   SOFTWARE                  type
+   FINGERPRINT               type
+   MS_APP_ID                 type_length_data
+   MS_IMPLEMENTATION_VERSION type_length_data
+   0xc003                    type
+   GOOG_NETWORK_INFO         type
+   0xdaba                    type
+```
+
+
 
 #### Truncated Fingerprints
 

mercury

@@ -41,7 +41,7 @@ _mercury()
     fi
 
     if [[ "$cur" == -* ]]; then
-        COMPREPLY=( $( compgen -W '--analysis --buffer --capture --certs-json --config --directory --dns-json --fingerprint --format --help --license --limit --metadata --nonselected-tcp-data --nonselected-udp-data --output-time --read --resources --select --stats --stats-limit --stats-time --tcp-reassembly --threads --user --verbose --version --write' -- "$cur") )
+        COMPREPLY=( $( compgen -W '--analysis --buffer --capture --certs-json --config --directory --dns-json --fingerprint --format --help --license --limit --metadata --nonselected-tcp-data --nonselected-udp-data --output-time --read --resources --select --stats --stats-limit --stats-time --reassembly --threads --user --verbose --version --write' -- "$cur") )
         # COMPREPLY=( $( compgen -W '$( _parse_help "$1" )' -- "$cur" ) )
         return 0
     fi

mercury.cfg

@@ -58,5 +58,5 @@ verbosity   = 0
 # set tls fingerprint format
 format      = tls/1
 
-# select tcp reassembly, for complete certificate chains
-tcp-reassembly
+# select reassembly, for complete certificate chains
+reassembly

src/Makefile.in

@@ -333,19 +333,19 @@ MSV ="-DMERCURY_SEMANTIC_VERSION=$(major),$(minor),$(patch)"
 .PHONY: increment-patchlevel increment-minor-version increment-major-version
 increment-patchlevel:
 	echo $(major).$(minor).$(shell expr $(patch) + 1) > ../VERSION
-	git add ../VERSION
+	git add ../VERSION ../doc/CHANGELOG.md
 	git commit -m "incrementing patchlevel"
 	git tag -a $(major).$(minor).$(shell expr $(patch) + 1) -m "\"patchlevel increment\""
 
 increment-minor-version:
 	echo $(major).$(shell expr $(minor) + 1).0 > ../VERSION
-	git add ../VERSION
+	git add ../VERSION ../doc/CHANGELOG.md
 	git commit -m "incrementing minor version"
 	git tag -a $(major).$(shell expr $(minor) + 1).0 -m "\"minor version increment\""
 
 increment-major-version:
 	echo $(shell expr $(major) + 1).0.0 > ../VERSION
-	git add ../VERSION
+	git add ../VERSION ../doc/CHANGELOG.md
 	git commit -m "incrementing major version"
 	git tag -a $(shell expr $(major) + 1).0.0 -m "\"major version increment\""
 

src/af_packet_v3.c

@@ -128,7 +128,7 @@ void ring_limits_init(struct ring_limits *rl, float frac);  // defined below
  * sig_close_flag and the packet worker threads will watch
  * sig_close_workers.
  */
-extern int sig_close_flag; /* Watched by the stats tracking thread, defined in signal_handling.c */
+extern volatile sig_atomic_t sig_close_flag; /* Watched by the stats tracking thread, defined in signal_handling.c */
 static int sig_close_workers = 0; /* Packet proccessing var */
 
 static double time_elapsed(struct timespec *ts) {
@@ -265,10 +265,14 @@ void *stats_thread_func(void *statst_arg) {
         exit(255);
     }
 
-    /**
+    /*
      * Enable all signals so that this thread shuts down first
      */
     enable_all_signals();
+    /* Block USR1 though since that's only for recovering
+     * stalled threads
+     */
+    disable_bt_signal();
 
     int duration = 0, socket_drops = 0, zero_drops = 0;
 
@@ -615,6 +619,18 @@ int af_packet_rx_ring_fanout_capture(struct thread_storage *thread_stor) {
 
     fprintf(stderr, "[PACKET PROCESSOR] Thread %d with pthread id %lu (PID %u) started...\n", thread_stor->tnum, thread_stor->tid, thread_stor->kpid);
 
+
+    /* Enable the bt signal for this thread which must be done before
+     * sigsetjmp since that saves the signal mask and the siglongjmp
+     * restores it.
+     *
+     * There is a brief window of time after we enable this signal
+     * but before we call sigsetjmp() where if USR1 is recieved
+     * the signal handler will be unable to locate the longjmp
+     * environment and will call abort().
+     */
+    enable_bt_signal();
+
     /* Save this execution context so that we can restore the thread back
      * to this point if it stalls during packet processing.
      */
@@ -628,6 +644,7 @@ int af_packet_rx_ring_fanout_capture(struct thread_storage *thread_stor) {
 
         __sync_synchronize(); /* enforce memory ordering */
         global_thread_stall[thread_stor->tnum].used = 1;
+
     } else {
         /* This branch of the if means a siglongjump was just performed
          * from our signal handler and execution is being restored
@@ -645,6 +662,7 @@ int af_packet_rx_ring_fanout_capture(struct thread_storage *thread_stor) {
         }
 
         fprintf(stderr, "[PACKET PROCESSOR] Thread %d with pthread id %lu (PID %u) resumed execution from stall...\n", thread_stor->tnum, thread_stor->tid, thread_stor->kpid);
+
     }
 
     /*

src/cert_analyze.cc

@@ -20,59 +20,10 @@
 #include "libmerc/x509.h"
 #include "libmerc/base64.h"
 #include "libmerc/rapidjson/document.h"
-#include <openssl/evp.h>
-
+#include "libmerc/crypto_engine.h"
 
 // certificate hashing
 
-class hasher {
-    EVP_MD_CTX *mdctx;
-
-public:
-
-    hasher() : mdctx{nullptr} { }
-
-    ~hasher() {
-        // EVP_MD_CTX_free() is preferred in v1.1.1, but unavailable in earlier versions
-        EVP_MD_CTX_destroy(mdctx);
-    }
-
-    constexpr static size_t output_size = 20;
-
-    void hash_buffer(const unsigned char *message, size_t message_len, unsigned char *digest, unsigned int digest_len) {
-
-        if ((unsigned int)EVP_MD_size(EVP_sha1()) > digest_len) {
-            handleErrors();
-        }
-
-        if (mdctx == NULL) {
-            // EVP_MD_CTX_new() is preferred in v1.1.1, but unavailable in earlier versions
-            if ((mdctx = EVP_MD_CTX_create()) == NULL) {
-                handleErrors();
-            }
-        }
-
-        if (1 != EVP_DigestInit_ex(mdctx, EVP_sha1(), NULL)) {
-            handleErrors();
-        }
-
-        if (1 != EVP_DigestUpdate(mdctx, message, message_len)) {
-            handleErrors();
-        }
-
-        unsigned int tmp_len;
-        if (1 != EVP_DigestFinal_ex(mdctx, digest, &tmp_len)) {
-            handleErrors();
-        }
-
-    }
-
-    void handleErrors() {
-        fprintf(stderr, "error: EVP hash failure\n");
-    }
-};
-
-
 void fprint_sha1_hash(FILE *f, const void *buffer, unsigned int len) {
 
     class hasher h;