Skip to content

Commit

Permalink
Add extra entropy checks and more precise(?) analysis.
Browse files Browse the repository at this point in the history 
Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
  • Loading branch information
@utoni
utoni committed Apr 18, 2024
1 parent 108b833 commit 2eca7c8
Show file tree
Hide file tree
Showing 103 changed files with 1,294 additions and 1,203 deletions.
11 changes: 9 additions & 2 deletions src/include/ndpi_main.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,15 @@ extern "C" {
int ndpi_is_printable_buffer(u_int8_t const * const buf, size_t len);
int ndpi_normalize_printable_string(char * const str, size_t len);
int ndpi_is_valid_hostname(char * const str, size_t len);
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy > 7.0f)
float ndpi_entropy(u_int8_t const * const buf, size_t len);

#define NDPI_ENTROPY_PLAINTEXT(entropy) (entropy < 4.941f)
#define NDPI_ENTROPY_EXECUTABLE(entropy) (entropy >= 4.941f)
#define NDPI_ENTROPY_EXECUTABLE_PACKED(entropy) (entropy >= 6.677f)
#define NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy) (entropy >= 7.174f)
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy >= 7.312f)
float ndpi_entropy(u_int8_t const * const buf, size_t len);
char *ndpi_entropy2str(float entropy, char *buf, size_t len);
void ndpi_entropy2risk(struct ndpi_flow_struct *flow);

#ifdef __cplusplus
}
Expand Down
2 changes: 1 addition & 1 deletion src/include/ndpi_typedefs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1207,7 +1207,7 @@ struct ndpi_flow_struct {
/* init parameter, internal used to set up timestamp,... */
u_int16_t guessed_protocol_id, guessed_protocol_id_by_ip, guessed_category, guessed_header_category;
u_int8_t l4_proto, protocol_id_already_guessed:1, fail_with_unknown:1,
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, _pad1: 1;
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, skip_entropy_check: 1;

u_int16_t num_dissector_calls;
ndpi_confidence_t confidence; /* ndpi_confidence_t */
Expand Down
37 changes: 12 additions & 25 deletions src/lib/ndpi_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4360,14 +4360,10 @@ static u_int16_t guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);

if(packet->payload_packet_len > sizeof(struct ndpi_icmphdr)) {
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));

if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
char str[32];

snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
if (flow->skip_entropy_check == 0) {
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
ndpi_entropy2risk(flow);
}

u_int16_t chksm = icmp4_checksum(packet->payload, packet->payload_packet_len);
Expand Down Expand Up @@ -8649,25 +8645,16 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
ndpi_search_shellscript(ndpi_str, flow);
}

if(flow->first_pkt_fully_encrypted == 0 &&
ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) == 0 &&
flow->packet_counter < 3)
{
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
char str[32];

snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
}
}
if(ret.app_protocol != NDPI_PROTOCOL_UNKNOWN &&
if(flow->skip_entropy_check == 0 &&
flow->first_pkt_fully_encrypted == 0 &&
flow->packet_counter < 5 &&
/* The following protocols do their own entropy calculation/classification. */
ret.app_protocol != NDPI_PROTOCOL_IP_ICMP &&
flow->entropy > 0.0f)
(flow->l4.tcp.http_stage <= 1 ||
ret.app_protocol != NDPI_PROTOCOL_HTTP))
{
flow->entropy = 0.0f;
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
ndpi_entropy2risk(flow);
}

return(ret);
Expand Down
92 changes: 92 additions & 0 deletions src/lib/ndpi_utils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2711,6 +2711,98 @@ float ndpi_entropy(u_int8_t const * const buf, size_t len) {
return entropy;
}

/* ******************************************************************** */

/* Losely implemented by: https://redirect.cs.umbc.edu/courses/graduate/CMSC691am/student%20talks/CMSC%20691%20Malware%20-%20Entropy%20Analysis%20Presentation.pdf */
char *ndpi_entropy2str(float entropy, char *buf, size_t len) {
if (buf == NULL) {
return NULL;
}

static const char entropy_fmtstr[] = "Entropy: %.3f (%s)";
if (NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted or Random");
} else if (NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted Executable");
} else if (NDPI_ENTROPY_EXECUTABLE_PACKED(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Compressed Executable");
} else if (NDPI_ENTROPY_EXECUTABLE(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Executable");
} else {
snprintf(buf, len, entropy_fmtstr, entropy, "Unknown");
}

return buf;
}

/* ******************************************************************** */

void ndpi_entropy2risk(struct ndpi_flow_struct *flow) {
char str[64];

if (NDPI_ENTROPY_PLAINTEXT(flow->entropy))
goto reset_risk;

if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_TLS ||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_QUIC ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_QUIC)
{
flow->skip_entropy_check = 1;
goto reset_risk;
}

if ((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_HTTP) &&
((flow->http.content_type != NULL &&
(strncmp(flow->http.content_type, "image/",
NDPI_STATICSTRING_LEN("image/")) == 0 ||
strncmp(flow->http.content_type, "video/",
NDPI_STATICSTRING_LEN("video/")) == 0 ||
strncmp(flow->http.content_type, "text/",
NDPI_STATICSTRING_LEN("text/")) == 0)) ||
(flow->http.request_content_type != NULL &&
(strncmp(flow->http.request_content_type, "image/",
NDPI_STATICSTRING_LEN("image/")) == 0 ||
strncmp(flow->http.request_content_type, "video/",
NDPI_STATICSTRING_LEN("video/")) == 0 ||
strncmp(flow->http.request_content_type, "text/",
NDPI_STATICSTRING_LEN("text/")) == 0))))
{
flow->skip_entropy_check = 1;
goto reset_risk;
}

if (flow->confidence != NDPI_CONFIDENCE_DPI &&
flow->confidence != NDPI_CONFIDENCE_DPI_CACHE) {
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
return;
}

if (ndpi_isset_risk(flow, NDPI_MALWARE_HOST_CONTACTED) ||
ndpi_isset_risk(flow, NDPI_BINARY_DATA_TRANSFER) ||
ndpi_isset_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER) ||
ndpi_isset_risk(flow, NDPI_POSSIBLE_EXPLOIT) ||
ndpi_isset_risk(flow, NDPI_HTTP_SUSPICIOUS_CONTENT) ||
ndpi_isset_risk(flow, NDPI_DNS_SUSPICIOUS_TRAFFIC) ||
ndpi_isset_risk(flow, NDPI_MALFORMED_PACKET) ||
(flow->category == NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT &&
(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_HTTP)) ||
flow->category == NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER ||
flow->category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED ||
flow->category == NDPI_PROTOCOL_CATEGORY_WEB)
{
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
return;
}

reset_risk:
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
}

/* ******************************************************************** */
static inline uint16_t get_n16bit(uint8_t const * cbuf) {
uint16_t r = ((uint16_t)cbuf[0]) | (((uint16_t)cbuf[1]) << 8);
Expand Down
5 changes: 5 additions & 0 deletions src/lib/protocols/http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,13 @@ static void ndpi_validate_http_content(struct ndpi_detection_module_struct *ndpi

if(len >= 8 /* 4 chars for \r\n\r\n and at least 4 charts for content guess */) {
double_ret += 4;
len -= 4;

ndpi_http_check_human_redeable_content(ndpi_struct, flow, double_ret, len);
if (flow->skip_entropy_check == 0) {
flow->entropy = ndpi_entropy(double_ret, len);
ndpi_entropy2risk(flow);
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion tests/cfgs/caches_cfg/result/teams.pcap.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ JA3 Host Stats:
11 TCP 192.168.1.6:60540 <-> 52.114.75.70:443 [proto: 91.250/TLS.Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 5][cat: Collaborative/15][14 pkts/5711 bytes <-> 10 pkts/8093 bytes][Goodput ratio: 83/92][0.13 sec][Hostname/SNI: eu-prod.asyncgw.teams.microsoft.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.173 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 8/9 32/32 13/14][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 408/809 1494/1506 517/688][TLSv1.2][JA3C: 74d5fa154a7fc0a7c655d8eaa34b89bf][JA4: t12d1312h2_8b80da21ef18_b00751acaffa][Plen Bins: 0,7,0,7,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,15,31,0,0]
12 TCP 192.168.1.6:60537 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 8][cat: Cloud/13][16 pkts/8418 bytes <-> 10 pkts/5367 bytes][Goodput ratio: 87/88][0.27 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: 0.221 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 14/27 46/46 20/20][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 526/537 1494/1506 639/623][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: a1674500365bdd882188db63730e69a2][JA4: t12d150700_0707305c9f76_0f3b2bcde21d][ServerNames: *.events.data.microsoft.com,events.data.microsoft.com,*.pipe.aria.microsoft.com,pipe.skype.com,*.pipe.skype.com,*.mobile.events.data.microsoft.com,mobile.events.data.microsoft.com,*.events.data.msn.com,events.data.msn.com][JA3S: ae4edc6faf64d08308082ad26be60767][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 4][Subject: CN=*.events.data.microsoft.com][Certificate SHA-1: 33:B3:B7:E9:DA:25:F5:A0:04:E9:63:87:B6:FB:54:77:DB:ED:27:EB][Safari][Validity: 2019-10-10 21:55:38 - 2021-10-10 21:55:38][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 7,7,7,0,0,0,7,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,7,0,0,31,15,0,0]
13 TCP 192.168.1.6:60555 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 11][cat: Cloud/13][18 pkts/5861 bytes <-> 13 pkts/7901 bytes][Goodput ratio: 80/89][2.79 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: -0.148 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 192/269 2443/2490 625/741][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 326/608 1494/1506 448/617][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: e4d448cdfe06dc1243c1eb026c74ac9a][JA4: t12d220700_0d4ca5d4ec72_3304d8368043][ServerNames: *.events.data.microsoft.com,events.data.microsoft.com,*.pipe.aria.microsoft.com,pipe.skype.com,*.pipe.skype.com,*.mobile.events.data.microsoft.com,mobile.events.data.microsoft.com,*.events.data.msn.com,events.data.msn.com][JA3S: 986571066668055ae9481cb84fda634a][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 4][Subject: CN=*.events.data.microsoft.com][Certificate SHA-1: 33:B3:B7:E9:DA:25:F5:A0:04:E9:63:87:B6:FB:54:77:DB:ED:27:EB][Firefox][Validity: 2019-10-10 21:55:38 - 2021-10-10 21:55:38][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,16,11,0,0,5,0,0,0,5,5,0,0,11,0,5,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,22,0,0]
14 UDP 192.168.1.6:51681 <-> 52.114.77.136:3478 [proto: 38/Skype_TeamsCall][IP: 276/Azure][Encrypted][Confidence: Match by port][DPI packets: 12][cat: VoIP/10][14 pkts/5838 bytes <-> 17 pkts/7907 bytes][Goodput ratio: 90/91][4.57 sec][bytes ratio: -0.151 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 347/256 2336/2336 693/595][Pkt Len c2s/s2c min/avg/max/stddev: 79/79 417/465 1243/1227 434/401][Risk: ** Susp Entropy **][Risk Score: 50][Risk Info: Entropy 7.83][PLAIN TEXT (TBHSWF)][Plen Bins: 0,36,0,0,0,12,6,0,3,6,0,0,0,3,0,0,0,0,0,0,0,0,0,6,6,0,0,0,0,0,3,0,3,3,0,0,0,9,0,0,0,0,0,0,0,0,0,0]
14 UDP 192.168.1.6:51681 <-> 52.114.77.136:3478 [proto: 38/Skype_TeamsCall][IP: 276/Azure][Encrypted][Confidence: Match by port][DPI packets: 12][cat: VoIP/10][14 pkts/5838 bytes <-> 17 pkts/7907 bytes][Goodput ratio: 90/91][4.57 sec][bytes ratio: -0.151 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 347/256 2336/2336 693/595][Pkt Len c2s/s2c min/avg/max/stddev: 79/79 417/465 1243/1227 434/401][Risk: ** Susp Entropy **][Risk Score: 50][Risk Info: Entropy: 6.927 (Compressed Executable)][PLAIN TEXT (TBHSWF)][Plen Bins: 0,36,0,0,0,12,6,0,3,6,0,0,0,3,0,0,0,0,0,0,0,0,0,6,6,0,0,0,0,0,3,0,3,3,0,0,0,9,0,0,0,0,0,0,0,0,0,0]
15 TCP 192.168.1.6:60547 <-> 52.114.88.59:443 [proto: 91.250/TLS.Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 14][cat: Collaborative/15][20 pkts/3926 bytes <-> 15 pkts/8828 bytes][Goodput ratio: 66/89][0.32 sec][Hostname/SNI: chatsvcagg.teams.microsoft.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.384 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 13/25 91/80 23/31][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 196/589 1494/1506 320/612][TLSv1.2][JA3C: ebf5e0e525258d7a8dcb54aa1564ecbd][JA4: t12d1311h2_8b80da21ef18_77989cba1f4a][Plen Bins: 0,21,10,5,0,5,10,5,0,0,0,5,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,5,21,0,0]
16 TCP 192.168.1.6:60565 <-> 52.114.108.8:443 [proto: 91.250/TLS.Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 5][cat: Collaborative/15][19 pkts/3306 bytes <-> 14 pkts/9053 bytes][Goodput ratio: 61/90][0.43 sec][Hostname/SNI: emea.ng.msg.teams.microsoft.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.465 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 27/12 276/54 68/17][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 174/647 1060/1506 238/633][TLSv1.2][JA3C: ebf5e0e525258d7a8dcb54aa1564ecbd][JA4: t12d1311h2_8b80da21ef18_77989cba1f4a][Plen Bins: 0,22,16,5,0,0,5,0,0,0,0,0,0,5,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,5,0,0,0,0,22,0,0]
17 TCP 192.168.1.6:60541 <-> 52.114.75.69:443 [proto: 91.125/TLS.Skype_Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 10][cat: VoIP/10][13 pkts/4051 bytes <-> 9 pkts/7973 bytes][Goodput ratio: 79/92][0.14 sec][Hostname/SNI: eu-api.asm.skype.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.326 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 10/11 31/36 14/16][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 312/886 1494/1506 422/676][TLSv1.2][JA3C: 74d5fa154a7fc0a7c655d8eaa34b89bf][JA4: t12d1312h2_8b80da21ef18_b00751acaffa][ServerNames: *.asm.skype.com][JA3S: 986571066668055ae9481cb84fda634a][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 1][Subject: CN=*.asm.skype.com][Certificate SHA-1: B9:41:1D:AE:56:09:68:D2:07:D0:69:E1:68:00:08:2B:EF:63:1E:48][Validity: 2019-05-07 12:50:03 - 2021-05-07 12:50:03][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,8,0,8,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,8,34,0,0]
Expand Down
Loading

0 comments on commit 2eca7c8

Please sign in to comment.