Add a new flow risk NDPI_ANONYMOUS_SUBSCRIBER

doc/flow_risks.rst

@@ -272,4 +272,9 @@ NDPI_HTTP_CRAWLER_BOT
 ===================================
 The risk is set whenever a crawler/bot/robot has been detected
 
+.. _Risk 045:
 
+NDPI_ANONYMOUS_SUBSCRIBER
+===================================
+The risk is set whenever the (source) ip address has been anonymized and it can't be used to identify the subscriber.
+Example: the flow is generated by an iCloud-private-relay exit node.

python/ndpi.py

@@ -336,6 +336,7 @@
   NDPI_PUNYCODE_IDN,
   NDPI_ERROR_CODE_DETECTED,
   NDPI_HTTP_CRAWLER_BOT,
+  NDPI_ANONYMOUS_SUBSCRIBER,
 
   /* Leave this as last member */
   NDPI_MAX_RISK

src/include/ndpi_typedefs.h

@@ -117,7 +117,8 @@ typedef enum {
   NDPI_PUNYCODE_IDN, /* https://en.wikipedia.org/wiki/Punycode */
   NDPI_ERROR_CODE_DETECTED,
   NDPI_HTTP_CRAWLER_BOT,
-
+  NDPI_ANONYMOUS_SUBSCRIBER,
+
   /* Leave this as last member */
   NDPI_MAX_RISK /* must be <= 63 due to (**) */
 } ndpi_risk_enum;
@@ -1044,6 +1045,7 @@ struct ndpi_detection_module_struct {
   ndpi_list *trusted_issuer_dn;
 
   void *ip_risk_mask_ptree;
+  void *ip_risk_ptree;
 
   struct {
     ndpi_automa hostnames, hostnames_shadow;
@@ -1186,7 +1188,7 @@ struct ndpi_flow_struct {
   char host_server_name[80];
 
   u_int8_t initial_binary_bytes[8], initial_binary_bytes_len;
-  u_int8_t risk_checked:1, ip_risk_mask_evaluated:1, host_risk_mask_evaluated:1, _notused:5;
+  u_int8_t risk_checked:1, ip_risk_mask_evaluated:1, host_risk_mask_evaluated:1, tree_risk_checked:1, _notused:4;
   ndpi_risk risk_mask; /* Stores the flow risk mask for flow peers */
   ndpi_risk risk; /* Issues found with this flow [bitmask of ndpi_risk] */
 
@@ -1403,6 +1405,8 @@ typedef enum
     ndpi_dont_load_google_list     = (1 << 10),
     ndpi_dont_load_google_cloud_list = (1 << 11),
     ndpi_dont_load_asn_lists       = (1 << 12),
+    ndpi_dont_load_icloud_private_relay_list  = (1 << 13),
+    ndpi_dont_init_risk_ptree      = (1 << 14),
   } ndpi_prefs;
 
 typedef struct {

src/lib/ndpi_main.c

@@ -63,6 +63,7 @@
 #include "ndpi_ms_skype_teams_match.c.inc"
 #include "ndpi_google_match.c.inc"
 #include "ndpi_google_cloud_match.c.inc"
+#include "ndpi_icloud_private_relay_match.c.inc"
 #include "ndpi_asn_telegram.c.inc"
 #include "ndpi_asn_apple.c.inc"
 #include "ndpi_asn_twitter.c.inc"
@@ -138,6 +139,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
   { NDPI_PUNYCODE_IDN,                          NDPI_RISK_LOW,    CLIENT_LOW_RISK_PERCENTAGE  },
   { NDPI_ERROR_CODE_DETECTED,                   NDPI_RISK_LOW,    CLIENT_LOW_RISK_PERCENTAGE  },
   { NDPI_HTTP_CRAWLER_BOT,                      NDPI_RISK_LOW,    CLIENT_LOW_RISK_PERCENTAGE  },
+  { NDPI_ANONYMOUS_SUBSCRIBER,                  NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE },
 
   /* Leave this as last member */
   { NDPI_MAX_RISK,                              NDPI_RISK_LOW,    CLIENT_FAIR_RISK_PERCENTAGE }
@@ -2051,6 +2053,23 @@ u_int16_t ndpi_network_port_ptree_match(struct ndpi_detection_module_struct *ndp
 
 /* ******************************************* */
 
+ndpi_risk_enum ndpi_network_risk_ptree_match(struct ndpi_detection_module_struct *ndpi_str,
+					     struct in_addr *pin /* network byte order */) {
+  ndpi_prefix_t prefix;
+  ndpi_patricia_node_t *node;
+
+  /* Make sure all in network byte order otherwise compares wont work */
+  ndpi_fill_prefix_v4(&prefix, pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->ip_risk_ptree)->maxbits);
+  node = ndpi_patricia_search_best(ndpi_str->ip_risk_ptree, &prefix);
+
+  if(node)
+    return((ndpi_risk_enum)node->value.u.uv32.user_value);
+
+  return(NDPI_NO_RISK);
+}
+
+/* ******************************************* */
+
 #if 0
 static u_int8_t tor_ptree_match(struct ndpi_detection_module_struct *ndpi_str, struct in_addr *pin) {
   return((ndpi_network_ptree_match(ndpi_str, pin) == NDPI_PROTOCOL_TOR) ? 1 : 0);
@@ -2151,6 +2170,10 @@ static void ndpi_init_ptree_ipv4(struct ndpi_detection_module_struct *ndpi_str,
 
     pin.s_addr = htonl(host_list[i].network);
     if((node = add_to_ptree(ptree, AF_INET, &pin, host_list[i].cidr /* bits */)) != NULL) {
+      /* Two main cases:
+         1) ip -> protocol: uv32.user_value = protocol; uv32.additional_user_value = 0;
+         2) ip -> risk: uv32.user_value = risk; uv32.additional_user_value = 0;
+      */
       node->value.u.uv32.user_value = host_list[i].value, node->value.u.uv32.additional_user_value = 0;
     }
   }
@@ -2447,6 +2470,14 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
 
   ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */);
 
+  if(!(prefs & ndpi_dont_init_risk_ptree)) {
+    if((ndpi_str->ip_risk_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL) {
+      if(!(prefs & ndpi_dont_load_icloud_private_relay_list)) {
+        ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_anonymous_subscriber_protocol_list);
+      }
+    }
+  }
+
   NDPI_BITMASK_RESET(ndpi_str->detection_bitmask);
 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
   ndpi_str->user_data = NULL;
@@ -2857,6 +2888,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
     if(ndpi_str->ip_risk_mask_ptree)
       ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->ip_risk_mask_ptree, free_ptree_data);
 
+    if(ndpi_str->ip_risk_ptree)
+      ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->ip_risk_ptree, free_ptree_data);
+
     if(ndpi_str->udpRoot != NULL)
       ndpi_tdestroy(ndpi_str->udpRoot, ndpi_free);
     if(ndpi_str->tcpRoot != NULL)
@@ -5899,6 +5933,27 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 
     flow->risk_checked = 1;
   }
+  if(!flow->tree_risk_checked) {
+    if(ndpi_str->ip_risk_ptree) {
+      /* TODO: ipv6 */
+      if(packet->iph &&
+         ndpi_is_public_ipv4(ntohl(packet->iph->saddr)) &&
+         ndpi_is_public_ipv4(ntohl(packet->iph->daddr))) {
+        struct in_addr addr;
+        ndpi_risk_enum net_risk;
+
+        addr.s_addr = packet->iph->saddr;
+        net_risk = ndpi_network_risk_ptree_match(ndpi_str, &addr);
+        if(net_risk == NDPI_NO_RISK) {
+          addr.s_addr = packet->iph->daddr;
+          net_risk = ndpi_network_risk_ptree_match(ndpi_str, &addr);
+        }
+        if(net_risk != NDPI_NO_RISK)
+          ndpi_set_risk(ndpi_str, flow, net_risk);
+      }
+    }
+    flow->tree_risk_checked = 1;
+  }
 
   ndpi_reconcile_protocols(ndpi_str, flow, &ret);
 

src/lib/ndpi_utils.c

@@ -1895,11 +1895,15 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
   case NDPI_ERROR_CODE_DETECTED:
     return("Error Code Detected");
     break;
-      
+
   case NDPI_HTTP_CRAWLER_BOT:
     return("Crawler/Bot Detected");
     break;
-
+
+  case NDPI_ANONYMOUS_SUBSCRIBER:
+    return("Anonymous Subscriber");
+    break;
+
   default:
     snprintf(buf, sizeof(buf), "%d", (int)risk);
     return(buf);

utils/icloud_private_relay_ip_addresses_download.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+cd "$(dirname "${0}")" || return
+
+DEST=../src/lib/ndpi_icloud_private_relay_match.c.inc
+TMP=/tmp/icloud.csv
+LIST=/tmp/icloud.list
+LIST_MERGED=/tmp/icloud.list_m
+ORIGIN="https://mask-api.icloud.com/egress-ip-ranges.csv"
+
+
+echo "(1) Downloading file..."
+http_response=$(curl -s -o "$TMP" -w "%{http_code}" ${ORIGIN})
+if [ "$http_response" != "200" ]; then
+    echo "Error $http_response: you probably need to update the list url!"
+    return
+fi
+
+echo "(2) Processing IP addresses..."
+
+# Note: the "grep -v :" is used to skip IPv6 addresses
+cut -d ',' -f 1 $TMP | grep -v ':' > $LIST
+./mergeipaddrlist.py $LIST > $LIST_MERGED
+./ipaddr2list.py $LIST_MERGED NDPI_ANONYMOUS_SUBSCRIBER > $DEST
+rm -f $TMP $LIST $LIST_MERGED
+
+echo "(3) iCloud Private Relay IPs are available in $DEST"
+
+
+

utils/ipaddr2list.py

@@ -3,6 +3,8 @@
 import sys
 import socket, struct
 
+# This scripts is mainly used to create "ip -> protocols" lists.
+# However it is also used to create "ip -> risk" lists
 proto = "NDPI_PROTOCOL_XYX"
 if len (sys.argv) < 2 :
     print("Usage: ipaddr2list.py <file> <protocol>")

wireshark/ndpi.lua

@@ -83,9 +83,10 @@ flow_risks[41] = ProtoField.bool("ndpi.flow_risk.cert_about_to_expire", "TLS cer
 flow_risks[42] = ProtoField.bool("ndpi.flow_risk.punycode_idn", "IDN Domain Name", num_bits_flow_risks, nil, bit(10), "nDPI Flow Risk: IDN Domain Name")
 flow_risks[43] = ProtoField.bool("ndpi.flow_risk.error_code_detected", "Error Code Detected", num_bits_flow_risks, nil, bit(11), "nDPI Flow Risk: Error Code Detected")
 flow_risks[44] = ProtoField.bool("ndpi.flow_risk.crawler_bot", "Crawler/Bot Detected", num_bits_flow_risks, nil, bit(12), "nDPI Flow Risk: Crawler/Bot Detected")
+flow_risks[45] = ProtoField.bool("ndpi.flow_risk.anonymous_subscriber", "Anonymous Subscriber", num_bits_flow_risks, nil, bit(13), "nDPI Flow Risk: Anonymous Subscriber")
 
 -- Last one: keep in sync the bitmask when adding new risks!!
-flow_risks[64] = ProtoField.new("Unused", "ndpi.flow_risk.unused", ftypes.UINT32, nil, base.HEX, bit(32) - bit(10))
+flow_risks[64] = ProtoField.new("Unused", "ndpi.flow_risk.unused", ftypes.UINT32, nil, base.HEX, bit(32) - bit(13))
 
 for _,v in pairs(flow_risks) do
   ndpi_fds[#ndpi_fds + 1] = v