Errors fixed

.github/workflows/build.yml

@@ -305,10 +305,10 @@ jobs:
             "uname -a &&
             lscpu | grep Endian
             "
-      - name: Configure and compile (no tests) using qemu for the specified architecture (s390x - big endian)
+      - name: Configure and compile using qemu for the specified architecture (s390x - big endian)
         if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x')
         uses: docker://multiarch/ubuntu-core:s390x-bionic
-        with: #./tests/do.sh disabled because we know we have some problems with big-endian machines
+        with:
           args: >
             bash -c
             "apt-get -y update &&
@@ -318,5 +318,6 @@ jobs:
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
             make -C python &&
+            ./tests/do.sh &&
             ./tests/do-unit.sh
             "

example/reader_util.c

@@ -544,6 +544,13 @@ void ndpi_workflow_free(struct ndpi_workflow * workflow) {
   ndpi_free(workflow);
 }
 
+static inline int cmp_n32(uint32_t a,uint32_t b) {
+	return a == b ? 0 : ntohl(a) < ntohl(b) ? -1:1;
+}
+static inline int cmp_n16(uint16_t a,uint16_t b) {
+	return a == b ? 0 : ntohs(a) < ntohs(b) ? -1:1;
+}
+
 /* ***************************************************** */
 
 int ndpi_workflow_node_cmp(const void *a, const void *b) {
@@ -557,29 +564,13 @@ int ndpi_workflow_node_cmp(const void *a, const void *b) {
   if(fa->vlan_id   < fb->vlan_id   ) return(-1); else { if(fa->vlan_id    > fb->vlan_id   ) return(1); }
   if(fa->protocol  < fb->protocol  ) return(-1); else { if(fa->protocol   > fb->protocol  ) return(1); }
 
-  if(
-     (
-      (fa->src_ip      == fb->src_ip  )
-      && (fa->src_port == fb->src_port)
-      && (fa->dst_ip   == fb->dst_ip  )
-      && (fa->dst_port == fb->dst_port)
-      )
-     ||
-     (
-      (fa->src_ip      == fb->dst_ip  )
-      && (fa->src_port == fb->dst_port)
-      && (fa->dst_ip   == fb->src_ip  )
-      && (fa->dst_port == fb->src_port)
-      )
-     )
-    return(0);
-
-  if(fa->src_ip   < fb->src_ip  ) return(-1); else { if(fa->src_ip   > fb->src_ip  ) return(1); }
-  if(fa->src_port < fb->src_port) return(-1); else { if(fa->src_port > fb->src_port) return(1); }
-  if(fa->dst_ip   < fb->dst_ip  ) return(-1); else { if(fa->dst_ip   > fb->dst_ip  ) return(1); }
-  if(fa->dst_port < fb->dst_port) return(-1); else { if(fa->dst_port > fb->dst_port) return(1); }
+  int r;
+  r = cmp_n32(fa->src_ip, fb->src_ip); if(r) return r;
+  r = cmp_n16(fa->src_port, fb->src_port) ; if(r) return r;
+  r = cmp_n32(fa->dst_ip, fb->dst_ip); if(r) return r;
+  r = cmp_n16(fa->dst_port, fb->dst_port);
 
-  return(0); /* notreached */
+  return(r);
 }
 
 /* ***************************************************** */
@@ -789,11 +780,17 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
   flow.protocol = iph->protocol, flow.vlan_id = vlan_id;
   flow.src_ip = iph->saddr, flow.dst_ip = iph->daddr;
   flow.src_port = htons(*sport), flow.dst_port = htons(*dport);
-  flow.hashval = hashval = flow.protocol + flow.src_ip + flow.dst_ip + flow.src_port + flow.dst_port;
+  flow.hashval = hashval = flow.protocol + ntohl(flow.src_ip) + ntohl(flow.dst_ip) 
+	  + ntohs(flow.src_port) + ntohs(flow.dst_port);
 
 #if 0
-  printf("hashval=%u [%u][%u][%u:%u][%u:%u]\n", hashval, flow.protocol, flow.vlan_id,
-	 flow.src_ip, flow.src_port, ntohs(flow.dst_ip), ntohs(flow.dst_port));
+  {
+  char ip1[48],ip2[48];
+       inet_ntop(AF_INET, &flow.src_ip, ip1, sizeof(ip1));
+       inet_ntop(AF_INET, &flow.dst_ip, ip2, sizeof(ip2));
+  printf("hashval=%u [%u][%u][%s:%u][%s:%u]\n", hashval, flow.protocol, flow.vlan_id,
+        ip1, ntohs(flow.src_port),  ip2, ntohs(flow.dst_port));
+  }
 #endif
 
   idx = hashval % workflow->prefs.num_roots;
@@ -905,24 +902,10 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
     struct ndpi_flow_info *rflow = *(struct ndpi_flow_info**)ret;
 
     if(is_changed) {
-      if(rflow->src_ip == iph->saddr
-	 && rflow->dst_ip == iph->daddr
-	 && rflow->src_port == htons(*sport)
-	 && rflow->dst_port == htons(*dport)
-	 )
-	*src_to_dst_direction = 0, rflow->bidirectional = 1;
-      else
-	*src_to_dst_direction = 1;
+	*src_to_dst_direction = 0, rflow->bidirectional |= 1;
     }
     else {
-      if(rflow->src_ip == iph->saddr
-	 && rflow->dst_ip == iph->daddr
-	 && rflow->src_port == htons(*sport)
-	 && rflow->dst_port == htons(*dport)
-	 )
 	*src_to_dst_direction = 1;
-      else
-	*src_to_dst_direction = 0, rflow->bidirectional = 1;
     }
     if(enable_flow_stats) {
       if(src_to_dst_direction) {

src/lib/ndpi_main.c

@@ -3064,9 +3064,8 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
 	      ndpi_set_risk(ndpi_str, flow, NDPI_SUSPICIOUS_ENTROPY);
 	    }
 
-	    struct ndpi_icmphdr * const icmphdr = (struct ndpi_icmphdr *)packet->payload;
 	    u_int16_t chksm = ndpi_calculate_icmp4_checksum(packet->payload, packet->payload_packet_len);
-	    if (icmphdr->checksum != chksm) {
+	    if (chksm) {
 	      ndpi_set_risk(ndpi_str, flow, NDPI_MALFORMED_PACKET);
 	    }
 	  }

src/lib/ndpi_utils.c

@@ -2300,24 +2300,26 @@ float ndpi_entropy(u_int8_t const * const buf, size_t len) {
 }
 
 /* ******************************************************************** */
+static inline uint16_t get_n16bit(uint8_t const * cbuf) {
+  uint16_t r = ((uint16_t)cbuf[0]) | (((uint16_t)cbuf[1]) << 8);
+  return r;
+}
 
-u_int16_t ndpi_calculate_icmp4_checksum(u_int8_t const * const buf, size_t len) {
-  u_int16_t const * sbuf = (u_int16_t *)buf;
+u_int16_t ndpi_calculate_icmp4_checksum(const u_int8_t * buf, size_t len) {
   u_int32_t checksum = 0;
 
   /*
    * The first two bytes of the icmp header are required.
    * The next two bytes is the checksum, which we want to ignore.
    */
-  checksum += *sbuf++; len -= 2; /* icmp->type, icmp->code */
-  sbuf++; len -= 2; /* icmp->checksum */
 
   for (; len > 1; len -= 2) {
-    checksum += *sbuf++;
+    checksum += get_n16bit(buf);
+    buf += 2;
   }
 
   if (len == 1) {
-    checksum += *(u_int8_t *)sbuf;
+    checksum += *buf;
   }
 
   checksum = (checksum >> 16) + (checksum & 0xFFFF);

tests/result/anydesk-2.pcap.out

@@ -8,8 +8,8 @@ AnyDesk	2083	346113	4
 
 JA3 Host Stats: 
 		 IP Address                  	 # JA3C     
-	1	 192.168.1.187            	 1      
-	2	 192.168.1.178            	 1      
+	1	 192.168.1.178            	 1      
+	2	 192.168.1.187            	 1      
 
 
 	1	TCP 192.168.1.187:54164 <-> 192.168.1.178:7070 [proto: 91.252/TLS.AnyDesk][Encrypted][Confidence: DPI][cat: RemoteAccess/12][509 pkts/226247 bytes <-> 1555 pkts/115282 bytes][Goodput ratio: 88/22][22.84 sec][bytes ratio: 0.325 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 48/14 2966/3021 229/106][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 444/74 1511/1514 475/47][Risk: ** Known Protocol on Non Standard Port **** TLS (probably) Not Carrying HTTPS **** Missing SNI TLS Extension **** Desktop/File Sharing Session **][Risk Score: 120][TLSv1.2][JA3C: 3f2fba0262b1a22b739126dfb2fe7a7d][JA3S: ee644a8a34c434abca4b737ec1d9efad][Subject: CN=AnyDesk Client, CN=AnyDesk Client][Certificate SHA-1: F8:4E:27:4E:F9:33:35:2F:1A:69:71:D5:02:6B:B8:72:EF:B7:BA:B0][Firefox][Cipher: TLS_DHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,64,6,1,3,1,1,1,0,1,1,0,0,1,1,0,3,0,0,0,0,0,3,1,0,1,1,0,1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,1,0,1,0,0]

tests/result/instagram.pcap.out

@@ -18,8 +18,8 @@ Instagram	3062	2617399	22
 
 JA3 Host Stats: 
 		 IP Address                  	 # JA3C     
-	1	 192.168.2.17             	 2      
-	2	 192.168.0.103            	 1      
+	1	 192.168.0.103            	 1      
+	2	 192.168.2.17             	 2      
 
 
 	1	TCP 192.168.2.17:49355 <-> 31.13.86.52:443 [proto: 91.211/TLS.Instagram][Encrypted][Confidence: DPI][cat: SocialNetwork/6][456 pkts/33086 bytes <-> 910 pkts/1277296 bytes][Goodput ratio: 9/95][14.29 sec][Hostname/SNI: scontent-mxp1-1.cdninstagram.com][ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.3 (Fizz)][bytes ratio: -0.950 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 38/1 10107/274 547/12][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 73/1404 657/1454 57/231][Risk: ** Possibly Malicious JA3 Fingerprint **][Risk Score: 50][TLSv1.3 (Fizz)][JA3C: 7a29c223fb122ec64d10f0a159e07996][JA3S: f4febc55ea12b31ae17cfb7e614afda8][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,98,0,0,0,0]

tests/result/quic_frags_ch_out_of_order_same_packet_craziness.pcapng.out

@@ -11,10 +11,10 @@ GoogleCloud	3	4176	3
 
 JA3 Host Stats: 
 		 IP Address                  	 # JA3C     
-	1	 147.196.90.42            	 1      
-	2	 168.144.64.5             	 1      
-	3	 52.187.20.175            	 1      
-	4	 159.117.176.124          	 1      
+	1	 52.187.20.175            	 1      
+	2	 159.117.176.124          	 1      
+	3	 168.144.64.5             	 1      
+	4	 147.196.90.42            	 1      
 
 
 	1	UDP 52.187.20.175:49880 -> 208.229.157.81:443 [proto: 188.276/QUIC.Azure][Encrypted][Confidence: DPI][cat: Cloud/13][4 pkts/5568 bytes -> 0 pkts/0 bytes][Goodput ratio: 97/0][2.12 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0]

tests/result/quic_interop_V.pcapng.out

@@ -12,8 +12,8 @@ Azure	36	39266	6
 
 JA3 Host Stats: 
 		 IP Address                  	 # JA3C     
-	1	 2001:b07:ac9:d5ae:a4d3:fe47:691e:807d 	 1      
-	2	 192.168.1.128            	 1      
+	1	 192.168.1.128            	 1      
+	2	 2001:b07:ac9:d5ae:a4d3:fe47:691e:807d 	 1      
 
 
 	1	UDP 192.168.1.128:34511 -> 131.159.24.198:443 [proto: 188/QUIC][Encrypted][Confidence: DPI][cat: Web/5][8 pkts/10352 bytes -> 0 pkts/0 bytes][Goodput ratio: 97/0][9.94 sec][Hostname/SNI: pandora.cm.in.tum.de][ALPN: hq-30;h3-30;hq-29;h3-29;hq-28;h3-28;hq-27;h3-27][TLS Supported Versions: TLSv1.3][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 150/0 1419/0 4800/0 1551/0][Pkt Len c2s/s2c min/avg/max/stddev: 1294/0 1294/0 1294/0 0/0][TLSv1.3][JA3C: 7d9e7f6dec1cb1dd8b79d72b1366b6cf][Firefox][PLAIN TEXT (SezYZO)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0]

tests/result/synscan.pcap.out

@@ -59,7 +59,7 @@ TargusDataspeed	2	116	2
 DNP3	2	116	2
 iSCSI	2	116	2
 
-	1	TCP 172.16.0.8:36050 -> 64.13.134.52:22 [proto: 92/SSH][Encrypted][Confidence: Match by port][cat: RemoteAccess/12][5 pkts/298 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][21.68 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	1	TCP 172.16.0.8:36050 <-> 64.13.134.52:22 [proto: 92/SSH][Encrypted][Confidence: Match by port][cat: RemoteAccess/12][1 pkts/58 bytes <-> 4 pkts/240 bytes][Goodput ratio: 0/0][21.68 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	2	TCP 172.16.0.8:36050 <-> 64.13.134.52:53 [proto: 5/DNS][ClearText][Confidence: Match by port][cat: Network/14][1 pkts/58 bytes <-> 4 pkts/240 bytes][Goodput ratio: 0/0][21.09 sec][::][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	3	TCP 172.16.0.8:36050 <-> 64.13.134.52:80 [proto: 7/HTTP][ClearText][Confidence: Match by port][cat: Web/5][1 pkts/58 bytes <-> 4 pkts/240 bytes][Goodput ratio: 0/0][21.27 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	4	TCP 172.16.0.8:36050 <-> 64.13.134.52:25 [proto: 3/SMTP][ClearText][Confidence: Match by port][cat: Email/3][1 pkts/58 bytes <-> 1 pkts/60 bytes][Goodput ratio: 0/0][0.06 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]