feat: Implement Tox network profiler

.gitignore

@@ -36,6 +36,7 @@ testing/data
 
 # Vim
 *.swp
+*.nvimlog
 
 # Object files
 *.o

CMakeLists.txt

@@ -306,6 +306,8 @@ set(toxcore_SOURCES
   toxcore/mem.h
   toxcore/mono_time.c
   toxcore/mono_time.h
+  toxcore/net_profile.c
+  toxcore/net_profile.h
   toxcore/net_crypto.c
   toxcore/net_crypto.h
   toxcore/network.c

auto_tests/CMakeLists.txt

@@ -67,6 +67,7 @@ auto_test(invalid_udp_proxy)
 auto_test(lan_discovery)
 auto_test(lossless_packet)
 auto_test(lossy_packet)
+auto_test(netprof)
 auto_test(network)
 auto_test(onion)
 auto_test(overflow_recvq)

auto_tests/TCP_test.c

@@ -111,12 +111,12 @@ static void test_basic(void)
 
     // Sending the handshake
     ck_assert_msg(net_send(ns, logger, sock, handshake, TCP_CLIENT_HANDSHAKE_SIZE - 1,
-                           &localhost) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
+                           &localhost, nullptr) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
                   "An attempt to send the initial handshake minus last byte failed.");
 
     do_tcp_server_delay(tcp_s, mono_time, 50);
 
-    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost) == 1,
+    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost, nullptr) == 1,
                   "The attempt to send the last byte of handshake failed.");
 
     free(handshake);
@@ -155,7 +155,7 @@ static void test_basic(void)
             msg_length = sizeof(r_req) - i;
         }
 
-        ck_assert_msg(net_send(ns, logger, sock, r_req + i, msg_length, &localhost) == msg_length,
+        ck_assert_msg(net_send(ns, logger, sock, r_req + i, msg_length, &localhost, nullptr) == msg_length,
                       "Failed to send request after completing the handshake.");
         i += msg_length;
 
@@ -234,12 +234,12 @@ static struct sec_TCP_con *new_tcp_con(const Logger *logger, const Memory *mem,
                   "Failed to encrypt the outgoing handshake.");
 
     ck_assert_msg(net_send(ns, logger, sock, handshake, TCP_CLIENT_HANDSHAKE_SIZE - 1,
-                           &localhost) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
+                           &localhost, nullptr) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
                   "Failed to send the first portion of the handshake to the TCP relay server.");
 
     do_tcp_server_delay(tcp_s, mono_time, 50);
 
-    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost) == 1,
+    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost, nullptr) == 1,
                   "Failed to send last byte of handshake.");
 
     do_tcp_server_delay(tcp_s, mono_time, 50);
@@ -283,7 +283,7 @@ static int write_packet_tcp_test_connection(const Logger *logger, struct sec_TCP
     localhost.ip = get_loopback();
     localhost.port = 0;
 
-    ck_assert_msg(net_send(con->ns, logger, con->sock, packet, packet_size, &localhost) == packet_size,
+    ck_assert_msg(net_send(con->ns, logger, con->sock, packet, packet_size, &localhost, nullptr) == packet_size,
                   "Failed to send a packet.");
     return 0;
 }
@@ -524,7 +524,7 @@ static void test_client(void)
     ip_port_tcp_s.port = net_htons(ports[random_u32(rng) % NUM_PORTS]);
     ip_port_tcp_s.ip = get_loopback();
 
-    TCP_Client_Connection *conn = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s, self_public_key, f_public_key, f_secret_key, nullptr);
+    TCP_Client_Connection *conn = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s, self_public_key, f_public_key, f_secret_key, nullptr, nullptr);
     // TCP sockets might need a moment before they can be written to.
     c_sleep(50);
     do_tcp_connection(logger, mono_time, conn, nullptr);
@@ -560,7 +560,7 @@ static void test_client(void)
     crypto_new_keypair(rng, f2_public_key, f2_secret_key);
     ip_port_tcp_s.port = net_htons(ports[random_u32(rng) % NUM_PORTS]);
     TCP_Client_Connection *conn2 = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s, self_public_key, f2_public_key,
-                                   f2_secret_key, nullptr);
+                                   f2_secret_key, nullptr, nullptr);
     c_sleep(50);
 
     // The client should call this function (defined earlier) during the routing process.
@@ -657,7 +657,7 @@ static void test_client_invalid(void)
     ip_port_tcp_s.port = net_htons(ports[random_u32(rng) % NUM_PORTS]);
     ip_port_tcp_s.ip = get_loopback();
     TCP_Client_Connection *conn = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s,
-                                  self_public_key, f_public_key, f_secret_key, nullptr);
+                                  self_public_key, f_public_key, f_secret_key, nullptr, nullptr);
 
     // Run the client's main loop but not the server.
     mono_time_update(mono_time);

auto_tests/netprof_test.c

@@ -0,0 +1,120 @@
+/** Auto Tests: basic network profile functionality test (UDP only)
+ * TODO(JFreegman): test TCP packets as well
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include "../toxcore/tox_private.h"
+#include "../toxcore/util.h"
+
+#include "auto_test_support.h"
+#include "check_compat.h"
+
+#define NUM_TOXES 2
+
+static void test_netprof(AutoTox *autotoxes)
+{
+    // Send some messages to create fake traffic
+    for (size_t i = 0; i < 256; ++i) {
+        for (uint32_t j = 0; j < NUM_TOXES; ++j) {
+            tox_friend_send_message(autotoxes[j].tox, 0, TOX_MESSAGE_TYPE_NORMAL, (const uint8_t *)"test", 4, nullptr);
+        }
+
+        iterate_all_wait(autotoxes, NUM_TOXES, ITERATION_INTERVAL);
+    }
+
+    // idle traffic for a while
+    for (size_t i = 0; i < 100; ++i) {
+        iterate_all_wait(autotoxes, NUM_TOXES, ITERATION_INTERVAL);
+    }
+
+    const Tox *tox1 = autotoxes[0].tox;
+
+    const unsigned long long UDP_count_sent1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+            TOX_NETPROF_DIRECTION_SENT);
+    const unsigned long long UDP_count_recv1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+            TOX_NETPROF_DIRECTION_RECV);
+    const unsigned long long TCP_count_sent1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+            TOX_NETPROF_DIRECTION_SENT);
+    const unsigned long long TCP_count_recv1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+            TOX_NETPROF_DIRECTION_RECV);
+
+    const unsigned long long UDP_bytes_sent1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+            TOX_NETPROF_DIRECTION_SENT);
+    const unsigned long long UDP_bytes_recv1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+            TOX_NETPROF_DIRECTION_RECV);
+    const unsigned long long TCP_bytes_sent1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+            TOX_NETPROF_DIRECTION_SENT);
+    const unsigned long long TCP_bytes_recv1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+            TOX_NETPROF_DIRECTION_RECV);
+
+    ck_assert(UDP_count_recv1 > 0 && UDP_count_sent1 > 0);
+    ck_assert(UDP_bytes_recv1 > 0 && UDP_bytes_sent1 > 0);
+
+    (void)TCP_count_sent1;
+    (void)TCP_bytes_sent1;
+    (void)TCP_bytes_recv1;
+    (void)TCP_count_recv1;
+
+    unsigned long long total_sent_count = 0;
+    unsigned long long total_recv_count = 0;
+    unsigned long long total_sent_bytes = 0;
+    unsigned long long total_recv_bytes = 0;
+
+    // tox1 makes sure the sum value of all packet ID's is equal to the totals
+    for (size_t i = 0; i < 256; ++i) {
+        // this id isn't valid for UDP packets but we still want to call the
+        // functions and make sure they return some non-zero value
+        if (i == TOX_NETPROF_PACKET_ID_TCP_DATA) {
+            ck_assert(tox_netprof_get_packet_id_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_SENT) > 0);
+            ck_assert(tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_SENT) > 0);
+            ck_assert(tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_SENT) > 0);
+            ck_assert(tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_RECV) > 0);
+            continue;
+        }
+
+        total_sent_count += tox_netprof_get_packet_id_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_SENT);
+        total_recv_count += tox_netprof_get_packet_id_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_RECV);
+
+        total_sent_bytes += tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_SENT);
+        total_recv_bytes += tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_RECV);
+    }
+
+    const unsigned long long total_packets = total_sent_count + total_recv_count;
+    ck_assert_msg(total_packets == UDP_count_sent1 + UDP_count_recv1,
+                  "%llu does not match %llu\n", total_packets, UDP_count_sent1 + UDP_count_recv1);
+
+    ck_assert_msg(total_sent_count == UDP_count_sent1, "%llu does not match %llu\n", total_sent_count, UDP_count_sent1);
+    ck_assert_msg(total_recv_count == UDP_count_recv1, "%llu does not match %llu\n", total_recv_count, UDP_count_recv1);
+
+
+    const unsigned long long total_bytes = total_sent_bytes + total_recv_bytes;
+    ck_assert_msg(total_bytes == UDP_bytes_sent1 + UDP_bytes_recv1,
+                  "%llu does not match %llu\n", total_bytes, UDP_bytes_sent1 + UDP_bytes_recv1);
+
+    ck_assert_msg(total_sent_bytes == UDP_bytes_sent1, "%llu does not match %llu\n", total_sent_bytes, UDP_bytes_sent1);
+    ck_assert_msg(total_recv_bytes == UDP_bytes_recv1, "%llu does not match %llu\n", total_recv_bytes, UDP_bytes_recv1);
+}
+
+int main(void)
+{
+    setvbuf(stdout, nullptr, _IONBF, 0);
+
+    Run_Auto_Options autotox_opts = default_run_auto_options();
+    autotox_opts.graph = GRAPH_COMPLETE;
+
+    run_auto_test(nullptr, NUM_TOXES, test_netprof, 0, &autotox_opts);
+
+    return 0;
+}
+
+#undef NUM_TOXES

auto_tests/onion_test.c

@@ -202,7 +202,7 @@ static int handle_test_4(void *object, const IP_Port *source, const uint8_t *pac
  * Use Onion_Path path to send data of length to dest.
  * Maximum length of data is ONION_MAX_DATA_SIZE.
  */
-static void send_onion_packet(const Networking_Core *net, const Random *rng, const Onion_Path *path, const IP_Port *dest, const uint8_t *data, uint16_t length)
+static void send_onion_packet(Networking_Core *net, const Random *rng, const Onion_Path *path, const IP_Port *dest, const uint8_t *data, uint16_t length)
 {
     uint8_t packet[ONION_MAX_PACKET_SIZE];
     const int len = create_onion_packet(rng, packet, sizeof(packet), path, dest, data, length);

other/bootstrap_node_packets.c

@@ -30,7 +30,7 @@ static int handle_info_request(void *object, const IP_Port *source, const uint8_
         return 1;
     }
 
-    const Networking_Core *nc = (const Networking_Core *)object;
+    Networking_Core *nc = (Networking_Core *)object;
 
     uint8_t data[1 + sizeof(bootstrap_version) + MAX_MOTD_LENGTH];
     data[0] = BOOTSTRAP_INFO_PACKET_ID;

toxcore/BUILD.bazel

@@ -299,6 +299,16 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "net_profile",
+    srcs = ["net_profile.c"],
+    hdrs = ["net_profile.h"],
+    deps = [
+        ":attributes",
+        ":ccompat",
+    ],
+)
+
 cc_library(
     name = "network",
     srcs = ["network.c"],
@@ -318,6 +328,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":util",
         "@libsodium",
         "@psocket",
@@ -570,6 +581,7 @@ cc_library(
         ":crypto_core",
         ":logger",
         ":mem",
+        ":net_profile",
         ":network",
     ],
 )
@@ -597,6 +609,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":onion",
         ":util",
@@ -618,6 +631,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":util",
     ],
@@ -640,6 +654,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":onion",
         ":util",
@@ -674,6 +689,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":util",
         "@pthread",
@@ -993,6 +1009,7 @@ cc_library(
         ":DHT",
         ":Messenger",
         ":TCP_client",
+        ":TCP_server",
         ":attributes",
         ":ccompat",
         ":crypto_core",
@@ -1003,6 +1020,7 @@ cc_library(
         ":mem",
         ":mono_time",
         ":net_crypto",
+        ":net_profile",
         ":network",
         ":onion_client",
         ":state",

toxcore/LAN_discovery.c

@@ -210,7 +210,7 @@ static Broadcast_Info *fetch_broadcast_info(const Network *ns)
  * @retval false on failure to find any valid broadcast target.
  */
 non_null()
-static bool send_broadcasts(const Networking_Core *net, const Broadcast_Info *broadcast, uint16_t port,
+static bool send_broadcasts(Networking_Core *net, const Broadcast_Info *broadcast, uint16_t port,
                             const uint8_t *data, uint16_t length)
 {
     if (broadcast->count == 0) {
@@ -339,7 +339,7 @@ bool ip_is_lan(const IP *ip)
     return false;
 }
 
-bool lan_discovery_send(const Networking_Core *net, const Broadcast_Info *broadcast, const uint8_t *dht_pk,
+bool lan_discovery_send(Networking_Core *net, const Broadcast_Info *broadcast, const uint8_t *dht_pk,
                         uint16_t port)
 {
     if (broadcast == nullptr) {

toxcore/LAN_discovery.h

@@ -25,7 +25,7 @@ typedef struct Broadcast_Info Broadcast_Info;
  * @return true on success, false on failure.
  */
 non_null()
-bool lan_discovery_send(const Networking_Core *net, const Broadcast_Info *broadcast, const uint8_t *dht_pk,
+bool lan_discovery_send(Networking_Core *net, const Broadcast_Info *broadcast, const uint8_t *dht_pk,
                         uint16_t port);
 
 /**

toxcore/Makefile.inc

@@ -74,6 +74,8 @@ libtoxcore_la_SOURCES = ../third_party/cmp/cmp.c \
                         ../toxcore/ping_array.c \
                         ../toxcore/net_crypto.h \
                         ../toxcore/net_crypto.c \
+                        ../toxcore/net_profile.c \
+                        ../toxcore/net_profile.h \
                         ../toxcore/friend_requests.h \
                         ../toxcore/friend_requests.c \
                         ../toxcore/LAN_discovery.h \

toxcore/TCP_client.c

@@ -20,6 +20,7 @@
 #include "logger.h"
 #include "mem.h"
 #include "mono_time.h"
+#include "net_profile.h"
 #include "network.h"
 #include "util.h"
 
@@ -582,7 +583,7 @@ void forwarding_handler(TCP_Client_Connection *con, forwarded_response_cb *forwa
 TCP_Client_Connection *new_tcp_connection(
     const Logger *logger, const Memory *mem, const Mono_Time *mono_time, const Random *rng, const Network *ns,
     const IP_Port *ip_port, const uint8_t *public_key, const uint8_t *self_public_key, const uint8_t *self_secret_key,
-    const TCP_Proxy_Info *proxy_info)
+    const TCP_Proxy_Info *proxy_info, Net_Profile *net_profile)
 {
     assert(logger != nullptr);
     assert(mem != nullptr);
@@ -634,6 +635,7 @@ TCP_Client_Connection *new_tcp_connection(
     temp->con.rng = rng;
     temp->con.sock = sock;
     temp->con.ip_port = *ip_port;
+    temp->con.net_profile = net_profile;
     memcpy(temp->public_key, public_key, CRYPTO_PUBLIC_KEY_SIZE);
     memcpy(temp->self_public_key, self_public_key, CRYPTO_PUBLIC_KEY_SIZE);
     encrypt_precompute(temp->public_key, self_secret_key, temp->con.shared_key);
@@ -819,6 +821,8 @@ static int handle_tcp_client_packet(const Logger *logger, TCP_Client_Connection
         return -1;
     }
 
+    netprof_record_packet(conn->con.net_profile, data[0], length, PACKET_DIRECTION_RECV);
+
     switch (data[0]) {
         case TCP_PACKET_ROUTING_RESPONSE:
             return handle_tcp_client_routing_response(conn, data, length);