[APM] Initial setup for trace stats test (#2712)

Co-authored-by: Iñigo López de Heredia <[email protected]>
Co-authored-by: Iñigo Lopez de Heredia <[email protected]>
Co-authored-by: Charles de Beauchesne <[email protected]>

README.md

@@ -1,6 +1,6 @@
 ## System tests
 
-Workbench designed to run advanced tests (integration, smoke, functionnal, fuzzing and performance)
+Workbench designed to run advanced tests (integration, smoke, functional, fuzzing and performance)
 
 ## Requirements
 

conftest.py

@@ -95,7 +95,7 @@ def pytest_configure(config):
             break
 
     if context.scenario is None:
-        pytest.exit(f"Scenario {config.option.scenario} does not exists", 1)
+        pytest.exit(f"Scenario {config.option.scenario} does not exist", 1)
 
     context.scenario.pytest_configure(config)
 
@@ -302,7 +302,7 @@ def pytest_collection_finish(session: pytest.Session):
         if not item.instance:  # item is a method bounded to a class
             continue
 
-        # the test metohd name is like test_xxxx
+        # the test method name is like test_xxxx
         # we replace the test_ by setup_, and call it if it exists
 
         setup_method_name = f"setup_{item.name[5:]}"

docs/weblog/README.md

@@ -6,7 +6,7 @@ A weblog is a web app that system uses to test the library. It mimics what would
 
 ## Disclaimer
 
-This document describes endpoints implemented on weblog. Though, it's not a complete description, and can contains mistakes. The source of truth are the test itself. If a weblog endpoint passes system tests, then you can consider it as ok. And if it does not passes it, then you must correct it, even if it's in line with this document.
+This document describes endpoints implemented on weblog. Though, it's not a complete description, and can contain mistakes. The source of truth are the test itself. If a weblog endpoint passes system tests, then you can consider it as ok. And if it does not passes it, then you must correct it, even if it's in line with this document.
 
 **You are strongly encouraged to help others by submitting corrections when you notice issues with this document.**
 
@@ -638,6 +638,11 @@ distributed tracing propagation headers.
 ### \[GET,POST\] /returnheaders
 This endpoint returns the headers received in order to be able to assert about distributed tracing propagation headers
 
+### \[GET\] /stats-unique
+The endpoint must accept a query string parameter `code`, which should be an integer. This parameter will be the status code of the response message, default to 200 OK.
+This endpoint is used for client-stats tests to provide a separate "resource" via the endpoint path `stats-unique` to disambiguate those tests from other
+stats generating tests.
+
 ### GET /healthcheck
 
 Returns a JSON dict, with those values :

manifests/cpp.yml

@@ -168,6 +168,8 @@ tests/:
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature
+    test_stats.py:
+      Test_Client_Stats: missing_feature
   test_config_consistency.py:
     Test_Config_ClientTagQueryString_Configured: missing_feature
     Test_Config_ClientTagQueryString_Empty: missing_feature (test can not capture span with the expected http.url tag)

manifests/dotnet.yml

@@ -365,6 +365,9 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented)
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.15.0
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
+  stats/:
+    test_miscs.py:
+      Test_Miscs: missing_feature
   test_config_consistency.py:
     Test_Config_ClientTagQueryString_Configured: missing_feature (configuration DNE)
     Test_Config_ClientTagQueryString_Empty: v2.53.0

manifests/java.yml

@@ -1231,6 +1231,8 @@ tests/:
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature
+    test_stats.py:
+      Test_Client_Stats: missing_feature
   test_the_test/:
     test_json_report.py:
       Test_Mock: v0.0.99

manifests/nodejs.yml

@@ -560,6 +560,11 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented)
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: *ref_5_16_0 #actual version unknown
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
+  stats/:
+    test_miscs.py:
+      Test_Miscs: missing_feature
+    test_stats.py:
+      Test_Client_Stats: missing_feature
   test_config_consistency.py:
     Test_Config_ClientTagQueryString_Configured: missing_feature (adding query string to http.url is not supported)
     Test_Config_ClientTagQueryString_Empty: missing_feature (removes query strings by default)

manifests/php.yml

@@ -316,6 +316,8 @@ tests/:
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature
+    test_stats.py:
+      Test_Client_Stats: missing_feature
   test_config_consistency.py:
     Test_Config_ClientTagQueryString_Configured: missing_feature (supports dd_trace_http_url_query_param_allowed instead)
     Test_Config_ClientTagQueryString_Empty: v1.2.0

manifests/python.yml

@@ -758,6 +758,9 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented)
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.8.0.dev
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature
+  stats/:
+    test_miscs.py:
+      Test_Miscs: missing_feature
   test_config_consistency.py:
     Test_Config_ClientTagQueryString_Configured: missing_feature (supports DD_HTPP_CLIENT_TAGS_QUERY_STRING instead)
     Test_Config_ClientTagQueryString_Empty: v2.12.0

manifests/ruby.yml

@@ -388,6 +388,8 @@ tests/:
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature
+    test_stats.py:
+      Test_Client_Stats: missing_feature
   test_config_consistency.py:
     Test_Config_ClientTagQueryString_Configured: missing_feature
     Test_Config_ClientTagQueryString_Empty: missing_feature (removes query string by default)

tests/stats/test_stats.py

@@ -0,0 +1,65 @@
+from utils import interfaces, weblog, features, scenarios, missing_feature, context, bug
+from utils.tools import logger
+
+"""
+Test scenarios we want:
+    * Generate N spans that will be aggregated together
+        - Must aggregate by:
+            - HTTP status code
+            - peer service tags (todo: can we just rely on the defaults?)
+        - Must have `is_trace_root` on trace root
+        - Must set peer tags
+        - Must have span_kind
+        
+Config:
+- apm_config.peer_tags_aggregation (we should see peer service tags and aggregation by them, note only works on client or producer kind)
+- apm_config.compute_stats_by_span_kind (span_kind will be set and we will calc stats on these spans even when not "top level")
+"""
+
+
+@features.client_side_stats_supported
+class Test_Client_Stats:
+    """Test client-side stats are compatible with Agent implementation"""
+
+    def setup_client_stats(self):
+        for _ in range(5):
+            weblog.get("/stats-unique")
+        for _ in range(3):
+            weblog.get("/stats-unique?code=204")
+
+    @bug(
+        context.weblog_variant in ("django-poc", "python3.12"), library="python", reason="APMSP-1375",
+    )
+    def test_client_stats(self):
+        stats_count = 0
+        for s in interfaces.agent.get_stats(resource="GET /stats-unique"):
+            stats_count += 1
+            logger.debug(f"asserting on {s}")
+            if s["HTTPStatusCode"] == 200:
+                assert 5 == s["Hits"], "expect 5 hits at 200 status code"
+                assert 5 == s["TopLevelHits"], "expect 5 top level hits at 200 status code"
+            elif s["HTTPStatusCode"] == 204:
+                assert 3 == s["Hits"], "expect 3 hits at 204 status code"
+                assert 3 == s["TopLevelHits"], "expect 3 top level hits at 204 status code"
+            else:
+                assert False, "Unexpected status code " + str(s["HTTPStatusCode"])
+            assert "weblog" == s["Service"], "expect weblog as service"
+            assert "web" == s["Type"], "expect 'web' type"
+        assert stats_count == 2, "expect 2 stats"
+
+    @missing_feature(
+        context.library in ("cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby"),
+        reason="Tracers have not implemented this feature yet.",
+    )
+    def test_is_trace_root(self):
+        """Test IsTraceRoot presence in stats.
+        Note: Once all tracers have implmented it and the test xpasses for all of them, we can move these
+        assertions to `test_client_stats` method."""
+        for s in interfaces.agent.get_stats(resource="GET /stats-unique"):
+            assert 1 == s["IsTraceRoot"]
+            assert "server" == s["SpanKind"]
+
+    @scenarios.everything_disabled
+    def test_disable(self):
+        requests = list(interfaces.library.get_data("/v0.6/stats"))
+        assert len(requests) == 0, "Stats should be disabled by default"

utils/_context/_scenarios/__init__.py

@@ -44,6 +44,7 @@ def all_endtoend_scenarios(test_object):
             "DD_DBM_PROPAGATION_MODE": "service",
             "DD_TRACE_STATS_COMPUTATION_ENABLED": "1",
             "DD_TRACE_FEATURES": "discovery",
+            "DD_TRACE_COMPUTE_STATS": "true",
         },
         include_postgres_db=True,
         scenario_groups=[ScenarioGroup.ESSENTIALS],

utils/build/docker/dotnet/weblog/Endpoints/StatsUniqEndpoint.cs

@@ -0,0 +1,23 @@
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Primitives;
+
+namespace weblog
+{
+    public class StatsUniqEndpoint : ISystemTestEndpoint
+    {
+        public void Register(Microsoft.AspNetCore.Routing.IEndpointRouteBuilder routeBuilder)
+        {
+            routeBuilder.MapGet("/stats-unique", async context =>
+            {
+                var stringStatus = context.Request.Query["code"];
+                var status = 200;
+                if (!StringValues.IsNullOrEmpty(stringStatus)) {
+                    status = int.Parse(stringStatus!);
+                }
+                context.Response.StatusCode = status;
+                await context.Response.CompleteAsync();
+            });
+        }
+    }
+}

utils/build/docker/golang/app/chi/main.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"strconv"
 	"time"
+
 	"weblog/internal/rasp"
 
 	"weblog/internal/common"
@@ -28,6 +29,16 @@ func main() {
 
 	mux := chi.NewRouter().With(chitrace.Middleware())
 
+	mux.HandleFunc("/stats-unique", func(w http.ResponseWriter, r *http.Request) {
+		if c := r.URL.Query().Get("code"); c != "" {
+			if code, err := strconv.Atoi(c); err == nil {
+				w.WriteHeader(code)
+				return
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	})
+
 	mux.HandleFunc("/waf", func(w http.ResponseWriter, r *http.Request) {
 		body, err := common.ParseBody(r)
 		if err == nil {

utils/build/docker/golang/app/echo/main.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+
 	"weblog/internal/common"
 	"weblog/internal/grpc"
 	"weblog/internal/rasp"
@@ -43,6 +44,26 @@ func main() {
 		return c.NoContent(http.StatusNotFound)
 	})
 
+	r.Any("/status", func(c echo.Context) error {
+		rCode := 200
+		if codeStr := c.Request().URL.Query().Get("code"); codeStr != "" {
+			if code, err := strconv.Atoi(codeStr); err == nil {
+				rCode = code
+			}
+		}
+		return c.NoContent(rCode)
+	})
+
+	r.Any("/stats-unique", func(c echo.Context) error {
+		rCode := 200
+		if codeStr := c.Request().URL.Query().Get("code"); codeStr != "" {
+			if code, err := strconv.Atoi(codeStr); err == nil {
+				rCode = code
+			}
+		}
+		return c.NoContent(rCode)
+	})
+
 	r.Any("/waf", waf)
 	r.Any("/waf/*", waf)
 

utils/build/docker/golang/app/gin/main.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+
 	"weblog/internal/common"
 	"weblog/internal/grpc"
 	"weblog/internal/rasp"
@@ -27,14 +28,23 @@ func main() {
 	r.Any("/", func(ctx *gin.Context) {
 		ctx.Writer.WriteHeader(http.StatusOK)
 	})
+	r.Any("/stats-unique", func(ctx *gin.Context) {
+		if c := ctx.Request.URL.Query().Get("code"); c != "" {
+			if code, err := strconv.Atoi(c); err == nil {
+				ctx.Writer.WriteHeader(code)
+				return
+			}
+		}
+		ctx.Writer.WriteHeader(http.StatusOK)
+	})
 
 	r.GET("/healthcheck", func(ctx *gin.Context) {
 		healthCheck, err := common.GetHealtchCheck()
 
 		if err != nil {
 			ctx.JSON(http.StatusInternalServerError, err)
 		}
-		
+
 		ctx.JSON(http.StatusOK, healthCheck)
 	})
 

utils/build/docker/golang/app/net-http/main.go

@@ -11,11 +11,13 @@ import (
 	"os"
 	"strconv"
 	"time"
+
 	"weblog/internal/common"
 	"weblog/internal/grpc"
 	"weblog/internal/rasp"
 
 	"github.com/Shopify/sarama"
+
 	saramatrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/Shopify/sarama"
 	"gopkg.in/DataDog/dd-trace-go.v1/datastreams"
 
@@ -49,21 +51,31 @@ func main() {
 		w.WriteHeader(http.StatusOK)
 	})
 
+	mux.HandleFunc("/stats-unique", func(w http.ResponseWriter, r *http.Request) {
+		if c := r.URL.Query().Get("code"); c != "" {
+			if code, err := strconv.Atoi(c); err == nil {
+				w.WriteHeader(code)
+				return
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	})
+
 	mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 
 		healthCheck, err := common.GetHealtchCheck()
-        if err != nil {
-            http.Error(w, "Can't get JSON data", http.StatusInternalServerError)
-        }
-
-        jsonData, err := json.Marshal(healthCheck)
-        if err != nil {
-            http.Error(w, "Can't build JSON data", http.StatusInternalServerError)
-            return
-        }
-
-        w.Header().Set("Content-Type", "application/json")
-        w.Write(jsonData)
+		if err != nil {
+			http.Error(w, "Can't get JSON data", http.StatusInternalServerError)
+		}
+
+		jsonData, err := json.Marshal(healthCheck)
+		if err != nil {
+			http.Error(w, "Can't build JSON data", http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(jsonData)
 	})
 
 	mux.HandleFunc("/waf", func(w http.ResponseWriter, r *http.Request) {

utils/build/docker/python/django/app/urls.py

@@ -254,6 +254,10 @@ def status_code(request, *args, **kwargs):
     return HttpResponse("OK, probably", status=int(request.GET.get("code", "200")))
 
 
+def stats_unique(request, *args, **kwargs):
+    return HttpResponse("OK, probably", status=int(request.GET.get("code", "200")))
+
+
 def identify(request):
     set_user(
         tracer,
@@ -725,6 +729,7 @@ def create_extra_service(request):
     path("createextraservice", create_extra_service),
     path("headers", headers),
     path("status", status_code),
+    path("stats-unique", stats_unique),
     path("identify", identify),
     path("users", users),
     path("identify-propagate", identify_propagate),

utils/build/docker/python/fastapi/main.py

@@ -286,6 +286,11 @@ async def status_code(code: int = 200):
     return PlainTextResponse("OK, probably", status_code=code)
 
 
+@app.get("/stats-unique")
+async def stats_unique(code: int = 200):
+    return PlainTextResponse("OK, probably", status_code=code)
+
+
 @app.get("/make_distant_call")
 def make_distant_call(url: str):
     response = requests.get(url)