diff --git a/README.md b/README.md index 1ab9f394aa..1e34031407 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## System tests -Workbench designed to run advanced tests (integration, smoke, functionnal, fuzzing and performance) +Workbench designed to run advanced tests (integration, smoke, functional, fuzzing and performance) ## Requirements diff --git a/conftest.py b/conftest.py index 7f7229aab4..c3a13ab3c9 100644 --- a/conftest.py +++ b/conftest.py @@ -95,7 +95,7 @@ def pytest_configure(config): break if context.scenario is None: - pytest.exit(f"Scenario {config.option.scenario} does not exists", 1) + pytest.exit(f"Scenario {config.option.scenario} does not exist", 1) context.scenario.pytest_configure(config) @@ -302,7 +302,7 @@ def pytest_collection_finish(session: pytest.Session): if not item.instance: # item is a method bounded to a class continue - # the test metohd name is like test_xxxx + # the test method name is like test_xxxx # we replace the test_ by setup_, and call it if it exists setup_method_name = f"setup_{item.name[5:]}" diff --git a/docs/weblog/README.md b/docs/weblog/README.md index 1826f31a8e..9d1dec1c66 100644 --- a/docs/weblog/README.md +++ b/docs/weblog/README.md @@ -6,7 +6,7 @@ A weblog is a web app that system uses to test the library. It mimics what would ## Disclaimer -This document describes endpoints implemented on weblog. Though, it's not a complete description, and can contains mistakes. The source of truth are the test itself. If a weblog endpoint passes system tests, then you can consider it as ok. And if it does not passes it, then you must correct it, even if it's in line with this document. +This document describes endpoints implemented on weblog. Though, it's not a complete description, and can contain mistakes. The source of truth are the test itself. If a weblog endpoint passes system tests, then you can consider it as ok. And if it does not passes it, then you must correct it, even if it's in line with this document. **You are strongly encouraged to help others by submitting corrections when you notice issues with this document.** @@ -638,6 +638,11 @@ distributed tracing propagation headers. ### \[GET,POST\] /returnheaders This endpoint returns the headers received in order to be able to assert about distributed tracing propagation headers +### \[GET\] /stats-unique +The endpoint must accept a query string parameter `code`, which should be an integer. This parameter will be the status code of the response message, default to 200 OK. +This endpoint is used for client-stats tests to provide a separate "resource" via the endpoint path `stats-unique` to disambiguate those tests from other +stats generating tests. + ### GET /healthcheck Returns a JSON dict, with those values : diff --git a/manifests/cpp.yml b/manifests/cpp.yml index cbfa9d4ff6..78afd69652 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -168,6 +168,8 @@ tests/: stats/: test_miscs.py: Test_Miscs: missing_feature + test_stats.py: + Test_Client_Stats: missing_feature test_config_consistency.py: Test_Config_ClientTagQueryString_Configured: missing_feature Test_Config_ClientTagQueryString_Empty: missing_feature (test can not capture span with the expected http.url tag) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 8cf197db44..fca0d53ced 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -365,6 +365,9 @@ tests/: Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented) Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.15.0 Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented) + stats/: + test_miscs.py: + Test_Miscs: missing_feature test_config_consistency.py: Test_Config_ClientTagQueryString_Configured: missing_feature (configuration DNE) Test_Config_ClientTagQueryString_Empty: v2.53.0 diff --git a/manifests/java.yml b/manifests/java.yml index f8d9cf8467..0aec17adb6 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1231,6 +1231,8 @@ tests/: stats/: test_miscs.py: Test_Miscs: missing_feature + test_stats.py: + Test_Client_Stats: missing_feature test_the_test/: test_json_report.py: Test_Mock: v0.0.99 diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 114f2b041f..5b0f1ab332 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -560,6 +560,11 @@ tests/: Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented) Test_RemoteConfigurationUpdateSequenceLiveDebugging: *ref_5_16_0 #actual version unknown Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented) + stats/: + test_miscs.py: + Test_Miscs: missing_feature + test_stats.py: + Test_Client_Stats: missing_feature test_config_consistency.py: Test_Config_ClientTagQueryString_Configured: missing_feature (adding query string to http.url is not supported) Test_Config_ClientTagQueryString_Empty: missing_feature (removes query strings by default) diff --git a/manifests/php.yml b/manifests/php.yml index c84030ff41..7849a4a614 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -316,6 +316,8 @@ tests/: stats/: test_miscs.py: Test_Miscs: missing_feature + test_stats.py: + Test_Client_Stats: missing_feature test_config_consistency.py: Test_Config_ClientTagQueryString_Configured: missing_feature (supports dd_trace_http_url_query_param_allowed instead) Test_Config_ClientTagQueryString_Empty: v1.2.0 diff --git a/manifests/python.yml b/manifests/python.yml index 479e83fb25..15f4a08909 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -758,6 +758,9 @@ tests/: Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented) Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.8.0.dev Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature + stats/: + test_miscs.py: + Test_Miscs: missing_feature test_config_consistency.py: Test_Config_ClientTagQueryString_Configured: missing_feature (supports DD_HTPP_CLIENT_TAGS_QUERY_STRING instead) Test_Config_ClientTagQueryString_Empty: v2.12.0 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index fafb221c0c..132ff4f0bd 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -388,6 +388,8 @@ tests/: stats/: test_miscs.py: Test_Miscs: missing_feature + test_stats.py: + Test_Client_Stats: missing_feature test_config_consistency.py: Test_Config_ClientTagQueryString_Configured: missing_feature Test_Config_ClientTagQueryString_Empty: missing_feature (removes query string by default) diff --git a/tests/stats/__init__.py b/tests/stats/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/stats/test_stats.py b/tests/stats/test_stats.py new file mode 100644 index 0000000000..b67e5d7d16 --- /dev/null +++ b/tests/stats/test_stats.py @@ -0,0 +1,65 @@ +from utils import interfaces, weblog, features, scenarios, missing_feature, context, bug +from utils.tools import logger + +""" +Test scenarios we want: + * Generate N spans that will be aggregated together + - Must aggregate by: + - HTTP status code + - peer service tags (todo: can we just rely on the defaults?) + - Must have `is_trace_root` on trace root + - Must set peer tags + - Must have span_kind + +Config: +- apm_config.peer_tags_aggregation (we should see peer service tags and aggregation by them, note only works on client or producer kind) +- apm_config.compute_stats_by_span_kind (span_kind will be set and we will calc stats on these spans even when not "top level") +""" + + +@features.client_side_stats_supported +class Test_Client_Stats: + """Test client-side stats are compatible with Agent implementation""" + + def setup_client_stats(self): + for _ in range(5): + weblog.get("/stats-unique") + for _ in range(3): + weblog.get("/stats-unique?code=204") + + @bug( + context.weblog_variant in ("django-poc", "python3.12"), library="python", reason="APMSP-1375", + ) + def test_client_stats(self): + stats_count = 0 + for s in interfaces.agent.get_stats(resource="GET /stats-unique"): + stats_count += 1 + logger.debug(f"asserting on {s}") + if s["HTTPStatusCode"] == 200: + assert 5 == s["Hits"], "expect 5 hits at 200 status code" + assert 5 == s["TopLevelHits"], "expect 5 top level hits at 200 status code" + elif s["HTTPStatusCode"] == 204: + assert 3 == s["Hits"], "expect 3 hits at 204 status code" + assert 3 == s["TopLevelHits"], "expect 3 top level hits at 204 status code" + else: + assert False, "Unexpected status code " + str(s["HTTPStatusCode"]) + assert "weblog" == s["Service"], "expect weblog as service" + assert "web" == s["Type"], "expect 'web' type" + assert stats_count == 2, "expect 2 stats" + + @missing_feature( + context.library in ("cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby"), + reason="Tracers have not implemented this feature yet.", + ) + def test_is_trace_root(self): + """Test IsTraceRoot presence in stats. + Note: Once all tracers have implmented it and the test xpasses for all of them, we can move these + assertions to `test_client_stats` method.""" + for s in interfaces.agent.get_stats(resource="GET /stats-unique"): + assert 1 == s["IsTraceRoot"] + assert "server" == s["SpanKind"] + + @scenarios.everything_disabled + def test_disable(self): + requests = list(interfaces.library.get_data("/v0.6/stats")) + assert len(requests) == 0, "Stats should be disabled by default" diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index a5582ac54d..7c3867e221 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -44,6 +44,7 @@ def all_endtoend_scenarios(test_object): "DD_DBM_PROPAGATION_MODE": "service", "DD_TRACE_STATS_COMPUTATION_ENABLED": "1", "DD_TRACE_FEATURES": "discovery", + "DD_TRACE_COMPUTE_STATS": "true", }, include_postgres_db=True, scenario_groups=[ScenarioGroup.ESSENTIALS], diff --git a/utils/build/docker/dotnet/weblog/Endpoints/StatsUniqEndpoint.cs b/utils/build/docker/dotnet/weblog/Endpoints/StatsUniqEndpoint.cs new file mode 100644 index 0000000000..2ea54b4737 --- /dev/null +++ b/utils/build/docker/dotnet/weblog/Endpoints/StatsUniqEndpoint.cs @@ -0,0 +1,23 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; + +namespace weblog +{ + public class StatsUniqEndpoint : ISystemTestEndpoint + { + public void Register(Microsoft.AspNetCore.Routing.IEndpointRouteBuilder routeBuilder) + { + routeBuilder.MapGet("/stats-unique", async context => + { + var stringStatus = context.Request.Query["code"]; + var status = 200; + if (!StringValues.IsNullOrEmpty(stringStatus)) { + status = int.Parse(stringStatus!); + } + context.Response.StatusCode = status; + await context.Response.CompleteAsync(); + }); + } + } +} diff --git a/utils/build/docker/golang/app/chi/main.go b/utils/build/docker/golang/app/chi/main.go index b8da109b40..792eaedb73 100644 --- a/utils/build/docker/golang/app/chi/main.go +++ b/utils/build/docker/golang/app/chi/main.go @@ -8,6 +8,7 @@ import ( "encoding/json" "strconv" "time" + "weblog/internal/rasp" "weblog/internal/common" @@ -28,6 +29,16 @@ func main() { mux := chi.NewRouter().With(chitrace.Middleware()) + mux.HandleFunc("/stats-unique", func(w http.ResponseWriter, r *http.Request) { + if c := r.URL.Query().Get("code"); c != "" { + if code, err := strconv.Atoi(c); err == nil { + w.WriteHeader(code) + return + } + } + w.WriteHeader(http.StatusOK) + }) + mux.HandleFunc("/waf", func(w http.ResponseWriter, r *http.Request) { body, err := common.ParseBody(r) if err == nil { diff --git a/utils/build/docker/golang/app/echo/main.go b/utils/build/docker/golang/app/echo/main.go index 0517ba115b..cf9e496f9d 100644 --- a/utils/build/docker/golang/app/echo/main.go +++ b/utils/build/docker/golang/app/echo/main.go @@ -5,6 +5,7 @@ import ( "net/http" "os" "strconv" + "weblog/internal/common" "weblog/internal/grpc" "weblog/internal/rasp" @@ -43,6 +44,26 @@ func main() { return c.NoContent(http.StatusNotFound) }) + r.Any("/status", func(c echo.Context) error { + rCode := 200 + if codeStr := c.Request().URL.Query().Get("code"); codeStr != "" { + if code, err := strconv.Atoi(codeStr); err == nil { + rCode = code + } + } + return c.NoContent(rCode) + }) + + r.Any("/stats-unique", func(c echo.Context) error { + rCode := 200 + if codeStr := c.Request().URL.Query().Get("code"); codeStr != "" { + if code, err := strconv.Atoi(codeStr); err == nil { + rCode = code + } + } + return c.NoContent(rCode) + }) + r.Any("/waf", waf) r.Any("/waf/*", waf) diff --git a/utils/build/docker/golang/app/gin/main.go b/utils/build/docker/golang/app/gin/main.go index 507c787959..57cec708e6 100644 --- a/utils/build/docker/golang/app/gin/main.go +++ b/utils/build/docker/golang/app/gin/main.go @@ -5,6 +5,7 @@ import ( "net/http" "os" "strconv" + "weblog/internal/common" "weblog/internal/grpc" "weblog/internal/rasp" @@ -27,6 +28,15 @@ func main() { r.Any("/", func(ctx *gin.Context) { ctx.Writer.WriteHeader(http.StatusOK) }) + r.Any("/stats-unique", func(ctx *gin.Context) { + if c := ctx.Request.URL.Query().Get("code"); c != "" { + if code, err := strconv.Atoi(c); err == nil { + ctx.Writer.WriteHeader(code) + return + } + } + ctx.Writer.WriteHeader(http.StatusOK) + }) r.GET("/healthcheck", func(ctx *gin.Context) { healthCheck, err := common.GetHealtchCheck() @@ -34,7 +44,7 @@ func main() { if err != nil { ctx.JSON(http.StatusInternalServerError, err) } - + ctx.JSON(http.StatusOK, healthCheck) }) diff --git a/utils/build/docker/golang/app/net-http/main.go b/utils/build/docker/golang/app/net-http/main.go index 6c5dfd4b3b..81890a8451 100644 --- a/utils/build/docker/golang/app/net-http/main.go +++ b/utils/build/docker/golang/app/net-http/main.go @@ -11,11 +11,13 @@ import ( "os" "strconv" "time" + "weblog/internal/common" "weblog/internal/grpc" "weblog/internal/rasp" "github.com/Shopify/sarama" + saramatrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/Shopify/sarama" "gopkg.in/DataDog/dd-trace-go.v1/datastreams" @@ -49,21 +51,31 @@ func main() { w.WriteHeader(http.StatusOK) }) + mux.HandleFunc("/stats-unique", func(w http.ResponseWriter, r *http.Request) { + if c := r.URL.Query().Get("code"); c != "" { + if code, err := strconv.Atoi(c); err == nil { + w.WriteHeader(code) + return + } + } + w.WriteHeader(http.StatusOK) + }) + mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) { healthCheck, err := common.GetHealtchCheck() - if err != nil { - http.Error(w, "Can't get JSON data", http.StatusInternalServerError) - } - - jsonData, err := json.Marshal(healthCheck) - if err != nil { - http.Error(w, "Can't build JSON data", http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.Write(jsonData) + if err != nil { + http.Error(w, "Can't get JSON data", http.StatusInternalServerError) + } + + jsonData, err := json.Marshal(healthCheck) + if err != nil { + http.Error(w, "Can't build JSON data", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(jsonData) }) mux.HandleFunc("/waf", func(w http.ResponseWriter, r *http.Request) { diff --git a/utils/build/docker/python/django/app/urls.py b/utils/build/docker/python/django/app/urls.py index 6e40ab8cb5..1aeee96aab 100644 --- a/utils/build/docker/python/django/app/urls.py +++ b/utils/build/docker/python/django/app/urls.py @@ -254,6 +254,10 @@ def status_code(request, *args, **kwargs): return HttpResponse("OK, probably", status=int(request.GET.get("code", "200"))) +def stats_unique(request, *args, **kwargs): + return HttpResponse("OK, probably", status=int(request.GET.get("code", "200"))) + + def identify(request): set_user( tracer, @@ -725,6 +729,7 @@ def create_extra_service(request): path("createextraservice", create_extra_service), path("headers", headers), path("status", status_code), + path("stats-unique", stats_unique), path("identify", identify), path("users", users), path("identify-propagate", identify_propagate), diff --git a/utils/build/docker/python/fastapi/main.py b/utils/build/docker/python/fastapi/main.py index b7777c967a..a8725b2b55 100644 --- a/utils/build/docker/python/fastapi/main.py +++ b/utils/build/docker/python/fastapi/main.py @@ -286,6 +286,11 @@ async def status_code(code: int = 200): return PlainTextResponse("OK, probably", status_code=code) +@app.get("/stats-unique") +async def stats_unique(code: int = 200): + return PlainTextResponse("OK, probably", status_code=code) + + @app.get("/make_distant_call") def make_distant_call(url: str): response = requests.get(url) diff --git a/utils/build/docker/python/flask/app.py b/utils/build/docker/python/flask/app.py index e00d12d189..4c2e0763a4 100644 --- a/utils/build/docker/python/flask/app.py +++ b/utils/build/docker/python/flask/app.py @@ -367,6 +367,12 @@ def status_code(): return Response("OK, probably", status=code) +@app.route("/stats-unique") +def stats_unique(): + code = flask_request.args.get("code", default=200, type=int) + return Response("OK, probably", status=code) + + @app.route("/make_distant_call") def make_distant_call(): url = flask_request.args["url"] diff --git a/utils/interfaces/_agent.py b/utils/interfaces/_agent.py index a03763c69a..c4d1f02801 100644 --- a/utils/interfaces/_agent.py +++ b/utils/interfaces/_agent.py @@ -141,3 +141,21 @@ def get_spans_list(self, request): def get_dsm_data(self): return self.get_data(path_filters="/api/v0.1/pipeline_stats") + + def get_stats(self, resource=""): + """Attempts to fetch the stats the agent will submit to the backend. + + When a valid request is given, then we filter the stats to the ones sampled + during that request's execution, and only return those. + """ + + for data in self.get_data(path_filters="/api/v0.2/stats"): + client_stats_payloads = data["request"]["content"]["Stats"] + + for client_stats_payload in client_stats_payloads: + for client_stats_buckets in client_stats_payload["Stats"]: + for client_grouped_stat in client_stats_buckets["Stats"]: + if resource == "": + yield client_grouped_stat + elif client_grouped_stat["Resource"] == resource: + yield client_grouped_stat diff --git a/utils/interfaces/schemas/agent/api/v0.2/stats-request.json b/utils/interfaces/schemas/agent/api/v0.2/stats-request.json index 8619b50321..702b689d0f 100644 --- a/utils/interfaces/schemas/agent/api/v0.2/stats-request.json +++ b/utils/interfaces/schemas/agent/api/v0.2/stats-request.json @@ -1,4 +1,41 @@ { - "$id": "/agent/api/v0.2/stats-request.json", - "type": "object" - } \ No newline at end of file + "$id": "/agent/api/v0.2/stats-request.json", + "type": "object", + "properties": { + "Stats": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Stats": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Start": { "type": "integer" }, + "Duration": { "type": "integer" }, + "Stats": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Service": { "type": "string" }, + "Name": { "type": "string" }, + "Resource": { "type": "string" } + }, + "required": ["Service", "Name", "Resource"] + } + } + }, + "required": ["Start", "Duration", "Stats"] + } + } + }, + "required": ["Stats"] + } + } + }, + "required": [ + "Stats" + ] +} \ No newline at end of file