Merge pull request #2563 from DataDog/szegedi/profiling-onboarding

PROF-9979: Add host system tests for DD_PROFILING_ENABLED=auto

.gitlab-ci.yml

@@ -40,9 +40,12 @@ onboarding_nodejs:
     TEST_LIBRARY: "nodejs"
   parallel:
       matrix:
+        - ONBOARDING_FILTER_ENV: [dev, prod]
+          ONBOARDING_FILTER_WEBLOG: [test-app-nodejs-profiling]
+          SCENARIO: [SIMPLE_HOST_AUTO_INJECTION_PROFILING]
         - ONBOARDING_FILTER_ENV: [dev, prod]
           ONBOARDING_FILTER_WEBLOG: [test-app-nodejs]
-          SCENARIO: [HOST_AUTO_INJECTION, HOST_AUTO_INJECTION_INSTALL_SCRIPT]
+          SCENARIO: [HOST_AUTO_INJECTION, HOST_AUTO_INJECTION_INSTALL_SCRIPT, HOST_AUTO_INJECTION_INSTALL_SCRIPT_PROFILING]
         - ONBOARDING_FILTER_ENV: [dev, prod]
           ONBOARDING_FILTER_WEBLOG: [test-shell-script]
           SCENARIO: [HOST_AUTO_INJECTION_BLOCK_LIST]

docs/scenarios/onboarding.md

@@ -262,8 +262,10 @@ The currently supported scenarios are the following:
 
 * **HOST_AUTO_INJECTION:** Onboarding Host Single Step Instrumentation scenario
 * **SIMPLE_HOST_AUTO_INJECTION:** Onboarding Host Single Step Instrumentation scenario (minimal test scenario)
+* **SIMPLE_HOST_AUTO_INJECTION_PROFILING:** Onboarding Host Single Step Instrumentation profiling scenario (minimal test scenario)
 * **HOST_AUTO_INJECTION_BLOCK_LIST:** Onboarding Host Single Step Instrumentation scenario: Test user defined blocking lists
 * **HOST_AUTO_INJECTION_INSTALL_SCRIPT:** Onboarding Host Single Step Instrumentation scenario using agent auto install script
+* **HOST_AUTO_INJECTION_INSTALL_SCRIPT_PROFILING:** Onboarding Host Single Step Instrumentation scenario using agent auto install script with enabling profiling
 * **CONTAINER_AUTO_INJECTION:** Onboarding Container Single Step Instrumentation scenario
 * **SIMPLE_CONTAINER_AUTO_INJECTION:** Onboarding Container Single Step Instrumentation scenario (minimal test scenario)
 * **CONTAINER_AUTO_INJECTION_INSTALL_SCRIPT:** Onboarding Container Single Step Instrumentation scenario using agent auto install script

tests/auto_inject/test_auto_inject_install.py

@@ -14,8 +14,8 @@
 
 
 class _AutoInjectBaseTest:
-    def _test_install(self, virtual_machine):
-        """ We can easily install agent and lib injection software from agent installation script. Given a  sample application we can enable tracing using local environment variables.  
+    def _test_install(self, virtual_machine, profile: bool = False):
+        """ We can easily install agent and lib injection software from agent installation script. Given a  sample application we can enable tracing using local environment variables.
             After starting application we can see application HTTP requests traces in the backend.
             Using the agent installation script we can install different versions of the software (release or beta) in different OS."""
         vm_ip = virtual_machine.ssh_config.hostname
@@ -36,7 +36,7 @@ def _test_install(self, virtual_machine):
             request_uuid = make_get_request(f"http://{vm_ip}:{vm_port}/")
 
         logger.info(f"Http request done with uuid: [{request_uuid}] for ip [{vm_ip}]")
-        wait_backend_trace_id(request_uuid, 120.0)
+        wait_backend_trace_id(request_uuid, 120.0, profile=profile)
 
     def execute_command(self, virtual_machine, command):
         # Env for the command
@@ -67,9 +67,9 @@ def execute_command(self, virtual_machine, command):
     def _test_uninstall(
         self, virtual_machine, stop_weblog_command, start_weblog_command, uninstall_command, install_command
     ):
-        """ We can unistall the auto injection software. We can start the app again 
+        """ We can unistall the auto injection software. We can start the app again
         The weblog app should work but no sending traces to the backend.
-        We can reinstall the auto inject software. The weblog app should be instrumented 
+        We can reinstall the auto inject software. The weblog app should be instrumented
         and reporting traces to the backend."""
         logger.info(f"Launching _test_uninstall for : [{virtual_machine.name}]")
 
@@ -159,6 +159,24 @@ def test_install(self, virtual_machine):
         logger.info(f"Done test_install for : [{virtual_machine.name}]")
 
 
+@features.host_auto_instrumentation_profiling
+@scenarios.simple_host_auto_injection_profiling
+class TestSimpleHostAutoInjectManualProfiling(_AutoInjectBaseTest):
+    def test_install(self, virtual_machine):
+        logger.info(f"Launching test_install for : [{virtual_machine.name}]...")
+        self._test_install(virtual_machine, profile=True)
+        logger.info(f"Done test_install for : [{virtual_machine.name}]")
+
+
+@features.host_auto_installation_script_profiling
+@scenarios.host_auto_injection_install_script_profiling
+class TestHostAutoInjectInstallScriptProfiling(_AutoInjectBaseTest):
+    def test_install(self, virtual_machine):
+        logger.info(f"Launching test_install for : [{virtual_machine.name}]...")
+        self._test_install(virtual_machine, profile=True)
+        logger.info(f"Done test_install for : [{virtual_machine.name}]")
+
+
 @features.host_auto_instrumentation
 @scenarios.host_auto_injection_ld_preload
 class TestHostAutoInjectManualLdPreload(_AutoInjectBaseTest):

utils/_context/_scenarios.py

@@ -1931,6 +1931,9 @@ def all_endtoend_scenarios(test_object):
     simple_host_auto_injection = HostAutoInjectionScenario(
         "SIMPLE_HOST_AUTO_INJECTION", "Onboarding Host Single Step Instrumentation scenario (minimal test scenario)",
     )
+    simple_host_auto_injection_profiling = HostAutoInjectionScenario(
+        "SIMPLE_HOST_AUTO_INJECTION_PROFILING", "Onboarding Host Single Step Instrumentation scenario with profiling",
+    )
     host_auto_injection_block_list = HostAutoInjectionScenario(
         "HOST_AUTO_INJECTION_BLOCK_LIST",
         "Onboarding Host Single Step Instrumentation scenario: Test user defined blocking lists",
@@ -1940,6 +1943,13 @@ def all_endtoend_scenarios(test_object):
         "Onboarding Host Single Step Instrumentation scenario using agent auto install script",
         vm_provision="host-auto-inject-install-script",
     )
+
+    host_auto_injection_install_script_profiling = HostAutoInjectionScenario(
+        "HOST_AUTO_INJECTION_INSTALL_SCRIPT_PROFILING",
+        "Onboarding Host Single Step Instrumentation scenario using agent auto install script with profiling",
+        vm_provision="host-auto-inject-install-script-profiling",
+    )
+
     # TODO Add the provision of this scenario to the default host scenario (when fixes are released)
     host_auto_injection_ld_preload = HostAutoInjectionScenario(
         "HOST_AUTO_INJECTION_LD_PRELOAD",

utils/_features.py

@@ -2197,6 +2197,16 @@ def debugger_expression_language(test_object):
         pytest.mark.features(feature_id=303)(test_object)
         return test_object
 
+    @staticmethod
+    def host_auto_instrumentation_profiling(test_object):
+        """
+        Host auto-instrumentation with profiling
+
+        https://feature-parity.us1.prod.dog/#/?feature=302
+        """
+        pytest.mark.features(feature_id=302)(test_object)
+        return test_object
+
     @staticmethod
     def appsec_standalone(test_object):
         """
@@ -2206,3 +2216,13 @@ def appsec_standalone(test_object):
         """
         pytest.mark.features(feature_id=305)(test_object)
         return test_object
+
+    @staticmethod
+    def host_auto_installation_script_profiling(test_object):
+        """
+        Profiling works when enabled through the agent installer script in Host environments
+
+        https://feature-parity.us1.prod.dog/#/?feature=306
+        """
+        pytest.mark.features(feature_id=306)(test_object)
+        return test_object

utils/build/virtual_machine/provisions/auto-inject/repositories/autoinstall/execute_install_script.sh

@@ -2,9 +2,11 @@
 
 #This script is needed only for this reason: https://datadoghq.atlassian.net/browse/AP-2165
 
-DD_APM_INSTRUMENTATION_ENABLED="$1"
+if [ -z "$INSTALLER_URL" ]; then
+    INSTALLER_URL="https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh"
+fi
 
-curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh --output install_script_agent7.sh
+curl -L $INSTALLER_URL --output install_script_agent7.sh
 chmod 755 install_script_agent7.sh
 # shellcheck disable=SC2154
 sed  "s/\"7\"/\"$DD_injection_major_version\"/g" install_script_agent7.sh > install_script_agent7_autoinject_temp.sh
@@ -17,5 +19,13 @@ if [ "$1" == "docker" ]; then
     DD_SITE=$DD_injection_repo_url DD_NO_AGENT_INSTALL=true ./install_script_agent7.sh
 fi
 # shellcheck disable=SC2154
-DD_SITE=$DD_injection_repo_url DD_REPO_URL=$DD_injection_repo_url DD_AGENT_DIST_CHANNEL=$DD_injection_dist_channel DD_AGENT_MAJOR_VERSION=$DD_injection_major_version DD_APM_INSTRUMENTATION_LANGUAGES="$DD_LANG" DD_APM_INSTRUMENTATION_ENABLED="$DD_APM_INSTRUMENTATION_ENABLED" ./install_script_agent7_autoinject.sh
+DD_SITE=$DD_injection_repo_url  \
+DD_REPO_URL=$DD_injection_repo_url \
+DD_AGENT_DIST_CHANNEL=$DD_injection_dist_channel \
+DD_AGENT_MAJOR_VERSION=$DD_injection_major_version \
+DD_APM_INSTRUMENTATION_LANGUAGES="$DD_LANG" \
+DD_APM_INSTRUMENTATION_ENABLED="$DD_APM_INSTRUMENTATION_ENABLED" \
+DD_PROFILING_ENABLED="$DD_PROFILING_ENABLED" \
+./install_script_agent7_autoinject.sh
+
 echo "lib-injection install done"

utils/build/virtual_machine/provisions/container-auto-inject-install-script/auto-inject_container_script.yml

@@ -14,6 +14,6 @@
     sudo mkdir -p /opt/datadog/apm/inject/run
     sudo chmod 777 /opt/datadog/apm/inject/run
     printf "DD_APM_RECEIVER_SOCKET=/opt/datadog/apm/inject/run/apm.socket\nDD_DOGSTATSD_SOCKET=/opt/datadog/apm/inject/run/dsd.socket\nDD_USE_DOGSTATSD=true\n" | sudo tee /var/run/datadog-installer/environment
-    bash execute_install_script.sh docker
+    DD_APM_INSTRUMENTATION_ENABLED=docker bash execute_install_script.sh
     sudo cp docker_config.yaml /etc/datadog-agent/inject/docker_config.yaml
     sudo cp debug_config.yaml /etc/datadog-agent/inject/debug_config.yaml

utils/build/virtual_machine/provisions/host-auto-inject-install-script-profiling/auto-inject_host_script.yml

@@ -0,0 +1,15 @@
+#Execute script installation for lib-injection packages using install script
+- os_type: linux
+  copy_files:
+      - name: copy-auto-install-script
+        local_path: utils/build/virtual_machine/provisions/auto-inject/repositories/autoinstall/execute_install_script.sh
+      - name: copy-tracer-debug-config
+        local_path: utils/build/virtual_machine/provisions/auto-inject/tracer_debug/debug_config.yaml
+  remote-command: |
+    printf "BOGUS=foo\n" | sudo tee -a /etc/environment
+    printf "DD_APM_RECEIVER_SOCKET=/opt/datadog/apm/inject/run/apm.socket\nDD_DOGSTATSD_SOCKET=/opt/datadog/apm/inject/run/dsd.socket\nDD_USE_DOGSTATSD=true\n" | sudo tee /var/run/datadog-installer/environment
+    DD_APM_INSTRUMENTATION_ENABLED=host DD_PROFILING_ENABLED=auto bash execute_install_script.sh
+    sudo mkdir -p /var/run/datadog-installer
+    sudo mkdir -p /opt/datadog/apm/inject/run
+    sudo chmod 777 /opt/datadog/apm/inject/run
+    sudo cp debug_config.yaml /etc/datadog-agent/inject/debug_config.yaml

utils/build/virtual_machine/provisions/host-auto-inject-install-script-profiling/provision.yml

@@ -0,0 +1,32 @@
+#Optional: Load the environment variables
+init-environment: !include utils/build/virtual_machine/provisions/auto-inject/auto-inject-environment.yml
+
+#Mandatory: Scripts to extract the installed/tested components (json {component1:version, component2:version})
+tested_components:
+  install: !include utils/build/virtual_machine/provisions/auto-inject/auto-inject-tested_components.yml
+
+#Mandatory: Steps to install provision 
+provision_steps:
+  - init-config #Very first machine actions, like disable auto updates
+  - prepare-repos #Configure the reporitories for install auto-injection
+  - install-agent #Install the agent (allways latest release)
+  - autoinjection_install_script #Install the auto-injection softaware 'datadog-apm-inject' and 'datadog-apm-library-$DD_LANG' using the agent install script
+
+init-config:
+  cache: true
+  install: !include utils/build/virtual_machine/provisions/auto-inject/auto-inject_init_vm_config.yml
+
+prepare-repos:
+  cache: true
+  install: !include utils/build/virtual_machine/provisions/auto-inject/auto-inject-prepare_repos.yml
+
+install-agent:
+  install:
+    - os_type: linux
+      remote-command: |
+        REPO_URL=$DD_agent_repo_url DD_AGENT_DIST_CHANNEL=$DD_agent_dist_channel DD_AGENT_MAJOR_VERSION=$DD_agent_major_version bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)"
+
+autoinjection_install_script:
+  install: !include utils/build/virtual_machine/provisions/host-auto-inject-install-script-profiling/auto-inject_host_script.yml
+
+

utils/build/virtual_machine/provisions/host-auto-inject-install-script/auto-inject_host_script.yml

@@ -7,7 +7,7 @@
         local_path: utils/build/virtual_machine/provisions/auto-inject/tracer_debug/debug_config.yaml
   remote-command: |
     printf "DD_APM_RECEIVER_SOCKET=/opt/datadog/apm/inject/run/apm.socket\nDD_DOGSTATSD_SOCKET=/opt/datadog/apm/inject/run/dsd.socket\nDD_USE_DOGSTATSD=true\n" | sudo tee /var/run/datadog-installer/environment
-    bash execute_install_script.sh host
+    DD_APM_INSTRUMENTATION_ENABLED=host bash execute_install_script.sh
     sudo mkdir -p /var/run/datadog-installer
     sudo mkdir -p /opt/datadog/apm/inject/run
     sudo chmod 777 /opt/datadog/apm/inject/run

utils/build/virtual_machine/weblogs/nodejs/provision_test-app-nodejs-profiling.yml

@@ -0,0 +1,35 @@
+lang_variant:
+    name: node16
+    version: 16
+    cache: true
+    install:
+      - os_type: linux
+        os_distro: deb
+        remote-command: curl -sL https://deb.nodesource.com/setup_16.x | sudo bash - && sudo apt-get install -y nodejs
+      - os_type: linux
+        os_distro: rpm
+        remote-command: |
+          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+          export NVM_DIR="$HOME/.nvm"
+          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
+          [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
+          nvm install 16.0.0
+          nvm use node
+          n=$(which node);n=${n%/bin/node}; chmod -R 755 $n/bin/*; sudo cp -r $n/{bin,lib,share} /usr/local
+
+weblog:
+    name: test-app-nodejs-profiling
+    excluded_os_branches: [amazon_linux2_dotnet6, amazon_linux2023_amd64]
+    install:
+      - os_type: linux
+        copy_files:
+          - name: copy-service
+            local_path: utils/build/virtual_machine/weblogs/nodejs/test-app-nodejs-profiling/test-app.service
+
+          - name: copy-run-weblog-script
+            local_path: utils/build/virtual_machine/weblogs/nodejs/test-app-nodejs-profiling/test-app-nodejs_run.sh
+
+          - name: copy-binary
+            local_path: lib-injection/build/docker/nodejs/sample-app/index.js
+
+        remote-command: sh test-app-nodejs_run.sh

utils/build/virtual_machine/weblogs/nodejs/test-app-nodejs-profiling/test-app-nodejs_run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+echo "START RUN APP"
+
+
+sudo sed -i "s/18080/5985/g" index.js 
+sudo cp index.js /home/datadog
+sudo cp test-app.service /etc/systemd/system/test-app.service
+sudo systemctl daemon-reload
+sudo systemctl enable test-app.service
+sudo systemctl start test-app.service
+sudo systemctl status test-app.service
+
+echo "RUN DONE"

utils/build/virtual_machine/weblogs/nodejs/test-app-nodejs-profiling/test-app.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=Nodejs Weblog App Service
+After=syslog.target network.target
+
+[Service]
+SuccessExitStatus=143
+
+User=datadog
+
+Type=simple
+
+Environment=DD_APM_INSTRUMENTATION_DEBUG=TRUE
+Environment=DD_PROFILING_ENABLED=auto
+Environment=DD_PROFILING_UPLOAD_PERIOD=10
+Environment=DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD=1500
+WorkingDirectory=/home/datadog
+ExecStart=/bin/bash -c 'node index.js'
+ExecStop=/bin/kill -15 $MAINPID
+
+[Install]
+WantedBy=multi-user.target

utils/build/virtual_machine/weblogs/nodejs/test-app-nodejs/test-app-nodejs_run.sh

@@ -10,6 +10,6 @@ sudo systemctl enable test-app.service
 sudo systemctl start test-app.service
 sudo systemctl status test-app.service
 sleep 5
-cat /home/datadog/app-std.out
+sudo cat /home/datadog/app-std.out
 
 echo "RUN DONE"

utils/build/virtual_machine/weblogs/nodejs/test-app-nodejs/test-app.service

@@ -11,9 +11,12 @@ Type=simple
 
 Environment=DD_APM_INSTRUMENTATION_DEBUG=TRUE
 Environment=DD_TRACE_DEBUG=true
+Environment=DD_PROFILING_UPLOAD_PERIOD=10
+Environment=DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD=1500
+EnvironmentFile=/etc/environment
 WorkingDirectory=/home/datadog
 ExecStart=/bin/bash -c 'node index.js >> /home/datadog/app-std.out 2>&1 </dev/null'
 ExecStop=/bin/kill -15 $MAINPID
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target