pants: Add system_user detection to pants-plugins/uses_services (…

…+ `mongo` detection improvements) (#6244)

CHANGELOG.rst

@@ -36,7 +36,7 @@ Added
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
-  #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241
+  #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

contrib/runners/orquesta_runner/tests/unit/BUILD

@@ -5,6 +5,18 @@ __defaults__(
 
 python_tests(
     name="tests",
+    overrides={
+        (
+            "test_basic.py",
+            "test_cancel.py",
+            "test_context.py",
+            "test_error_handling.py",
+            "test_pause_and_resume.py",
+            "test_with_items.py",
+        ): dict(
+            uses=["system_user"],
+        ),
+    },
 )
 
 python_test_utils(

pants-plugins/uses_services/BUILD

@@ -17,5 +17,6 @@ python_tests(
     #    "mongo_rules_test.py": dict(uses=["mongo"]),
     #    "rabbitmq_rules_test.py": dict(uses=["rabbitmq"]),
     #    "redis_rules_test.py": dict(uses=["redis"]),
+    #    "system_user_test.py": dict(uses=["system_user"]),
     # },
 )

pants-plugins/uses_services/mongo_rules.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 
 from dataclasses import dataclass
 from textwrap import dedent
@@ -20,6 +19,7 @@
     PytestPluginSetupRequest,
     PytestPluginSetup,
 )
+from pants.backend.python.subsystems.pytest import PyTest
 from pants.backend.python.util_rules.pex import (
     PexRequest,
     PexRequirements,
@@ -64,9 +64,12 @@ class UsesMongoRequest:
     db_host: str = "127.0.0.1"  # localhost in test_db.DbConnectionTestCase
     db_port: int = 27017
     # db_name is "st2" in test_db.DbConnectionTestCase
-    db_name: str = f"st2-test{os.environ.get('ST2TESTS_PARALLEL_SLOT', '')}"
+    db_name: str = "st2-test{}"  # {} will be replaced by test slot (a format string)
+
     db_connection_timeout: int = 3000
 
+    execution_slot_var: str = "ST2TESTS_PARALLEL_SLOT"
+
 
 @dataclass(frozen=True)
 class MongoIsRunning:
@@ -87,7 +90,7 @@ def is_applicable(cls, target: Target) -> bool:
     level=LogLevel.DEBUG,
 )
 async def mongo_is_running_for_pytest(
-    request: PytestUsesMongoRequest,
+    request: PytestUsesMongoRequest, pytest: PyTest
 ) -> PytestPluginSetup:
     # TODO: delete these comments once the Makefile becomes irrelevant.
     #       the comments explore how the Makefile prepares to run and runs tests
@@ -104,7 +107,10 @@ async def mongo_is_running_for_pytest(
     #            nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit
 
     # this will raise an error if mongo is not running
-    _ = await Get(MongoIsRunning, UsesMongoRequest())
+    _ = await Get(
+        MongoIsRunning,
+        UsesMongoRequest(execution_slot_var=pytest.execution_slot_var or ""),
+    )
 
     return PytestPluginSetup()
 
@@ -145,8 +151,10 @@ async def mongo_is_running(
                 str(request.db_port),
                 request.db_name,
                 str(request.db_connection_timeout),
+                request.execution_slot_var,
             ),
             input_digest=script_digest,
+            execution_slot_variable=request.execution_slot_var,
             description="Checking to see if Mongo is up and accessible.",
             # this can change from run to run, so don't cache results.
             cache_scope=ProcessCacheScope.PER_SESSION,

pants-plugins/uses_services/register.py

@@ -16,7 +16,13 @@
     PythonTestsGeneratorTarget,
 )
 
-from uses_services import mongo_rules, platform_rules, rabbitmq_rules, redis_rules
+from uses_services import (
+    mongo_rules,
+    platform_rules,
+    rabbitmq_rules,
+    redis_rules,
+    system_user_rules,
+)
 from uses_services.target_types import UsesServicesField
 
 
@@ -28,4 +34,5 @@ def rules():
         *mongo_rules.rules(),
         *rabbitmq_rules.rules(),
         *redis_rules.rules(),
+        *system_user_rules.rules(),
     ]

pants-plugins/uses_services/scripts/has_system_user.py

@@ -0,0 +1,43 @@
+# Copyright 2024 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+import pwd
+import sys
+
+
+def _has_system_user(system_user: str) -> bool:
+    """Make sure the system_user exists.
+
+    This should not import the st2 code as it should be self-contained.
+    """
+    try:
+        pwd.getpwnam(system_user)
+    except KeyError:
+        # put current user (for use in error msg instructions)
+        print(pwd.getpwuid(os.getuid()).pw_name)
+        return False
+    print(system_user)
+    return True
+
+
+if __name__ == "__main__":
+    args = dict((k, v) for k, v in enumerate(sys.argv))
+
+    system_user = args.get(1, "stanley")
+
+    is_running = _has_system_user(system_user)
+    exit_code = 0 if is_running else 1
+    sys.exit(exit_code)

pants-plugins/uses_services/scripts/is_mongo_running.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import sys
 
 
@@ -48,9 +49,12 @@ def _is_mongo_running(
     args = dict((k, v) for k, v in enumerate(sys.argv))
     db_host = args.get(1, "127.0.0.1")
     db_port = args.get(2, 27017)
-    db_name = args.get(3, "st2-test")
+    db_name = args.get(3, "st2-test{}")
     connection_timeout_ms = args.get(4, 3000)
 
+    slot_var = args.get(5, "ST2TESTS_PARALLEL_SLOT")
+    db_name = db_name.format(os.environ.get(slot_var) or "")
+
     is_running = _is_mongo_running(
         db_host, int(db_port), db_name, int(connection_timeout_ms)
     )

pants-plugins/uses_services/system_user_rules.py

@@ -0,0 +1,158 @@
+# Copyright 2024 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from dataclasses import dataclass
+from textwrap import dedent
+
+from pants.backend.python.goals.pytest_runner import (
+    PytestPluginSetupRequest,
+    PytestPluginSetup,
+)
+from pants.backend.python.target_types import Executable
+from pants.backend.python.util_rules.pex import (
+    PexRequest,
+    VenvPex,
+    VenvPexProcess,
+    rules as pex_rules,
+)
+from pants.core.goals.test import TestExtraEnv
+from pants.engine.fs import CreateDigest, Digest, FileContent
+from pants.engine.rules import collect_rules, Get, rule
+from pants.engine.process import FallibleProcessResult, ProcessCacheScope
+from pants.engine.target import Target
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+
+from uses_services.exceptions import ServiceMissingError
+from uses_services.platform_rules import Platform
+from uses_services.scripts.has_system_user import (
+    __file__ as has_system_user_full_path,
+)
+from uses_services.target_types import UsesServicesField
+
+
+@dataclass(frozen=True)
+class UsesSystemUserRequest:
+    """One or more targets need the system_user (like stanley) using these settings.
+
+    The system_user attributes represent the system_user.user settings from st2.conf.
+    In st2 code, they come from:
+        oslo_config.cfg.CONF.system_user.user
+    """
+
+    system_user: str = "stanley"
+
+
+@dataclass(frozen=True)
+class HasSystemUser:
+    pass
+
+
+class PytestUsesSystemUserRequest(PytestPluginSetupRequest):
+    @classmethod
+    def is_applicable(cls, target: Target) -> bool:
+        if not target.has_field(UsesServicesField):
+            return False
+        uses = target.get(UsesServicesField).value
+        return uses is not None and "system_user" in uses
+
+
+@rule(
+    desc="Ensure system_user is present before running tests.",
+    level=LogLevel.DEBUG,
+)
+async def has_system_user_for_pytest(
+    request: PytestUsesSystemUserRequest,
+    test_extra_env: TestExtraEnv,
+) -> PytestPluginSetup:
+    system_user = test_extra_env.env.get("ST2TESTS_SYSTEM_USER", "stanley")
+
+    # this will raise an error if system_user is not present
+    _ = await Get(HasSystemUser, UsesSystemUserRequest(system_user=system_user))
+
+    return PytestPluginSetup()
+
+
+@rule(
+    desc="Test to see if system_user is present.",
+    level=LogLevel.DEBUG,
+)
+async def has_system_user(
+    request: UsesSystemUserRequest, platform: Platform
+) -> HasSystemUser:
+    script_path = "./has_system_user.py"
+
+    # pants is already watching this directory as it is under a source root.
+    # So, we don't need to double watch with PathGlobs, just open it.
+    with open(has_system_user_full_path, "rb") as script_file:
+        script_contents = script_file.read()
+
+    script_digest = await Get(
+        Digest, CreateDigest([FileContent(script_path, script_contents)])
+    )
+    script_pex = await Get(
+        VenvPex,
+        PexRequest(
+            output_filename="script.pex",
+            internal_only=True,
+            sources=script_digest,
+            main=Executable(script_path),
+        ),
+    )
+
+    result = await Get(
+        FallibleProcessResult,
+        VenvPexProcess(
+            script_pex,
+            argv=(request.system_user,),
+            description="Checking to see if system_user is present.",
+            # this can change from run to run, so don't cache results.
+            cache_scope=ProcessCacheScope.PER_SESSION,
+            level=LogLevel.DEBUG,
+        ),
+    )
+    has_user = result.exit_code == 0
+
+    if has_user:
+        return HasSystemUser()
+
+    current_user = result.stdout.decode().strip()
+
+    # system_user is not present, so raise an error with instructions.
+    raise ServiceMissingError(
+        service="system_user",
+        platform=platform,
+        msg=dedent(
+            f"""\
+            The system_user ({request.system_user}) does not seem to be present!
+
+            Please export the ST2TESTS_SYSTEM_USER env var to specify which user
+            tests should use as the system_user. This user must be present on
+            your system.
+
+            To use your current user ({current_user}) as the system_user, run:
+
+            export ST2TESTS_SYSTEM_USER=$(id -un)
+            """
+        ),
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(PytestPluginSetupRequest, PytestUsesSystemUserRequest),
+        *pex_rules(),
+    ]