diff --git a/modules/beta/emrstudio-on-eks/app.py b/modules/beta/emrstudio-on-eks/app.py index b4b06eaa..590a1678 100644 --- a/modules/beta/emrstudio-on-eks/app.py +++ b/modules/beta/emrstudio-on-eks/app.py @@ -1,27 +1,26 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -# type: ignore - import json import os import aws_cdk -from aws_cdk import App +import cdk_nag -from rbac_stack import EmrEksRbacStack -from studio_stack import StudioLiveStack +from rbac_stack import EmrEksRbacStack # type: ignore[attr-defined] +from studio_stack import StudioLiveStack # type: ignore[attr-defined] -deployment_name = os.getenv("ADDF_DEPLOYMENT_NAME", "") -module_name = os.getenv("ADDF_MODULE_NAME", "") +project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "") +deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "") +module_name = os.getenv("SEEDFARMER_MODULE_NAME", "") def _param(name: str) -> str: - return f"ADDF_PARAMETER_{name}" + return f"SEEDFARMER_PARAMETER_{name}" vpc_id = os.getenv(_param("VPC_ID")) # required -private_subnet_ids = json.loads(os.getenv(_param("PRIVATE_SUBNET_IDS"))) # required +private_subnet_ids = json.loads(os.getenv(_param("PRIVATE_SUBNET_IDS"))) # type: ignore[arg-type] # required if not vpc_id: raise ValueError("missing input parameter vpc-id") @@ -37,15 +36,16 @@ def _param(name: str) -> str: sso_username = os.getenv(_param("SSO_USERNAME")) # required emr_eks_namespace = os.getenv(_param("EMR_EKS_NAMESPACE"), "emr-studio") -app = App() +app = aws_cdk.App() eks_stack = EmrEksRbacStack( scope=app, - id=f"addf-{deployment_name}-{module_name}-rbac", + id=f"{project_name}-{deployment_name}-{module_name}-rbac", env=aws_cdk.Environment( account=os.environ["CDK_DEFAULT_ACCOUNT"], region=os.environ["CDK_DEFAULT_REGION"], ), + project=project_name, deployment=deployment_name, module=module_name, eks_cluster_name=eks_cluster_name, @@ -57,11 +57,12 @@ def _param(name: str) -> str: emr_studio = StudioLiveStack( app, - id=f"addf-{deployment_name}-{module_name}", + id=f"{project_name}-{deployment_name}-{module_name}", env=aws_cdk.Environment( account=os.environ["CDK_DEFAULT_ACCOUNT"], region=os.environ["CDK_DEFAULT_REGION"], ), + project=project_name, deployment=deployment_name, module=module_name, vpc_id=vpc_id, @@ -73,4 +74,6 @@ def _param(name: str) -> str: sso_username=sso_username, ) +aws_cdk.Aspects.of(app).add(cdk_nag.AwsSolutionsChecks(log_ignores=True)) + app.synth(force=True) diff --git a/modules/beta/emrstudio-on-eks/deployspec.yaml b/modules/beta/emrstudio-on-eks/deployspec.yaml index b75180fa..1d1d8fd2 100644 --- a/modules/beta/emrstudio-on-eks/deployspec.yaml +++ b/modules/beta/emrstudio-on-eks/deployspec.yaml @@ -1,33 +1,33 @@ +publishGenericEnvVariables: true deploy: phases: install: commands: - - npm install -g aws-cdk@2.82.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.155.0 + - pip install -r requirements.txt build: commands: - - aws iam create-service-linked-role --aws-service-name emr-containers.amazonaws.com || true - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-rbac - - export EMR_SERVICE_ACCOUNT_ROLE=arn:aws:iam::${AWS_ACCOUNT_ID}:role/AWSServiceRoleForAmazonEMRContainers - # Assume the EKS_CLUSTER_ADMIN_ROLE and add our new service account role as a user in the aws_auth ConfigMap - - eval $(aws sts assume-role --role-arn ${ADDF_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"') - - | - eksctl get iamidentitymapping --cluster ${ADDF_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} \ - && echo "IAM Identity Mapping already found" \ - || eksctl create iamidentitymapping --cluster ${ADDF_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} --username emr-containers - # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will faile - - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME} - - export ADDF_MODULE_METADATA=$(python -c "import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])") + - aws iam create-service-linked-role --aws-service-name emr-containers.amazonaws.com || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}-rbac + - export EMR_SERVICE_ACCOUNT_ROLE=arn:aws:iam::${AWS_ACCOUNT_ID}:role/AWSServiceRoleForAmazonEMRContainers + # Assume the EKS_CLUSTER_ADMIN_ROLE and add our new service account role as a user in the aws_auth ConfigMap + - eval $(aws sts assume-role --role-arn ${SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"') + - | + eksctl get iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} \ + && echo "IAM Identity Mapping already found" \ + || eksctl create iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} --username emr-containers + # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will faile + - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME} + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - - npm install -g aws-cdk@2.82.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.155.0 + - pip install -r requirements.txt build: commands: - - python cleanup.py ${ADDF_DEPLOYMENT_NAME} ${ADDF_MODULE_NAME} - - cdk destroy --force --app "python app.py" addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME} - - cdk destroy --force --app "python app.py" addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-rbac - + - python cleanup.py ${SEEDFARMER_DEPLOYMENT_NAME} ${SEEDFARMER_MODULE_NAME} + - cdk destroy --force --app "python app.py" ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME} + - cdk destroy --force --app "python app.py" ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}-rbac diff --git a/modules/beta/emrstudio-on-eks/pyproject.toml b/modules/beta/emrstudio-on-eks/pyproject.toml index c40c190b..2440ed76 100644 --- a/modules/beta/emrstudio-on-eks/pyproject.toml +++ b/modules/beta/emrstudio-on-eks/pyproject.toml @@ -24,10 +24,10 @@ fixable = ["ALL"] omit = ["tests/*"] [tool.mypy] -python_version = 3.8 +python_version = "3.8" strict = true ignore_missing_imports = true -allow_untyped_decorators = true +disallow_untyped_decorators = false exclude = "codeseeder.out/|tests/" [tool.pytest.ini_options] diff --git a/modules/beta/emrstudio-on-eks/rbac_stack.py b/modules/beta/emrstudio-on-eks/rbac_stack.py index 83f1c5d3..cfd8a760 100755 --- a/modules/beta/emrstudio-on-eks/rbac_stack.py +++ b/modules/beta/emrstudio-on-eks/rbac_stack.py @@ -6,8 +6,7 @@ import logging from typing import Any, cast -import cdk_nag -from aws_cdk import Aspects, CfnJson, Stack, Tags +from aws_cdk import CfnJson, Stack, Tags from aws_cdk import aws_eks as eks from aws_cdk import aws_iam as iam from aws_cdk.lambda_layer_kubectl_v29 import KubectlV29Layer @@ -28,6 +27,7 @@ def __init__( scope: Construct, id: str, *, + project: str, deployment: str, module: str, eks_cluster_name: str, @@ -40,12 +40,12 @@ def __init__( super().__init__( scope, id, - description="This stack deploys EMR Studio RBAC Configuration for ADDF", + description="This stack deploys EMR Studio RBAC Configuration", **kwargs, ) - Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"addf-{deployment}") + Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"{project}-{deployment}") - dep_mod = f"addf-{deployment}-{module}" + dep_mod = f"{project}-{deployment}-{module}" dep_mod = dep_mod[:27] # Import EKS Cluster @@ -270,8 +270,6 @@ def __init__( ) ) - Aspects.of(self).add(cdk_nag.AwsSolutionsChecks()) - NagSuppressions.add_stack_suppressions( self, apply_to_nested_stacks=True, @@ -284,5 +282,9 @@ def __init__( "id": "AwsSolutions-IAM5", "reason": "Resource access restriced to ADDF resources", }, + { + "id": "AwsSolutions-L1", + "reason": "Not creating the Lambda directly", + }, ], ) diff --git a/modules/beta/emrstudio-on-eks/requirements.in b/modules/beta/emrstudio-on-eks/requirements.in index 790021b8..e4fb132a 100644 --- a/modules/beta/emrstudio-on-eks/requirements.in +++ b/modules/beta/emrstudio-on-eks/requirements.in @@ -1,8 +1,8 @@ -aws-cdk-lib~=2.128.0 +aws-cdk-lib~=2.155.0 aws-cdk.lambda-layer-kubectl-v29==2.0.0 -cdk-nag==2.12.29 -constructs==10.0.91 -boto3~=1.24.0 -pyOpenSSL==24.0.0 -cryptography~=42.0.0 +cdk-nag==2.28.195 +constructs==10.3.0 +boto3~=1.35.11 +pyOpenSSL~=24.2.1 +cryptography~=43.0.1 urllib3>=1.26.19 diff --git a/modules/beta/emrstudio-on-eks/requirements.txt b/modules/beta/emrstudio-on-eks/requirements.txt index 03672912..9d976a83 100644 --- a/modules/beta/emrstudio-on-eks/requirements.txt +++ b/modules/beta/emrstudio-on-eks/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements.txt requirements.in @@ -14,48 +14,49 @@ aws-cdk-asset-kubectl-v20==2.1.2 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.0.3 # via aws-cdk-lib +aws-cdk-cloud-assembly-schema==36.0.24 + # via aws-cdk-lib aws-cdk-lambda-layer-kubectl-v29==2.0.0 # via -r requirements.in -aws-cdk-lib==2.128.0 +aws-cdk-lib==2.155.0 # via # -r requirements.in # aws-cdk-lambda-layer-kubectl-v29 # cdk-nag -boto3==1.24.96 +boto3==1.35.11 # via -r requirements.in -botocore==1.27.96 +botocore==1.35.11 # via # boto3 # s3transfer cattrs==23.2.3 # via jsii -cdk-nag==2.12.29 +cdk-nag==2.28.195 # via -r requirements.in cffi==1.16.0 # via cryptography -constructs==10.0.91 +constructs==10.3.0 # via # -r requirements.in # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib # cdk-nag -cryptography==42.0.8 +cryptography==43.0.1 # via # -r requirements.in # pyopenssl -exceptiongroup==1.2.1 - # via cattrs importlib-resources==6.4.0 # via jsii jmespath==1.0.1 # via # boto3 # botocore -jsii==1.100.0 +jsii==1.103.1 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-cloud-assembly-schema # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib # cdk-nag @@ -65,6 +66,7 @@ publication==0.0.3 # aws-cdk-asset-awscli-v1 # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-cloud-assembly-schema # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib # cdk-nag @@ -72,13 +74,13 @@ publication==0.0.3 # jsii pycparser==2.22 # via cffi -pyopenssl==24.0.0 +pyopenssl==24.2.1 # via -r requirements.in python-dateutil==2.9.0.post0 # via # botocore # jsii -s3transfer==0.6.2 +s3transfer==0.10.2 # via boto3 six==1.16.0 # via python-dateutil @@ -87,13 +89,14 @@ typeguard==2.13.3 # aws-cdk-asset-awscli-v1 # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-cloud-assembly-schema # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib + # cdk-nag + # constructs # jsii typing-extensions==4.12.2 - # via - # cattrs - # jsii + # via jsii urllib3==1.26.19 # via # -r requirements.in diff --git a/modules/beta/emrstudio-on-eks/studio_stack.py b/modules/beta/emrstudio-on-eks/studio_stack.py index 33c243f3..15ebc159 100644 --- a/modules/beta/emrstudio-on-eks/studio_stack.py +++ b/modules/beta/emrstudio-on-eks/studio_stack.py @@ -6,8 +6,7 @@ import random from typing import List, cast -import cdk_nag -from aws_cdk import Aspects, CfnOutput, Stack, Tags +from aws_cdk import CfnOutput, Stack, Tags from aws_cdk import aws_ec2 as ec2 from aws_cdk import aws_emr as emr from aws_cdk import aws_emrcontainers as emrc @@ -30,6 +29,7 @@ def __init__( self, scope: Construct, id: str, + project: str, deployment: str, module: str, vpc_id: str, @@ -41,11 +41,11 @@ def __init__( sso_username: str, **kwargs, ) -> None: - super().__init__(scope, id, description="This stack deploys EMR Studio for ADDF", **kwargs) - dep_mod = f"addf-{deployment}-{module}" + super().__init__(scope, id, description="This stack deploys EMR Studio", **kwargs) + dep_mod = f"{project}-{deployment}-{module}" dep_mod = dep_mod[:27] - Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"addf-{deployment}") + Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"{project}-{deployment}") # EMR virtual cluster self.emr_vc = emrc.CfnVirtualCluster( @@ -454,8 +454,6 @@ def __init__( studio_id=self.studio.attr_studio_id, ) - Aspects.of(self).add(cdk_nag.AwsSolutionsChecks()) - NagSuppressions.add_stack_suppressions( self, apply_to_nested_stacks=True, @@ -468,6 +466,10 @@ def __init__( "id": "AwsSolutions-IAM5", "reason": "Resource access restriced to ADDF resources", }, + { + "id": "AwsSolutions-L1", + "reason": "Not creating the Lambda directly", + }, ], ) diff --git a/modules/beta/emrstudio-on-eks/tests/test_app.py b/modules/beta/emrstudio-on-eks/tests/test_app.py new file mode 100644 index 00000000..fcafa3a8 --- /dev/null +++ b/modules/beta/emrstudio-on-eks/tests/test_app.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import sys +from unittest import mock + +import pytest + + +@pytest.fixture(scope="function", autouse=True) +def stack_defaults(): + with mock.patch.dict(os.environ, {}, clear=True): + os.environ["SEEDFARMER_PROJECT_NAME"] = "test-project" + os.environ["SEEDFARMER_DEPLOYMENT_NAME"] = "test-deployment" + os.environ["SEEDFARMER_MODULE_NAME"] = "test-module" + + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + + os.environ["SEEDFARMER_PARAMETER_VPC_ID"] = "vpc-123" + os.environ["SEEDFARMER_PARAMETER_PRIVATE_SUBNET_IDS"] = json.dumps(["subnet-123", "subnet-456"]) + + os.environ["SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME"] = "test-cluster" + os.environ["SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN"] = "arn:aws:iam::111111111111:role/test-role" + os.environ["SEEDFARMER_PARAMETER_EKS_OIDC_ARN"] = ( + "arn:aws:iam::111111111111:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/XXXXXX" + ) + os.environ["SEEDFARMER_PARAMETER_EKS_OPENID_ISSUER"] = "oidc.eks.us-east-1.amazonaws.com/id/XXXXXX" + os.environ["SEEDFARMER_PARAMETER_ARTIFACT_BUCKET_NAME"] = "test-bucket" + os.environ["SEEDFARMER_PARAMETER_SSO_USERNAME"] = "sso-username" + + # Unload the app import so that subsequent tests don't reuse + if "app" in sys.modules: + del sys.modules["app"] + + yield + + +def test_app(stack_defaults): + import app # noqa: F401 diff --git a/modules/beta/emrstudio-on-eks/tests/test_stack.py b/modules/beta/emrstudio-on-eks/tests/test_stack.py index eb154c97..471718e0 100644 --- a/modules/beta/emrstudio-on-eks/tests/test_stack.py +++ b/modules/beta/emrstudio-on-eks/tests/test_stack.py @@ -1,2 +1,84 @@ -def test_placeholder() -> None: - return None +import aws_cdk as cdk +import cdk_nag +import pytest +from aws_cdk.assertions import Annotations, Match, Template + + +@pytest.fixture(scope="function") +def app() -> cdk.App: + return cdk.App() + + +@pytest.fixture(scope="function") +def rbac_stack(app: cdk.App) -> cdk.Stack: + from rbac_stack import EmrEksRbacStack + + project_name = "test-project" + dep_name = "test-deployment" + mod_name = "test-module" + app_prefix = f"{project_name}-{dep_name}-{mod_name}-rbac" + + return EmrEksRbacStack( + scope=app, + id=app_prefix, + project=project_name, + deployment=dep_name, + module=mod_name, + env=cdk.Environment( + account="111111111111", + region="us-east-1", + ), + eks_cluster_name="test-cluster", + eks_admin_role_arn="arn:aws:iam::111111111111:role/test-role", + eks_oidc_arn="arn:aws:iam::111111111111:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/XXXXXX", + eks_openid_issuer="oidc.eks.us-east-1.amazonaws.com/id/XXXXXX", + emr_namespace="emr-studio", + ) + + +@pytest.fixture(scope="function") +def studio_stack(app: cdk.App) -> cdk.Stack: + from studio_stack import StudioLiveStack + + project_name = "test-project" + dep_name = "test-deployment" + mod_name = "test-module" + app_prefix = f"{project_name}-{dep_name}-{mod_name}" + + return StudioLiveStack( + scope=app, + id=app_prefix, + project=project_name, + deployment=dep_name, + module=mod_name, + env=cdk.Environment( + account="111111111111", + region="us-east-1", + ), + vpc_id="vpc-123", + private_subnet_ids=["subnet12", "subnet34"], + artifact_bucket_name="test-bucket", + eks_cluster_name="test-cluster", + execution_role_arn="arn:aws:iam::111111111111:role/test-role", + emr_namespace="test-namespace", + sso_username="test-user", + ) + + +def test_synthesize_stack(rbac_stack: cdk.Stack, studio_stack: cdk.Stack) -> None: + Template.from_stack(rbac_stack) + Template.from_stack(studio_stack) + + +def test_no_cdk_nag_errors(rbac_stack: cdk.Stack, studio_stack: cdk.Stack) -> None: + cdk.Aspects.of(rbac_stack).add(cdk_nag.AwsSolutionsChecks()) + cdk.Aspects.of(studio_stack).add(cdk_nag.AwsSolutionsChecks()) + + nag_errors = Annotations.from_stack(rbac_stack).find_error( + "*", + Match.string_like_regexp(r"AwsSolutions-.*"), + ) + Annotations.from_stack(studio_stack).find_error( + "*", + Match.string_like_regexp(r"AwsSolutions-.*"), + ) + assert not nag_errors, f"Found {len(nag_errors)} CDK nag errors" diff --git a/modules/integration/eks-to-opensearch/app.py b/modules/integration/eks-to-opensearch/app.py index 8ca81f32..c6fede95 100644 --- a/modules/integration/eks-to-opensearch/app.py +++ b/modules/integration/eks-to-opensearch/app.py @@ -8,12 +8,13 @@ from stack import EksOpenSearchIntegrationStack -deployment_name = os.getenv("ADDF_DEPLOYMENT_NAME", "") -module_name = os.getenv("ADDF_MODULE_NAME", "") +project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "") +deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "") +module_name = os.getenv("SEEDFARMER_MODULE_NAME", "") def _param(name: str) -> str: - return f"ADDF_PARAMETER_{name}" + return f"SEEDFARMER_PARAMETER_{name}" opensearch_sg_id = os.getenv(_param("OPENSEARCH_SG_ID"), "") @@ -27,11 +28,12 @@ def _param(name: str) -> str: stack = EksOpenSearchIntegrationStack( scope=app, - id=f"addf-{deployment_name}-{module_name}", + id=f"{project_name}-{deployment_name}-{module_name}", env=aws_cdk.Environment( account=os.environ["CDK_DEFAULT_ACCOUNT"], region=os.environ["CDK_DEFAULT_REGION"], ), + project=project_name, deployment=deployment_name, module=module_name, opensearch_sg_id=opensearch_sg_id, diff --git a/modules/integration/eks-to-opensearch/deployspec.yaml b/modules/integration/eks-to-opensearch/deployspec.yaml index 22329a07..9d3978cb 100644 --- a/modules/integration/eks-to-opensearch/deployspec.yaml +++ b/modules/integration/eks-to-opensearch/deployspec.yaml @@ -1,21 +1,20 @@ +publishGenericEnvVariables: true deploy: phases: install: commands: - - npm install -g aws-cdk@2.82.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.155.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - - export ADDF_MODULE_METADATA=$(python -c "import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])") + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - - npm install -g aws-cdk@2.82.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.155.0 + - pip install -r requirements.txt build: commands: - - cdk destroy --force --app "python app.py" - - + - cdk destroy --force --app "python app.py" diff --git a/modules/integration/eks-to-opensearch/modulestack.template b/modules/integration/eks-to-opensearch/modulestack.template deleted file mode 100644 index 6421798f..00000000 --- a/modules/integration/eks-to-opensearch/modulestack.template +++ /dev/null @@ -1,23 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 -Description: This stack deploys a Module specific IAM permissions - -Parameters: - DeploymentName: - Type: String - Description: The name of the deployment - ModuleName: - Type: String - Description: The name of the Module - RoleName: - Type: String - Description: The name of the IAM Role - -Resources: - Policy: - Type: 'AWS::IAM::Policy' - Properties: - PolicyDocument: - Statement: - Version: 2012-10-17 - PolicyName: "eks-to-os-modulespecific-policy" - Roles: [!Ref RoleName] diff --git a/modules/integration/eks-to-opensearch/pyproject.toml b/modules/integration/eks-to-opensearch/pyproject.toml index 4b7c2546..4dc63d76 100644 --- a/modules/integration/eks-to-opensearch/pyproject.toml +++ b/modules/integration/eks-to-opensearch/pyproject.toml @@ -24,14 +24,14 @@ fixable = ["ALL"] omit = ["tests/*"] [tool.mypy] -python_version = 3.8 +python_version = "3.8" strict = true ignore_missing_imports = true -allow_untyped_decorators = true -exclude = "codeseeder.out/|example" +disallow_untyped_decorators = false +exclude = "codeseeder.out/|example|tests" [tool.pytest.ini_options] addopts = "-v --cov=. --cov-report term" pythonpath = [ "." -] \ No newline at end of file +] diff --git a/modules/integration/eks-to-opensearch/requirements.in b/modules/integration/eks-to-opensearch/requirements.in index f6053a79..37c2adb1 100644 --- a/modules/integration/eks-to-opensearch/requirements.in +++ b/modules/integration/eks-to-opensearch/requirements.in @@ -1,4 +1,4 @@ -aws-cdk-lib~=2.128.0 -aws-cdk.lambda-layer-kubectl-v29==2.0.0 -cdk-nag==2.12.29 -constructs==10.0.91 +aws-cdk-lib~=2.155.0 +aws-cdk.lambda-layer-kubectl-v29~=2.0.0 +cdk-nag~=2.28.195 +constructs~=10.3.0 diff --git a/modules/integration/eks-to-opensearch/requirements.txt b/modules/integration/eks-to-opensearch/requirements.txt index 29679b3f..24c4e3d5 100644 --- a/modules/integration/eks-to-opensearch/requirements.txt +++ b/modules/integration/eks-to-opensearch/requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # -# pip-compile requirements.in +# pip-compile --output-file=requirements.txt requirements.in # attrs==23.2.0 # via @@ -14,32 +14,33 @@ aws-cdk-asset-kubectl-v20==2.1.2 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.0.3 # via aws-cdk-lib +aws-cdk-cloud-assembly-schema==36.0.24 + # via aws-cdk-lib aws-cdk-lambda-layer-kubectl-v29==2.0.0 # via -r requirements.in -aws-cdk-lib==2.128.0 +aws-cdk-lib==2.155.0 # via # -r requirements.in # aws-cdk-lambda-layer-kubectl-v29 # cdk-nag cattrs==23.2.3 # via jsii -cdk-nag==2.12.29 +cdk-nag==2.28.195 # via -r requirements.in -constructs==10.0.91 +constructs==10.3.0 # via # -r requirements.in # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib # cdk-nag -exceptiongroup==1.2.1 - # via cattrs importlib-resources==6.4.0 # via jsii -jsii==1.99.0 +jsii==1.103.1 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-cloud-assembly-schema # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib # cdk-nag @@ -49,6 +50,7 @@ publication==0.0.3 # aws-cdk-asset-awscli-v1 # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-cloud-assembly-schema # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib # cdk-nag @@ -63,10 +65,11 @@ typeguard==2.13.3 # aws-cdk-asset-awscli-v1 # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-cloud-assembly-schema # aws-cdk-lambda-layer-kubectl-v29 # aws-cdk-lib + # cdk-nag + # constructs # jsii typing-extensions==4.12.1 - # via - # cattrs - # jsii + # via jsii diff --git a/modules/integration/eks-to-opensearch/stack.py b/modules/integration/eks-to-opensearch/stack.py index ff349b01..dae7f058 100755 --- a/modules/integration/eks-to-opensearch/stack.py +++ b/modules/integration/eks-to-opensearch/stack.py @@ -23,6 +23,7 @@ def __init__( scope: Construct, id: str, *, + project: str, deployment: str, module: str, opensearch_sg_id: str, @@ -36,12 +37,12 @@ def __init__( super().__init__( scope, id, - description="This stack integrates EKS Cluster with Opensearch cluster for ADDF", + description="This stack integrates EKS Cluster with Opensearch cluster", **kwargs, ) - Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"addf-{deployment}") + Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"{project}-{deployment}") - dep_mod = f"addf-{deployment}-{module}" + dep_mod = f"{project}-{deployment}-{module}" dep_mod = dep_mod[:27] # Import OpenSearch Domain @@ -108,23 +109,23 @@ def __init__( ) fluentbit_chart.node.add_dependency(fluentbit_service_account) - Aspects.of(self).add(cdk_nag.AwsSolutionsChecks()) + Aspects.of(self).add(cdk_nag.AwsSolutionsChecks(log_ignores=True)) NagSuppressions.add_stack_suppressions( self, apply_to_nested_stacks=True, suppressions=[ NagPackSuppression( - **{ - "id": "AwsSolutions-IAM4", - "reason": "Managed Policies are for service account roles only", - } + id="AwsSolutions-IAM4", + reason="Managed Policies are for service account roles only", ), NagPackSuppression( - **{ - "id": "AwsSolutions-IAM5", - "reason": "Resource access restriced to ADDF resources", - } + id="AwsSolutions-IAM5", + reason="Resource access restriced to ADDF resources", + ), + NagPackSuppression( + id="AwsSolutions-L1", + reason="Not creating the Lambda directly", ), ], ) diff --git a/modules/integration/eks-to-opensearch/tests/test_app.py b/modules/integration/eks-to-opensearch/tests/test_app.py new file mode 100644 index 00000000..7003eaf1 --- /dev/null +++ b/modules/integration/eks-to-opensearch/tests/test_app.py @@ -0,0 +1,38 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +from unittest import mock + +import pytest + + +@pytest.fixture(scope="function", autouse=True) +def stack_defaults(): + with mock.patch.dict(os.environ, {}, clear=True): + os.environ["SEEDFARMER_PROJECT_NAME"] = "test-project" + os.environ["SEEDFARMER_DEPLOYMENT_NAME"] = "test-deployment" + os.environ["SEEDFARMER_MODULE_NAME"] = "test-module" + + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + + os.environ["SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME"] = "test-cluster" + os.environ["SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN"] = "arn:aws:iam::111111111111:role/test-role" + os.environ["SEEDFARMER_PARAMETER_EKS_CLUSTER_SG_ID"] = "sg-xxx" + os.environ["SEEDFARMER_PARAMETER_OPENSEARCH_SG_ID"] = "sg-yyy" + os.environ["SEEDFARMER_PARAMETER_OPENSEARCH_DOMAIN_ENDPOINT"] = "xxxxxxx.us-east-1.es.amazonaws.com" + os.environ["SEEDFARMER_PARAMETER_EKS_OIDC_ARN"] = ( + "arn:aws:iam::111111111111:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/XXXXXX" + ) + + # Unload the app import so that subsequent tests don't reuse + if "app" in sys.modules: + del sys.modules["app"] + + yield + + +def test_app(stack_defaults): + import app # noqa: F401 diff --git a/modules/integration/eks-to-opensearch/tests/test_stack.py b/modules/integration/eks-to-opensearch/tests/test_stack.py index eb154c97..918ea6bf 100644 --- a/modules/integration/eks-to-opensearch/tests/test_stack.py +++ b/modules/integration/eks-to-opensearch/tests/test_stack.py @@ -1,2 +1,51 @@ -def test_placeholder() -> None: - return None +import aws_cdk as cdk +import cdk_nag +import pytest +from aws_cdk.assertions import Annotations, Match, Template + + +@pytest.fixture(scope="function") +def app() -> cdk.App: + return cdk.App() + + +@pytest.fixture(scope="function") +def stack(app: cdk.App) -> cdk.Stack: + from stack import EksOpenSearchIntegrationStack + + project_name = "test-project" + dep_name = "test-deployment" + mod_name = "test-module" + app_prefix = f"{project_name}-{dep_name}-{mod_name}" + + return EksOpenSearchIntegrationStack( + scope=app, + id=app_prefix, + project=project_name, + deployment=dep_name, + module=mod_name, + env=cdk.Environment( + account="111111111111", + region="us-east-1", + ), + opensearch_sg_id="sg-xxx", + opensearch_domain_endpoint="xxxxxxx.us-east-1.es.amazonaws.com", + eks_cluster_name="test-cluster", + eks_admin_role_arn="arn:aws:iam::111111111111:role/test-role", + eks_cluster_sg_id="sg-yyy", + eks_oidc_arn="arn:aws:iam::111111111111:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/XXXXXX", + ) + + +def test_synthesize_stack(stack: cdk.Stack) -> None: + Template.from_stack(stack) + + +def test_no_cdk_nag_errors(stack: cdk.Stack) -> None: + cdk.Aspects.of(stack).add(cdk_nag.AwsSolutionsChecks()) + + nag_errors = Annotations.from_stack(stack).find_error( + "*", + Match.string_like_regexp(r"AwsSolutions-.*"), + ) + assert not nag_errors, f"Found {len(nag_errors)} CDK nag errors"