fix: Update dependencies for eks-to-opensearch and `emrstudio-to-ek…

…s` (#584)
awslabs · Sep 4, 2024 · 33aea27 · 33aea27
1 parent 055e45a
commit 33aea27
Show file tree

Hide file tree

Showing 18 changed files with 345 additions and 142 deletions.
diff --git a/modules/beta/emrstudio-on-eks/app.py b/modules/beta/emrstudio-on-eks/app.py
@@ -1,27 +1,26 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# type: ignore
-
 import json
 import os
 
 import aws_cdk
-from aws_cdk import App
+import cdk_nag
 
-from rbac_stack import EmrEksRbacStack
-from studio_stack import StudioLiveStack
+from rbac_stack import EmrEksRbacStack  # type: ignore[attr-defined]
+from studio_stack import StudioLiveStack  # type: ignore[attr-defined]
 
-deployment_name = os.getenv("ADDF_DEPLOYMENT_NAME", "")
-module_name = os.getenv("ADDF_MODULE_NAME", "")
+project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "")
+deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "")
+module_name = os.getenv("SEEDFARMER_MODULE_NAME", "")
 
 
 def _param(name: str) -> str:
-    return f"ADDF_PARAMETER_{name}"
+    return f"SEEDFARMER_PARAMETER_{name}"
 
 
 vpc_id = os.getenv(_param("VPC_ID"))  # required
-private_subnet_ids = json.loads(os.getenv(_param("PRIVATE_SUBNET_IDS")))  # required
+private_subnet_ids = json.loads(os.getenv(_param("PRIVATE_SUBNET_IDS")))  # type: ignore[arg-type]  # required
 
 if not vpc_id:
     raise ValueError("missing input parameter vpc-id")
@@ -37,15 +36,16 @@ def _param(name: str) -> str:
 sso_username = os.getenv(_param("SSO_USERNAME"))  # required
 emr_eks_namespace = os.getenv(_param("EMR_EKS_NAMESPACE"), "emr-studio")
 
-app = App()
+app = aws_cdk.App()
 
 eks_stack = EmrEksRbacStack(
     scope=app,
-    id=f"addf-{deployment_name}-{module_name}-rbac",
+    id=f"{project_name}-{deployment_name}-{module_name}-rbac",
     env=aws_cdk.Environment(
         account=os.environ["CDK_DEFAULT_ACCOUNT"],
         region=os.environ["CDK_DEFAULT_REGION"],
     ),
+    project=project_name,
     deployment=deployment_name,
     module=module_name,
     eks_cluster_name=eks_cluster_name,
@@ -57,11 +57,12 @@ def _param(name: str) -> str:
 
 emr_studio = StudioLiveStack(
     app,
-    id=f"addf-{deployment_name}-{module_name}",
+    id=f"{project_name}-{deployment_name}-{module_name}",
     env=aws_cdk.Environment(
         account=os.environ["CDK_DEFAULT_ACCOUNT"],
         region=os.environ["CDK_DEFAULT_REGION"],
     ),
+    project=project_name,
     deployment=deployment_name,
     module=module_name,
     vpc_id=vpc_id,
@@ -73,4 +74,6 @@ def _param(name: str) -> str:
     sso_username=sso_username,
 )
 
+aws_cdk.Aspects.of(app).add(cdk_nag.AwsSolutionsChecks(log_ignores=True))
+
 app.synth(force=True)
diff --git a/modules/beta/emrstudio-on-eks/deployspec.yaml b/modules/beta/emrstudio-on-eks/deployspec.yaml
@@ -1,33 +1,33 @@
+publishGenericEnvVariables: true
 deploy:
   phases:
     install:
       commands:
-      - npm install -g aws-cdk@2.82.0
-      - pip install -r requirements.txt
+        - npm install -g aws-cdk@2.155.0
+        - pip install -r requirements.txt
     build:
       commands:
-      - aws iam create-service-linked-role --aws-service-name emr-containers.amazonaws.com || true
-      - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-rbac
-      - export EMR_SERVICE_ACCOUNT_ROLE=arn:aws:iam::${AWS_ACCOUNT_ID}:role/AWSServiceRoleForAmazonEMRContainers
-      # Assume the EKS_CLUSTER_ADMIN_ROLE and add our new service account role as a user in the aws_auth ConfigMap
-      - eval $(aws sts assume-role --role-arn ${ADDF_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"')
-      - |
-        eksctl get iamidentitymapping --cluster ${ADDF_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} \
-        && echo "IAM Identity Mapping already found" \
-        || eksctl create iamidentitymapping --cluster ${ADDF_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} --username emr-containers
-      # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will faile
-      - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN
-      - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}
-      - export ADDF_MODULE_METADATA=$(python -c "import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])")
+        - aws iam create-service-linked-role --aws-service-name emr-containers.amazonaws.com || true
+        - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}-rbac
+        - export EMR_SERVICE_ACCOUNT_ROLE=arn:aws:iam::${AWS_ACCOUNT_ID}:role/AWSServiceRoleForAmazonEMRContainers
+        # Assume the EKS_CLUSTER_ADMIN_ROLE and add our new service account role as a user in the aws_auth ConfigMap
+        - eval $(aws sts assume-role --role-arn ${SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"')
+        - |
+          eksctl get iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} \
+          && echo "IAM Identity Mapping already found" \
+          || eksctl create iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EMR_SERVICE_ACCOUNT_ROLE} --username emr-containers
+        # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will faile
+        - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN
+        - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}
+        - seedfarmer metadata convert -f cdk-exports.json || true
 destroy:
   phases:
     install:
       commands:
-      - npm install -g aws-cdk@2.82.0
-      - pip install -r requirements.txt
+        - npm install -g aws-cdk@2.155.0
+        - pip install -r requirements.txt
     build:
       commands:
-      - python cleanup.py ${ADDF_DEPLOYMENT_NAME} ${ADDF_MODULE_NAME}
-      - cdk destroy --force --app "python app.py" addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}
-      - cdk destroy --force --app "python app.py" addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-rbac
-
+        - python cleanup.py ${SEEDFARMER_DEPLOYMENT_NAME} ${SEEDFARMER_MODULE_NAME}
+        - cdk destroy --force --app "python app.py" ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}
+        - cdk destroy --force --app "python app.py" ${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}-rbac
diff --git a/modules/beta/emrstudio-on-eks/pyproject.toml b/modules/beta/emrstudio-on-eks/pyproject.toml
@@ -24,10 +24,10 @@ fixable = ["ALL"]
 omit = ["tests/*"]
 
 [tool.mypy]
-python_version = 3.8
+python_version = "3.8"
 strict = true
 ignore_missing_imports = true
-allow_untyped_decorators = true
+disallow_untyped_decorators = false
 exclude = "codeseeder.out/|tests/"
 
 [tool.pytest.ini_options]

diff --git a/modules/beta/emrstudio-on-eks/rbac_stack.py b/modules/beta/emrstudio-on-eks/rbac_stack.py
@@ -6,8 +6,7 @@
 import logging
 from typing import Any, cast
 
-import cdk_nag
-from aws_cdk import Aspects, CfnJson, Stack, Tags
+from aws_cdk import CfnJson, Stack, Tags
 from aws_cdk import aws_eks as eks
 from aws_cdk import aws_iam as iam
 from aws_cdk.lambda_layer_kubectl_v29 import KubectlV29Layer
@@ -28,6 +27,7 @@ def __init__(
         scope: Construct,
         id: str,
         *,
+        project: str,
         deployment: str,
         module: str,
         eks_cluster_name: str,
@@ -40,12 +40,12 @@ def __init__(
         super().__init__(
             scope,
             id,
-            description="This stack deploys EMR Studio RBAC Configuration for ADDF",
+            description="This stack deploys EMR Studio RBAC Configuration",
             **kwargs,
         )
-        Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"addf-{deployment}")
+        Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"{project}-{deployment}")
 
-        dep_mod = f"addf-{deployment}-{module}"
+        dep_mod = f"{project}-{deployment}-{module}"
         dep_mod = dep_mod[:27]
 
         # Import EKS Cluster
@@ -270,8 +270,6 @@ def __init__(
             )
         )
 
-        Aspects.of(self).add(cdk_nag.AwsSolutionsChecks())
-
         NagSuppressions.add_stack_suppressions(
             self,
             apply_to_nested_stacks=True,
@@ -284,5 +282,9 @@ def __init__(
                     "id": "AwsSolutions-IAM5",
                     "reason": "Resource access restriced to ADDF resources",
                 },
+                {
+                    "id": "AwsSolutions-L1",
+                    "reason": "Not creating the Lambda directly",
+                },
             ],
         )
diff --git a/modules/beta/emrstudio-on-eks/requirements.in b/modules/beta/emrstudio-on-eks/requirements.in
@@ -1,8 +1,8 @@
-aws-cdk-lib~=2.128.0
+aws-cdk-lib~=2.155.0
 aws-cdk.lambda-layer-kubectl-v29==2.0.0
-cdk-nag==2.12.29
-constructs==10.0.91
-boto3~=1.24.0
-pyOpenSSL==24.0.0
-cryptography~=42.0.0
+cdk-nag==2.28.195
+constructs==10.3.0
+boto3~=1.35.11
+pyOpenSSL~=24.2.1
+cryptography~=43.0.1
 urllib3>=1.26.19
diff --git a/modules/beta/emrstudio-on-eks/requirements.txt b/modules/beta/emrstudio-on-eks/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile --output-file=requirements.txt requirements.in
@@ -14,48 +14,49 @@ aws-cdk-asset-kubectl-v20==2.1.2
     # via aws-cdk-lib
 aws-cdk-asset-node-proxy-agent-v6==2.0.3
     # via aws-cdk-lib
+aws-cdk-cloud-assembly-schema==36.0.24
+    # via aws-cdk-lib
 aws-cdk-lambda-layer-kubectl-v29==2.0.0
     # via -r requirements.in
-aws-cdk-lib==2.128.0
+aws-cdk-lib==2.155.0
     # via
     #   -r requirements.in
     #   aws-cdk-lambda-layer-kubectl-v29
     #   cdk-nag
-boto3==1.24.96
+boto3==1.35.11
     # via -r requirements.in
-botocore==1.27.96
+botocore==1.35.11
     # via
     #   boto3
     #   s3transfer
 cattrs==23.2.3
     # via jsii
-cdk-nag==2.12.29
+cdk-nag==2.28.195
     # via -r requirements.in
 cffi==1.16.0
     # via cryptography
-constructs==10.0.91
+constructs==10.3.0
     # via
     #   -r requirements.in
     #   aws-cdk-lambda-layer-kubectl-v29
     #   aws-cdk-lib
     #   cdk-nag
-cryptography==42.0.8
+cryptography==43.0.1
     # via
     #   -r requirements.in
     #   pyopenssl
-exceptiongroup==1.2.1
-    # via cattrs
 importlib-resources==6.4.0
     # via jsii
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jsii==1.100.0
+jsii==1.103.1
     # via
     #   aws-cdk-asset-awscli-v1
     #   aws-cdk-asset-kubectl-v20
     #   aws-cdk-asset-node-proxy-agent-v6
+    #   aws-cdk-cloud-assembly-schema
     #   aws-cdk-lambda-layer-kubectl-v29
     #   aws-cdk-lib
     #   cdk-nag
@@ -65,20 +66,21 @@ publication==0.0.3
     #   aws-cdk-asset-awscli-v1
     #   aws-cdk-asset-kubectl-v20
     #   aws-cdk-asset-node-proxy-agent-v6
+    #   aws-cdk-cloud-assembly-schema
     #   aws-cdk-lambda-layer-kubectl-v29
     #   aws-cdk-lib
     #   cdk-nag
     #   constructs
     #   jsii
 pycparser==2.22
     # via cffi
-pyopenssl==24.0.0
+pyopenssl==24.2.1
     # via -r requirements.in
 python-dateutil==2.9.0.post0
     # via
     #   botocore
     #   jsii
-s3transfer==0.6.2
+s3transfer==0.10.2
     # via boto3
 six==1.16.0
     # via python-dateutil
@@ -87,13 +89,14 @@ typeguard==2.13.3
     #   aws-cdk-asset-awscli-v1
     #   aws-cdk-asset-kubectl-v20
     #   aws-cdk-asset-node-proxy-agent-v6
+    #   aws-cdk-cloud-assembly-schema
     #   aws-cdk-lambda-layer-kubectl-v29
     #   aws-cdk-lib
+    #   cdk-nag
+    #   constructs
     #   jsii
 typing-extensions==4.12.2
-    # via
-    #   cattrs
-    #   jsii
+    # via jsii
 urllib3==1.26.19
     # via
     #   -r requirements.in

diff --git a/modules/beta/emrstudio-on-eks/studio_stack.py b/modules/beta/emrstudio-on-eks/studio_stack.py
@@ -6,8 +6,7 @@
 import random
 from typing import List, cast
 
-import cdk_nag
-from aws_cdk import Aspects, CfnOutput, Stack, Tags
+from aws_cdk import CfnOutput, Stack, Tags
 from aws_cdk import aws_ec2 as ec2
 from aws_cdk import aws_emr as emr
 from aws_cdk import aws_emrcontainers as emrc
@@ -30,6 +29,7 @@ def __init__(
         self,
         scope: Construct,
         id: str,
+        project: str,
         deployment: str,
         module: str,
         vpc_id: str,
@@ -41,11 +41,11 @@ def __init__(
         sso_username: str,
         **kwargs,
     ) -> None:
-        super().__init__(scope, id, description="This stack deploys EMR Studio for ADDF", **kwargs)
-        dep_mod = f"addf-{deployment}-{module}"
+        super().__init__(scope, id, description="This stack deploys EMR Studio", **kwargs)
+        dep_mod = f"{project}-{deployment}-{module}"
         dep_mod = dep_mod[:27]
 
-        Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"addf-{deployment}")
+        Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=f"{project}-{deployment}")
 
         # EMR virtual cluster
         self.emr_vc = emrc.CfnVirtualCluster(
@@ -454,8 +454,6 @@ def __init__(
             studio_id=self.studio.attr_studio_id,
         )
 
-        Aspects.of(self).add(cdk_nag.AwsSolutionsChecks())
-
         NagSuppressions.add_stack_suppressions(
             self,
             apply_to_nested_stacks=True,
@@ -468,6 +466,10 @@ def __init__(
                     "id": "AwsSolutions-IAM5",
                     "reason": "Resource access restriced to ADDF resources",
                 },
+                {
+                    "id": "AwsSolutions-L1",
+                    "reason": "Not creating the Lambda directly",
+                },
             ],
         )