feat: Add SFN driven rosbag-image-pipeline module

manifests/sfn-example/aws-analysis-modules.yaml

@@ -0,0 +1,143 @@
+name: rip
+path: modules/analysis/rosbag-image-pipeline-sfn
+parameters:
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: VpcId
+  - name: private-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: PrivateSubnetIds
+  - name: image-topics
+    value:
+      - /flir_adk/rgb_front_left/image_raw
+      - /flir_adk/rgb_front_right/image_raw
+  - name: sensor-topics
+    value:
+      - /vehicle/gps/fix
+      - /vehicle/gps/time
+      - /vehicle/gps/vel
+      - /imu_raw
+  - name: desired-encoding
+    value: bgr8
+  - name: lane-detection-job-concurrency
+    value: 20
+  - name: lane-detection-instance-type
+    value: ml.m5.2xlarge
+  - name: lane-detection-image-uri
+    valueFrom:
+      moduleMetadata:
+        group: docker-images
+        name: lane-detection
+        key: ImageUri
+  - name: lane-detection-iam-role
+    valueFrom:
+      moduleMetadata:
+        group: docker-images
+        name: lane-detection
+        key: ExecutionRole
+  - name: object-detection-job-concurrency
+    value: 30
+  - name: object-detection-instance-type
+    value: ml.m5.xlarge
+  - name: object-detection-image-uri
+    valueFrom:
+      moduleMetadata:
+        group: docker-images
+        name: object-detection
+        key: ImageUri
+  - name: object-detection-iam-role
+    valueFrom:
+      moduleMetadata:
+        group: docker-images
+        name: object-detection
+        key: ExecutionRole
+  - name: parquet-batch-job-def-arn
+    valueFrom:
+      moduleMetadata:
+        group: docker-images
+        name: ros-to-parquet
+        key: JobDefinitionArn
+  - name: png-batch-job-def-arn
+    valueFrom:
+      moduleMetadata:
+        group: docker-images
+        name: ros-to-png
+        key: JobDefinitionArn
+  - name: rosbag-scene-metadata-table
+    valueFrom:
+      moduleMetadata:
+        group: core
+        name: metadata-storage
+        key: RosbagSceneMetadataTable
+  - name: source-bucket
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: RawBucketName
+  - name: intermediate-bucket
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: IntermediateBucketName
+  - name: logs-bucket-name
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: LogsBucketName
+  - name: artifacts-bucket-name
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: ArtifactsBucketName
+  - name: full-access-policy-arn
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: FullAccessPolicyArn
+  - name: on-demand-job-queue-arn
+    valueFrom:
+      moduleMetadata:
+        group: core
+        name: batch-compute
+        key: OnDemandJobQueueArn
+  - name: spot-job-queue-arn
+    valueFrom:
+      moduleMetadata:
+        group: core
+        name: batch-compute
+        key: SpotJobQueueArn
+  - name: fargate-job-queue-arn
+    valueFrom:
+      moduleMetadata:
+        group: core
+        name: batch-compute
+        key: FargateJobQueueArn
+  - name: emr-job-exec-role
+    valueFrom:
+      moduleMetadata:
+        group: core
+        name: emr-serverless
+        key: EmrJobExecutionRoleArn
+  - name: emr-app-id
+    valueFrom:
+      moduleMetadata:
+        group: core
+        name: emr-serverless
+        key: EmrApplicationId
+  - name: rosbag-scene-metadata-table
+    valueFrom:
+      moduleMetadata:
+        group: core
+        name: metadata-storage
+        key: RosbagSceneMetadataTable

manifests/sfn-example/core-modules.yaml

@@ -0,0 +1,91 @@
+name: batch-compute
+path: git::https://github.com/awslabs/idf-modules.git//modules/compute/aws-batch/?ref=main
+parameters:
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: VpcId
+  - name: private-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: PrivateSubnetIds
+  - name: batch-compute
+    value:
+      batch_compute_config:
+        - env_name: ng1
+          compute_type: ON_DEMAND
+          max_vcpus: 4800
+          desired_vcpus: 0
+          order: 1
+          instance_types:
+             - "m5.xlarge"
+        - env_name: ng2
+          max_vcpus: 4800
+          desired_vcpus: 0
+          compute_type: SPOT
+          order: 1
+          # instance_types: #if not set, the code defaults to "optimal"
+          #   - "m5.large"
+        - env_name: ng3
+          max_vcpus: 4800
+          desired_vcpus: 0
+          compute_type: FARGATE
+          order: 1
+---
+name: metadata-storage
+path: git::https://github.com/awslabs/autonomous-driving-data-framework.git//modules/core/metadata-storage/?ref=main
+parameters:
+  - name: glue-db-suffix
+    value: vsidata
+  - name: rosbag-bagfile-table-suffix
+    value: Rosbag-BagFile-Metadata
+  - name: rosbag-scene-table-suffix
+    value: Rosbag-Scene-Metadata
+---
+name: opensearch
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/opensearch/?ref=main
+parameters:
+  - name: retention-type
+    value: DESTROY
+  - name: opensearch_data_nodes
+    value: 1
+  - name: opensearch_data_nodes_instance_type
+    value: r6g.large.search
+  - name: opensearch_master_nodes
+    value: 0
+  - name: opensearch_master_nodes_instance_type
+    value: r6g.large.search
+  - name: opensearch_ebs_volume_size
+    value: 50
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: VpcId
+  - name: private-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: PrivateSubnetIds
+---
+name: emr-serverless
+path: git::https://github.com/awslabs/idf-modules.git//modules/compute/emr-serverless/?ref=main
+parameters:
+  - name: private-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: PrivateSubnetIds
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: networking
+        key: VpcId

manifests/sfn-example/deployment.yaml

@@ -0,0 +1,21 @@
+name: sfn-example
+toolchainRegion: us-west-2
+forceDependencyRedeploy: True
+groups:
+  - name: optionals
+    path: manifests/sfn-example/optional-modules.yaml
+  - name: core
+    path: manifests/sfn-example/core-modules.yaml
+  - name: docker-images
+    path: manifests/sfn-example/docker-image-modules.yaml
+  - name: analysis
+    path: manifests/sfn-example/aws-analysis-modules.yaml
+targetAccountMappings:
+  - alias: primary
+    accountId: 123456789101
+    default: true
+    parametersGlobal:
+      dockerCredentialsSecret: aws-addf-docker-credentials
+    regionMappings:
+      - region: us-west-2
+        default: true

manifests/sfn-example/docker-image-modules.yaml

@@ -0,0 +1,71 @@
+name: ros-to-parquet
+path: git::https://github.com/awslabs/autonomous-driving-data-framework.git//modules/sensor-extraction/ros-to-parquet/?ref=main
+parameters:
+  - name: platform
+    value: FARGATE
+  - name: retries
+    value: 1
+  - name: timeout-seconds
+    value: 1800
+  - name: vcpus
+    value: 2
+  - name: memory-mib
+    value: 8192
+  - name: removal-policy
+    value: DESTROY
+  - name: full-access-policy-arn
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: FullAccessPolicyArn
+---
+name: ros-to-png
+path: git::https://github.com/awslabs/autonomous-driving-data-framework.git//modules/sensor-extraction/ros-to-png/?ref=main
+parameters:
+  - name: platform
+    value: FARGATE
+  - name: retries
+    value: 1
+  - name: timeout-seconds
+    value: 1800
+  - name: vcpus
+    value: 2
+  - name: memory-mib
+    value: 8192
+  - name: resized-width
+    value: 1280
+  - name: resized-height
+    value: 720
+  - name: removal-policy
+    value: DESTROY
+  - name: full-access-policy-arn
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: FullAccessPolicyArn
+---
+name: object-detection
+path: git::https://github.com/awslabs/autonomous-driving-data-framework.git//modules/post-processing/yolo-object-detection/?ref=main
+parameters:
+  - name: removal-policy
+    value: DESTROY
+  - name: full-access-policy-arn
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: FullAccessPolicyArn
+---
+name: lane-detection
+path: git::https://github.com/awslabs/autonomous-driving-data-framework.git//modules/post-processing/yolop-lane-detection/?ref=main
+parameters:
+  - name: removal-policy
+    value: DESTROY
+  - name: full-access-policy-arn
+    valueFrom:
+      moduleMetadata:
+        group: optionals
+        name: datalake-buckets
+        key: FullAccessPolicyArn

manifests/sfn-example/optional-modules.yaml

@@ -0,0 +1,13 @@
+name: networking
+path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk/?ref=main
+parameters:
+  - name: internet-accessible
+    value: true
+---
+name: datalake-buckets
+path: git::https://github.com/awslabs/autonomous-driving-data-framework.git//modules/optionals/datalake-buckets/?ref=main
+parameters:
+  - name: encryption-type
+    value: SSE
+  - name: retention-policy
+    value: DESTROY