Merge pull request #167 from brooklyn-data/add_support_for_dbt_spark

Add support for dbt-spark adapter
brooklyn-data · Aug 11, 2022 · e368c93 · e368c93
2 parents f554a04 + 5c2faaa
commit e368c93
Show file tree

Hide file tree

Showing 20 changed files with 46 additions and 17 deletions.
diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # dbt Artifacts Package
 This package builds a mart of tables and views describing the project it is installed in. In pre V1 versions of the package, the artifacts dbt produces were uploaded to the warehouse, hence the name of the package. That's no longer the case, but the name has stuck!
 
-The package currently supports Databricks and Snowflake.
+The package currently supports Databricks, Spark and Snowflake adapters.
 
 Models included:
 
@@ -28,7 +28,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/
 ```
 packages:
   - package: brooklyn-data/dbt_artifacts
-    version: 1.0.0
+    version: 1.1.0
 ```
 
 2. Run `dbt deps` to install the package
@@ -146,6 +146,8 @@ tox -e integration_snowflake # For the Snowflake tests
 tox -e integration_databricks # For the Databricks tests
 ```
 
+The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac, DBT_ENV_SPARK_DRIVER_PATH should be set to `/Library/simba/spark/lib/libsparkodbc_sbu.dylib`. Spark tests have not yet been added to the integration tests.
+
 ### SQLFluff
 
 We use SQLFluff to keep SQL style consistent. A GitHub action automatically tests pull requests and adds annotations where there are failures. SQLFluff can also be run locally with `tox`. To install tox, we recommend using `pipx`.

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'dbt_artifacts'
-version: '1.0.0'
+version: '1.1.0'
 config-version: 2
 require-dbt-version: ">=1.0.0"
 profile: "dbt_artifacts"

diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh
@@ -13,6 +13,8 @@ export DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE=
 export DBT_ENV_SECRET_DATABRICKS_HOST=
 export DBT_ENV_SECRET_DATABRICKS_HTTP_PATH=
 export DBT_ENV_SECRET_DATABRICKS_TOKEN=
+export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dylib on a Mac
+export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path
 
 # dbt environment variables, change these
 export DBT_CLOUD_PROJECT_ID=

diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml
@@ -25,3 +25,12 @@ dbt_artifacts:
       http_path: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_HTTP_PATH') }}"
       token: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_TOKEN') }}"
       threads: 8
+    spark:
+      type: spark
+      method: odbc
+      schema: dbt_artifacts_test_commit_spark_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }}
+      host: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_HOST') }}"
+      driver: "{{ env_var('DBT_ENV_SPARK_DRIVER_PATH') }}"
+      endpoint: "{{ env_var('DBT_ENV_SPARK_ENDPOINT') }}"
+      token: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_TOKEN') }}"
+      threads: 8
diff --git a/integration_test_project/snapshots/snapshot.sql b/integration_test_project/snapshots/snapshot.sql
@@ -1,10 +1,14 @@
+-- Exclude this from Spark tests as it complains:
+-- Snapshot functionality requires file_format be set to 'delta' or 'hudi'
+-- It's not possible to dynamically change the file_format depending on target
 {% snapshot my_snapshot %}
 {{
     config(
         strategy='check',
         unique_key='id',
         target_schema='snapshot',
         check_cols=['id', 'fruit'],
+        tags="snapshot"
     )
 }}
 

diff --git a/macros/column_identifier.sql b/macros/column_identifier.sql
@@ -10,6 +10,6 @@
     ${{ column_index }}
 {%- endmacro %}
 
-{% macro databricks__column_identifier(column_index) -%}
+{% macro spark__column_identifier(column_index) -%}
     col{{ column_index }}
 {%- endmacro %}
diff --git a/macros/create_exposures_table_if_not_exists.sql b/macros/create_exposures_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_invocations_table_if_not_exists.sql b/macros/create_invocations_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         dbt_version STRING,

diff --git a/macros/create_model_executions_table_if_not_exists.sql b/macros/create_model_executions_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_models_table_if_not_exists.sql b/macros/create_models_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_seed_executions_table_if_not_exists.sql b/macros/create_seed_executions_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_seeds_table_if_not_exists.sql b/macros/create_seeds_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_snapshot_executions_table_if_not_exists.sql b/macros/create_snapshot_executions_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create or replace table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_snapshots_table_if_not_exists.sql b/macros/create_snapshots_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_sources_table_if_not_exists.sql b/macros/create_sources_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_test_executions_table_if_not_exists.sql b/macros/create_test_executions_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/create_tests_table_if_not_exists.sql b/macros/create_tests_table_if_not_exists.sql
@@ -16,7 +16,7 @@
 
 {%- endmacro %}
 
-{% macro databricks__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+{% macro spark__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{schema_name}}.{{table_name}} (
         command_invocation_id STRING,
         node_id STRING,

diff --git a/macros/insert_into_metadata_table.sql b/macros/insert_into_metadata_table.sql
@@ -2,7 +2,7 @@
     {{ return(adapter.dispatch('insert_into_metadata_table', 'dbt_artifacts')(database_name, schema_name, table_name, content)) }}
 {%- endmacro %}
 
-{% macro databricks__insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
+{% macro spark__insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
     {% set insert_into_table_query %}
     insert into {{ schema_name }}.{{ table_name }}
     {{ content }}

diff --git a/tox.ini b/tox.ini
@@ -84,6 +84,8 @@ passenv =
     DBT_ENV_SECRET_DATABRICKS_HOST
     DBT_ENV_SECRET_DATABRICKS_HTTP_PATH
     DBT_ENV_SECRET_DATABRICKS_TOKEN
+    DBT_ENV_SPARK_DRIVER_PATH
+    DBT_ENV_SPARK_ENDPOINT
     DBT_CLOUD_PROJECT_ID
     DBT_CLOUD_JOB_ID
     DBT_CLOUD_RUN_ID
@@ -128,3 +130,13 @@ commands =
     dbt run-operation create_dbt_artifacts_tables --target databricks
     dbt build --exclude dbt_artifacts --target databricks
     dbt build -s dbt_artifacts --target databricks
+
+[testenv:integration_spark]
+changedir = integration_test_project
+deps = dbt-spark[ODBC]~=1.1.0
+commands =
+    dbt deps
+    dbt run-operation create_dbt_artifacts_tables --target spark
+    dbt build --exclude dbt_artifacts tag:snapshot --target spark
+    dbt build -s dbt_artifacts --target spark
+