From 786cefd4a6e66e802340bbbe9f90f579a79cd307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=9E=B8?= Date: Thu, 8 Aug 2024 17:19:17 +0800 Subject: [PATCH] Support serde properties after 'store by' in hive. --- .../java/com/alibaba/druid/sql/SQLUtils.java | 11 -- .../hive/parser/HiveCreateTableParser.java | 49 ++--- .../hive/visitor/HiveOutputVisitor.java | 170 ++++++++++++++++++ .../sql/repository/SchemaRepository.java | 11 -- .../SchemaResolveVisitorFactory.java | 3 +- .../sql/visitor/SQLASTOutputVisitor.java | 163 ----------------- .../druid/sql/visitor/SQLASTVisitor.java | 8 - .../druid/sql/visitor/SchemaStatVisitor.java | 6 - core/src/test/resources/bvt/parser/hive/1.txt | 23 +++ 9 files changed, 221 insertions(+), 223 deletions(-) diff --git a/core/src/main/java/com/alibaba/druid/sql/SQLUtils.java b/core/src/main/java/com/alibaba/druid/sql/SQLUtils.java index 4d3bdca239..c0b20985a8 100644 --- a/core/src/main/java/com/alibaba/druid/sql/SQLUtils.java +++ b/core/src/main/java/com/alibaba/druid/sql/SQLUtils.java @@ -29,7 +29,6 @@ import com.alibaba.druid.sql.dialect.h2.visitor.H2SchemaStatVisitor; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitorAdapter; import com.alibaba.druid.sql.dialect.hive.visitor.HiveOutputVisitor; import com.alibaba.druid.sql.dialect.hive.visitor.HiveSchemaStatVisitor; @@ -1572,11 +1571,6 @@ public boolean visit(SQLCreateTableStatement x) { return false; } - @Override - public boolean visit(HiveCreateTableStatement x) { - return false; - } - @Override public boolean visit(OdpsCreateTableStatement x) { return false; @@ -1629,11 +1623,6 @@ public boolean visit(SQLCreateTableStatement x) { return false; } - @Override - public boolean visit(HiveCreateTableStatement x) { - return false; - } - @Override public boolean visit(SQLInsertStatement x) { if (filter == null || filter.test(x)) { diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java b/core/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java index e3a8ab3b7c..5b3a3e4257 100644 --- a/core/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java @@ -100,6 +100,30 @@ protected void createTableQuery(SQLCreateTableStatement stmt) { } } + protected void parseCreateTableWithSerderPropertie(HiveCreateTableStatement stmt) { + if (lexer.token() == Token.WITH) { + lexer.nextToken(); + acceptIdentifier("SERDEPROPERTIES"); + + accept(Token.LPAREN); + + for (; ; ) { + String key = lexer.stringVal(); + lexer.nextToken(); + accept(Token.EQ); + SQLExpr value = this.exprParser.primary(); + stmt.getSerdeProperties().put(key, value); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } + + accept(Token.RPAREN); + } + } + protected void parseCreateTableRest(SQLCreateTableStatement createTable) { HiveCreateTableStatement stmt = (HiveCreateTableStatement) createTable; if (lexer.nextIfIdentifier(FnvHash.Constants.ENGINE)) { @@ -242,6 +266,8 @@ protected void parseCreateTableRest(SQLCreateTableStatement createTable) { accept(Token.BY); SQLName name = this.exprParser.name(); stmt.setStoredBy(name); + + parseCreateTableWithSerderPropertie(stmt); } else { accept(Token.AS); @@ -392,28 +418,7 @@ private void parseSortedBy(HiveCreateTableStatement stmt) { private void parseRowFormat(HiveCreateTableStatement stmt) { SQLExternalRecordFormat format = this.getExprParser().parseRowFormat(); stmt.setRowFormat(format); - - if (lexer.token() == Token.WITH) { - lexer.nextToken(); - acceptIdentifier("SERDEPROPERTIES"); - - accept(Token.LPAREN); - - for (; ; ) { - String name = lexer.stringVal(); - lexer.nextToken(); - accept(Token.EQ); - SQLExpr value = this.exprParser.primary(); - stmt.getSerdeProperties().put(name, value); - if (lexer.token() == Token.COMMA) { - lexer.nextToken(); - continue; - } - break; - } - - accept(Token.RPAREN); - } + parseCreateTableWithSerderPropertie(stmt); } @Override diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java b/core/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java index 215ca99dcd..b037c5e707 100644 --- a/core/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java @@ -16,22 +16,27 @@ package com.alibaba.druid.sql.dialect.hive.visitor; import com.alibaba.druid.DbType; +import com.alibaba.druid.sql.ast.SQLAdhocTableSource; import com.alibaba.druid.sql.ast.SQLCommentHint; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; +import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.hive.ast.HiveAddJarStatement; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement; import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement; import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateFunctionStatement; +import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.hive.stmt.HiveLoadDataStatement; import com.alibaba.druid.sql.dialect.hive.stmt.HiveMsckRepairStatement; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; import java.util.List; +import java.util.Map; public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVisitor { { @@ -511,4 +516,169 @@ protected void printTableOptionsPrefix(SQLCreateTableStatement x) { incrementIndent(); println(); } + + @Override + public boolean visit(SQLCreateTableStatement x) { + printCreateTable((HiveCreateTableStatement) x, true); + return false; + } + protected void printCreateTable(HiveCreateTableStatement x, boolean printSelect) { + final SQLObject parent = x.getParent(); + + if (x.hasBeforeComment()) { + printlnComments(x.getBeforeCommentsDirect()); + } + + if (parent instanceof SQLAdhocTableSource) { + // skip + } else { + print0(ucase ? "CREATE " : "create "); + } + + printCreateTableFeatures(x); + + print0(ucase ? "TABLE " : "table "); + + if (x.isIfNotExists()) { + print0(ucase ? "IF NOT EXISTS " : "if not exists "); + } + + printTableSourceExpr(x.getName()); + + printTableElements(x.getTableElementList()); + + SQLExprTableSource inherits = x.getInherits(); + if (inherits != null) { + print0(ucase ? " INHERITS (" : " inherits ("); + inherits.accept(this); + print(')'); + } + + SQLExpr using = x.getUsing(); + if (using != null) { + println(); + print0(ucase ? "USING " : "using "); + using.accept(this); + } + + printComment(x.getComment()); + + List mappedBy = x.getMappedBy(); + if (mappedBy != null && mappedBy.size() > 0) { + println(); + print0(ucase ? "MAPPED BY (" : "mapped by ("); + printAndAccept(mappedBy, ", "); + print0(ucase ? ")" : ")"); + } + + printPartitionedBy(x); + + List clusteredBy = x.getClusteredBy(); + if (clusteredBy.size() > 0) { + println(); + print0(ucase ? "CLUSTERED BY (" : "clustered by ("); + printAndAccept(clusteredBy, ","); + print(')'); + } + List sortedBy = x.getSortedBy(); + if (sortedBy.size() > 0) { + println(); + print0(ucase ? "SORTED BY (" : "sorted by ("); + printAndAccept(sortedBy, ", "); + print(')'); + } + int buckets = x.getBuckets(); + if (buckets > 0) { + println(); + print0(ucase ? "INTO " : "into "); + print(buckets); + print0(ucase ? " BUCKETS" : " buckets"); + } + List skewedBy = x.getSkewedBy(); + if (skewedBy.size() > 0) { + println(); + print0(ucase ? "SKEWED BY (" : "skewed by ("); + printAndAccept(skewedBy, ","); + print(')'); + + List skewedByOn = x.getSkewedByOn(); + if (skewedByOn.size() > 0) { + print0(ucase ? " ON (" : " on ("); + printAndAccept(skewedByOn, ","); + print(')'); + } + if (x.isSkewedByStoreAsDirectories()) { + print(ucase ? " STORED AS DIRECTORIES" : " stored as directories"); + } + } + + SQLExternalRecordFormat format = x.getRowFormat(); + SQLExpr storedBy = x.getStoredBy(); + if (format != null) { + println(); + print0(ucase ? "ROW FORMAT" : "row format"); + if (format.getSerde() == null) { + print0(ucase ? " DELIMITED" : " delimited "); + } + visit(format); + if (storedBy == null) { + printSerdeProperties(x.getSerdeProperties()); + } + } + + printCreateTableLike(x); + + SQLExpr storedAs = x.getStoredAs(); + if (storedAs != null) { + println(); + if (x.isLbracketUse()) { + print("["); + } + print0(ucase ? "STORED AS" : "stored as"); + if (storedAs instanceof SQLIdentifierExpr) { + print(' '); + printExpr(storedAs, parameterized); + } else { + incrementIndent(); + println(); + printExpr(storedAs, parameterized); + decrementIndent(); + } + + if (x.isRbracketUse()) { + print("]"); + } + } + + if (storedBy != null) { + println(); + print0(ucase ? "STORED BY " : "STORED by "); + printExpr(storedBy, parameterized); + Map serdeProperties = x.getSerdeProperties(); + printSerdeProperties(serdeProperties); + } + + SQLExpr location = x.getLocation(); + if (location != null) { + println(); + print0(ucase ? "LOCATION " : "location "); + printExpr(location, parameterized); + } + + printTableOptions(x); + printLifeCycle(x.getLifeCycle()); + + SQLSelect select = x.getSelect(); + if (printSelect && select != null) { + println(); + if (x.isLikeQuery()) { // for dla + print0(ucase ? "LIKE" : "like"); + } else { + print0(ucase ? "AS" : "as"); + } + + println(); + visit(select); + } + } } diff --git a/core/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java b/core/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java index 570ceeaefb..e36d4b9072 100644 --- a/core/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java +++ b/core/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java @@ -27,7 +27,6 @@ import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; import com.alibaba.druid.sql.ast.statement.*; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlCreateTableStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; @@ -713,11 +712,6 @@ public boolean visit(SQLCreateSequenceStatement x) { return false; } - public boolean visit(HiveCreateTableStatement x) { - acceptCreateTable(x); - return false; - } - public boolean visit(MySqlCreateTableStatement x) { acceptCreateTable(x); return false; @@ -916,11 +910,6 @@ public boolean visit(SQLCreateTableStatement x) { return false; } - public boolean visit(HiveCreateTableStatement x) { - acceptCreateTable(x); - return false; - } - public boolean visit(SQLDropTableStatement x) { acceptDropTable(x); return false; diff --git a/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java b/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java index 3b4396e833..0d2bb4ec1c 100644 --- a/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java +++ b/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java @@ -25,7 +25,6 @@ import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement; import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitorAdapter; import com.alibaba.druid.sql.dialect.mysql.ast.MysqlForeignKey; import com.alibaba.druid.sql.dialect.mysql.ast.clause.MySqlCursorDeclareStatement; @@ -526,7 +525,7 @@ public boolean visit(SQLSelectItem x) { return true; } - public boolean visit(HiveCreateTableStatement x) { + public boolean visit(SQLCreateTableStatement x) { resolve(this, x); return false; } diff --git a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 93db7a4054..7c8c8bb581 100644 --- a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -26,7 +26,6 @@ import com.alibaba.druid.sql.ast.statement.SQLMergeStatement.MergeInsertClause; import com.alibaba.druid.sql.ast.statement.SQLMergeStatement.MergeUpdateClause; import com.alibaba.druid.sql.dialect.hive.ast.HiveInputOutputFormat; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.mysql.ast.MySqlPrimaryKey; import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlOrderingExpr; import com.alibaba.druid.sql.dialect.mysql.ast.statement.*; @@ -11079,168 +11078,6 @@ public boolean visit(SQLAdhocTableSource x) { return false; } - @Override - public boolean visit(HiveCreateTableStatement x) { - printCreateTable(x, true); - - return false; - } - - protected void printCreateTable(HiveCreateTableStatement x, boolean printSelect) { - final SQLObject parent = x.getParent(); - - if (x.hasBeforeComment()) { - printlnComments(x.getBeforeCommentsDirect()); - } - - if (parent instanceof SQLAdhocTableSource) { - // skip - } else { - print0(ucase ? "CREATE " : "create "); - } - - printCreateTableFeatures(x); - - print0(ucase ? "TABLE " : "table "); - - if (x.isIfNotExists()) { - print0(ucase ? "IF NOT EXISTS " : "if not exists "); - } - - printTableSourceExpr(x.getName()); - - printTableElements(x.getTableElementList()); - - SQLExprTableSource inherits = x.getInherits(); - if (inherits != null) { - print0(ucase ? " INHERITS (" : " inherits ("); - inherits.accept(this); - print(')'); - } - - SQLExpr using = x.getUsing(); - if (using != null) { - println(); - print0(ucase ? "USING " : "using "); - using.accept(this); - } - - printComment(x.getComment()); - - List mappedBy = x.getMappedBy(); - if (mappedBy != null && mappedBy.size() > 0) { - println(); - print0(ucase ? "MAPPED BY (" : "mapped by ("); - printAndAccept(mappedBy, ", "); - print0(ucase ? ")" : ")"); - } - - printPartitionedBy(x); - - List clusteredBy = x.getClusteredBy(); - if (clusteredBy.size() > 0) { - println(); - print0(ucase ? "CLUSTERED BY (" : "clustered by ("); - printAndAccept(clusteredBy, ","); - print(')'); - } - List sortedBy = x.getSortedBy(); - if (sortedBy.size() > 0) { - println(); - print0(ucase ? "SORTED BY (" : "sorted by ("); - printAndAccept(sortedBy, ", "); - print(')'); - } - int buckets = x.getBuckets(); - if (buckets > 0) { - println(); - print0(ucase ? "INTO " : "into "); - print(buckets); - print0(ucase ? " BUCKETS" : " buckets"); - } - List skewedBy = x.getSkewedBy(); - if (skewedBy.size() > 0) { - println(); - print0(ucase ? "SKEWED BY (" : "skewed by ("); - printAndAccept(skewedBy, ","); - print(')'); - - List skewedByOn = x.getSkewedByOn(); - if (skewedByOn.size() > 0) { - print0(ucase ? " ON (" : " on ("); - printAndAccept(skewedByOn, ","); - print(')'); - } - if (x.isSkewedByStoreAsDirectories()) { - print(ucase ? " STORED AS DIRECTORIES" : " stored as directories"); - } - } - - SQLExternalRecordFormat format = x.getRowFormat(); - if (format != null) { - println(); - print0(ucase ? "ROW FORMAT" : "row format"); - if (format.getSerde() == null) { - print0(ucase ? " DELIMITED" : " delimited "); - } - visit(format); - } - Map serdeProperties = x.getSerdeProperties(); - printSerdeProperties(serdeProperties); - printCreateTableLike(x); - - SQLExpr storedAs = x.getStoredAs(); - if (storedAs != null) { - println(); - if (x.isLbracketUse()) { - print("["); - } - print0(ucase ? "STORED AS" : "stored as"); - if (storedAs instanceof SQLIdentifierExpr) { - print(' '); - printExpr(storedAs, parameterized); - } else { - incrementIndent(); - println(); - printExpr(storedAs, parameterized); - decrementIndent(); - } - - if (x.isRbracketUse()) { - print("]"); - } - } - SQLExpr storedBy = x.getStoredBy(); - if (storedBy != null) { - println(); - print0(ucase ? " STORED BY " : " STORED by "); - printExpr(storedBy, parameterized); - } - - SQLExpr location = x.getLocation(); - if (location != null) { - println(); - print0(ucase ? "LOCATION " : "location "); - printExpr(location, parameterized); - } - - printTableOptions(x); - printLifeCycle(x.getLifeCycle()); - - SQLSelect select = x.getSelect(); - if (printSelect && select != null) { - println(); - if (x.isLikeQuery()) { // for dla - print0(ucase ? "LIKE" : "like"); - } else { - print0(ucase ? "AS" : "as"); - } - - println(); - visit(select); - } - } - protected void printCreateTableLike(SQLCreateTableStatement x) { SQLExprTableSource like = x.getLike(); if (like == null) { diff --git a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index 7df05fbca2..b71a28fcb5 100644 --- a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -21,7 +21,6 @@ import com.alibaba.druid.sql.ast.expr.*; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.hive.ast.HiveInputOutputFormat; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlKillStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.SQLAlterResourceGroupStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.SQLCreateResourceGroupStatement; @@ -2200,13 +2199,6 @@ default boolean visit(SQLAdhocTableSource x) { default void endVisit(SQLAdhocTableSource x) { } - default boolean visit(HiveCreateTableStatement x) { - return true; - } - - default void endVisit(HiveCreateTableStatement x) { - } - default boolean visit(HiveInputOutputFormat x) { return true; } diff --git a/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java b/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java index ec9adc0239..e786924613 100644 --- a/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java @@ -22,7 +22,6 @@ import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert; import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlExpr; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; import com.alibaba.druid.sql.dialect.oracle.ast.expr.OracleExpr; @@ -3190,11 +3189,6 @@ public boolean visit(SQLAlterTableArchivePartition x) { return true; } - @Override - public boolean visit(HiveCreateTableStatement x) { - return visit((SQLCreateTableStatement) x); - } - @Override public boolean visit(SQLCopyFromStatement x) { SQLExprTableSource table = x.getTable(); diff --git a/core/src/test/resources/bvt/parser/hive/1.txt b/core/src/test/resources/bvt/parser/hive/1.txt index 08b598cbbf..3571f19b29 100644 --- a/core/src/test/resources/bvt/parser/hive/1.txt +++ b/core/src/test/resources/bvt/parser/hive/1.txt @@ -44,4 +44,27 @@ LOCATION 'hdfs://dc/user/hive/warehouse/ods.db/ods_dc_test_df' TBLPROPERTIES ( 'parquet.compression' = 'snappy', 'transient_lastDdlTime' = '1603362950' +) +------------------------------------------------------------------------------------------------------------------------ +create external table if not exists test.test ( + key string, +) row format serde 'org.apache.hadoop.hive.hbase.HBaseSerDe' stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +with + serdeproperties ( + 'hbase.columns.mapping' = ':key,1111' + ) tblproperties ( + 'hbase.table.name' = '11111' + ) +-------------------- +CREATE EXTERNAL TABLE IF NOT EXISTS test.test ( + key string +) +ROW FORMAT + SERDE 'org.apache.hadoop.hive.hbase.HBaseSerDe' +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + 'hbase.columns.mapping' = ':key,1111' +) +TBLPROPERTIES ( + 'hbase.table.name' = '11111' ) \ No newline at end of file