Skip to content

Commit

Permalink
Support serde properties after 'store by' in hive.
Browse files Browse the repository at this point in the history
  • Loading branch information
@lingo-xp @wenshao
lingo-xp authored and wenshao committed Aug 8, 2024
1 parent db28875 commit 786cefd
Show file tree
Hide file tree
Showing 9 changed files with 221 additions and 223 deletions.
11 changes: 0 additions & 11 deletions core/src/main/java/com/alibaba/druid/sql/SQLUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import com.alibaba.druid.sql.dialect.h2.visitor.H2SchemaStatVisitor;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitorAdapter;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveOutputVisitor;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveSchemaStatVisitor;
Expand Down Expand Up @@ -1572,11 +1571,6 @@ public boolean visit(SQLCreateTableStatement x) {
return false;
}

@Override
public boolean visit(HiveCreateTableStatement x) {
return false;
}

@Override
public boolean visit(OdpsCreateTableStatement x) {
return false;
Expand Down Expand Up @@ -1629,11 +1623,6 @@ public boolean visit(SQLCreateTableStatement x) {
return false;
}

@Override
public boolean visit(HiveCreateTableStatement x) {
return false;
}

@Override
public boolean visit(SQLInsertStatement x) {
if (filter == null || filter.test(x)) {
Expand Down
49 changes: 27 additions & 22 deletions core/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,30 @@ protected void createTableQuery(SQLCreateTableStatement stmt) {
}
}

protected void parseCreateTableWithSerderPropertie(HiveCreateTableStatement stmt) {
if (lexer.token() == Token.WITH) {
lexer.nextToken();
acceptIdentifier("SERDEPROPERTIES");

accept(Token.LPAREN);

for (; ; ) {
String key = lexer.stringVal();
lexer.nextToken();
accept(Token.EQ);
SQLExpr value = this.exprParser.primary();
stmt.getSerdeProperties().put(key, value);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
}
break;
}

accept(Token.RPAREN);
}
}

protected void parseCreateTableRest(SQLCreateTableStatement createTable) {
HiveCreateTableStatement stmt = (HiveCreateTableStatement) createTable;
if (lexer.nextIfIdentifier(FnvHash.Constants.ENGINE)) {
Expand Down Expand Up @@ -242,6 +266,8 @@ protected void parseCreateTableRest(SQLCreateTableStatement createTable) {
accept(Token.BY);
SQLName name = this.exprParser.name();
stmt.setStoredBy(name);

parseCreateTableWithSerderPropertie(stmt);
} else {
accept(Token.AS);

Expand Down Expand Up @@ -392,28 +418,7 @@ private void parseSortedBy(HiveCreateTableStatement stmt) {
private void parseRowFormat(HiveCreateTableStatement stmt) {
SQLExternalRecordFormat format = this.getExprParser().parseRowFormat();
stmt.setRowFormat(format);

if (lexer.token() == Token.WITH) {
lexer.nextToken();
acceptIdentifier("SERDEPROPERTIES");

accept(Token.LPAREN);

for (; ; ) {
String name = lexer.stringVal();
lexer.nextToken();
accept(Token.EQ);
SQLExpr value = this.exprParser.primary();
stmt.getSerdeProperties().put(name, value);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
}
break;
}

accept(Token.RPAREN);
}
parseCreateTableWithSerderPropertie(stmt);
}

@Override
Expand Down
170 changes: 170 additions & 0 deletions core/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,27 @@
package com.alibaba.druid.sql.dialect.hive.visitor;

import com.alibaba.druid.DbType;
import com.alibaba.druid.sql.ast.SQLAdhocTableSource;
import com.alibaba.druid.sql.ast.SQLCommentHint;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.hive.ast.HiveAddJarStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateFunctionStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveLoadDataStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveMsckRepairStatement;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;

import java.util.List;
import java.util.Map;

public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVisitor {
{
Expand Down Expand Up @@ -511,4 +516,169 @@ protected void printTableOptionsPrefix(SQLCreateTableStatement x) {
incrementIndent();
println();
}

@Override
public boolean visit(SQLCreateTableStatement x) {
printCreateTable((HiveCreateTableStatement) x, true);
return false;
}
protected void printCreateTable(HiveCreateTableStatement x, boolean printSelect) {
final SQLObject parent = x.getParent();

if (x.hasBeforeComment()) {
printlnComments(x.getBeforeCommentsDirect());
}

if (parent instanceof SQLAdhocTableSource) {
// skip
} else {
print0(ucase ? "CREATE " : "create ");
}

printCreateTableFeatures(x);

print0(ucase ? "TABLE " : "table ");

if (x.isIfNotExists()) {
print0(ucase ? "IF NOT EXISTS " : "if not exists ");
}

printTableSourceExpr(x.getName());

printTableElements(x.getTableElementList());

SQLExprTableSource inherits = x.getInherits();
if (inherits != null) {
print0(ucase ? " INHERITS (" : " inherits (");
inherits.accept(this);
print(')');
}

SQLExpr using = x.getUsing();
if (using != null) {
println();
print0(ucase ? "USING " : "using ");
using.accept(this);
}

printComment(x.getComment());

List<SQLAssignItem> mappedBy = x.getMappedBy();
if (mappedBy != null && mappedBy.size() > 0) {
println();
print0(ucase ? "MAPPED BY (" : "mapped by (");
printAndAccept(mappedBy, ", ");
print0(ucase ? ")" : ")");
}

printPartitionedBy(x);

List<SQLSelectOrderByItem> clusteredBy = x.getClusteredBy();
if (clusteredBy.size() > 0) {
println();
print0(ucase ? "CLUSTERED BY (" : "clustered by (");
printAndAccept(clusteredBy, ",");
print(')');
}
List<SQLSelectOrderByItem> sortedBy = x.getSortedBy();
if (sortedBy.size() > 0) {
println();
print0(ucase ? "SORTED BY (" : "sorted by (");
printAndAccept(sortedBy, ", ");
print(')');
}
int buckets = x.getBuckets();
if (buckets > 0) {
println();
print0(ucase ? "INTO " : "into ");
print(buckets);
print0(ucase ? " BUCKETS" : " buckets");
}
List<SQLExpr> skewedBy = x.getSkewedBy();
if (skewedBy.size() > 0) {
println();
print0(ucase ? "SKEWED BY (" : "skewed by (");
printAndAccept(skewedBy, ",");
print(')');

List<SQLExpr> skewedByOn = x.getSkewedByOn();
if (skewedByOn.size() > 0) {
print0(ucase ? " ON (" : " on (");
printAndAccept(skewedByOn, ",");
print(')');
}
if (x.isSkewedByStoreAsDirectories()) {
print(ucase ? " STORED AS DIRECTORIES" : " stored as directories");
}
}

SQLExternalRecordFormat format = x.getRowFormat();
SQLExpr storedBy = x.getStoredBy();
if (format != null) {
println();
print0(ucase ? "ROW FORMAT" : "row format");
if (format.getSerde() == null) {
print0(ucase ? " DELIMITED" : " delimited ");
}
visit(format);
if (storedBy == null) {
printSerdeProperties(x.getSerdeProperties());
}
}

printCreateTableLike(x);

SQLExpr storedAs = x.getStoredAs();
if (storedAs != null) {
println();
if (x.isLbracketUse()) {
print("[");
}
print0(ucase ? "STORED AS" : "stored as");
if (storedAs instanceof SQLIdentifierExpr) {
print(' ');
printExpr(storedAs, parameterized);
} else {
incrementIndent();
println();
printExpr(storedAs, parameterized);
decrementIndent();
}

if (x.isRbracketUse()) {
print("]");
}
}

if (storedBy != null) {
println();
print0(ucase ? "STORED BY " : "STORED by ");
printExpr(storedBy, parameterized);
Map<String, SQLObject> serdeProperties = x.getSerdeProperties();
printSerdeProperties(serdeProperties);
}

SQLExpr location = x.getLocation();
if (location != null) {
println();
print0(ucase ? "LOCATION " : "location ");
printExpr(location, parameterized);
}

printTableOptions(x);
printLifeCycle(x.getLifeCycle());

SQLSelect select = x.getSelect();
if (printSelect && select != null) {
println();
if (x.isLikeQuery()) { // for dla
print0(ucase ? "LIKE" : "like");
} else {
print0(ucase ? "AS" : "as");
}

println();
visit(select);
}
}
}
11 changes: 0 additions & 11 deletions core/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter;
Expand Down Expand Up @@ -713,11 +712,6 @@ public boolean visit(SQLCreateSequenceStatement x) {
return false;
}

public boolean visit(HiveCreateTableStatement x) {
acceptCreateTable(x);
return false;
}

public boolean visit(MySqlCreateTableStatement x) {
acceptCreateTable(x);
return false;
Expand Down Expand Up @@ -916,11 +910,6 @@ public boolean visit(SQLCreateTableStatement x) {
return false;
}

public boolean visit(HiveCreateTableStatement x) {
acceptCreateTable(x);
return false;
}

public boolean visit(SQLDropTableStatement x) {
acceptDropTable(x);
return false;
Expand Down
3 changes: 1 addition & 2 deletions core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitorAdapter;
import com.alibaba.druid.sql.dialect.mysql.ast.MysqlForeignKey;
import com.alibaba.druid.sql.dialect.mysql.ast.clause.MySqlCursorDeclareStatement;
Expand Down Expand Up @@ -526,7 +525,7 @@ public boolean visit(SQLSelectItem x) {
return true;
}

public boolean visit(HiveCreateTableStatement x) {
public boolean visit(SQLCreateTableStatement x) {
resolve(this, x);
return false;
}
Expand Down
Loading

0 comments on commit 786cefd

Please sign in to comment.