From e88a25ba6b28733cf19eb1f013ddb4ed425b4b4c Mon Sep 17 00:00:00 2001 From: burhanxz Date: Fri, 29 Apr 2022 15:12:49 +0800 Subject: [PATCH 1/4] fix ssl enabled protocol --- .../src/main/java/com/alibaba/polardbx/ssl/SslConstant.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslConstant.java b/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslConstant.java index 83a42de1b..39ca34208 100644 --- a/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslConstant.java +++ b/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslConstant.java @@ -41,7 +41,7 @@ public class SslConstant { SSLContext sslContext = SSLContext.getInstance("TLS"); sslContext.init(null, null, null); sslSocket = (SSLSocket) sslContext.getSocketFactory().createSocket(); - String[] enabledProtocols = sslSocket.getEnabledProtocols(); + enabledProtocols = sslSocket.getEnabledProtocols(); if (enabledProtocols != null && enabledProtocols.length > 0) { String enabledProtocol = enabledProtocols[0]; PROTOCOL = enabledProtocol; From 4912817383a6b7c4852c7ef378d8fd7ce4e443b6 Mon Sep 17 00:00:00 2001 From: burhanxz Date: Fri, 29 Apr 2022 15:13:13 +0800 Subject: [PATCH 2/4] fix Fix pem file base64 decode. --- .../main/java/com/alibaba/polardbx/ssl/SslContextFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslContextFactory.java b/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslContextFactory.java index 0a6e1c727..8b5202f02 100644 --- a/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslContextFactory.java +++ b/polardbx-net/src/main/java/com/alibaba/polardbx/ssl/SslContextFactory.java @@ -214,7 +214,7 @@ protected static byte[] parseFileContent(byte[] pem, String beginDelimiter, Stri String data = new String(pem); String[] tokens = data.split(beginDelimiter); tokens = tokens[1].split(endDelimiter); - return Base64.getDecoder().decode(tokens[0]); + return Base64.getDecoder().decode(tokens[0].replaceAll("[\r\n]", "")); } protected static RSAPrivateKey generatePrivateKeyFromDER(byte[] keyBytes) throws InvalidKeySpecException, From 8d0ec7fd9e09462c1564b03ee2207e196cb7d2ab Mon Sep 17 00:00:00 2001 From: burhanxz Date: Fri, 29 Apr 2022 15:16:27 +0800 Subject: [PATCH 3/4] use Wisp2 --- polardbx-server/src/main/bin/startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polardbx-server/src/main/bin/startup.sh b/polardbx-server/src/main/bin/startup.sh index 78abac437..e54fc0fb1 100644 --- a/polardbx-server/src/main/bin/startup.sh +++ b/polardbx-server/src/main/bin/startup.sh @@ -412,7 +412,7 @@ fi #2.6.32-220.23.2.al.ali1.1.alios6.x86_64 not support Wisp2 if [ "$wisp" == "wisp" ] && [ "$KERNEL_VERSION" != "2.6.32-220.23.2.al.ali1.1.alios6.x86_64" ]; then - JAVA_OPTS="$JAVA_OPTS -XX:+UseWisp2" + JAVA_OPTS="$JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseWisp2" fi # in docker container, limit cpu cores From 93094e5f9e8d99782fb5ba7749f5b3ad68608f14 Mon Sep 17 00:00:00 2001 From: burhanxz Date: Fri, 29 Apr 2022 15:16:48 +0800 Subject: [PATCH 4/4] support create file storage --- docs/zh_CN/quickstart-oss.md | 58 +++----- .../calcite/rel/ddl/CreateFileStorage.java | 49 +++++++ .../calcite/sql/SqlCreateFileStorage.java | 98 ++++++++++++++ .../java/org/apache/calcite/sql/SqlKind.java | 9 +- .../sql/validate/SqlValidatorImpl.java | 4 +- .../calcite/sql2rel/SqlToRelConverter.java | 11 ++ .../charset/AbstractCharsetHandler.java | 6 +- .../common/charset/CollationName.java | 2 +- .../common/exception/code/ErrorCode.java | 3 +- .../main/resources/res/ErrorCode.properties | 3 +- .../common/charset/CharsetFactoryTest.java | 4 +- .../factory/CreateFileStorageJobFactory.java | 76 +++++++++++ .../factory/DropPartitionTableJobFactory.java | 8 +- .../factory/RecycleOssTableJobFactory.java | 4 - .../job/factory/RenameTableJobFactory.java | 25 ++-- .../task/basic/oss/CreateFileStorageTask.java | 124 ++++++++++++++++++ .../ddl/job/task/basic/oss/OSSTaskUtils.java | 37 ++++++ .../ddl/LogicalCreateFileStorageHandler.java | 49 +++++++ .../handler/ddl/LogicalDropTableHandler.java | 2 - .../handler/CommandHandlerFactoryMyImp.java | 6 + .../gms/engine/FileStorageInfoKey.java | 37 ++++++ .../gms/engine/FileStorageInfoRecord.java | 1 + .../optimizer/core/rel/ToDrdsRelVisitor.java | 4 + .../rel/ddl/LogicalCreateFileStorage.java | 21 +++ .../visitor/FastSqlToCalciteNodeVisitor.java | 21 +++ .../optimizer/utils/OptimizerUtils.java | 1 + .../statement/CreateFileStorageStatement.java | 62 +++++++++ .../mysql/parser/MySqlStatementParser.java | 34 +++++ .../mysql/visitor/MySqlASTVisitor.java | 5 + .../mysql/visitor/MySqlASTVisitorAdapter.java | 11 ++ .../mysql/visitor/MySqlOutputVisitor.java | 22 ++++ .../visitor/MySqlParameterizedVisitor.java | 11 ++ .../mysql/visitor/MySqlSchemaStatVisitor.java | 11 ++ 33 files changed, 747 insertions(+), 72 deletions(-) create mode 100644 polardbx-calcite/src/main/java/org/apache/calcite/rel/ddl/CreateFileStorage.java create mode 100644 polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateFileStorage.java create mode 100644 polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java create mode 100644 polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java create mode 100644 polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalCreateFileStorageHandler.java create mode 100644 polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoKey.java create mode 100644 polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ddl/LogicalCreateFileStorage.java create mode 100644 polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/ast/statement/CreateFileStorageStatement.java diff --git a/docs/zh_CN/quickstart-oss.md b/docs/zh_CN/quickstart-oss.md index 4da79d3df..151402f3f 100644 --- a/docs/zh_CN/quickstart-oss.md +++ b/docs/zh_CN/quickstart-oss.md @@ -3,68 +3,38 @@ - 首先源码编译安装 PolarDB-X，可参考[源码编译部署](quickstart-development.md) - 或者通过K8S构建出 PolarDB-X集群，可参考[K8S部署](http://www.github.com/ApsaraDB/galaxykube/blob/main/docs/zh/deploy/quick-start.md) -安装完成后通过如下命令进行OSS存储初始化: +安装完成后连接PolarDB-X集群，通过如下SQL命令进行OSS存储初始化: -- -e: 初始化存储引擎类型 -- -P: 之前准备的dnPasswordKey -- --uri: OSS存储uri -- --ep: OSS存储 end-point -- --ak: Access Key -- --sk: Secret Key - -注（关于--ep参数）：在阿里云OSS产品界面，通过“对象存储-Bucket列表-bucket-概览”可以获取到一组endpoint（地域节点）信息。仅当在相同Region的ECS部署环境下，使用经典网络访问或vpc网络访问endpoint。否则使用外网访问endpoint。 例子： ``` -bin/startup.sh \ - -e OSS \ - --uri oss://oss-bucket/ \ - --ep oss-cn-huhehaote.aliyuncs.com \ - --ak akakakakakakakakakakakakakak \ - --sk sksksksksksksksksksksksksksk \ - -P asdf1234ghjk5678 +create filestorage oss with ('file_uri' = 'oss://oss-bucket-name/', 'endpoint'='oss-endpoint', 'access_key_id'='your_ak', 'access_key_secret'='your_sk'); ``` -注1：启动实例后，通过执行show file storage语句来验证是否配置成功。 - -注2：startup.sh脚本位于CN安装bin目录下。 +- file_uri: OSS存储uri +- endpoint: OSS存储 end-point +- access_key_id: Access Key +- access_key_secret: Secret Key -如果你是通过K8S构建, 请通过下面方式登陆进入CN节点，获取dnPasswordKey并进行初始化 -``` -// 获取CN节点POD -kubectl get pods | grep cn +注1:（关于endpoint参数）：在阿里云OSS产品界面，通过“对象存储-Bucket列表-bucket-概览”可以获取到一组endpoint（地域节点）信息。仅当在相同Region的ECS部署环境下，使用经典网络访问或vpc网络访问endpoint。否则使用外网访问endpoint。 -// 登陆任意一个CN节点 -kubectl exec -it [your-cn-pod-name] bash -// 获取dnPasswordKey -env | grep dnPasswordKey - -// startup.sh位于/home/admin/drds-server/bin目录下 -cd /home/admin/drds-server/bin -``` +注2：通过执行show file storage语句来验证是否配置成功。 # 本地磁盘存储配置 对于没有OSS但希望体验用户，可以通过本地磁盘模拟OSS -注意：本地磁盘为CN节点本地磁盘，如果你是通过K8S方式部署，请限制CN节点个数为1个 - -安装完成后通过如下命令进行本地磁盘存储初始化: +注意：file_uri为CN节点能访问的数据目录路径，如果你无法保证多个CN能够共同访问这一数据目录路径，请限制CN节点个数为1个。 -- -e: 初始化存储引擎类型 -- -P: 之前准备的dnPasswordKey -- --uri: 本地磁盘uri，即本地存储目录。 +通过如下命令进行本地磁盘存储初始化: 例子： ``` -bin/startup.sh \ - -e LOCAL_DISK \ - --uri file:///tmp/local-dir \ - -P asdf1234ghjk5678 +create filestorage local_disk with ('file_uri' = 'file://tmp/orc/'); ``` -存储引擎改为`engine = 'local_disk'`即可使用本地磁盘存储，例如： +建表时存储引擎改为`engine = 'local_disk'`即可使用本地磁盘存储，例如： ``` create table sbtest1 like sysbench.sbtest1 engine = 'local_disk' archive_mode = 'loading'; ``` @@ -72,7 +42,7 @@ create table sbtest1 like sysbench.sbtest1 engine = 'local_disk' archive_mode = # 开始体验 -例子中的存储引擎都会以`engine = 'oss'`展示，如果你使用的是本地存储请修改为`engine = 'local_disk'` +注：例子中的存储引擎都会以`engine = 'oss'`展示，如果你使用的是本地存储请修改为`engine = 'local_disk'` ## Hello World ``` @@ -103,6 +73,8 @@ select * from oss_t1; ## TTL Innodb数据自动过期并归档到OSS存储上示例 +注：例子中的存储引擎都会以`engine = 'oss'`展示，如果你使用的是本地存储请修改为`engine = 'local_disk'` + ``` create database ttl_test partition_mode = 'auto'; use ttl_test; diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/rel/ddl/CreateFileStorage.java b/polardbx-calcite/src/main/java/org/apache/calcite/rel/ddl/CreateFileStorage.java new file mode 100644 index 000000000..9608045c5 --- /dev/null +++ b/polardbx-calcite/src/main/java/org/apache/calcite/rel/ddl/CreateFileStorage.java @@ -0,0 +1,49 @@ +package org.apache.calcite.rel.ddl; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.DDL; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlDdl; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.parser.SqlParserPos; + +import java.util.List; +import java.util.Map; + +public class CreateFileStorage extends DDL { + private final String engineName; + private final Map with; + + protected CreateFileStorage(RelOptCluster cluster, RelTraitSet traits, SqlDdl ddl, + RelDataType rowType, String engineName, Map with) { + super(cluster, traits, ddl, rowType); + this.engineName = engineName; + this.with = with; + this.sqlNode = ddl; + this.setTableName(new SqlIdentifier(engineName, SqlParserPos.ZERO)); + } + + public static CreateFileStorage create(RelOptCluster cluster, RelTraitSet traits, SqlDdl ddl, + RelDataType rowType, String engineName, Map with) { + + return new CreateFileStorage(cluster, traits, ddl, rowType, engineName, with); + } + + @Override + public CreateFileStorage copy( + RelTraitSet traitSet, List inputs) { + assert traitSet.containsIfApplicable(Convention.NONE); + return new CreateFileStorage(this.getCluster(), traitSet, this.ddl, rowType, engineName, with); + } + + public String getEngineName() { + return engineName; + } + + public Map getWith() { + return with; + } +} diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateFileStorage.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateFileStorage.java new file mode 100644 index 000000000..017dfd229 --- /dev/null +++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateFileStorage.java @@ -0,0 +1,98 @@ +package org.apache.calcite.sql; + +import com.google.common.collect.ImmutableList; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelDataTypeFieldImpl; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.util.Pair; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SqlCreateFileStorage extends SqlDdl { + private static final SqlOperator OPERATOR = new SqlCreateFileStorageOperator(); + + private SqlIdentifier engineName; + private List> with; + + public SqlCreateFileStorage(SqlParserPos pos, + SqlIdentifier engineName, + List> with) { + super(OPERATOR, pos); + this.engineName = engineName; + this.with = with; + } + + public SqlIdentifier getEngineName() { + return engineName; + } + + public SqlCreateFileStorage setEngineName(SqlIdentifier engineName) { + this.engineName = engineName; + return this; + } + + public List> getWith() { + return with; + } + + public Map getWithValue() { + Map map = new HashMap<>(); + for (Pair pair : with) { + map.put(pair.left.toString().toUpperCase(), pair.right.toString()); + } + return map; + } + + public SqlCreateFileStorage setWith( + List> with) { + this.with = with; + return this; + } + + @Override + public List getOperandList() { + return ImmutableList.of(); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + final SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.SELECT, "CREATE FILESTORAGE", ""); + + engineName.unparse(writer, leftPrec, rightPrec); + + final SqlWriter.Frame withFrame = writer.startList(SqlWriter.FrameTypeEnum.SELECT, " WITH (", ")"); + + for (Pair pair : with) { + pair.left.unparse(writer, leftPrec, rightPrec); + writer.sep("="); + pair.right.unparse(writer, leftPrec, rightPrec); + } + + writer.endList(withFrame); + + writer.endList(frame); + } + + public static class SqlCreateFileStorageOperator extends SqlSpecialOperator { + + public SqlCreateFileStorageOperator(){ + super("CREATE FILESTORAGE", SqlKind.CREATE_FILESTORAGE); + } + + @Override + public RelDataType deriveType(SqlValidator validator, SqlValidatorScope scope, SqlCall call) { + final RelDataTypeFactory typeFactory = validator.getTypeFactory(); + final RelDataType columnType = typeFactory.createSqlType(SqlTypeName.CHAR); + + return typeFactory.createStructType(ImmutableList.of((RelDataTypeField) new RelDataTypeFieldImpl("CREATE FILESTORAGE RESULT", + 0, columnType))); + } + } +} diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlKind.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlKind.java index 895fb092f..439382f34 100644 --- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -1665,6 +1665,11 @@ public enum SqlKind { */ ALTER_TABLEGROUP, + /** + * file storage management: create fileStorage + */ + CREATE_FILESTORAGE, + /** * file storage management: alter fileStorage */ @@ -1911,13 +1916,13 @@ public enum SqlKind { DROP_DATABASE, MOVE_DATABASE, CHECK_GLOBAL_INDEX, ALTER_TABLEGROUP, CREATE_TABLEGROUP, CHANGE_CONSENSUS_ROLE, ALTER_SYSTEM_SET_CONFIG, ALTER_TABLE_SET_TABLEGROUP, - REFRESH_TOPOLOGY, DROP_TABLEGROUP, ALTER_FILESTORAGE, DROP_FILESTORAGE + REFRESH_TOPOLOGY, DROP_TABLEGROUP, ALTER_FILESTORAGE, DROP_FILESTORAGE, CREATE_FILESTORAGE )); public static final EnumSet DDL_SUPPORTED_BY_NEW_ENGINE = EnumSet.of(RENAME_TABLE, TRUNCATE_TABLE, DROP_TABLE, CREATE_INDEX, DROP_INDEX, ALTER_TABLE, CREATE_TABLE, ALTER_TABLEGROUP, ALTER_TABLE_SET_TABLEGROUP, REFRESH_TOPOLOGY, CHECK_GLOBAL_INDEX, ALTER_RULE, - MOVE_DATABASE, ALTER_FILESTORAGE, DROP_FILESTORAGE); + MOVE_DATABASE, ALTER_FILESTORAGE, DROP_FILESTORAGE, CREATE_FILESTORAGE); public static final EnumSet SUPPORT_DDL = EnumSet.of(CREATE_TABLE, ALTER_TABLE, DROP_TABLE, diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 10f50e5c6..20ff585ef 100644 --- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -987,7 +987,8 @@ private SqlNode validateScopedExpression( registerQuery(scope, null, outermostNode, outermostNode, null, false); } if (topNode.getKind() == SqlKind.CREATE_TABLE || topNode.getKind() == SqlKind.DROP_TABLE - || topNode.getKind() == SqlKind.DROP_VIEW || topNode.getKind() == SqlKind.DROP_FILESTORAGE) { + || topNode.getKind() == SqlKind.DROP_VIEW || topNode.getKind() == SqlKind.DROP_FILESTORAGE + || topNode.getKind() == SqlKind.CREATE_FILESTORAGE) { if (topNode.getKind() == SqlKind.CREATE_TABLE) { outermostNode.validate(this, scope); } @@ -3929,6 +3930,7 @@ private void registerQuery( case IDENTIFIER: case ALTER_FILESTORAGE: case DROP_FILESTORAGE: + case CREATE_FILESTORAGE: setValidatedNodeType(node, RelOptUtil.createDmlRowType( SqlKind.INSERT, typeFactory)); break; diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 792230b12..da615bb1f 100644 --- a/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -88,6 +88,7 @@ import org.apache.calcite.rel.ddl.AlterTableSetTableGroup; import org.apache.calcite.rel.ddl.ChangeConsensusRole; import org.apache.calcite.rel.ddl.CreateDatabase; +import org.apache.calcite.rel.ddl.CreateFileStorage; import org.apache.calcite.rel.ddl.CreateIndex; import org.apache.calcite.rel.ddl.CreateTable; import org.apache.calcite.rel.ddl.CreateTableGroup; @@ -183,6 +184,7 @@ import org.apache.calcite.sql.SqlChangeConsensusRole; import org.apache.calcite.sql.SqlCharStringLiteral; import org.apache.calcite.sql.SqlCreateDatabase; +import org.apache.calcite.sql.SqlCreateFileStorage; import org.apache.calcite.sql.SqlCreateIndex; import org.apache.calcite.sql.SqlCreateTable; import org.apache.calcite.sql.SqlCreateTableGroup; @@ -3367,6 +3369,8 @@ protected RelRoot convertQueryRecursive(SqlNode query, boolean top, RelDataType return RelRoot.of(convertUnArchive((SqlUnArchive) query), kind); case DROP_FILESTORAGE: return RelRoot.of(convertDropFileStorage((SqlDropFileStorage) query), kind); + case CREATE_FILESTORAGE: + return RelRoot.of(convertCreateFileStorage((SqlCreateFileStorage) query), kind); default: if (kind.belongsTo(SqlKind.DAL)) { return RelRoot.of(convertDal((SqlDal) query), kind); @@ -3480,6 +3484,13 @@ private RelNode convertDropFileStorage(SqlDropFileStorage query) { targetRowType, query.getName().toString()); } + private RelNode convertCreateFileStorage(SqlCreateFileStorage query) { + final RelDataType targetRowType = validator.getValidatedNodeType(query); + assert targetRowType != null; + return CreateFileStorage.create(getCluster(), getCluster().traitSetOf(Convention.NONE), query, + targetRowType, query.getEngineName().toString(), query.getWithValue()); + } + private RelNode convertAlterTableGroup(SqlAlterTableGroup query) {; Map partRexInfoCtx = getRexInfoFromSqlAlterSpec(query.getAlters()); query.setPartRexInfoCtx(partRexInfoCtx); diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/AbstractCharsetHandler.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/AbstractCharsetHandler.java index 4592b548f..4b9c677cc 100644 --- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/AbstractCharsetHandler.java +++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/AbstractCharsetHandler.java @@ -34,13 +34,11 @@ public abstract class AbstractCharsetHandler implements CharsetHandler { private static final long SIGNED_MAX_LONG = 0x7fffffffffffffffL; protected Charset charset; - protected CharsetEncoder encoder; protected CollationHandler collationHandler; protected CollationName collationName; AbstractCharsetHandler(Charset charset, CollationName collationName) { this.charset = charset; - this.encoder = charset.newEncoder(); this.collationName = collationName; this.collationHandler = null; } @@ -80,10 +78,12 @@ public Slice encode(String unicodeChars) throws CharacterCodingException { return Slices.EMPTY_SLICE; } - encoder.reset(); CharBuffer charBuffer = CharBuffer.wrap(unicodeChars); + // For un-mappable characters, throw error. + CharsetEncoder encoder = charset.newEncoder(); ByteBuffer buffer = encoder.encode(charBuffer); + if (!buffer.hasRemaining()) { return Slices.EMPTY_SLICE; } diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java index 3c3eea11d..0f9b7102e 100644 --- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java +++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java @@ -321,7 +321,7 @@ public enum CollationName { public static ImmutableList POLAR_DB_X_IMPLEMENTED_COLLATION_NAMES = ImmutableList.of( // for utf8 - UTF8_GENERAL_CI, UTF8_BIN, UTF8_UNICODE_CI, UTF8_GENERAL_MYSQL500_CI, + UTF8_GENERAL_CI, UTF8_BIN, UTF8_UNICODE_CI, // for utf8mb UTF8MB4_GENERAL_CI, UTF8MB4_BIN, UTF8MB4_UNICODE_CI, diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java index 2f8aecf3e..963eb0711 100644 --- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java +++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java @@ -642,7 +642,8 @@ public enum ErrorCode { ERR_UNEXPECTED_REL_TREE(ErrorType.OSS, 11010), ERR_EXECUTE_ON_OSS(ErrorType.Executor, 11011), ERR_FILE_STORAGE_READ_ONLY(ErrorType.OSS, 11012), - ERR_OSS_CONNECT(ErrorType.OSS, 11013); + ERR_OSS_CONNECT(ErrorType.OSS, 11013), + ERR_FILE_STORAGE_EXISTS(ErrorType.OSS, 11014); private int code; diff --git a/polardbx-common/src/main/resources/res/ErrorCode.properties b/polardbx-common/src/main/resources/res/ErrorCode.properties index 19890a20d..d1a801eef 100644 --- a/polardbx-common/src/main/resources/res/ErrorCode.properties +++ b/polardbx-common/src/main/resources/res/ErrorCode.properties @@ -279,4 +279,5 @@ ERR_BACK_FILL_CHECK={0} ERR_UNARCHIVE_FIRST=There is some table archived to oss table, please execute "{0}" first. ERR_EXECUTE_ON_OSS={0} ERR_OSS_FORMAT={0} -ERR_OSS_CONNECT={0} \ No newline at end of file +ERR_OSS_CONNECT={0} +ERR_FILE_STORAGE_EXISTS={0} \ No newline at end of file diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/charset/CharsetFactoryTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/charset/CharsetFactoryTest.java index f0100cf7e..8be147273 100644 --- a/polardbx-common/src/test/java/com/alibaba/polardbx/common/charset/CharsetFactoryTest.java +++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/charset/CharsetFactoryTest.java @@ -16,8 +16,6 @@ package com.alibaba.polardbx.common.charset; -import com.alibaba.polardbx.common.charset.CharsetFactory; -import com.alibaba.polardbx.common.charset.CollationName; import com.alibaba.polardbx.common.collation.CollationHandler; import org.junit.Assert; import org.junit.Test; @@ -25,7 +23,7 @@ public class CharsetFactoryTest { @Test public void test() { - for (CollationName collation : CollationName.values()) { + for (CollationName collation : CollationName.POLAR_DB_X_IMPLEMENTED_COLLATION_NAMES) { CollationHandler collationHandler = CharsetFactory.INSTANCE.createCollationHandler(collation); Assert.assertEquals(collationHandler.getName(), collation); Assert.assertEquals(collationHandler.getCharsetHandler().getName(), CollationName.getCharsetOf(collation)); diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java new file mode 100644 index 000000000..b0730419a --- /dev/null +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java @@ -0,0 +1,76 @@ +package com.alibaba.polardbx.executor.ddl.job.factory; + +import com.alibaba.polardbx.common.Engine; +import com.alibaba.polardbx.common.exception.TddlRuntimeException; +import com.alibaba.polardbx.common.exception.code.ErrorCode; +import com.alibaba.polardbx.common.utils.logger.Logger; +import com.alibaba.polardbx.common.utils.logger.LoggerFactory; +import com.alibaba.polardbx.executor.ddl.job.task.basic.oss.CreateFileStorageTask; +import com.alibaba.polardbx.executor.ddl.job.task.basic.oss.OSSTaskUtils; +import com.alibaba.polardbx.executor.ddl.newengine.job.DdlJobFactory; +import com.alibaba.polardbx.executor.ddl.newengine.job.ExecutableDdlJob; +import com.alibaba.polardbx.gms.engine.FileStorageInfoKey; +import com.alibaba.polardbx.optimizer.context.ExecutionContext; +import org.eclipse.jetty.util.StringUtil; + +import java.util.Map; +import java.util.Set; + +public class CreateFileStorageJobFactory extends DdlJobFactory { + private static final Logger logger = LoggerFactory.getLogger("oss"); + + private ExecutionContext executionContext; + private Engine engine; + private Map items; + + public CreateFileStorageJobFactory( + Engine engine, Map items, + ExecutionContext executionContext) { + this.executionContext = executionContext; + this.engine = engine; + this.items = items; + } + + @Override + protected void validate() { + if (!items.containsKey(FileStorageInfoKey.FILE_URI)) { + throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "Should contain FILE_URI in with!"); + } + if (StringUtil.endsWithIgnoreCase(engine.name(), "OSS")) { + if (!items.containsKey(FileStorageInfoKey.ENDPOINT)) { + throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "Should contain ENDPOINT in with!"); + } + + // check endpoint + String endpointValue = items.get(FileStorageInfoKey.ENDPOINT); + if (!OSSTaskUtils.checkEndpoint(endpointValue)) { + throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "bad ENDPOINT value in with!"); + } + + if (!items.containsKey(FileStorageInfoKey.ACCESS_KEY_ID)) { + throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "Should contain ACCESS_KEY_ID in with!"); + } + if (!items.containsKey(FileStorageInfoKey.ACCESS_KEY_SECRET)) { + throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, + "Should contain ACCESS_KEY_SECRET in with!"); + } + } + } + + @Override + protected ExecutableDdlJob doCreate() { + ExecutableDdlJob executableDdlJob = new ExecutableDdlJob(); + executableDdlJob.addTask(new CreateFileStorageTask(executionContext.getSchemaName(), engine.name(), items)); + return executableDdlJob; + } + + @Override + protected void excludeResources(Set resources) { + + } + + @Override + protected void sharedResources(Set resources) { + + } +} diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/DropPartitionTableJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/DropPartitionTableJobFactory.java index 89931d53a..c5516f3e0 100644 --- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/DropPartitionTableJobFactory.java +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/DropPartitionTableJobFactory.java @@ -112,7 +112,9 @@ protected ExecutableDdlJob doCreate() { tasks.add(updateTableRemoveTsTask); } tasks.add(phyDdlTask); - tasks.add(cdcDdlMarkTask); + if (!Engine.isFileStore(engine)) { + tasks.add(cdcDdlMarkTask); + } tasks.add(removeMetaTask); if (syncTableGroup != null) { tasks.add(syncTableGroup); @@ -127,7 +129,9 @@ protected ExecutableDdlJob doCreate() { executableDdlJob.setStoreTableLocalityTask(dropLocality); executableDdlJob.setDropTableHideTableMetaTask(dropTableHideTableMetaTask); executableDdlJob.setPhyDdlTask(phyDdlTask); - executableDdlJob.setCdcDdlMarkTask(cdcDdlMarkTask); + if (!Engine.isFileStore(engine)) { + executableDdlJob.setCdcDdlMarkTask(cdcDdlMarkTask); + } executableDdlJob.setRemoveMetaTask(removeMetaTask); executableDdlJob.setTableSyncTask(tableSyncTask); diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RecycleOssTableJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RecycleOssTableJobFactory.java index b1f06a26b..96e5d1e97 100644 --- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RecycleOssTableJobFactory.java +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RecycleOssTableJobFactory.java @@ -16,7 +16,6 @@ package com.alibaba.polardbx.executor.ddl.job.factory; - import com.alibaba.polardbx.executor.ddl.job.converter.PhysicalPlanData; import com.alibaba.polardbx.executor.ddl.job.task.basic.DropTableHideTableMetaTask; import com.alibaba.polardbx.executor.ddl.job.task.basic.RenamePartitionTablePhyDdlTask; @@ -26,7 +25,6 @@ import com.alibaba.polardbx.executor.ddl.job.task.basic.RenameTableUpdateMetaTask; import com.alibaba.polardbx.executor.ddl.job.task.basic.RenameTableValidateTask; import com.alibaba.polardbx.executor.ddl.job.task.basic.oss.UnBindingArchiveTableMetaByArchiveTableTask; -import com.alibaba.polardbx.executor.ddl.job.task.cdc.CdcDdlMarkTask; import com.alibaba.polardbx.executor.ddl.job.validator.GsiValidator; import com.alibaba.polardbx.executor.ddl.newengine.job.DdlJobFactory; import com.alibaba.polardbx.executor.ddl.newengine.job.DdlTask; @@ -65,7 +63,6 @@ protected ExecutableDdlJob doCreate() { DdlTask unBindingTask = new UnBindingArchiveTableMetaByArchiveTableTask(schemaName, logicalTableName); DdlTask addMetaTask = new RenameTableAddMetaTask(schemaName, logicalTableName, newLogicalTableName); - DdlTask cdcDdlMarkTask = new CdcDdlMarkTask(schemaName, physicalPlanData); DdlTask phyDdlTask; boolean isNewPartitionDb = DbInfoManager.getInstance().isNewPartitionDb(schemaName); @@ -84,7 +81,6 @@ protected ExecutableDdlJob doCreate() { unBindingTask, addMetaTask, phyDdlTask, - cdcDdlMarkTask, updateMetaTask, hideTableMetaTask, syncTask diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RenameTableJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RenameTableJobFactory.java index 80f73f731..76a1b402b 100644 --- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RenameTableJobFactory.java +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/RenameTableJobFactory.java @@ -16,6 +16,7 @@ package com.alibaba.polardbx.executor.ddl.job.factory; +import com.alibaba.polardbx.common.Engine; import com.alibaba.polardbx.executor.ddl.job.converter.PhysicalPlanData; import com.alibaba.polardbx.executor.ddl.job.task.basic.RenamePartitionTablePhyDdlTask; import com.alibaba.polardbx.executor.ddl.job.task.basic.RenameTableAddMetaTask; @@ -29,9 +30,11 @@ import com.alibaba.polardbx.executor.ddl.newengine.job.DdlTask; import com.alibaba.polardbx.executor.ddl.newengine.job.ExecutableDdlJob; import com.alibaba.polardbx.gms.topology.DbInfoManager; +import com.alibaba.polardbx.optimizer.OptimizerContext; import com.alibaba.polardbx.optimizer.context.ExecutionContext; -import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.List; import java.util.Set; public class RenameTableJobFactory extends DdlJobFactory { @@ -71,15 +74,19 @@ protected ExecutableDdlJob doCreate() { DdlTask updateMetaTask = new RenameTableUpdateMetaTask(schemaName, logicalTableName, newLogicalTableName); DdlTask syncTask = new RenameTableSyncTask(schemaName, logicalTableName, newLogicalTableName); + List taskList = new ArrayList<>(); + taskList.add(validateTask); + taskList.add(addMetaTask); + taskList.add(phyDdlTask); + Engine engine = OptimizerContext.getContext(schemaName).getLatestSchemaManager().getTable(logicalTableName).getEngine(); + if (!Engine.isFileStore(engine)) { + taskList.add(cdcDdlMarkTask); + } + taskList.add(updateMetaTask); + taskList.add(syncTask); + ExecutableDdlJob executableDdlJob = new ExecutableDdlJob(); - executableDdlJob.addSequentialTasks(Lists.newArrayList( - validateTask, - addMetaTask, - phyDdlTask, - cdcDdlMarkTask, - updateMetaTask, - syncTask - )); + executableDdlJob.addSequentialTasks(taskList); executableDdlJob.labelAsHead(validateTask); return executableDdlJob; } diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java new file mode 100644 index 000000000..34fe65973 --- /dev/null +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java @@ -0,0 +1,124 @@ +package com.alibaba.polardbx.executor.ddl.job.task.basic.oss; + +import com.alibaba.fastjson.annotation.JSONCreator; +import com.alibaba.polardbx.common.Engine; +import com.alibaba.polardbx.common.exception.TddlRuntimeException; +import com.alibaba.polardbx.common.exception.code.ErrorCode; +import com.alibaba.polardbx.common.utils.GeneralUtil; +import com.alibaba.polardbx.executor.ddl.job.task.BaseGmsTask; +import com.alibaba.polardbx.executor.ddl.job.task.util.TaskName; +import com.alibaba.polardbx.gms.engine.CachePolicy; +import com.alibaba.polardbx.gms.engine.DeletePolicy; +import com.alibaba.polardbx.gms.engine.FileStorageInfoAccessor; +import com.alibaba.polardbx.gms.engine.FileStorageInfoKey; +import com.alibaba.polardbx.gms.engine.FileStorageInfoRecord; +import com.alibaba.polardbx.gms.engine.FileSystemManager; +import com.alibaba.polardbx.gms.engine.FileSystemUtils; +import com.alibaba.polardbx.gms.listener.impl.MetaDbDataIdBuilder; +import com.alibaba.polardbx.gms.metadb.table.TableInfoManager; +import com.alibaba.polardbx.gms.topology.ConfigListenerAccessor; +import com.alibaba.polardbx.gms.util.PasswdUtil; +import com.alibaba.polardbx.optimizer.context.ExecutionContext; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.google.common.collect.ImmutableList; +import lombok.Getter; +import org.apache.hadoop.fs.FileSystem; + +import java.io.IOException; +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Getter +@TaskName(name = "CreateFileStorageTask") +public class CreateFileStorageTask extends BaseGmsTask { + private String engineName; + private Map items; + + @JSONCreator + public CreateFileStorageTask(String schemaName, String engineName, Map items) { + super(schemaName, null); + this.engineName = engineName; + this.items = items; + onExceptionTryRollback(); + } + + @Override + protected void executeImpl(Connection metaDbConnection, ExecutionContext executionContext) { + FileStorageInfoAccessor fileStorageInfoAccessor = new FileStorageInfoAccessor(); + fileStorageInfoAccessor.setConnection(metaDbConnection); + + FileStorageInfoRecord record1 = new FileStorageInfoRecord(); + + record1.instId = ""; + record1.engine = engineName; + record1.fileSystemConf = ""; + record1.priority = 1; + record1.regionId = ""; + record1.availableZoneId = ""; + record1.cachePolicy = CachePolicy.META_AND_DATA_CACHE.getValue(); + record1.deletePolicy = DeletePolicy.MASTER_ONLY.getValue(); + record1.status = 1; + String uri = items.get(FileStorageInfoKey.FILE_URI).trim(); + if (!uri.endsWith("/")) { + uri = uri + "/"; + } + record1.fileUri = uri; + if (Engine.OSS.name().equalsIgnoreCase(record1.engine)) { + record1.externalEndpoint = items.get(FileStorageInfoKey.ENDPOINT); + record1.internalClassicEndpoint = items.get(FileStorageInfoKey.ENDPOINT); + record1.internalVpcEndpoint = items.get(FileStorageInfoKey.ENDPOINT); + record1.accessKeyId = items.get(FileStorageInfoKey.ACCESS_KEY_ID); + record1.accessKeySecret = PasswdUtil.encrypt(items.get(FileStorageInfoKey.ACCESS_KEY_SECRET)); + + // check the endpoint is right + int wait = 10; + List unexpectedErrors = new ArrayList<>(); + ExecutorService executor = Executors.newFixedThreadPool(1); + Future future = null; + try (FileSystem master = FileSystemManager.buildFileSystem(record1)) { + future = executor.submit(() -> { + try { + master.exists(FileSystemUtils.buildPath(master, "1.orc")); + } catch (Exception e) { + unexpectedErrors.add(e.getMessage()); + } + }); + future.get(wait, TimeUnit.SECONDS); + } catch (TimeoutException ex) { + // check the endpoint is right + throw new TddlRuntimeException(ErrorCode.ERR_OSS_CONNECT, + "Bad Endpoint value! Failed to connect to oss in " + wait + " seconds!"); + } catch (IOException | InterruptedException | ExecutionException e) { + unexpectedErrors.add(e.getMessage()); + } finally { + if (future != null) { + future.cancel(true); + } + if (!unexpectedErrors.isEmpty()) { + throw new TddlRuntimeException(ErrorCode.ERR_OSS_CONNECT, unexpectedErrors.get(0)); + } + } + } + + if (fileStorageInfoAccessor.query(Engine.of(engineName)).size() != 0) { + throw new TddlRuntimeException(ErrorCode.ERR_FILE_STORAGE_EXISTS, String.format("FileStorage %s already exists", engineName)); + } + fileStorageInfoAccessor.insertIgnore(ImmutableList.of(record1)); + + ConfigListenerAccessor configListenerAccessor = new ConfigListenerAccessor(); + configListenerAccessor.setConnection(metaDbConnection); + configListenerAccessor.updateOpVersion(MetaDbDataIdBuilder.getFileStorageInfoDataId()); + } + + protected void updateTableVersion(Connection metaDbConnection) { + } +} diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java index a2eb4f563..2d510486b 100644 --- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java @@ -45,6 +45,43 @@ import java.util.Set; public class OSSTaskUtils { + private static final ImmutableList ENDPOINTS = ImmutableList.of( + "oss-cn-hangzhou.aliyuncs.com", "oss-cn-hangzhou-internal.aliyuncs.com", + "oss-cn-shanghai.aliyuncs.com", "oss-cn-shanghai-internal.aliyuncs.com", + "oss-cn-nanjing.aliyuncs.com", "oss-cn-nanjing-internal.aliyuncs.com", + "oss-cn-qingdao.aliyuncs.com", "oss-cn-qingdao-internal.aliyuncs.com", + "oss-cn-beijing.aliyuncs.com", "oss-cn-beijing-internal.aliyuncs.com", + "oss-cn-zhangjiakou", "oss-cn-zhangjiakou.aliyuncs.com", + "oss-cn-huhehaote.aliyuncs.com", "oss-cn-huhehaote-internal.aliyuncs.com", + "oss-cn-wulanchabu.aliyuncs.com", "oss-cn-wulanchabu-internal.aliyuncs.com", + "oss-cn-shenzhen.aliyuncs.com", "oss-cn-shenzhen-internal.aliyuncs.com", + "oss-cn-heyuan.aliyuncs.com", "oss-cn-heyuan-internal.aliyuncs.com", + "oss-cn-guangzhou.aliyuncs.com", "oss-cn-guangzhou-internal.aliyuncs.com", + "oss-cn-chengdu.aliyuncs.com", "oss-cn-chengdu-internal.aliyuncs.com", + "oss-cn-hongkong.aliyuncs.com", "oss-cn-hongkong-internal.aliyuncs.com", + "oss-us-west-1.aliyuncs.com", "oss-us-west-1-internal.aliyuncs.com", + "oss-us-east-1.aliyuncs.com", "oss-us-east-1-internal.aliyuncs.com", + "oss-ap-northeast-1.aliyuncs.com", "oss-ap-northeast-1-internal.aliyuncs.com", + "oss-ap-northeast-2.aliyuncs.com", "oss-ap-northeast-2-internal.aliyuncs.com", + "oss-ap-southeast-1.aliyuncs.com", "oss-ap-southeast-1-internal.aliyuncs.com", + "oss-ap-southeast-2.aliyuncs.com", "oss-ap-southeast-2-internal.aliyuncs.com", + "oss-ap-southeast-3.aliyuncs.com", "oss-ap-southeast-3-internal.aliyuncs.com", + "oss-ap-southeast-5.aliyuncs.com", "oss-ap-southeast-5-internal.aliyuncs.com", + "oss-ap-southeast-6.aliyuncs.com", "oss-ap-southeast-6-internal.aliyuncs.com", + "oss-ap-southeast-7.aliyuncs.com", "oss-ap-southeast-7-internal.aliyuncs.com", + "oss-ap-south-1.aliyuncs.com", "oss-ap-south-1-internal.aliyuncs.com", + "oss-eu-central-1.aliyuncs.com", "oss-eu-central-1-internal.aliyuncs.com", + "oss-eu-west-1.aliyuncs.com", "oss-eu-west-1-internal.aliyuncs.com", + "oss-me-east-1.aliyuncs.com", "oss-me-east-1-internal.aliyuncs.com" + ); + + public static boolean checkEndpoint(String endpoint) { + if (endpoint == null) { + return false; + } + return ENDPOINTS.contains(endpoint); + } + public static Pair getSingleTopology(String sourceLogicalSchema, String sourceLogicalTable, TableMeta sourceTableMeta) { Pair singlePhySchemaAndTable = null; boolean isSingle = diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalCreateFileStorageHandler.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalCreateFileStorageHandler.java new file mode 100644 index 000000000..ea3ca7ce3 --- /dev/null +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalCreateFileStorageHandler.java @@ -0,0 +1,49 @@ +package com.alibaba.polardbx.executor.handler.ddl; + +import com.alibaba.polardbx.common.Engine; +import com.alibaba.polardbx.common.exception.TddlRuntimeException; +import com.alibaba.polardbx.common.exception.code.ErrorCode; +import com.alibaba.polardbx.executor.ddl.job.factory.CreateFileStorageJobFactory; +import com.alibaba.polardbx.executor.ddl.newengine.job.DdlJob; +import com.alibaba.polardbx.executor.spi.IRepository; +import com.alibaba.polardbx.gms.engine.FileStorageInfoKey; +import com.alibaba.polardbx.optimizer.context.ExecutionContext; +import com.alibaba.polardbx.optimizer.core.rel.ddl.BaseDdlOperation; +import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCreateFileStorage; +import org.apache.calcite.rel.ddl.CreateFileStorage; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +public class LogicalCreateFileStorageHandler extends LogicalCommonDdlHandler { + public LogicalCreateFileStorageHandler(IRepository repo) { + super(repo); + } + + @Override + protected DdlJob buildDdlJob(BaseDdlOperation logicalDdlPlan, ExecutionContext executionContext) { + LogicalCreateFileStorage logicalCreateFileStorage = (LogicalCreateFileStorage) logicalDdlPlan; + CreateFileStorage createFileStorage = logicalCreateFileStorage.getCreateFileStorage(); + + Engine engine = Engine.of(createFileStorage.getEngineName()); + if (engine == null || !Engine.isFileStore(engine)) { + throw new TddlRuntimeException(ErrorCode.ERR_NOT_SUPPORT, + "invalid engine : " + createFileStorage.getEngineName()); + } + + // check fileStorageInfoKey + Map with = new HashMap<>(); + for (Map.Entry e : createFileStorage.getWith().entrySet()) { + String key = e.getKey(); + String value = e.getValue(); + FileStorageInfoKey fileStorageInfoKey; + if ((fileStorageInfoKey = FileStorageInfoKey.of(key)) == null) { + throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "error key: " + key); + } + with.put(fileStorageInfoKey, value); + } + + return new CreateFileStorageJobFactory(engine, with, executionContext).create(); + } +} diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalDropTableHandler.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalDropTableHandler.java index 029b7d1c4..6c898cd3d 100644 --- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalDropTableHandler.java +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/ddl/LogicalDropTableHandler.java @@ -210,8 +210,6 @@ private DdlJob buildRecycleFileStorageTableJob(LogicalDropTable logicalDropTable SqlIdentifier targetTableNode = sourceTableNode.setName(sourceTableNode.names.size() - 1, fileStorageBinName); SqlNode sqlRenameTable = new SqlRenameTable(targetTableNode, sourceTableNode, SqlParserPos.ZERO); - executionContext.getDdlContext() - .setDdlStmt(CdcTruncateWithRecycleMarkTask.CDC_RECYCLE_HINTS + sqlRenameTable.toString()); RenameTable renameTable = RenameTable.create(logicalDropTable.getCluster(), sqlRenameTable, sourceTableNode, targetTableNode); diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/repo/mysql/handler/CommandHandlerFactoryMyImp.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/repo/mysql/handler/CommandHandlerFactoryMyImp.java index 83c76f0e1..9d9f27368 100644 --- a/polardbx-executor/src/main/java/com/alibaba/polardbx/repo/mysql/handler/CommandHandlerFactoryMyImp.java +++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/repo/mysql/handler/CommandHandlerFactoryMyImp.java @@ -77,6 +77,7 @@ import com.alibaba.polardbx.executor.handler.ddl.LogicalCheckGsiHandler; import com.alibaba.polardbx.executor.handler.ddl.LogicalCommonDdlHandler; import com.alibaba.polardbx.executor.handler.ddl.LogicalCreateDatabaseHandler; +import com.alibaba.polardbx.executor.handler.ddl.LogicalCreateFileStorageHandler; import com.alibaba.polardbx.executor.handler.ddl.LogicalCreateIndexHandler; import com.alibaba.polardbx.executor.handler.ddl.LogicalCreateTableHandler; import com.alibaba.polardbx.executor.handler.ddl.LogicalCreateViewHandler; @@ -137,6 +138,7 @@ import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalChangeConsensusLeader; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCheckGsi; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCreateDatabase; +import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCreateFileStorage; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCreateIndex; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCreateTable; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCreateTableGroup; @@ -204,6 +206,7 @@ public CommandHandlerFactoryMyImp(IRepository repo) { LOGICAL_ALTER_TABLE_HANDLER = new LogicalAlterTableHandler(repo); LOGICAL_ALTER_FILESTORAGE_HANDLER = new LogicalAlterFileStoragHandler(repo); LOGICAL_DROP_FILESTORAGE_HANDLER = new LogicalDropFileStorageHandler(repo); + LOGICAL_CREATE_FILESTORAGE_HANDLER = new LogicalCreateFileStorageHandler(repo); LOGICAL_RENAME_TABLE_HANDLER = new LogicalRenameTableHandler(repo); LOGICAL_TRUNCATE_TABLE_HANDLER = new LogicalTruncateTableHandler(repo); LOGICAL_DROP_TABLE_HANDLER = new LogicalDropTableHandler(repo); @@ -348,6 +351,7 @@ public CommandHandlerFactoryMyImp(IRepository repo) { private final LogicalCommonDdlHandler LOGICAL_ALTER_TABLE_HANDLER; private final LogicalCommonDdlHandler LOGICAL_ALTER_FILESTORAGE_HANDLER; private final LogicalCommonDdlHandler LOGICAL_DROP_FILESTORAGE_HANDLER; + private final LogicalCommonDdlHandler LOGICAL_CREATE_FILESTORAGE_HANDLER; private final LogicalCommonDdlHandler LOGICAL_ALTER_TABLE_REPARTITION_HANDLER; private final LogicalCommonDdlHandler LOGICAL_RENAME_TABLE_HANDLER; private final LogicalCommonDdlHandler LOGICAL_TRUNCATE_TABLE_HANDLER; @@ -620,6 +624,8 @@ public PlanHandler getCommandHandler(RelNode logicalPlan, ExecutionContext execu return LOGICAL_ALTER_FILESTORAGE_HANDLER; } else if (logicalPlan instanceof LogicalDropFileStorage) { return LOGICAL_DROP_FILESTORAGE_HANDLER; + } else if (logicalPlan instanceof LogicalCreateFileStorage) { + return LOGICAL_CREATE_FILESTORAGE_HANDLER; } else if (logicalPlan instanceof PhyQueryOperation) { return PHY_QUERY_HANDLER; } else if (logicalPlan instanceof LogicalAlterTableGroupSplitPartition) { diff --git a/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoKey.java b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoKey.java new file mode 100644 index 000000000..364a81b7d --- /dev/null +++ b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoKey.java @@ -0,0 +1,37 @@ +package com.alibaba.polardbx.gms.engine; + +import com.alibaba.polardbx.common.utils.TStringUtil; + +public enum FileStorageInfoKey { + ENGINE, + ENDPOINT, + + FILE_URI, + FILE_SYSTEM_CONF, + + ACCESS_KEY_ID, + ACCESS_KEY_SECRET, + + PRIORITY, + + REGION_ID, + AVAILABLE_ZONE_ID, + + CACHE_POLICY, + DELETE_POLICY, + STATUS, + + ENDPOINT_ORDINAL; + + public static FileStorageInfoKey of(String key) { + if (TStringUtil.isEmpty(key)) { + return null; + } + for (FileStorageInfoKey record : values()) { + if (record.name().equalsIgnoreCase(key)) { + return record; + } + } + return null; + } +} diff --git a/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoRecord.java b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoRecord.java index 792064d9c..d9820b044 100644 --- a/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoRecord.java +++ b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/engine/FileStorageInfoRecord.java @@ -18,6 +18,7 @@ import com.alibaba.polardbx.common.jdbc.ParameterContext; import com.alibaba.polardbx.common.jdbc.ParameterMethod; +import com.alibaba.polardbx.common.utils.TStringUtil; import com.alibaba.polardbx.gms.metadb.record.SystemTableRecord; import com.alibaba.polardbx.gms.util.MetaDbUtil; import lombok.Data; diff --git a/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ToDrdsRelVisitor.java b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ToDrdsRelVisitor.java index 44f075f91..38b677660 100644 --- a/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ToDrdsRelVisitor.java +++ b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ToDrdsRelVisitor.java @@ -45,6 +45,7 @@ import com.alibaba.polardbx.optimizer.core.rel.dal.PhyDal; import com.alibaba.polardbx.optimizer.core.rel.dal.PhyShow; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalAlterFileStorage; +import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalCreateFileStorage; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalDropFileStorage; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalAlterRule; import com.alibaba.polardbx.optimizer.core.rel.ddl.LogicalAlterSystemSetConfig; @@ -124,6 +125,7 @@ import org.apache.calcite.rel.ddl.AlterTableSetTableGroup; import org.apache.calcite.rel.ddl.ChangeConsensusRole; import org.apache.calcite.rel.ddl.CreateDatabase; +import org.apache.calcite.rel.ddl.CreateFileStorage; import org.apache.calcite.rel.ddl.CreateIndex; import org.apache.calcite.rel.ddl.CreateTable; import org.apache.calcite.rel.ddl.CreateTableGroup; @@ -851,6 +853,8 @@ private RelNode convertToLogicalDdlPlan(DDL ddl) { return LogicalAlterFileStorage.create(ddl); } else if (ddl instanceof DropFileStorage) { return LogicalDropFileStorage.create(ddl); + } else if (ddl instanceof CreateFileStorage) { + return LogicalCreateFileStorage.create(ddl); } else { throw new TddlRuntimeException(ErrorCode.ERR_DDL_JOB_UNSUPPORTED, "operation " + ddl.getSqlNode().getKind()); diff --git a/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ddl/LogicalCreateFileStorage.java b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ddl/LogicalCreateFileStorage.java new file mode 100644 index 000000000..16464fc38 --- /dev/null +++ b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/core/rel/ddl/LogicalCreateFileStorage.java @@ -0,0 +1,21 @@ +package com.alibaba.polardbx.optimizer.core.rel.ddl; + +import org.apache.calcite.rel.core.DDL; +import org.apache.calcite.rel.ddl.CreateFileStorage; + +public class LogicalCreateFileStorage extends BaseDdlOperation { + private CreateFileStorage createFileStorage; + + public LogicalCreateFileStorage(DDL ddl) { + super(ddl); + this.createFileStorage = (CreateFileStorage) relDdl; + } + + public CreateFileStorage getCreateFileStorage() { + return createFileStorage; + } + + public static LogicalCreateFileStorage create(DDL ddl) { + return new LogicalCreateFileStorage(ddl); + } +} diff --git a/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/parse/visitor/FastSqlToCalciteNodeVisitor.java b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/parse/visitor/FastSqlToCalciteNodeVisitor.java index 197e0daee..1d6276e04 100644 --- a/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/parse/visitor/FastSqlToCalciteNodeVisitor.java +++ b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/parse/visitor/FastSqlToCalciteNodeVisitor.java @@ -200,6 +200,7 @@ import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlCharExpr; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOutFileExpr; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlUserName; +import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CreateFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableAsOfTimeStamp; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableBroadcast; @@ -437,6 +438,7 @@ import org.apache.calcite.sql.SqlConvertToCharacterSet; import org.apache.calcite.sql.SqlCreateCclRule; import org.apache.calcite.sql.SqlCreateCclTrigger; +import org.apache.calcite.sql.SqlCreateFileStorage; import org.apache.calcite.sql.SqlCreateIndex; import org.apache.calcite.sql.SqlCreateIndex.SqlIndexAlgorithmType; import org.apache.calcite.sql.SqlCreateIndex.SqlIndexConstraintType; @@ -7897,6 +7899,25 @@ public boolean visit(DrdsClearCclRulesStatement x) { return false; } + @Override + public boolean visit(CreateFileStorageStatement x) { + SqlIdentifier engineName = (SqlIdentifier) convertToSqlNode(x.getEngineName()); + + List withValue = x.getWithValue(); + List> with = new ArrayList<>(withValue.size()); + for (SQLAssignItem sqlAssignItem : withValue) { + SqlIdentifier target = + (SqlIdentifier) convertToSqlNode(sqlAssignItem.getTarget()); + SqlIdentifier value = + (SqlIdentifier) convertToSqlNode(sqlAssignItem.getValue()); + with.add(Pair.of(target, value)); + } + SqlCreateFileStorage sqlCreateFileStorage = new SqlCreateFileStorage(SqlParserPos.ZERO, engineName, with); + sqlNode = sqlCreateFileStorage; + return false; + } + + @Override public boolean visit(DrdsCreateScheduleStatement x) { diff --git a/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/utils/OptimizerUtils.java b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/utils/OptimizerUtils.java index 184429f76..8097c6cc0 100644 --- a/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/utils/OptimizerUtils.java +++ b/polardbx-optimizer/src/main/java/com/alibaba/polardbx/optimizer/utils/OptimizerUtils.java @@ -147,6 +147,7 @@ public static boolean supportedSqlKind(SqlNode ast) { case DROP_SCHEDULE: case ALTER_FILESTORAGE: case DROP_FILESTORAGE: + case CREATE_FILESTORAGE: return true; default: if (ast.isA(SqlKind.DAL)) { diff --git a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/ast/statement/CreateFileStorageStatement.java b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/ast/statement/CreateFileStorageStatement.java new file mode 100644 index 000000000..cca64aee4 --- /dev/null +++ b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/ast/statement/CreateFileStorageStatement.java @@ -0,0 +1,62 @@ +package com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement; + +import com.alibaba.polardbx.druid.sql.ast.SQLName; +import com.alibaba.polardbx.druid.sql.ast.statement.SQLAssignItem; +import com.alibaba.polardbx.druid.sql.ast.statement.SQLCreateStatement; +import com.alibaba.polardbx.druid.sql.dialect.mysql.visitor.MySqlASTVisitor; +import com.google.common.collect.ImmutableMap; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CreateFileStorageStatement extends MySqlStatementImpl implements SQLCreateStatement { + private SQLName engineName; + private List withValue; + + public CreateFileStorageStatement() { + this.withValue = new ArrayList(); + } + + public void accept0(MySqlASTVisitor visitor) { + if (visitor.visit(this)) { + if (this.engineName != null) { + this.engineName.accept(visitor); + } + if (this.withValue != null) { + for (SQLAssignItem item : withValue) { + item.accept(visitor); + } + } + } + visitor.endVisit(this); + } + + @Override + public CreateFileStorageStatement clone() { + CreateFileStorageStatement x = new CreateFileStorageStatement(); + if (this.engineName != null) { + x.engineName = this.engineName.clone(); + } + if (this.withValue != null) { + for (SQLAssignItem item : withValue) { + x.withValue.add(item.clone()); + } + } + return x; + } + + public SQLName getEngineName() { + return engineName; + } + + public CreateFileStorageStatement setEngineName(SQLName engineName) { + this.engineName = engineName; + return this; + } + + public List getWithValue() { + return withValue; + } +} diff --git a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/parser/MySqlStatementParser.java b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/parser/MySqlStatementParser.java index 93aa4352f..9723d1817 100644 --- a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/parser/MySqlStatementParser.java +++ b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/parser/MySqlStatementParser.java @@ -196,6 +196,7 @@ import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.clause.MySqlSelectIntoStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlUserName; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CobarShowStatus; +import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CreateFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableAsOfTimeStamp; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableBroadcast; @@ -895,6 +896,11 @@ public SQLStatement parseCreate() { return parseCreateSchedule(); } + if (lexer.identifierEquals("FileStorage")) { + lexer.reset(mark); + return parseCreateFileStorage(); + } + throw new ParserException("TODO " + lexer.info()); } @@ -10076,6 +10082,34 @@ public SQLStatement parseCreateCclTrigger() { return stmt; } + public SQLStatement parseCreateFileStorage() { + accept(Token.CREATE); + acceptIdentifier("FileStorage"); + + CreateFileStorageStatement stmt = new CreateFileStorageStatement(); + + stmt.setEngineName(this.exprParser.name()); + if (lexer.token() == Token.WITH) { + lexer.nextToken(); + accept(Token.LPAREN); + + for (;;) { + SQLName key = this.exprParser.name(); + accept(Token.EQ); + SQLName value = this.exprParser.name(); + stmt.getWithValue().add(new SQLAssignItem(key, value)); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } + + accept(Token.RPAREN); + } + return stmt; + } + public SQLStatement parseCreateSchedule() { accept(Token.CREATE); acceptIdentifier("SCHEDULE"); diff --git a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitor.java b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitor.java index b309be71d..ac2ad04e5 100644 --- a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitor.java +++ b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitor.java @@ -50,6 +50,7 @@ import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOutFileExpr; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlUserName; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CobarShowStatus; +import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CreateFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableAsOfTimeStamp; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableBroadcast; @@ -411,6 +412,10 @@ public interface MySqlASTVisitor extends SQLASTVisitor { void endVisit(DrdsDropScheduleStatement x); + boolean visit(CreateFileStorageStatement x); + + void endVisit(CreateFileStorageStatement x); + boolean visit(DrdsDropScheduleStatement x); void endVisit(DrdsDropCclRuleStatement x); diff --git a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitorAdapter.java b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitorAdapter.java index 12cee5160..fe1269468 100644 --- a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitorAdapter.java +++ b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlASTVisitorAdapter.java @@ -50,6 +50,7 @@ import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOutFileExpr; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlUserName; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CobarShowStatus; +import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CreateFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableAsOfTimeStamp; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableBroadcast; @@ -622,6 +623,16 @@ public boolean visit(DrdsCreateScheduleStatement x) { return false; } + @Override + public void endVisit(CreateFileStorageStatement x) { + + } + + @Override + public boolean visit(CreateFileStorageStatement x) { + return false; + } + @Override public void endVisit(DrdsDropScheduleStatement x) { diff --git a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java index 445b3f7dd..4bb450e9a 100644 --- a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java +++ b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java @@ -128,6 +128,7 @@ import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOutFileExpr; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlUserName; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CobarShowStatus; +import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CreateFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableAsOfTimeStamp; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableBroadcast; @@ -2594,6 +2595,27 @@ public void endVisit(DrdsDropScheduleStatement x) { } + @Override + public boolean visit(CreateFileStorageStatement x) { + print0(ucase ? "CREATE FILESTORAGE " : "create filestorage "); + x.getEngineName().accept(this); + + List withValue = x.getWithValue(); + if (withValue.size() > 0) { + println(); + print0(ucase ? "WITH (" : "with ("); + printAndAccept(withValue, ", "); + print(')'); + } + + return false; + } + + @Override + public void endVisit(CreateFileStorageStatement x) { + + } + @Override public boolean visit(DrdsDropScheduleStatement x) { print0(ucase ? "DROP SCHEDULE " : "drop schedule "); diff --git a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlParameterizedVisitor.java b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlParameterizedVisitor.java index d045ce94f..62f19bcfe 100644 --- a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlParameterizedVisitor.java +++ b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlParameterizedVisitor.java @@ -41,6 +41,7 @@ import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOutFileExpr; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlUserName; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CobarShowStatus; +import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CreateFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableAsOfTimeStamp; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableBroadcast; @@ -657,6 +658,16 @@ public void endVisit(DrdsDropCclRuleStatement x) { } + @Override + public boolean visit(CreateFileStorageStatement x) { + return false; + } + + @Override + public void endVisit(CreateFileStorageStatement x) { + + } + @Override public boolean visit(DrdsDropCclRuleStatement x) { return false; diff --git a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlSchemaStatVisitor.java b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlSchemaStatVisitor.java index 2d12f27c8..0fbd33932 100644 --- a/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlSchemaStatVisitor.java +++ b/polardbx-parser/src/main/java/com/alibaba/polardbx/druid/sql/dialect/mysql/visitor/MySqlSchemaStatVisitor.java @@ -58,6 +58,7 @@ import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOutFileExpr; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlUserName; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CobarShowStatus; +import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.CreateFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterFileStorageStatement; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableAsOfTimeStamp; import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.DrdsAlterTableBroadcast; @@ -711,6 +712,16 @@ public boolean visit(DrdsCreateScheduleStatement x) { return false; } + @Override + public void endVisit(CreateFileStorageStatement x) { + + } + + @Override + public boolean visit(CreateFileStorageStatement x) { + return false; + } + @Override public void endVisit(DrdsDropScheduleStatement x) {