Skip to content

Commit

Permalink
perf: 优化插入MySQL时脚本参数过长的报错信息,使用validation校验脚本长度 TencentBlueKing#3374
Browse files Browse the repository at this point in the history
  • Loading branch information
wuyzh39 committed Jan 21, 2025
1 parent b9c1459 commit ab53e59
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 45 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,28 @@ public static byte[] decodeContentToByte(String content) {
throw e;
}
}

/**
* 根据BASE64编码后的字符串计算原始字节流长度,不用decode字符串
*
* @param encodedContent 编码后的字符串
* @return 原始字符串的长度
*/
public static int calOriginBytesLength(String encodedContent) {
if (StringUtils.isEmpty(encodedContent)) {
return 0;
}

// 最多只会填充两个 =
int fillCount = 0;
for (int i = 0; i <= 1 && encodedContent.length() - 1 - i >= 0; i++) {
if (encodedContent.charAt(encodedContent.length() - 1 - i) == '=') {
fillCount++;
} else {
break;
}
}

return encodedContent.length() * 3 / 4 - fillCount;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
*
* Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
*
* License for BK-JOB蓝鲸智云作业平台:
* --------------------------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
* to permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
* THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/

package com.tencent.bk.job.common.util;

import org.junit.jupiter.api.Test;

import java.util.Random;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class Base64UtilTest {

@Test
void testCalOriginBytesLength() {
for (int i = 0; i < 100; i++) {
String originStr = genRandomString();
assertCal(originStr);
}
String specialStrWithChinese = "# 在当前脚本执行时,第一行输出当前时间和进程ID,详见上面函数:job_get_now\n"
+ "job_start\n"
+ "\n"
+ "###### 作业平台中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值\n"
+ "###### 如果返回值为0,则认为此脚本执行成功,如果非0,则认为脚本执行失败\n"
+ "###### 可在此处开始编写您的脚本逻辑代码";
assertCal(specialStrWithChinese);

}

private void assertCal(String originStr) {

String encodedStr = Base64Util.encodeContentToStr(originStr);
int calLengthByUtil = Base64Util.calOriginBytesLength(encodedStr);

byte[] bytes = Base64Util.decodeContentToByte(encodedStr);
int originBytesLength = bytes.length;

System.out.println("originStr:" + originStr);
System.out.println("originBytesLength:" + originBytesLength);
System.out.println("calLengthByUtil:" + calLengthByUtil);

assertEquals(originBytesLength, calLengthByUtil,
"bytes length should be " + originBytesLength + ", but is " + calLengthByUtil);
}

private String genRandomString() {
final int ASCII_LOW = 32;
final int ASCII_HIGH = 126;
final int SHORTEST = 1;
final int LONGEST = 500;
Random rand = new Random();
int length = rand.nextInt(LONGEST - SHORTEST) + SHORTEST;
StringBuilder sb = new StringBuilder(length);
for (int i = 0; i < length; i++) {
int randomAscii = ASCII_LOW + rand.nextInt(ASCII_HIGH - ASCII_LOW + 1);
sb.append((char) randomAscii);
}
return sb.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,27 +27,27 @@
import lombok.Getter;

/**
* 这里暂时只记录字符串形式的MySQL类型,用于获取各类型的最大长度
* 这里暂时只记录TEXT形式的MySQL类型,用于获取各类型的最大长度。
* 因为在MySQL中,TEXT类型的存储是以字节流的长度为限制的,而char/varchar则是以字符串的长度为限制的,所以校验长度合法的逻辑是不太一样的。
*
*/
@Getter
public enum MySQLDataType {
CHAR("CHAR", 255L),
VARCHAR("VARCHAR", 65535L),
public enum MySQLTextDataType {
TINYTEXT("TINYTEXT", 255L),
TEXT("TEXT", 65535L),
MEDIUMTEXT("MEDIUMTEXT", 16777215L),
LONGTEXT("LONGTEXT", 4294967295L);

private final String value;
private final Long maximumLength;
private final Long maximumLength; // MySQL能存的最大字节数

MySQLDataType(String value, Long maximumLength) {
MySQLTextDataType(String value, Long maximumLength) {
this.value = value;
this.maximumLength = maximumLength;
}

public static MySQLDataType valOf(String value) {
for (MySQLDataType type : MySQLDataType.values()) {
public static MySQLTextDataType valOf(String value) {
for (MySQLTextDataType type : MySQLTextDataType.values()) {
if (type.value.equals(value)) {
return type;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
package com.tencent.bk.job.common.validation;


import com.tencent.bk.job.common.constant.MySQLDataType;
import com.tencent.bk.job.common.constant.MySQLTextDataType;
import com.tencent.bk.job.common.util.Base64Util;
import lombok.extern.slf4j.Slf4j;

import javax.validation.Constraint;
Expand All @@ -35,6 +36,7 @@
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import java.nio.charset.StandardCharsets;

import static java.lang.annotation.ElementType.ANNOTATION_TYPE;
import static java.lang.annotation.ElementType.FIELD;
Expand All @@ -44,13 +46,16 @@
import static java.lang.annotation.RetentionPolicy.RUNTIME;

/**
* 通过在MySQL定义的字段类型判断是否过长
* 通过在MySQL定义的TEXT一族字段类型(TINYTEXT/TEXT/MEDIUMTEXT/LONGTEXT)判断是否过长
* 因为在MySQL中,TEXT类型的存储是以字节流的长度为限制的,而char/varchar则是以字符串的长度为限制的,所以校验长度合法的逻辑是不太一样的
* 因此这个校验器只校验TEXT类型的字节流长度,若需要校验char/varchar类型的字段长度,请用str.length()校验
*
*/
@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE})
@Constraint(validatedBy = NotExceedMySQLFieldLength.FieldLengthValidator.class)
@Constraint(validatedBy = NotExceedMySQLTextFieldLength.FieldLengthValidator.class)
@Documented
@Retention(RUNTIME)
public @interface NotExceedMySQLFieldLength {
public @interface NotExceedMySQLTextFieldLength {

String message() default "{fieldName} {validation.constraints.NotExceedMySQLFieldLength.message}";

Expand All @@ -60,29 +65,39 @@

String fieldName();

MySQLDataType fieldType();
MySQLTextDataType fieldType();

boolean base64();

@Slf4j
class FieldLengthValidator implements ConstraintValidator<NotExceedMySQLFieldLength, String> {
class FieldLengthValidator implements ConstraintValidator<NotExceedMySQLTextFieldLength, String> {

MySQLDataType fieldType = null;
MySQLTextDataType fieldType = null;
boolean useBase64 = false;

@Override
public void initialize(NotExceedMySQLFieldLength constraintAnnotation) {
public void initialize(NotExceedMySQLTextFieldLength constraintAnnotation) {
fieldType = constraintAnnotation.fieldType();
useBase64 = constraintAnnotation.base64();
}

@Override
public boolean isValid(String scriptContent, ConstraintValidatorContext constraintValidatorContext) {

log.debug("[Validate MySQLFieldLength] field type: {}, content length: {}, maximum length: {}]",
fieldType.getValue(), scriptContent.length(), fieldType.getMaximumLength());

if (fieldType == null || scriptContent == null) {
public boolean isValid(String value, ConstraintValidatorContext constraintValidatorContext) {
if (value == null || fieldType == null) {
return true;
}

return scriptContent.length() <= fieldType.getMaximumLength();
int currentLength;
if (useBase64) {
currentLength = Base64Util.calOriginBytesLength(value);
} else {
currentLength = value.getBytes(StandardCharsets.UTF_8).length;
}

log.debug("[Validate MySQLFieldLength] field type: {}, current length: {}, maximum length: {}]",
fieldType.getValue(), currentLength, fieldType.getMaximumLength());

return currentLength <= fieldType.getMaximumLength();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import com.tencent.bk.job.common.constant.JobConstants;
import com.tencent.bk.job.common.constant.MySQLDataType;
import com.tencent.bk.job.common.constant.MySQLTextDataType;
import com.tencent.bk.job.common.esb.model.EsbAppScopeReq;
import com.tencent.bk.job.common.esb.model.job.EsbIpDTO;
import com.tencent.bk.job.common.esb.model.job.EsbServerDTO;
import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength;
import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.validator.constraints.Range;
Expand All @@ -54,9 +54,10 @@ public class EsbFastExecuteScriptRequest extends EsbAppScopeReq {
* "脚本内容,BASE64编码
*/
@JsonProperty("script_content")
@NotExceedMySQLFieldLength(
@NotExceedMySQLTextFieldLength(
fieldName = "scriptContent",
fieldType = MySQLDataType.MEDIUMTEXT
fieldType = MySQLTextDataType.MEDIUMTEXT,
base64 = true
)
private String content;

Expand All @@ -75,9 +76,10 @@ public class EsbFastExecuteScriptRequest extends EsbAppScopeReq {
* 脚本参数, BASE64编码
*/
@JsonProperty("script_param")
@NotExceedMySQLFieldLength(
@NotExceedMySQLTextFieldLength(
fieldName = "scriptParam",
fieldType = MySQLDataType.TEXT
fieldType = MySQLTextDataType.TEXT,
base64 = true
)
private String scriptParam;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import com.tencent.bk.job.common.constant.JobConstants;
import com.tencent.bk.job.common.constant.MySQLDataType;
import com.tencent.bk.job.common.constant.MySQLTextDataType;
import com.tencent.bk.job.common.esb.model.EsbAppScopeReq;
import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteTargetDTO;
import com.tencent.bk.job.common.validation.CheckEnum;
import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength;
import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength;
import com.tencent.bk.job.common.validation.ValidationGroups;
import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO;
import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.validator.EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider;
Expand Down Expand Up @@ -63,9 +63,10 @@ public class EsbBkCIPluginFastExecuteScriptRequest extends EsbAppScopeReq {
* 脚本内容,BASE64编码
*/
@JsonProperty("script_content")
@NotExceedMySQLFieldLength(
@NotExceedMySQLTextFieldLength(
fieldName = "scriptContent",
fieldType = MySQLDataType.MEDIUMTEXT
fieldType = MySQLTextDataType.MEDIUMTEXT,
base64 = true
)
private String content;

Expand Down Expand Up @@ -97,6 +98,11 @@ public class EsbBkCIPluginFastExecuteScriptRequest extends EsbAppScopeReq {
* 脚本参数, BASE64编码
*/
@JsonProperty("script_param")
@NotExceedMySQLTextFieldLength(
fieldName = "scriptParam",
fieldType = MySQLTextDataType.TEXT,
base64 = true
)
private String scriptParam;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import com.tencent.bk.job.common.constant.JobConstants;
import com.tencent.bk.job.common.constant.MySQLDataType;
import com.tencent.bk.job.common.constant.MySQLTextDataType;
import com.tencent.bk.job.common.esb.model.EsbAppScopeReq;
import com.tencent.bk.job.common.esb.model.job.EsbIpDTO;
import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO;
import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength;
import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength;
import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO;
import lombok.Getter;
import lombok.Setter;
Expand All @@ -56,9 +56,10 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq {
* "脚本内容,BASE64编码
*/
@JsonProperty("script_content")
@NotExceedMySQLFieldLength(
@NotExceedMySQLTextFieldLength(
fieldName = "scriptContent",
fieldType = MySQLDataType.MEDIUMTEXT
fieldType = MySQLTextDataType.MEDIUMTEXT,
base64 = true
)
private String content;

Expand All @@ -84,9 +85,10 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq {
* 脚本参数, BASE64编码
*/
@JsonProperty("script_param")
@NotExceedMySQLFieldLength(
@NotExceedMySQLTextFieldLength(
fieldName = "scriptParam",
fieldType = MySQLDataType.TEXT
fieldType = MySQLTextDataType.TEXT,
base64 = true
)
private String scriptParam;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
import com.tencent.bk.job.common.annotation.CompatibleImplementation;
import com.tencent.bk.job.common.constant.CompatibleType;
import com.tencent.bk.job.common.constant.JobConstants;
import com.tencent.bk.job.common.constant.MySQLDataType;
import com.tencent.bk.job.common.constant.MySQLTextDataType;
import com.tencent.bk.job.common.model.vo.TaskTargetVO;
import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength;
import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength;
import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
Expand All @@ -54,9 +54,10 @@ public class WebFastExecuteScriptRequest {
* 脚本内容
*/
@ApiModelProperty(value = "脚本内容,BASE64编码,当手动录入的时候使用此参数")
@NotExceedMySQLFieldLength(
@NotExceedMySQLTextFieldLength(
fieldName = "scriptContent",
fieldType = MySQLDataType.MEDIUMTEXT
fieldType = MySQLTextDataType.MEDIUMTEXT,
base64 = true
)
private String content;

Expand Down Expand Up @@ -88,9 +89,10 @@ public class WebFastExecuteScriptRequest {
* 脚本参数
*/
@ApiModelProperty(value = "脚本参数")
@NotExceedMySQLFieldLength(
@NotExceedMySQLTextFieldLength(
fieldName = "scriptParam",
fieldType = MySQLDataType.TEXT
fieldType = MySQLTextDataType.TEXT,
base64 = true
)
private String scriptParam;

Expand Down

0 comments on commit ab53e59

Please sign in to comment.