diff --git a/collector/pom.xml b/collector/pom.xml index 70deccc..92b9b41 100644 --- a/collector/pom.xml +++ b/collector/pom.xml @@ -103,6 +103,12 @@ sshd-core 2.8.0 + + + com.microsoft.sqlserver + mssql-jdbc + 10.2.0.jre8 + \ No newline at end of file diff --git a/collector/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java b/collector/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java index c4c74f5..0e852f4 100644 --- a/collector/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java +++ b/collector/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java @@ -56,7 +56,9 @@ public class JdbcCommonCollect extends AbstractCollect { int timeout = 3000; try { // 获取查询语句超时时间 - timeout = Integer.parseInt(jdbcProtocol.getTimeout()); + if (jdbcProtocol.getTimeout() != null) { + timeout = Integer.parseInt(jdbcProtocol.getTimeout()); + } } catch (Exception e) { log.warn(e.getMessage()); } @@ -140,7 +142,9 @@ public class JdbcCommonCollect extends AbstractCollect { Connection connection = DriverManager.getConnection(url, username, password); statement = connection.createStatement(); // 设置查询超时时间10秒 - statement.setQueryTimeout(timeout); + int timeoutSecond = timeout / 1000; + timeoutSecond = timeoutSecond <= 0 ? 1 : timeoutSecond; + statement.setQueryTimeout(timeoutSecond); // 设置查询最大行数1000行 statement.setMaxRows(1000); JdbcConnect jdbcConnect = new JdbcConnect(connection); @@ -200,7 +204,7 @@ public class JdbcCommonCollect extends AbstractCollect { HashMap values = new HashMap<>(columns.size()); while (resultSet.next()) { if (resultSet.getString(1) != null) { - values.put(resultSet.getString(1).toLowerCase(), resultSet.getString(2)); + values.put(resultSet.getString(1).toLowerCase().trim(), resultSet.getString(2)); } } CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); @@ -277,6 +281,10 @@ public class JdbcCommonCollect extends AbstractCollect { url = "jdbc:postgresql://" + jdbcProtocol.getHost() + ":" + jdbcProtocol.getPort() + "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase()); break; + case "sqlserver": + url = "jdbc:sqlserver://" + jdbcProtocol.getHost() + ":" + jdbcProtocol.getPort() + + ";" + (jdbcProtocol.getDatabase() == null ? "" : "DatabaseName=" + jdbcProtocol.getDatabase()); + break; default: throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform()); diff --git a/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java b/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java index 54c1a31..93ad92f 100644 --- a/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java +++ b/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -172,14 +173,23 @@ public class MetricsCollect implements Runnable, Comparable { if (metrics.getCalculates() == null) { metrics.setCalculates(Collections.emptyList()); } + // eg: database_pages=Database pages 非常规映射 + Map fieldAliasMap = new HashMap<>(8); Map fieldExpressionMap = metrics.getCalculates() .stream() .map(cal -> { int splitIndex = cal.indexOf("="); String field = cal.substring(0, splitIndex); String expressionStr = cal.substring(splitIndex + 1); - Expression expression = AviatorEvaluator.compile(expressionStr, true); + Expression expression = null; + try { + expression = AviatorEvaluator.compile(expressionStr, true); + } catch (Exception e) { + fieldAliasMap.put(field, expressionStr); + return null; + } return new Object[]{field, expression}; }) + .filter(Objects::nonNull) .collect(Collectors.toMap(arr -> (String)arr[0], arr -> (Expression) arr[1])); List fields = metrics.getFields(); @@ -226,7 +236,12 @@ public class MetricsCollect implements Runnable, Comparable { } } else { // 不存在 则映射别名值 - value = aliasFieldValueMap.get(realField); + String aliasField = fieldAliasMap.get(realField); + if (aliasField != null) { + value = aliasFieldValueMap.get(aliasField); + } else { + value = aliasFieldValueMap.get(realField); + } } if (value == null) { value = CommonConstants.NULL_VALUE; diff --git a/manager/src/main/resources/define/app/sqlserver.yml b/manager/src/main/resources/define/app/sqlserver.yml new file mode 100644 index 0000000..f4eacc9 --- /dev/null +++ b/manager/src/main/resources/define/app/sqlserver.yml @@ -0,0 +1,136 @@ +category: db +app: sqlserver +name: + zh-CN: SqlServer数据库 + en-US: SqlServer DB +# 参数映射map. type是参数类型: 0-number数字, 1-string明文字符串, 2-secret加密字符串 +# 强制固定必须参数 - host +configmap: + - key: host + type: 1 + - key: port + type: 0 + - key: username + type: 1 + - key: password + type: 2 + - key: database + type: 1 + - key: timeout + type: 0 + - key: url + type: 1 +# 指标组列表 +metrics: + - name: basic + # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集 + # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度 + priority: 0 + # 指标组中的具体监控指标 + fields: + # 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位 + - field: machine_name + type: 1 + instance: true + - field: server_name + type: 1 + - field: version + type: 1 + - field: edition + type: 1 + - field: start_time + type: 1 + protocol: jdbc + jdbc: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + platform: sqlserver + username: ^_^username^_^ + password: ^_^password^_^ + database: ^_^database^_^ + timeout: ^_^timeout^_^ + # SQL查询方式： oneRow, multiRow, columns + queryType: oneRow + # sql + sql: SELECT SERVERPROPERTY('MachineName') AS machine_name, SERVERPROPERTY('ServerName') AS server_name, SERVERPROPERTY('ProductVersion') AS version, SERVERPROPERTY('Edition') AS edition, sqlserver_start_time AS start_time FROM sys.dm_os_sys_info; + url: ^_^url^_^ + + - name: performance_counters + priority: 1 + fields: + # 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位 + - field: database_pages + type: 0 + - field: target_pages + type: 0 + - field: page_life_expectancy + type: 0 + - field: buffer_cache_hit_ratio + type: 0 + - field: checkpoint_pages_sec + type: 0 + - field: page_reads_sec + type: 0 + - field: page_writes_sec + type: 0 + # (非必须)监控指标别名，与上面的指标名映射。用于采集接口数据字段不直接是最终指标名称,需要此别名做映射转换 + aliasFields: + - Database pages + - Target pages + - Page life expectancy + - Buffer cache hit ratio + - Checkpoint pages/sec + - Page reads/sec + - Page writes/sec + # (非必须)指标计算表达式,与上面的别名一起作用,计算出最终需要的指标值 + # eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime + calculates: + - database_pages=Database pages + - target_pages=Target pages + - page_life_expectancy=Page life expectancy + - buffer_cache_hit_ratio=Buffer cache hit ratio + - checkpoint_pages_sec=Checkpoint pages/sec + - page_reads_sec=Page reads/sec + - page_writes_sec=Page writes/sec + protocol: jdbc + jdbc: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + platform: sqlserver + username: ^_^username^_^ + password: ^_^password^_^ + database: ^_^database^_^ + timeout: ^_^timeout^_^ + # SQL查询方式： oneRow, multiRow, columns + queryType: columns + # sql + sql: select counter_name, cntr_value from sys.dm_os_performance_counters where object_name = 'SQLServer:Buffer Manager'; + url: ^_^url^_^ + + - name: connection + priority: 1 + fields: + # 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位 + - field: connection + type: 0 + unit: 连接数 + protocol: jdbc + jdbc: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + platform: sqlserver + username: ^_^username^_^ + password: ^_^password^_^ + database: ^_^database^_^ + timeout: ^_^timeout^_^ + # SQL查询方式： oneRow, multiRow, columns + queryType: oneRow + # sql + sql: SELECT cntr_value as connection FROM sys.dm_os_performance_counters WHERE object_name = 'SQLServer:General Statistics' AND counter_name = 'User Connections'; + url: ^_^url^_^ \ No newline at end of file diff --git a/manager/src/main/resources/define/param/mariadb.yml b/manager/src/main/resources/define/param/mariadb.yml index 6c436d5..9443d1c 100644 --- a/manager/src/main/resources/define/param/mariadb.yml +++ b/manager/src/main/resources/define/param/mariadb.yml @@ -9,7 +9,7 @@ param: type: number range: '[0,65535]' required: true - defaultValue: 80 + defaultValue: 3306 placeholder: '请输入端口' - field: timeout name: 查询超时时间 diff --git a/manager/src/main/resources/define/param/mysql.yml b/manager/src/main/resources/define/param/mysql.yml index 791188c..3cd86f2 100644 --- a/manager/src/main/resources/define/param/mysql.yml +++ b/manager/src/main/resources/define/param/mysql.yml @@ -9,7 +9,7 @@ param: type: number range: '[0,65535]' required: true - defaultValue: 80 + defaultValue: 3306 placeholder: '请输入端口' - field: timeout name: 查询超时时间 diff --git a/manager/src/main/resources/define/param/sqlserver.yml b/manager/src/main/resources/define/param/sqlserver.yml new file mode 100644 index 0000000..d69707f --- /dev/null +++ b/manager/src/main/resources/define/param/sqlserver.yml @@ -0,0 +1,36 @@ +app: sqlserver +param: + - field: host + name: 主机Host + type: host + required: true + - field: port + name: 端口 + type: number + range: '[0,65535]' + required: true + defaultValue: 1433 + placeholder: '请输入端口' + - field: timeout + name: 查询超时时间 + type: number + required: false + defaultValue: 3000 + placeholder: '查询超时时间' + - field: database + name: 数据库名称 + type: text + required: false + - field: username + name: 用户名 + type: text + limit: 20 + required: false + - field: password + name: 密码 + type: password + required: false + - field: url + name: URL + type: text + required: false \ No newline at end of file diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/TdEngineDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/TdEngineDataStorage.java index db54e86..dc9ad56 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/TdEngineDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/TdEngineDataStorage.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; /** * influxdb存储采集数据 @@ -39,6 +40,7 @@ public class TdEngineDataStorage implements DisposableBean { private HikariDataSource hikariDataSource; private WarehouseWorkerPool workerPool; private MetricsDataExporter dataExporter; + private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')"); private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO %s USING %s TAGS (%s) VALUES %s"; private static final String CREATE_SUPER_TABLE_SQL = "CREATE STABLE IF NOT EXISTS %s %s TAGS (monitor BIGINT)"; private static final String NO_SUPER_TABLE_ERROR = "Table does not exist"; @@ -200,8 +202,9 @@ public class TdEngineDataStorage implements DisposableBean { } private String formatStringValue(String value){ - return value.replaceAll("(\\\\)|(')","\\\\$0"); + return SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0"); } + @Override public void destroy() throws Exception { if (hikariDataSource != null) {