From 4ae1145d1900ee061ee4837a85982c86c4143b13 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Fri, 3 Dec 2021 13:41:17 +0800 Subject: [PATCH] =?UTF-8?q?[monitor]=20=E6=94=AF=E6=8C=81=E7=BD=91?= =?UTF-8?q?=E7=AB=99=E7=9B=91=E6=8E=A7=E7=B1=BB=E5=9E=8B=EF=BC=8C=E9=99=8D?= =?UTF-8?q?=E4=BD=8E=E9=9B=AA=E8=8A=B1=E7=AE=97=E6=B3=95UUID=E6=9C=80?= =?UTF-8?q?=E5=A4=A7=E5=80=BC=E8=A7=A3=E5=86=B3ts=20json=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=A4=A7=E6=95=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/usthe/collector/Collector.java | 19 ------- .../collect/http/HttpCollectImpl.java | 45 ++++++++++++--- .../collector/dispatch/DispatchConstants.java | 4 ++ .../collector/util/CollectorConstants.java | 11 ++++ .../common/util/SnowFlakeIdGenerator.java | 2 +- .../usthe/common/util/SnowFlakeIdWorker.java | 5 +- .../common/util/SnowFlakeIdGeneratorTest.java | 22 ++++++++ .../main/resources/define/app/A-example.yml | 6 +- manager/src/main/resources/define/app/api.yml | 56 +++++++++++++++++++ .../src/main/resources/define/param/api.yml | 47 ++++++++++++++++ manager/src/main/resources/sureness.yml | 1 + .../monitor-new/monitor-new.component.ts | 2 +- web-app/src/assets/tmp/app-data.json | 4 +- web-app/src/assets/tmp/i18n/zh-CN.json | 1 + 14 files changed, 190 insertions(+), 35 deletions(-) create mode 100644 collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java create mode 100644 common/src/test/java/com/usthe/common/util/SnowFlakeIdGeneratorTest.java create mode 100644 manager/src/main/resources/define/app/api.yml create mode 100644 manager/src/main/resources/define/param/api.yml diff --git a/collector/server/src/main/java/com/usthe/collector/Collector.java b/collector/server/src/main/java/com/usthe/collector/Collector.java index ac67dc5..275ec54 100644 --- a/collector/server/src/main/java/com/usthe/collector/Collector.java +++ b/collector/server/src/main/java/com/usthe/collector/Collector.java @@ -5,10 +5,6 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.InetAddress; - /** * collector start * @author tomsun28 @@ -19,20 +15,5 @@ import java.net.InetAddress; public class Collector { public static void main(String[] args) { SpringApplication.run(Collector.class, args); -// DatagramSocket s = new DatagramSocket(); -// -// /** 2、提供数据，封装打包 ---DatagramPacket(byte[] buf, int length, InetAddress address, int port) */ -// -// byte[] bs = "正在使用UDP发送--我是数据! ".getBytes(); -// DatagramPacket dp = new DatagramPacket(bs, bs.length, InetAddress.getByName("192.168.1.189"), 8070); -// -// /** 3、使用send发送 */ -// try { -// s.send(dp); -// s.receive(dp) -// } catch (IOException e) { -// System.out.println("发送失败： "); -// e.printStackTrace(); -// } } } diff --git a/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java b/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java index d51d420..73fcc07 100644 --- a/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java +++ b/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java @@ -7,6 +7,7 @@ import com.google.gson.JsonParser; import com.usthe.collector.collect.AbstractCollect; import com.usthe.collector.common.http.HttpClientPool; import com.usthe.collector.dispatch.DispatchConstants; +import com.usthe.collector.util.CollectorConstants; import com.usthe.collector.util.JsonPathParser; import com.usthe.common.entity.job.Metrics; import com.usthe.common.entity.job.protocol.HttpProtocol; @@ -56,6 +57,7 @@ public class HttpCollectImpl extends AbstractCollect { @Override public void collect(CollectRep.MetricsData.Builder builder, long appId, String app, Metrics metrics) { + long startTime = System.currentTimeMillis(); // 简单校验必有参数 if (metrics == null || metrics.getHttp() == null) { builder.setCode(CollectRep.Code.FAIL); @@ -84,18 +86,21 @@ public class HttpCollectImpl extends AbstractCollect { builder.setMsg("statusCode: " + statusCode + ",entity empty."); return; } + Long responseTime = System.currentTimeMillis() - startTime; String parseType = metrics.getHttp().getParseType(); try { if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) { - parseResponseByDefault(resp, metrics.getAliasFields(), builder); + parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime); } else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) { - parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder); + parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime); } else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) { parseResponseByPrometheus(resp, metrics.getAliasFields(), metrics.getHttp(), builder); } else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) { parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder); + } else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){ + parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime); } else { - parseResponseByDefault(resp, metrics.getAliasFields(), builder); + parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime); } } catch (Exception e) { log.info("parse error: {}.", e.getMessage(), e); @@ -140,13 +145,26 @@ public class HttpCollectImpl extends AbstractCollect { } } + private void parseResponseByWebsite(String resp, List aliasFields, + CollectRep.MetricsData.Builder builder, Long responseTime) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + // todo resp 网站关键字监测 + for (String alias : aliasFields) { + if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(responseTime.toString()); + } else { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + } + } + builder.addValues(valueRowBuilder.build()); + } + private void parseResponseByXmlPath(String resp, List aliasFields, HttpProtocol http, CollectRep.MetricsData.Builder builder) { - } private void parseResponseByJsonPath(String resp, List aliasFields, HttpProtocol http, - CollectRep.MetricsData.Builder builder) { + CollectRep.MetricsData.Builder builder, Long responseTime) { List> results = JsonPathParser.parseContentWithJsonPath(resp,http. getParseScript()); for (Map stringMap : results) { CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); @@ -155,7 +173,11 @@ public class HttpCollectImpl extends AbstractCollect { if (value != null) { valueRowBuilder.addColumns(String.valueOf(value)); } else { - valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(responseTime.toString()); + } else { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + } } } builder.addValues(valueRowBuilder.build()); @@ -167,7 +189,8 @@ public class HttpCollectImpl extends AbstractCollect { } - private void parseResponseByDefault(String resp, List aliasFields, CollectRep.MetricsData.Builder builder) { + private void parseResponseByDefault(String resp, List aliasFields, + CollectRep.MetricsData.Builder builder, Long responseTime) { JsonElement element = JsonParser.parseString(resp); if (element.isJsonArray()) { JsonArray array = element.getAsJsonArray(); @@ -181,7 +204,11 @@ public class HttpCollectImpl extends AbstractCollect { String value = valueElement.getAsString(); valueRowBuilder.addColumns(value); } else { - valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(responseTime.toString()); + } else { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + } } } builder.addValues(valueRowBuilder.build()); @@ -283,6 +310,8 @@ public class HttpCollectImpl extends AbstractCollect { } else if (DispatchConstants.PARSE_PROMETHEUS.equals(httpProtocol.getParseType())) { requestBuilder.addHeader(HttpHeaders.ACCEPT, DispatchConstants.PARSE_PROMETHEUS_ACCEPT); requestBuilder.addHeader(HttpHeaders.ACCEPT_ENCODING, "gzip"); + } else { + requestBuilder.addHeader(HttpHeaders.ACCEPT, "*/*"); } // 判断是否使用Bearer Token认证 diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java index 2ca936b..b311d90 100644 --- a/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java @@ -56,6 +56,10 @@ public interface DispatchConstants { * 解析方式 自定义xml path */ String PARSE_XML_PATH = "xmlPath"; + /** + * 解析方式 网站可用性监控规则 提供responseTime指标 + */ + String PARSE_WEBSITE = "website"; /** * 解析方式 prometheus规则 */ diff --git a/collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java b/collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java new file mode 100644 index 0000000..176acd3 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java @@ -0,0 +1,11 @@ +package com.usthe.collector.util; + +/** + * collector 常量 + * @author tom + * @date 2021/12/3 12:15 + */ +public interface CollectorConstants { + + String RESPONSE_TIME = "responseTime"; +} diff --git a/common/src/main/java/com/usthe/common/util/SnowFlakeIdGenerator.java b/common/src/main/java/com/usthe/common/util/SnowFlakeIdGenerator.java index 6c1d235..ed9d768 100644 --- a/common/src/main/java/com/usthe/common/util/SnowFlakeIdGenerator.java +++ b/common/src/main/java/com/usthe/common/util/SnowFlakeIdGenerator.java @@ -10,7 +10,7 @@ public class SnowFlakeIdGenerator { private final static SnowFlakeIdWorker ID_WORKER; static { - ID_WORKER = new SnowFlakeIdWorker(1, 0); + ID_WORKER = new SnowFlakeIdWorker(0); } public static long generateId() { diff --git a/common/src/main/java/com/usthe/common/util/SnowFlakeIdWorker.java b/common/src/main/java/com/usthe/common/util/SnowFlakeIdWorker.java index 8c5a1ab..4a7f847 100644 --- a/common/src/main/java/com/usthe/common/util/SnowFlakeIdWorker.java +++ b/common/src/main/java/com/usthe/common/util/SnowFlakeIdWorker.java @@ -2,6 +2,7 @@ package com.usthe.common.util; /** * 雪花算法生成器实例 + * 注意 由于前端JS TS 在json解析大数会造成精度丢失 UUID 不能超过 9007199254740991（16位） * @author from https://www.cnblogs.com/vchar/p/14857677.html * @date 2021/11/10 10:58 */ @@ -15,12 +16,12 @@ public class SnowFlakeIdWorker { /** * 机器 ID 所占的位数 */ - private static final long WORKER_ID_BITS = 5L; + private static final long WORKER_ID_BITS = 2L; /** * 数据标识 ID 所占的位数 */ - private static final long DATA_CENTER_ID_BITS = 5L; + private static final long DATA_CENTER_ID_BITS = 4L; /** * 支持的最大机器ID，最大为31 diff --git a/common/src/test/java/com/usthe/common/util/SnowFlakeIdGeneratorTest.java b/common/src/test/java/com/usthe/common/util/SnowFlakeIdGeneratorTest.java new file mode 100644 index 0000000..3ff484c --- /dev/null +++ b/common/src/test/java/com/usthe/common/util/SnowFlakeIdGeneratorTest.java @@ -0,0 +1,22 @@ +package com.usthe.common.util; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author tom + * @date 2021/12/3 13:28 + */ +class SnowFlakeIdGeneratorTest { + + @Test + void generateId() { + // 注意 由于前端JS TS 在json解析大数会造成精度丢失 UUID 不能超过 9007199254740991（16位） + for (int i = 0; i < 1000; i++) { + long id = SnowFlakeIdGenerator.generateId(); + Assertions.assertTrue(id < 9007199254740991L); + } + } +} \ No newline at end of file diff --git a/manager/src/main/resources/define/app/A-example.yml b/manager/src/main/resources/define/app/A-example.yml index 5245a15..bf38f34 100644 --- a/manager/src/main/resources/define/app/A-example.yml +++ b/manager/src/main/resources/define/app/A-example.yml @@ -13,7 +13,8 @@ configmap: type: 2 # 指标组列表 metrics: - # 第一个监控指标组 cpu +# 第一个监控指标组 cpu +# 注意：内置监控指标有 (responseTime - 响应时间) - name: cpu # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集 # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度 @@ -74,7 +75,8 @@ metrics: type: Basic Auth basicAuthUsername: ^_^username^_^ basicAuthPassword: ^_^password^_^ - # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,xmlPath-xmlPath脚本,prometheus-Prometheus数据规则 + # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控 + # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则 parseType: jsonPath parseScript: '$' diff --git a/manager/src/main/resources/define/app/api.yml b/manager/src/main/resources/define/app/api.yml new file mode 100644 index 0000000..67d061a --- /dev/null +++ b/manager/src/main/resources/define/app/api.yml @@ -0,0 +1,56 @@ +# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws... +app: api +# 参数映射map. type是参数类型: 0-number数字, 1-string明文字符串, 2-secret加密字符串 +# 强制固定必须参数 - host +configmap: + - key: host + type: 1 + - key: port + type: 0 + - key: uri + type: 1 + - key: method + type: 1 + - key: ssl + type: 1 + - key: username + type: 1 + - key: password + type: 2 +# 指标组列表 +metrics: + # 第一个监控指标组 cpu + # 注意：内置监控指标有 (responseTime - 响应时间) + - name: summary + # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集 + # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度 + priority: 0 + # 指标组中的具体监控指标 + fields: + # 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位 + - field: responseTime + type: 0 + unit: ms +# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http +# 当protocol为http协议时具体的采集配置 + http: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + # url请求接口路径 + url: ^_^uri^_^ + # 请求方式 GET POST PUT DELETE PATCH + method: ^_^method^_^ + # 是否启用ssl/tls,即是http还是https,默认false + ssl: false + # 认证 + authorization: + # 认证方式: Basic Auth, Digest Auth, Bearer Token + type: Basic Auth + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控 + # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则 + parseType: website \ No newline at end of file diff --git a/manager/src/main/resources/define/param/api.yml b/manager/src/main/resources/define/param/api.yml new file mode 100644 index 0000000..90b670d --- /dev/null +++ b/manager/src/main/resources/define/param/api.yml @@ -0,0 +1,47 @@ +# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws... +app: api +# 强制固定必须参数 - host(ipv4,ipv6,域名) +param: + # field-字段名称标识符 + - field: host + # name-参数字段显示名称 + name: 主机Host + # type-字段类型,样式(大部分映射input标签type属性) + type: host + # 是否是必输项 true-必填 false-可选 + required: true + - field: port + name: 端口 + type: number + # 当type为number时,用range表示范围 + range: '[0,65535]' + required: true + - field: uri + name: URI路径 + type: text + # 当type为text时,用limit表示字符串限制大小 + limit: 100 + required: true + - field: method + name: 请求方式 + type: text + # 当type为text时,用limit表示字符串限制大小 + limit: 20 + required: true + - field: username + name: 用户名 + type: text + # 当type为text时,用limit表示字符串限制大小 + limit: 20 + required: false + - field: password + name: 密码 + type: password + required: false + - field: ssl + name: 使用SSL + type: boolean + required: true + # 当type为boolean时,前端用switch展示开关 + # 当type为radio单选框,checkbox复选框时,option表示可选项值列表 + # option: Yes,No \ No newline at end of file diff --git a/manager/src/main/resources/sureness.yml b/manager/src/main/resources/sureness.yml index e37518d..217dc0c 100644 --- a/manager/src/main/resources/sureness.yml +++ b/manager/src/main/resources/sureness.yml @@ -10,6 +10,7 @@ resourceRole: # load api resource which do not need be protected, means them need be excluded. # these api resource can be access by everyone excludedResource: + - /**===* - /account/auth/form===post - /**/*.html===get - /**/*.js===get diff --git a/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.ts b/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.ts index c9251fc..766eb04 100644 --- a/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.ts +++ b/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.ts @@ -102,7 +102,7 @@ export class MonitorNewComponent implements OnInit { "params": this.params }; this.isSpinning = true; - this.monitorSvc.newMonitor(detectMonitor) + this.monitorSvc.detectMonitor(detectMonitor) .subscribe(message => { this.isSpinning = false; if (message.code === 0) { diff --git a/web-app/src/assets/tmp/app-data.json b/web-app/src/assets/tmp/app-data.json index b6e53d5..fbba51f 100644 --- a/web-app/src/assets/tmp/app-data.json +++ b/web-app/src/assets/tmp/app-data.json @@ -35,8 +35,8 @@ "icon": "anticon-cloud", "children": [ { - "text": "http", - "link": "/monitors?app=http", + "text": "api", + "link": "/monitors?app=api", "i18n": "monitor.app.http" }, { diff --git a/web-app/src/assets/tmp/i18n/zh-CN.json b/web-app/src/assets/tmp/i18n/zh-CN.json index bd0dce5..4bd9596 100644 --- a/web-app/src/assets/tmp/i18n/zh-CN.json +++ b/web-app/src/assets/tmp/i18n/zh-CN.json @@ -28,6 +28,7 @@ }, "app": { "": "监控类型", + "api": "HTTP API", "http": "HTTP API", "ping": "PING连通性", "telnet": "Telnet端口可用性",