From 018db2a14f231d12e6a918cc95dfdcc7fd3f9687 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Wed, 6 Apr 2022 10:26:33 +0800 Subject: [PATCH] [manager]feature:[website api]monitor support keyword match (#72) --- .../collect/http/HttpCollectImpl.java | 22 ++++++++---- .../com/usthe/collector/util/CollectUtil.java | 35 +++++++++++++++++++ .../collector/util/CollectorConstants.java | 2 ++ .../entity/job/protocol/HttpProtocol.java | 5 ++- manager/src/main/resources/define/app/api.yml | 8 ++++- .../src/main/resources/define/app/website.yml | 10 ++++-- .../src/main/resources/define/param/api.yml | 5 +++ .../main/resources/define/param/website.yml | 5 +++ 8 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 collector/src/main/java/com/usthe/collector/util/CollectUtil.java diff --git a/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java b/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java index 50d19ec..cb11c6e 100644 --- a/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java +++ b/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java @@ -7,6 +7,7 @@ import com.google.gson.JsonParser; import com.usthe.collector.collect.AbstractCollect; import com.usthe.collector.collect.common.http.CommonHttpClient; import com.usthe.collector.dispatch.DispatchConstants; +import com.usthe.collector.util.CollectUtil; import com.usthe.collector.util.CollectorConstants; import com.usthe.collector.util.JsonPathParser; import com.usthe.common.entity.job.Metrics; @@ -100,7 +101,7 @@ public class HttpCollectImpl extends AbstractCollect { String parseType = metrics.getHttp().getParseType(); try { if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) { - parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime); + parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime); } else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) { parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime); } else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) { @@ -108,11 +109,11 @@ public class HttpCollectImpl extends AbstractCollect { } else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) { parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder); } else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){ - parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime); + parseResponseByWebsite(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime); } else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) { parseResponseBySiteMap(resp, metrics.getAliasFields(), builder); } else { - parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime); + parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime); } } catch (Exception e) { log.info("parse error: {}.", e.getMessage(), e); @@ -169,13 +170,16 @@ public class HttpCollectImpl extends AbstractCollect { } } - private void parseResponseByWebsite(String resp, List aliasFields, + private void parseResponseByWebsite(String resp, List aliasFields, HttpProtocol http, CollectRep.MetricsData.Builder builder, Long responseTime) { CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - // todo resp 网站关键字监测 + // 网站关键词数量监测 + int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword()); for (String alias : aliasFields) { if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { valueRowBuilder.addColumns(responseTime.toString()); + } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(Integer.toString(keywordNum)); } else { valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); } @@ -277,6 +281,7 @@ public class HttpCollectImpl extends AbstractCollect { private void parseResponseByJsonPath(String resp, List aliasFields, HttpProtocol http, CollectRep.MetricsData.Builder builder, Long responseTime) { List> results = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript()); + int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword()); for (Map stringMap : results) { CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); for (String alias : aliasFields) { @@ -286,6 +291,8 @@ public class HttpCollectImpl extends AbstractCollect { } else { if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { valueRowBuilder.addColumns(responseTime.toString()); + } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(Integer.toString(keywordNum)); } else { valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); } @@ -300,9 +307,10 @@ public class HttpCollectImpl extends AbstractCollect { } - private void parseResponseByDefault(String resp, List aliasFields, + private void parseResponseByDefault(String resp, List aliasFields, HttpProtocol http, CollectRep.MetricsData.Builder builder, Long responseTime) { JsonElement element = JsonParser.parseString(resp); + int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword()); if (element.isJsonArray()) { JsonArray array = element.getAsJsonArray(); for (JsonElement jsonElement : array) { @@ -317,6 +325,8 @@ public class HttpCollectImpl extends AbstractCollect { } else { if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { valueRowBuilder.addColumns(responseTime.toString()); + } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(Integer.toString(keywordNum)); } else { valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); } diff --git a/collector/src/main/java/com/usthe/collector/util/CollectUtil.java b/collector/src/main/java/com/usthe/collector/util/CollectUtil.java new file mode 100644 index 0000000..b23a2d2 --- /dev/null +++ b/collector/src/main/java/com/usthe/collector/util/CollectUtil.java @@ -0,0 +1,35 @@ +package com.usthe.collector.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 采集器工具类 + * @author tom + * @date 2022/4/6 09:35 + */ +public class CollectUtil { + + /** + * 关键字匹配计数 + * @param content 内容 + * @param keyword 关键字 + * @return 匹配次数 + */ + public static int countMatchKeyword(String content, String keyword) { + if (content == null || "".equals(content) || keyword == null || "".equals(keyword.trim())) { + return 0; + } + try { + Pattern pattern = Pattern.compile(keyword); + Matcher matcher = pattern.matcher(content); + int count = 0; + while (matcher.find()) { + count++; + } + return count; + } catch (Exception e) { + return 0; + } + } +} diff --git a/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java b/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java index 689532d..288af68 100644 --- a/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java +++ b/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java @@ -9,6 +9,8 @@ public interface CollectorConstants { String RESPONSE_TIME = "responseTime"; + String KEYWORD = "keyword"; + String STATUS_CODE = "statusCode"; String ERROR_MSG = "errorMsg"; diff --git a/common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java b/common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java index 9cb7c8f..40c51cb 100644 --- a/common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java +++ b/common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java @@ -49,7 +49,6 @@ public class HttpProtocol { * http请求携带的请求体 */ private String payload; - /** * 认证信息 */ @@ -66,6 +65,10 @@ public class HttpProtocol { * 数据解析脚本 当解析方式为 jsonPath or xmlPath时存在 */ private String parseScript; + /** + * 内容关键字 + */ + private String keyword; /** * 认证信息 diff --git a/manager/src/main/resources/define/app/api.yml b/manager/src/main/resources/define/app/api.yml index 207da24..a7a2ef9 100644 --- a/manager/src/main/resources/define/app/api.yml +++ b/manager/src/main/resources/define/app/api.yml @@ -32,6 +32,8 @@ configmap: type: 3 - key: params type: 3 + - key: keyword + type: 1 # 指标组列表 metrics: # 第一个监控指标组 cpu @@ -46,6 +48,9 @@ metrics: - field: responseTime type: 0 unit: ms + - field: keyword + type: 0 + unit: 次数 # 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk protocol: http # 当protocol为http协议时具体的采集配置 @@ -78,4 +83,5 @@ metrics: digestAuthPassword: ^_^password^_^ # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控 # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则 - parseType: website \ No newline at end of file + parseType: website + keyword: ^_^keyword^_^ \ No newline at end of file diff --git a/manager/src/main/resources/define/app/website.yml b/manager/src/main/resources/define/app/website.yml index a30d7fb..f115187 100644 --- a/manager/src/main/resources/define/app/website.yml +++ b/manager/src/main/resources/define/app/website.yml @@ -18,10 +18,12 @@ configmap: type: 1 - key: password type: 2 + - key: keyword + type: 1 # 指标组列表 metrics: # 第一个监控指标组 cpu - # 注意：内置监控指标有 (responseTime - 响应时间) + # 注意：内置监控指标有 (responseTime - 响应时间, keyword - 关键字数量) - name: summary # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集 # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度 @@ -32,6 +34,9 @@ metrics: - field: responseTime type: 0 unit: ms + - field: keyword + type: 0 + unit: 次数 # 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk protocol: http # 当protocol为http协议时具体的采集配置 @@ -55,4 +60,5 @@ metrics: digestAuthPassword: ^_^password^_^ # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控 # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则 - parseType: website \ No newline at end of file + parseType: website + keyword: ^_^keyword^_^ \ No newline at end of file diff --git a/manager/src/main/resources/define/param/api.yml b/manager/src/main/resources/define/param/api.yml index 6bda131..95af32e 100644 --- a/manager/src/main/resources/define/param/api.yml +++ b/manager/src/main/resources/define/param/api.yml @@ -90,3 +90,8 @@ param: type: password required: false hide: true + - field: keyword + name: 关键字 + type: text + required: false + hide: true diff --git a/manager/src/main/resources/define/param/website.yml b/manager/src/main/resources/define/param/website.yml index 1390158..e16caed 100644 --- a/manager/src/main/resources/define/param/website.yml +++ b/manager/src/main/resources/define/param/website.yml @@ -51,4 +51,9 @@ param: name: 密码 type: password required: false + hide: true + - field: keyword + name: 关键字 + type: text + required: false hide: true \ No newline at end of file