diff --git a/collector/src/main/java/com/usthe/collector/dispatch/export/MetricsDataExporter.java b/collector/src/main/java/com/usthe/collector/dispatch/export/MetricsDataExporter.java index 64ed79c..5d6d89d 100644 --- a/collector/src/main/java/com/usthe/collector/dispatch/export/MetricsDataExporter.java +++ b/collector/src/main/java/com/usthe/collector/dispatch/export/MetricsDataExporter.java @@ -19,12 +19,12 @@ public class MetricsDataExporter implements DisposableBean { private final LinkedBlockingQueue metricsDataToAlertQueue; private final LinkedBlockingQueue metricsDataToPersistentStorageQueue; - private final LinkedBlockingQueue metricsDataToWarehouseRedisQueue; + private final LinkedBlockingQueue metricsDataToMemoryStorageQueue; public MetricsDataExporter() { metricsDataToAlertQueue = new LinkedBlockingQueue<>(); metricsDataToPersistentStorageQueue = new LinkedBlockingQueue<>(); - metricsDataToWarehouseRedisQueue = new LinkedBlockingQueue<>(); + metricsDataToMemoryStorageQueue = new LinkedBlockingQueue<>(); } public CollectRep.MetricsData pollAlertMetricsData() throws InterruptedException { @@ -35,8 +35,8 @@ public class MetricsDataExporter implements DisposableBean { return metricsDataToAlertQueue.poll(2, TimeUnit.SECONDS); } - public CollectRep.MetricsData pollWarehouseRedisMetricsData() throws InterruptedException { - return metricsDataToWarehouseRedisQueue.poll(2, TimeUnit.SECONDS); + public CollectRep.MetricsData pollMemoryStorageMetricsData() throws InterruptedException { + return metricsDataToMemoryStorageQueue.poll(2, TimeUnit.SECONDS); } /** @@ -46,7 +46,7 @@ public class MetricsDataExporter implements DisposableBean { public void send(CollectRep.MetricsData metricsData) { metricsDataToAlertQueue.offer(metricsData); metricsDataToPersistentStorageQueue.offer(metricsData); - metricsDataToWarehouseRedisQueue.offer(metricsData); + metricsDataToMemoryStorageQueue.offer(metricsData); } @Override diff --git a/manager/pom.xml b/manager/pom.xml index 98e172e..c18bc26 100644 --- a/manager/pom.xml +++ b/manager/pom.xml @@ -52,12 +52,6 @@ spring-boot-configuration-processor true - - - org.springframework.cloud - spring-cloud-starter-openfeign - 3.0.5 - org.springframework.boot @@ -67,15 +61,6 @@ org.springframework.boot spring-boot-starter-data-jpa - - - org.springframework.boot - spring-boot-starter-cache - - - org.springframework.boot - spring-boot-starter-data-redis - mysql diff --git a/manager/src/main/java/com/usthe/manager/Manager.java b/manager/src/main/java/com/usthe/manager/Manager.java index 76da45b..a096887 100644 --- a/manager/src/main/java/com/usthe/manager/Manager.java +++ b/manager/src/main/java/com/usthe/manager/Manager.java @@ -3,7 +3,6 @@ package com.usthe.manager; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; /** @@ -12,7 +11,6 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories; */ @SpringBootApplication -@EnableFeignClients(basePackages = {"com.usthe"}) @EnableJpaRepositories(basePackages = {"com.usthe"}) @EntityScan(basePackages = {"com.usthe"}) public class Manager { diff --git a/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java b/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java index b606de5..d5122d9 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java +++ b/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java @@ -8,7 +8,7 @@ import com.usthe.common.entity.dto.Value; import com.usthe.common.entity.dto.ValueRow; import com.usthe.common.entity.message.CollectRep; import com.usthe.common.util.CommonConstants; -import com.usthe.warehouse.store.RedisDataStorage; +import com.usthe.warehouse.store.MemoryDataStorage; import com.usthe.warehouse.store.TdEngineDataStorage; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -40,7 +40,7 @@ public class MetricsDataController { private static final Integer METRIC_FULL_LENGTH = 3; @Autowired - private RedisDataStorage redisDataStorage; + private MemoryDataStorage memoryDataStorage; @Autowired private TdEngineDataStorage tdEngineDataStorage; @@ -52,7 +52,7 @@ public class MetricsDataController { @PathVariable Long monitorId, @ApiParam(value = "监控指标组", example = "cpu") @PathVariable String metrics) { - CollectRep.MetricsData redisData = redisDataStorage.getCurrentMetricsData(monitorId, metrics); + CollectRep.MetricsData redisData = memoryDataStorage.getCurrentMetricsData(monitorId, metrics); if (redisData == null) { return ResponseEntity.ok().body(new Message<>("query metrics data is empty")); } diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/MemoryDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/MemoryDataStorage.java new file mode 100644 index 0000000..95225b1 --- /dev/null +++ b/warehouse/src/main/java/com/usthe/warehouse/store/MemoryDataStorage.java @@ -0,0 +1,77 @@ +package com.usthe.warehouse.store; + +import com.usthe.collector.dispatch.export.MetricsDataExporter; +import com.usthe.common.entity.message.CollectRep; +import com.usthe.warehouse.WarehouseProperties; +import com.usthe.warehouse.WarehouseWorkerPool; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; +import org.springframework.lang.NonNull; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * redis存储采集实时数据 + * @author tom + * @date 2021/11/25 10:26 + */ +@Configuration +@AutoConfigureAfter(value = {WarehouseProperties.class}) +@ConditionalOnProperty(prefix = "warehouse.store.memory", + name = "enabled", havingValue = "true", matchIfMissing = true) +@Slf4j +public class MemoryDataStorage implements DisposableBean { + + private Map metricsDataMap; + private WarehouseWorkerPool workerPool; + private MetricsDataExporter dataExporter; + + public MemoryDataStorage(WarehouseWorkerPool workerPool, MetricsDataExporter dataExporter) { + metricsDataMap = new ConcurrentHashMap<>(1024); + this.workerPool = workerPool; + this.dataExporter = dataExporter; + startStorageData(); + } + + public CollectRep.MetricsData getCurrentMetricsData(@NonNull Long monitorId, @NonNull String metric) { + String hashKey = monitorId + metric; + return metricsDataMap.get(hashKey); + } + + private void startStorageData() { + Runnable runnable = () -> { + Thread.currentThread().setName("warehouse-memory-data-storage"); + while (!Thread.currentThread().isInterrupted()) { + try { + CollectRep.MetricsData metricsData = dataExporter.pollMemoryStorageMetricsData(); + if (metricsData != null) { + saveData(metricsData); + } + } catch (InterruptedException e) { + log.error(e.getMessage()); + } + } + }; + workerPool.executeJob(runnable); + } + + private void saveData(CollectRep.MetricsData metricsData) { + String hashKey = metricsData.getId() + metricsData.getMetrics(); + if (metricsData.getValuesList().isEmpty()) { + log.info("[warehouse] redis flush metrics data {} is null, ignore.", hashKey); + return; + } + metricsDataMap.put(hashKey, metricsData); + } + + @Override + public void destroy() throws Exception { + if (metricsDataMap != null) { + metricsDataMap.clear(); + } + } +} diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/RedisDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/RedisDataStorage.java index 10a3d4f..af1f9b9 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/RedisDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/RedisDataStorage.java @@ -27,7 +27,7 @@ import java.time.temporal.ChronoUnit; @Configuration @AutoConfigureAfter(value = {WarehouseProperties.class}) @ConditionalOnProperty(prefix = "warehouse.store.redis", - name = "enabled", havingValue = "true", matchIfMissing = true) + name = "enabled", havingValue = "true", matchIfMissing = false) @Slf4j public class RedisDataStorage implements DisposableBean { @@ -54,7 +54,7 @@ public class RedisDataStorage implements DisposableBean { Thread.currentThread().setName("warehouse-redis-data-storage"); while (!Thread.currentThread().isInterrupted()) { try { - CollectRep.MetricsData metricsData = dataExporter.pollWarehouseRedisMetricsData(); + CollectRep.MetricsData metricsData = dataExporter.pollMemoryStorageMetricsData(); if (metricsData != null) { saveData(metricsData); } diff --git a/warehouse/src/main/resources/META-INF/spring.factories b/warehouse/src/main/resources/META-INF/spring.factories index 41bfa91..9ba0a9e 100644 --- a/warehouse/src/main/resources/META-INF/spring.factories +++ b/warehouse/src/main/resources/META-INF/spring.factories @@ -2,6 +2,6 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ com.usthe.warehouse.WarehouseProperties,\ com.usthe.warehouse.MetricsDataQueue,\ com.usthe.warehouse.WarehouseWorkerPool,\ -com.usthe.warehouse.store.RedisDataStorage,\ +com.usthe.warehouse.store.MemoryDataStorage,\ com.usthe.warehouse.store.TdEngineDataStorage,\ com.usthe.warehouse.controller.MetricsDataController \ No newline at end of file diff --git a/web-app/src/assets/app-data.json b/web-app/src/assets/app-data.json index feabff2..92f166d 100644 --- a/web-app/src/assets/app-data.json +++ b/web-app/src/assets/app-data.json @@ -105,20 +105,6 @@ "link": "/alert/notice" } ] - }, - { - "text": "More", - "i18n": "menu.extras", - "group": true, - "hideInBreadcrumb": true, - "children": [ - { - "text": "Settings", - "link": "/account/settings", - "i18n": "menu.extras.setting", - "icon": "anticon-setting" - } - ] } ] }