基于Grafana和Prometheus的监视系统(3):jav

作者: 奔跑的Libra | 来源:发表于2018-06-27 16:19 被阅读1035次

基于Grafana和Prometheus的监视系统(3):java客户端使用

0.基本说明

如何在代码中进行指标埋点: prometheus java client的使用
如何生成jvm 指标数据: hotspot expoter的使用
在spring boot 中进行指标采集
非指标数据的采集: Elasticsearch的使用

1. java client的使用

[https://github.com/prometheus/client_java]
使用方式和log4j的使用很相似

 <!-- The client -->
        <dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient</artifactId>
            <version>0.4.0</version>
        </dependency>
        <!-- Hotspot JVM metrics-->
        <dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient_hotspot</artifactId>
            <version>0.4.0</version>
        </dependency>
        <!-- Exposition HTTPServer-->
        <dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient_httpserver</artifactId>
            <version>0.4.0</version>
        </dependency>
        <!-- Pushgateway exposition-->
        <dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient_pushgateway</artifactId>
            <version>0.4.0</version>
        </dependency>

class A {
 static CollectorRegistry registry = new CollectorRegistry();
    // 请求总数统计
    static final  Counter counterRequest = Counter.build().name("TsHistoryData_RequestTotal").help("xx").
            labelNames("request").register(registry);
    // 请求时间
    static final Gauge costTime = Gauge.build().name("TsHistoryData_CostTime").help("xx").
            labelNames("id").register(registry);
    // 直方图, 统计在某个bucket时间的请求的个数.(linearBuckets | exponentialBuckets)
    static final Histogram requestLatency_hgm = Histogram.build()
            .name("TsHistoryData_RequestLatency_hgm").exponentialBuckets(0.5,2,10).help("Request latency in seconds.").register(registry);
    static final Summary requestLatency_suy = Summary.build()
            .name("TsHistoryData_RequestLatency_suy").
                    quantile(0.1, 0.01).
                    quantile(0.3, 0.01).
                    quantile(0.5,0.01).
                    quantile(0.7, 0.01).
                    quantile(0.9, 0.01).
                    quantile(0.95, 0.01).help("Request latency in seconds.").register(registry);
 void function1() {
        String requestId = System.currentTimeMillis() + "";
        counterRequest.labels("request").inc();
        Gauge.Timer costTimeTimer = costTime.labels(requestId).startTimer();
        Histogram.Timer requestTimer = requestLatency_hgm.startTimer();
        Summary.Timer requestTimer_suy = requestLatency_suy.startTimer();
        ......
        costTimeTimer.setDuration();
        requestTimer.observeDuration();
        requestTimer_suy.observeDuration();
        Monitor.pushMetricToPushGateway(registry, "TS_HISTORY_DATA");
  }
}

2. hotspot expoter

对于程序运行时的jvm指标进行监控

public static void jvmExport() throws Exception {
        DefaultExports.initialize();
        InetSocketAddress address = new InetSocketAddress("192.168.1.222", 9201);
        Server server = new Server(address);
        ServletContextHandler context = new ServletContextHandler();
        context.setContextPath("/");
        server.setHandler(context);
        context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics");
        server.start();
        // server.join();
    }

3.在spring boot中的使用

1.添加Maven的依赖

<!-- Hotspot JVM metrics-->
        <dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient_hotspot</artifactId>
            <version>0.4.0</version>
        </dependency>
        <dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient_spring_boot</artifactId>
            <version>0.4.0</version>
        </dependency>
        <!-- Exposition servlet -->
        <dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient_servlet</artifactId>
            <version>0.4.0</version>
        </dependency>

启用Prometheus Metrics Endpoint
添加注解@EnablePrometheusEndpoint启用Prometheus Endpoint,这里同时使用了simpleclient_hotspot中提供的DefaultExporter,该Exporter展示当前应用JVM的相关信息

@SpringBootApplication
@EnablePrometheusEndpoint
public class CoreApplication extends WebMvcConfigurerAdapter implements CommandLineRunner {

    public static void main(String[] args) {
        SpringApplication springApplication = new SpringApplication(CoreApplication.class);
        springApplication.run(args);
    }

    @Override
    public void run(String... strings) throws Exception {
        DefaultExports.initialize();
    }
}

向外暴露指标的接口

@Configuration
public class MonitoringConfig {

    @Bean
    ServletRegistrationBean servletRegistrationBean() {

        return new ServletRegistrationBean(new MetricsServlet(), "/metrics");
    }
}

这样访问 localhost:8080/metrics 可以看到jvm相关的指标.

3.添加拦截器，为监控埋点做准备
除了获取应用JVM相关的状态以外，我们还可能需要添加一些自定义的监控Metrics实现对系统性能，以及业务状态进行采集，以提供日后优化的相关支撑数据。首先我们使用拦截器处理对应用的所有请求。

继承WebMvcConfigurerAdapter类，复写addInterceptors方法，对所有请求/**添加拦截器

@SpringBootApplication
@EnablePrometheusEndpoint
public class CoreApplication extends WebMvcConfigurerAdapter implements CommandLineRunner {

    @Override
    public void addInterceptors(InterceptorRegistry registry) {
        registry.addInterceptor(new PrometheusMetricsInterceptor()).addPathPatterns("/**");
    }
}

PrometheusMetricsInterceptor集成HandlerInterceptorAdapter，通过复写父方法，实现对请求处理前/处理完成的处理。

@Component
public class PrometheusMetricsInterceptor extends HandlerInterceptorAdapter {
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
    return super.preHandle(request, response, handler);
}

@Override
public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {
    super.afterCompletion(request, response, handler, ex);
}
}

4.自定义指标

@Component
public class PrometheusMetricsInterceptor extends HandlerInterceptorAdapter {

    static final Counter requestCounter = Counter.build()
            .name("module_core_http_requests_total").labelNames("path", "method", "code")
            .help("Total requests.").register();

    static final Gauge inprogressRequests = Gauge.build()
            .name("module_core_http_inprogress_requests").labelNames("path", "method", "code")
            .help("Inprogress requests.").register();

    static final Gauge requestTime = Gauge.build()
            .name("module_core_http_requests_costTime").labelNames("path", "method", "code")
            .help("requests cost time.").register();

    static final Histogram requestLatencyHistogram = Histogram.build().labelNames("path", "method", "code")
            .name("module_core_http_requests_latency_seconds_histogram").help("Request latency in seconds.")
            .register();

    static final Summary requestLatency = Summary.build()
            .name("module_core_http_requests_latency_seconds_summary")
            .quantile(0.5, 0.05)
            .quantile(0.9, 0.01)
            .labelNames("path", "method", "code")
            .help("Request latency in seconds.").register();
    private Histogram.Timer histogramRequestTimer;

    private Summary.Timer summaryTimer;

    private Gauge.Timer gaugeTimer;

    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
        String requestURI = request.getRequestURI();
        String method = request.getMethod();
        int status = response.getStatus();
        inprogressRequests.labels(requestURI, method, String.valueOf(status)).inc();
        histogramRequestTimer = requestLatencyHistogram.labels(requestURI, method, String.valueOf(status)).startTimer();
        summaryTimer = requestLatency.labels(requestURI, method, String.valueOf(status)).startTimer();
        gaugeTimer = requestTime.labels(requestURI, method, String.valueOf(status)).startTimer();
        return super.preHandle(request, response, handler);
    }

    @Override
    public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {

        String requestURI = request.getRequestURI();
        String method = request.getMethod();
        int status = response.getStatus();

        requestCounter.labels(requestURI, method, String.valueOf(status)).inc();
        inprogressRequests.labels(requestURI, method, String.valueOf(status)).dec();
        histogramRequestTimer.observeDuration();
        summaryTimer.observeDuration();
        gaugeTimer.setDuration();
        super.afterCompletion(request, response, handler, ex);
    }
}

5.使用Collector暴露业务指标
除了在拦截器中使用Prometheus提供的Counter,Summary,Gauage等构造监控指标以外，我们还可以通过自定义的Collector实现对相关业务指标的暴露

@SpringBootApplication
@EnablePrometheusEndpoint
public class CoreApplication extends WebMvcConfigurerAdapter implements CommandLineRunner {
@Autowired
private CustomExporter customExporter;

...省略的代码

@Override
public void run(String... args) throws Exception {
    ...省略的代码
    customExporter.register();
}
}

CustomExporter集成自io.prometheus.client.Collector，在调用Collector的register()方法后，当访问/metrics时，则会自动从Collector的collection()方法中获取采集到的监控指标。

由于这里CustomExporter存在于Spring的IOC容器当中，这里可以直接访问业务代码，返回需要的业务相关的指标。

@Component
public class CustomExporter extends Collector {
    @Override
    public List<MetricFamilySamples> collect() {
        List<MetricFamilySamples> mfs = new ArrayList<>();

        GaugeMetricFamily labeledGauge =
                new GaugeMetricFamily("module_core_custom_metrics", "custom metrics", Collections.singletonList("labelname"));


        labeledGauge.addMetric(Collections.singletonList("labelvalue"), 1);

        mfs.add(labeledGauge);
        return mfs;
    }
}

4. ES的使用

docker compose [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html]
java client

<dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>6.0.1</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch -->
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>6.0.0</version>
        </dependency>

public class EsConfig {

    private static RestHighLevelClient client;
    public  static RestHighLevelClient init() throws Exception{
        client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("192.168.1.223", 9200, "http")
                        ));
        return client;
    }
    public static void close() throws Exception{
        client.close();
    }

}

 try {
            client = EsConfig.init();

            Map<String, Object> jsonMap = new HashMap<>();
            jsonMap.put("Id", requestId);
            jsonMap.put("Date", new Date());
            jsonMap.put("Value", tt);
            IndexRequest indexRequest = new IndexRequest("ts_data_history", "doc", requestId)
                    .source(jsonMap);
            IndexResponse indexResponse = client.index(indexRequest);
            client.close();
        } catch (Exception e) {
            e.printStackTrace();
        }

基于Grafana和Prometheus的监视系统(3):jav