Storm

UI api

链接：http://storm.apache.org/releases/1.2.2/STORM-UI-REST-API.html

/api/v1/cluster/configuration (GET)

返回cluster的配置：

  {
    "dev.zookeeper.path": "/tmp/dev-storm-zookeeper",
    "topology.tick.tuple.freq.secs": null,
    "topology.builtin.metrics.bucket.size.secs": 60,
    "topology.fall.back.on.java.serialization": true,
    "topology.max.error.report.per.interval": 5,
    "zmq.linger.millis": 5000,
    "topology.skip.missing.kryo.registrations": false,
    "storm.messaging.netty.client_worker_threads": 1,
    "ui.childopts": "-Xmx768m",
    "storm.zookeeper.session.timeout": 20000,
    "nimbus.reassign": true,
    "topology.trident.batch.emit.interval.millis": 500,
    "storm.messaging.netty.flush.check.interval.ms": 10,
    "nimbus.monitor.freq.secs": 10,
    "logviewer.childopts": "-Xmx128m",
    "java.library.path": "/usr/local/lib:/opt/local/lib:/usr/lib",
    "topology.executor.send.buffer.size": 1024,
    }

/api/v1/cluster/summary (GET)

返回cluster的总结：比如nimbus的正常运行时间，supervisor 的数目等。关键词：集群的内存，cpu

 {
    "stormVersion": "0.9.2-incubating-SNAPSHOT",
    "supervisors": 1,
    "slotsTotal": 4,
    "slotsUsed": 3,
    "slotsFree": 1,
    "executorsTotal": 28,
    "tasksTotal": 28,
    "schedulerDisplayResource": true,
    "totalMem": 4096.0,
    "totalCpu": 400.0,
    "availMem": 1024.0,
    "availCPU": 250.0,
    "memAssignedPercentUtil": 75.0,
    "cpuAssignedPercentUtil": 37.5
    }

/api/v1/supervisor/summary (GET)

返回supervisor 的概要信息。关键词：supervisor的内存，cpu

{
    "supervisors": [
        {
            "id": "0b879808-2a26-442b-8f7d-23101e0c3696",
            "host": "10.11.1.7",
            "uptime": "5m 58s",
            "uptimeSeconds": 358,
            "slotsTotal": 4,
            "slotsUsed": 3,
            "totalMem": 3000,
            "totalCpu": 400,
            "usedMem": 1280,
            "usedCPU": 160
        }
    ],
    "schedulerDisplayResource": true
}

/api/v1/nimbus/summary (GET)

返回nimbus 的概要信息。关键词：nimubs的运行时间

{
    "nimbuses":[
        {
            "host":"192.168.202.1",
            "port":6627,
            "nimbusLogLink":"http:\/\/192.168.202.1:8000\/log?file=nimbus.log",
            "status":Leader,
            "version":"0.10.0-SNAPSHOT",
            "nimbusUpTime":"3m 33s",
            "nimbusUpTimeSeconds":"213"
        }
    ]
}

/api/v1/history/summary (GET)

返回正在运行的topology的ID列表。

{
    "topo-history":[
        "wc6-1-1446571009",
        "wc8-2-1446587178"
     ]
}

/api/v1/supervisor (GET)

返回每个supervisor的详细信息。关键词：supervisor的内存，cpu，运行时间，worker的运行时间，cpu分配

{
    "supervisors": [{ 
        "totalMem": 4096.0, 
        "host":"192.168.10.237",
        "id":"bdfe8eff-f1d8-4bce-81f5-9d3ae1bf432e-169.254.129.212",
        "uptime":"7m 8s",
        "totalCpu":400.0,
        "usedCpu":495.0,
        "usedMem":3432.0,
        "slotsUsed":2,
        "version":"0.10.1",
        "slotsTotal":4,
        "uptimeSeconds":428
    }],
    "schedulerDisplayResource":true,
    "workers":[{
        "topologyName":"ras",
        "topologyId":"ras-4-1460229987",
        "host":"192.168.10.237",
        "supervisorId":"bdfe8eff-f1d8-4bce-81f5-9d3ae1bf432e-169.254.129.212",
        "assignedMemOnHeap":704.0,
        "uptime":"2m 47s",
        "uptimeSeconds":167,
        "port":6707,
        "workerLogLink":"http:\/\/192.168.10.237:8000\/log?file=ras-4-1460229987%2F6707%2Fworker.log",
        "componentNumTasks": {
            "word":5
        },
        "executorsTotal":8,
        "assignedCpu":130.0,
        "assignedMemOffHeap":80.0
    },
    {
        "topologyName":"ras",
        "topologyId":"ras-4-1460229987",
        "host":"192.168.10.237",
        "supervisorId":"bdfe8eff-f1d8-4bce-81f5-9d3ae1bf432e-169.254.129.212",
        "assignedMemOnHeap":904.0,
        "uptime":"2m 53s",
        "port":6706,
        "workerLogLink":"http:\/\/192.168.10.237:8000\/log?file=ras-4-1460229987%2F6706%2Fworker.log",
        "componentNumTasks":{
            "exclaim2":2,
            "exclaim1":3,
            "word":5
        },
        "executorsTotal":10,
        "uptimeSeconds":173,
        "assignedCpu":165.0,
        "assignedMemOffHeap":80.0
    }]
}

/api/v1/topology/summary (GET)

返回所有的topology 的概要。关键词：运行时间，资源的配置

{
    "topologies": [
        {
            "id": "WordCount3-1-1402960825",
            "name": "WordCount3",
            "status": "ACTIVE",
            "uptime": "6m 5s",
            "uptimeSeconds": 365,
            "tasksTotal": 28,
            "workersTotal": 3,
            "executorsTotal": 28,
            "replicationCount": 1,
            "requestedMemOnHeap": 640,
            "requestedMemOffHeap": 128,
            "requestedTotalMem": 768,
            "requestedCpu": 80,
            "assignedMemOnHeap": 640,
            "assignedMemOffHeap": 128,
            "assignedTotalMem": 768,
            "assignedCpu": 80
        }
    ],
    "schedulerDisplayResource": true
}

/api/v1/topology-workers/:id (GET)

返回woker的host和port信息。

{
    "hostPortList":[
            {
                "host":"192.168.202.2",
                "port":6701
            },
            {
                "host":"192.168.202.2",
                "port":6702
            },
            {
                "host":"192.168.202.3",
                "port":6700
            }
        ],
    "logviewerPort":8000
}

/api/v1/topology/:id (GET)（最全）

返回topology的信息和统计，使用拓扑的ID。参考输入：

 1. http://ui-daemon-host-name:8080/api/v1/topology/WordCount3-1-1402960825
 2. http://ui-daemon-host-name:8080/api/v1/topology/WordCount3-1-1402960825?sys=1
 3. http://ui-daemon-host-name:8080/api/v1/topology/WordCount3-1-1402960825?window=600

参数：
id：拓扑的id
window：默认的值是所有时间，秒为单位的测试metrics的时间
sys：可选1或者0，默认是0。包括系统统计部分的响应。
关键词：topology的统计信息：ack，fail，latency，CPU，内存分配情况，worker，spout的详细统计信息

 {
    "name": "WordCount3",
    "id": "WordCount3-1-1402960825",
    "workersTotal": 3,
    "window": "600",
    "status": "ACTIVE",
    "tasksTotal": 28,
    "executorsTotal": 28,
    "uptime": "29m 19s",
    "uptimeSeconds": 1759,
    "msgTimeout": 30,
    "windowHint": "10m 0s",
    "schedulerDisplayResource": true,
    "topologyStats": [
        {
            "windowPretty": "10m 0s",
            "window": "600",
            "emitted": 397960,
            "transferred": 213380,
            "completeLatency": "0.000",
            "acked": 213460,
            "failed": 0
        },
        {
            "windowPretty": "3h 0m 0s",
            "window": "10800",
            "emitted": 1190260,
            "transferred": 638260,
            "completeLatency": "0.000",
            "acked": 638280,
            "failed": 0
        },
        {
            "windowPretty": "1d 0h 0m 0s",
            "window": "86400",
            "emitted": 1190260,
            "transferred": 638260,
            "completeLatency": "0.000",
            "acked": 638280,
            "failed": 0
        },
        {
            "windowPretty": "All time",
            "window": ":all-time",
            "emitted": 1190260,
            "transferred": 638260,
            "completeLatency": "0.000",
            "acked": 638280,
            "failed": 0
        }
    ],
    "workers":[{
        "topologyName":"WordCount3",
        "topologyId":"WordCount3-1-1402960825",
        "host":"192.168.10.237",
        "supervisorId":"bdfe8eff-f1d8-4bce-81f5-9d3ae1bf432e-169.254.129.212",
        "uptime":"2m 47s",
        "uptimeSeconds":167,
        "port":6707,
        "workerLogLink":"http:\/\/192.168.10.237:8000\/log?file=WordCount3-1-1402960825%2F6707%2Fworker.log",
        "componentNumTasks": {
            "spout":5
        },
        "executorsTotal":8,
        "assignedMemOnHeap":704.0,
        "assignedCpu":130.0,
        "assignedMemOffHeap":80.0
    }],
    "spouts": [
        {
            "executors": 5,
            "emitted": 28880,
            "completeLatency": "0.000",
            "transferred": 28880,
            "acked": 0,
            "spoutId": "spout",
            "tasks": 5,
            "lastError": "",
            "errorLapsedSecs": null,
            "failed": 0
        }
    ],
        "bolts": [
        {
            "executors": 12,
            "emitted": 184580,
            "transferred": 0,
            "acked": 184640,
            "executeLatency": "0.048",
            "tasks": 12,
            "executed": 184620,
            "processLatency": "0.043",
            "boltId": "count",
            "lastError": "",
            "errorLapsedSecs": null,
            "capacity": "0.003",
            "failed": 0
        },
        {
            "executors": 8,
            "emitted": 184500,
            "transferred": 184500,
            "acked": 28820,
            "executeLatency": "0.024",
            "tasks": 8,
            "executed": 28780,
            "processLatency": "2.112",
            "boltId": "split",
            "lastError": "",
            "errorLapsedSecs": null,
            "capacity": "0.000",
            "failed": 0
        }
    ],
    "configuration": {
        "storm.id": "WordCount3-1-1402960825",
        "dev.zookeeper.path": "/tmp/dev-storm-zookeeper",
        "topology.tick.tuple.freq.secs": null,
        "topology.builtin.metrics.bucket.size.secs": 60,
        "topology.fall.back.on.java.serialization": true,
        "topology.max.error.report.per.interval": 5,
        "zmq.linger.millis": 5000,
        "topology.skip.missing.kryo.registrations": false,
        "storm.messaging.netty.client_worker_threads": 1,
        "ui.childopts": "-Xmx768m",
        "storm.zookeeper.session.timeout": 20000,
        "nimbus.reassign": true,
        "topology.trident.batch.emit.interval.millis": 500,
        "storm.messaging.netty.flush.check.interval.ms": 10,
        "nimbus.monitor.freq.secs": 10,
        "logviewer.childopts": "-Xmx128m",
        "java.library.path": "/usr/local/lib:/opt/local/lib:/usr/lib",
        "topology.executor.send.buffer.size": 1024,
        "storm.local.dir": "storm-local",
        "storm.messaging.netty.buffer_size": 5242880,
        "supervisor.worker.start.timeout.secs": 120,
        "topology.enable.message.timeouts": true,
        "nimbus.cleanup.inbox.freq.secs": 600,
        "nimbus.inbox.jar.expiration.secs": 3600,
        "drpc.worker.threads": 64,
        "topology.worker.shared.thread.pool.size": 4,
        "nimbus.seeds": [
            "hw10843.local"
        ],
        "storm.messaging.netty.min_wait_ms": 100,
        "storm.zookeeper.port": 2181,
        "transactional.zookeeper.port": null,
        "topology.executor.receive.buffer.size": 1024,
        "transactional.zookeeper.servers": null,
        "storm.zookeeper.root": "/storm",
        "storm.zookeeper.retry.intervalceiling.millis": 30000,
        "supervisor.enable": true,
        "storm.messaging.netty.server_worker_threads": 1
    },
    "replicationCount": 1
}

/api/v1/topology/:id/metrics

topology的metrics的详细部分。每个组件的每条流。

{
    "window":":all-time",
    "window-hint":"All time",
    "spouts":[
        {
            "id":"spout",
            "emitted":[
                {
                    "stream_id":"__metrics",
                    "value":20
                },
                {
                    "stream_id":"default",
                    "value":17350280
                },
                {
                    "stream_id":"__ack_init",
                    "value":17328160
                },
                {
                    "stream_id":"__system",
                    "value":20
                }
            ],
            "transferred":[
                {
                    "stream_id":"__metrics",
                    "value":20
                },
                {
                    "stream_id":"default",
                    "value":17350280
                },
                {
                    "stream_id":"__ack_init",
                    "value":17328160
                },
                {
                    "stream_id":"__system",
                    "value":0
                }
            ],
            "acked":[
                {
                    "stream_id":"default",
                    "value":17339180
                }
            ],
            "failed":[

            ],
            "complete_ms_avg":[
                {
                    "stream_id":"default",
                    "value":"920.497"
                }
            ]
        }
    ],
    "bolts":[
        {
            "id":"count",
            "emitted":[
                {
                    "stream_id":"__metrics",
                    "value":120
                },
                {
                    "stream_id":"default",
                    "value":190748180
                },
                {
                    "stream_id":"__ack_ack",
                    "value":190718100
                },
                {
                    "stream_id":"__system",
                    "value":20
                }
            ],
            "transferred":[
                {
                    "stream_id":"__metrics",
                    "value":120
                },
                {
                    "stream_id":"default",
                    "value":0
                },
                {
                    "stream_id":"__ack_ack",
                    "value":190718100
                },
                {
                    "stream_id":"__system",
                    "value":0
                }
            ],
            "acked":[
                {
                    "component_id":"split",
                    "stream_id":"default",
                    "value":190733160
                }
            ],
            "failed":[

            ],
            "process_ms_avg":[
                {
                    "component_id":"split",
                    "stream_id":"default",
                    "value":"0.004"
                }
            ],
            "executed":[
                {
                    "component_id":"split",
                    "stream_id":"default",
                    "value":190733140
                }
            ],
            "executed_ms_avg":[
                {
                    "component_id":"split",
                    "stream_id":"default",
                    "value":"0.005"
                }
            ]
        },
        {
            "id":"split",
            "emitted":[
                {
                    "stream_id":"__metrics",
                    "value":60
                },
                {
                    "stream_id":"default",
                    "value":190754740
                },
                {
                    "stream_id":"__ack_ack",
                    "value":17317580
                },
                {
                    "stream_id":"__system",
                    "value":20
                }
            ],
            "transferred":[
                {
                    "stream_id":"__metrics",
                    "value":60
                },
                {
                    "stream_id":"default",
                    "value":190754740
                },
                {
                    "stream_id":"__ack_ack",
                    "value":17317580
                },
                {
                    "stream_id":"__system",
                    "value":0
                }
            ],
            "acked":[
                {
                    "component_id":"spout",
                    "stream_id":"default",
                    "value":17339180
                }
            ],
            "failed":[

            ],
            "process_ms_avg":[
                {
                    "component_id":"spout",
                    "stream_id":"default",
                    "value":"0.051"
                }
            ],
            "executed":[
                {
                    "component_id":"spout",
                    "stream_id":"default",
                    "value":17339240
                }
            ],
            "executed_ms_avg":[
                {
                    "component_id":"spout",
                    "stream_id":"default",
                    "value":"0.052"
                }
            ]
        }
    ]
}

/api/v1/topology/:id/component/:component (GET)

精确到组件的metrics。

Profiling and Debugging GET Operations

可以对topology进行重启，kill等操作。