[elasticsearch] python에서 간단 elasticsearch 모니터링

간단하게 elasticsearch가 정상적으로 동작하는지 체크할 수 있는 python 코드 예시이다.

먼저 elasticsearch python 모듈을 설치한다.

$ pip install elasticsearch

alive check가 없어서 dummy index를 생성했다가 삭제하는 코드를 추가한다.

from elasticsearch import Elasticsearch

es = Elasticsearch(host=host, port=9200)

..

    try:

        es.indices.create(index='test-index', ignore=400)

        es.indices.delete(index='test-index', ignore=[400, 404])

    except :

        put_err('server down', es_host[count])

elasticsearch rest api 중 _stats은 현재 es 서버 상황을 볼 수 있다.

$curl -XGET 'http://장비이름:9200/_nodes/stats'

{"cluster_name":"google_es","nodes":{"kpG92KHDSoO7NhvfvCetog":{"timestamp":1458295650270,"name":"green045","transport_address":"inet[/172.17.50.245:9300]","host":"green045.kr3.iwilab.com","ip":["inet[/172.17.50.245:9300]","NONE"],"attributes":{"master":"true"},"indices":{"docs":{"count":2049610,"deleted":0},"store":{"size_in_bytes":2780497067,"throttle_time_in_millis":0},"indexing":{"index_total":82828641,"index_time_in_millis":33104647,"index_current":5373,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":536},"get":{"total":4,"time_in_millis":0,"exists_total":1,"exists_time_in_millis":0,"missing_total":3,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":89666361,"query_time_in_millis":42660274,"query_current":0,"fetch_total":89666355,"fetch_time_in_millis":5548098,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2674,"total_time_in_millis":22264700,"total_docs":111622903,"total_size_in_bytes":300312239738},"refresh":{"total":17402,"total_time_in_millis":22551946},"flush":{"total":1609,"total_time_in_millis":6980833},"warmer":{"current":0,"total":261,"total_time_in_millis":53},"filter_cache":{"memory_size_in_bytes":478164,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":802760,"evictions":9},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":92612272},"segments":{"count":34,"memory_in_bytes":124820108,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":2048000,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":17},"suggest":{"total":16763,"time_in_millis":6103,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0}},"os":{"timestamp":1458295650270,"uptime_in_millis":18157670,"load_average":[0.0,0.0,0.0],"cpu":{"sys":0,"user":0,"idle":99,"usage":0,"stolen":0},"mem":{"free_in_bytes":6580514816,"used_in_bytes":27076222976,"free_percent":29,"used_percent":70,"actual_free_in_bytes":9924431872,"actual_used_in_bytes":23732305920},"swap":{"used_in_bytes":0,"free_in_bytes":10737414144}},"process":{"timestamp":1458295650271,"open_file_descriptors":289,"cpu":{"percent":0,"sys_in_millis":7528750,"user_in_millis":162504230,"total_in_millis":170032980},"mem":{"resident_in_bytes":22813118464,"share_in_bytes":120283136,"total_virtual_in_bytes":27931869184}},"jvm":{"timestamp":1458295650271,"uptime_in_millis":10310773097,"mem":{"heap_used_in_bytes":13092004720,"heap_used_percent":61,"heap_committed_in_bytes":21405106176,"heap_max_in_bytes":21405106176,"non_heap_used_in_bytes":110885664,"non_heap_committed_in_bytes":113668096,"pools":{"young":{"used_in_bytes":529111904,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":3899608,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":12558993208,"max_in_bytes":20776943616,"peak_used_in_bytes":15920147088,"peak_max_in_bytes":20776943616}}},"threads":{"count":611,"peak_count":614},"gc":{"collectors":{"young":{"collection_count":78537,"collection_time_in_millis":2869917},"old":{"collection_count":140,"collection_time_in_millis":8780}}},"buffer_pools":{"direct":{"count":15882,"used_in_bytes":283727750,"total_capacity_in_bytes":283727750},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}}},"thread_pool":{"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":194262},"index":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":4},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":4,"completed":9120},"suggest":{"threads":256,"queue":0,"active":0,"rejected":0,"largest":256,"completed":6541558},"generic":{"threads":5,"queue":0,"active":0,"rejected":0,"largest":5,"completed":1242222},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":3,"completed":20792},"search":{"threads":256,"queue":0,"active":0,"rejected":0,"largest":256,"completed":179332719},"flush":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":4,"completed":8768},"optimize":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":2955615},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":4,"completed":26901},"bulk":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":12738},"snapshot":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":2,"completed":2157}},"network":{"tcp":{"active_opens":1491570,"passive_opens":4946581,"curr_estab":82,"in_segs":443092500,"out_segs":317298535,"retrans_segs":38975,"estab_resets":1384179,"attempt_fails":3585983,"in_errs":0,"out_rsts":38442}},"fs":{"timestamp":1458295650271,"total":{"total_in_bytes":284037365760,"free_in_bytes":180158369792,"available_in_bytes":165723308032,"disk_reads":74699,"disk_writes":12144405,"disk_io_op":12219104,"disk_read_size_in_bytes":2125903872,"disk_write_size_in_bytes":689377091584,"disk_io_size_in_bytes":691502995456,"disk_queue":"0","disk_service_time":"0"},"data":[{"path":"/home/elasticsearch/data/google_es/nodes/0","mount":"/","dev":"/dev/sda2","total_in_bytes":284037365760,"free_in_bytes":180158369792,"available_in_bytes":165723308032,"disk_reads":74699,"disk_writes":12144405,"disk_io_op":12219104,"disk_read_size_in_bytes":2125903872,"disk_write_size_in_bytes":689377091584,"disk_io_size_in_bytes":691502995456,"disk_queue":"0","disk_service_time":"0"}]},"transport":{"server_open":13,"rx_count":6,"rx_size_in_bytes":1860,"tx_count":6,"tx_size_in_bytes":1860},"http":{"current_open":54,"total_opened":1080551},"breakers":{"request":{"limit_size_in_bytes":8562042470,"limit_size":"7.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":12843063705,"limit_size":"11.9gb","estimated_size_in_bytes":802760,"estimated_size":"783.9kb","overhead":1.03,"tripped":0},"parent":{"limit_size_in_bytes":14983574323,"limit_size":"13.9gb","estimated_size_in_bytes":802760,"estimated_size":"783.9kb","overhead":1.0,"tripped":0}}}}}

cpu, thread, memory, disk, merge  정보를 확인할 수 있다. 

python으로 하면 대충 이렇게 개발할 수 있다.

from elasticsearch import Elasticsearch

es = Elasticsearch(host="salmon001.dakao.io", port=9200)

es.nodes.stats(node_id="salmon001")

stat = es.nodes.stats(node_id="salmon001")['nodes']

..