Elastic Stack 7.x 初识 — Jevic

Elastic Stack 7.x 初识

2019/11/06 ELK

概述

RDBMS Elasticsearch
Table Index(Type)
Row Document
Column Filed
Schema Mapping
SQL DSL

关键特性

安装配置

Elasticsearch

  • 此处使用的版本为: 7.4.2
  • 提示: OpenJDK 已经内置 $ES_PATH/jdk JAVA (JVM)
系统配置
[root@jevic ~]# cat /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
* soft nproc unlimited
* hard nproc unlimited
es soft memlock unlimited
es hard memlock unlimited

[root@jevic ~]# cat /etc/security/limits.d/20-nproc.conf
# Default limit for number of user's processes to prevent
# accidental fork bombs.
# See rhbz #432903 for reasoning.

*          soft    nproc     102400
root       soft    nproc     unlimited
elasticsearch.yml
cluster.name: JevicTestDB
node.name: node194
node.master: true
node.data: true
path.data: /es-data1/data
bootstrap.memory_lock: true
network.host: 192.168.0.194
http.port: 9200
discovery.zen.minimum_master_nodes: 2
discovery.seed_hosts: ["node194", "node198", "node204"]
#discovery.seed_providers: unicast_hosts.txt
## 重试间隔
discovery.find_peers_interval: 1s
cluster.initial_master_nodes: ["node194", "node198", "node204"]
### sql插件支持
http.cors.enabled: true
http.cors.allow-origin: "*"
#### 必须停用xpack sql才可以使用elasticsearch-sql
xpack.sql.enabled: false
### 开启认证:
### 初始化认证: $ES_PATH/bin/elasticsearch-setup-passwords interactive
#xpack.security.enabled: true

Kibana

server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
### 开启认证
#xpack.security.enabled: true
#elasticsearch.username: "kibana"
#elasticsearch.password: "123456"

Logstash

  • 收集 NGINX 日志
日志格式
log_format  json '{"cip":"$remote_addr",'
                            '"timestamp":"$time_iso8601",'
                            '"rtime":$request_time,'
                            '"upres_time":$upstream_response_time,'
                            '"sbyte":$body_bytes_sent,'
                            '"host":"$http_host",'
                            '"request":"$request",'
                            '"scheme":"$scheme",'
                            '"length":"$content_length",'
                            '"server":"$upstream_addr",'
                            '"method":"$request_method",'
                            '"status":$status}';
pipe 配置
input{
    file {
        path => [ "/var/log/nginx/es_access.log" ]
        start_position => "beginning"
        codec => "json"
    }
}

filter {
  date {
    match => [ "timestamp", "ISO8601" ]
    target => "@timestamp"
  }
  mutate {
    split => { "request" => " " }
    add_field => { "url" => "%{[request][1]}"}
  }
  mutate {
    remove_field => [ "path", "tags", "request"]
  }
}

output {
    #elasticsearch {
    #    hosts => [ "192.168.0.194:9200" ]
    #    index => "nginx-%{+YYYY.MM.dd}"
    #}
     stdout {
	       codec => rubydebug
     }
}

扩展阅读

Search

    Post Directory