cluster.name: my-application     #集群名称，若相同，且是同一网段会自动加入
node.name: node-1                #当前节点名称
node.attr.rack: r1               #
network.host: 127.0.0.1          #默认情况下，Elastic只允许本机访问
                                 #若要远程访问，取消注释并修改值为0.0.0.0

-Xms1g     #最小堆内存
-Xmx1g     #最大堆内存
#两个值最好一致，且为机器物理内存的一半到2/3，也不能太小

-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
#垃圾回收算法，官方已经优化过了，不用调整

PUT /megacorp/employee/1
{
    "first_name" : "John",
    "last_name" :  "Smith",
    "age" :        25,
    "about" :      "I love to go rock climbing",
    "interests": [ "sports", "music" ]
}

{
    "cluster_name":     "elasticsearch",
    "status":           "green",
    "timed_out":        false,
......
}

curl -H "Content-Type: application/json" -X PUT localhost:9200/blogs/_settings -d '
{
    "number_of_replicas": 2
}'

curl -H "Content-Type: application/json" -X PUT localhost:9200/blogs/article/1 -d '
{
    "title":   "article1",
    "content": "article1"
}'
# -H设置内容类型，要设为JSON格式
# -X设置请求类型，设为PUT
# -d设置请求数据

{
    "_index": "blogs",
    "_type": "article",
    "_id": "1",
    "_version": 1,
    "result": "created",
    "_shards":{
        "total": 2,        #目前总共的分片数
        "successful": 1,
        "failed": 0
    },
    "_seq_no": 0,
    "_primary_term": 1
}

{
    "_index": "blogs",
    "_type": "article",
    "_id": "1",
    "_version": 1,
    "found": true,
    "_source": {
        "title": "article1",
        "content": "article1"
    }
}

{
  ......
  "hits" : {
    "total" : 2,
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "blogs",
        "_type" : "article",
        "_id" : "2",
        "_score" : 1.0,
        "_source" : {
          "title" : "article2",
          "content" : "article2"
        }
      },
      ......
    ]
  }
}

curl localhost:9200/_search?q=title:article2
{
    ......
    "hits":{"total":1,"max_score":0.2876821,"hits":[{"_index":"blogs","_type":"article","_id":"2","_score":0.2876821,"_source":
{
    "title":   "article2",
    "content": "article2" }}]}}

curl localhost:9200/_search -X GET -H "Content-Type: application/json" -d '
{
    "query":{
        "match":{           #使用了match类型查询
            "title": "article1"
        }
    }
}'

# 已经alias curl_lo_g='curl -X GET -H 'Content-Type:application/json''
# export LO_ES='localhost:9200'

curl_lo_g ${LO_ES}/_all/employee/_search -d '
{
  "query": {
    "bool": {
      "filter": {
        "range": {
          "age": {
            "gt": "30"   #要加上双引号
          }
        }
      },                 #这里有逗号
      "must": {
        "match": {
          "hobby": "swimming"
        }
      }
    }
  }
}'

curl_lo_g ${LO_ES}/_all/employee/_search -d '
{
  "query": {
    "match": {
      "hobby": "climbing"
    }
  },
  "highlight": {      #只需要添加highlight搜索即可
    "fields": {       #fields指定高亮的字段
      "hobby": {}     #需要高亮搜索的字段
    }
  }
}'

......
{"_index":"tech","_type":"employee","_id":"3","_score":0.2876821,"_source":
{
  "name": "wangwu",
  "age": "26",
  "address": "yangzhou",
  "hobby": ["swimming", "climbing"]
},
"highlight":{             #标出高亮部分
  "hobby":[
    "<em>climbing</em>"   #高亮部分由HTML标签<em>封装
                          #告诉浏览器把其中的文本表示为强调的内容
                          #通常为斜体
  ]}}]}}

logstash
    -n NAME          指定logstash的node.name，若不指定默认是当前的主机名
    -f CONFIG_PATH   从特定文件或目录加载logstash配置
    -e CONFIG_STRING 使用给定的字符串作为配置数据（与配置文件的语法相同）
                     默认输入：“input {stdin {type => stdin}}”
                     默认输出：“output {stdout {codec => rubydebug}}“
                     若直接使用默认，则-e "" 即可（不能什么都不加）
    --field-reference-parser MODE 在解析字段引用时使用的模式
    字段引用解析器用于扩展管道配置中的字段引用，能更好地处理非法和模糊输入
    可用的MODE值：1. LEGACY：LEGACY解析器，不发出警告
                2. COMPAT：COMPAT解析器，对每个不同的模糊或非法语法输入警告一次（默认使用）
                3.STRICT：STRICT解析器，模糊或非法语法输入会引发插件运行时异常
    --modules MODULES   加载logstash模块，不能与'-e'或'-f'一起使用
                        因为会覆盖在logstash.yml中加载的模块
    两种写法：--modules module1 --modules module2 ...
            --modules=module1,module2
    -M MODULES_VARIABLE Load variables for module template.
                                  Multiple instances of '-M' or
                                  '--modules.variable' are supported.
                                  Ignored if '--modules' flag is not used.
                                  Should be in the format of
                                  '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
                                  as in
                                  '-M "example.var.filter.mutate.fieldname=fieldvalue"'
    --setup                       Load index template into Elasticsearch, and saved searches,
                                  index-pattern, visualizations, and dashboards into Kibana when
                                  running modules.
                                   (default: false)
    --cloud.id CLOUD_ID           Sets the elasticsearch and kibana host settings for
                                  module connections in Elastic Cloud.
                                  Your Elastic Cloud User interface or the Cloud support
                                  team should provide this.
                                  Add an optional label prefix '<label>:' to help you
                                  identify multiple cloud.ids.
                                  e.g. 'staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
    --cloud.auth CLOUD_AUTH       Sets the elasticsearch and kibana username and password
                                  for module connections in Elastic Cloud
                                  e.g. 'username:<password>'
    --pipeline.id ID              Sets the ID of the pipeline.
                                   (default: "main")
    -w COUNT  指定pipeline worker数量（即线程数），默认1
    --experimental-java-execution (Experimental) Use new Java execution engine.
                                   (default: false)
    -b, --pipeline.batch.size SIZE Size of batches the pipeline is to work in.
                                   (default: 125)
    -u, --pipeline.batch.delay DELAY_IN_MS When creating pipeline batches, how long to wait while polling
                                  for the next event.
                                   (default: 50)
    --pipeline.unsafe_shutdown    Force logstash to exit during shutdown even
                                  if there are still inflight events in memory.
                                  By default, logstash will refuse to quit until all
                                  received events have been pushed to the outputs.
                                   (default: false)
    --path.data PATH   数据存储目录。插件需要能访问该目录，默认安装目录下的data/
    -p, --path.plugins PATH   插件目录，可指定多个Plugins are expected to be in a specific directory hierarchy:
                                  'PATH/logstash/TYPE/NAME.rb' where TYPE is 'inputs' 'filters', 'outputs' or 'codecs' and NAME is the name of the plugin.
                                   (default: [])
    -l PATH      指定Logstash的日志目录，默认安装目录下的logs/
    --log.level LEVEL             设置日志等级（fatal/error/warn/info/debug/trace），默认info
    --config.debug                Print the compiled config ruby code out as a debug log (you must also have --log.level=debug enabled).
                                  WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
                                  in plaintext passwords appearing in your logs!
                                   (default: false)
    -i, --interactive SHELL       Drop to shell instead of running as normal.
                                  Valid shells are "irb" and "pry"

    -t           检查logstash配置文件语法是否正常
    -r           自动检测配置文件是否变动，若变动自动重载
    --config.reload.interval RELOAD_INTERVAL How frequently to poll the configuration location
                                  for changes, in seconds.
                                   (default: 3000000000)
    --http.host HTTP_HOST         Web API binding host (default: "127.0.0.1")
    --http.port HTTP_PORT         Web API http port (default: 9600..9700)
    --log.format FORMAT           Specify if Logstash should write its own logs in JSON form (one
                                  event per line) or in plain text (using Ruby's Object#inspect)
                                   (default: "plain")
    --path.settings SETTINGS_DIR  包含logstash.yml的目录，可通过LS_SETTINGS_DIR环境变量配置
    默认"/usr/local/logstash6.4/config"
    --verbose 相当于设置日志等级为info
    --debug   相当于设置日志等级为debug
    --quiet   相当于设置日志等级为info

node.name             #节点名，默认为主机名

#path参数
path.data             #数据存放目录，默认为安装目录的data/
path.config           #logstash.yml路径
path.plugins          #插件的路径
#插件应该位于特定的目录层次结构中：PATH/logstash/TYPE/NAME.rb
#其中TYPE的值可以是inputs，filters，outputs或codecs，NAME是插件的名称

#http参数
http.port             #监听的http主机端口，默认为9600
http.host             #监听的http主机IP，默认为127.0.0.1

#日志log参数
log.level             #日志等级（fatal/error/warn/info/debug/trace），默认info
log.format            #日志格式，默认为plain格式
path.logs             #日志的路径，默认安装目录的logs/

#pipeline参数（关于pipeline的配置可专门存放在一个配置文件中，如pipeline.yml）
pipeline.id           #pipeline的ID，默认为main
pipeline.workers      #管道的worker数量，默认为CPU的核数
pipeline.batch.size   #单个工作线程将从输入收集的最大事件数，默认125。增大此值会加大内存开销，还需调整JVM堆内存参数
pipeline.batch.delay  #event被调度到worker前等待的时间，单位毫秒，默认50
pipeline.unsafe_shutdown  #在关闭时立刻退出，会导致数据丢失。
        #默认为false，完全退出前会将数据处理好并输出到屏幕再退出

#config参数
config.string         #pipeline配置
config.test_and_exit  #检查配置是否有效，然后退出。默认为false不检查
config.reload.automatic  #定期检查配置是否已更改，并在配置发生更改时重新加载配置。默认false不检查
config.reload.interval  #检查配置是否变动的时间间隔，需要上一条开启。默认3s
config.debug          #是否开启调试日志消息。若开启还需要将log.level的值设为debug。默认false
                      #注：日志消息可能会包含明文密码
config.support_escapes #是否开启转义字符，即启用\转义。默认false

modules               #设置模块

#queue参数
queue.type            #用于事件缓冲的队列模型（model to use for event buffering），有以下两种，默认memory。
       memory：传统基于内存的队列  persisted：基于磁盘的响应队列（disk-based ACKed queueing）
path.queue            #启用持久队列（即上一项值为persisted）时存储数据文件的目录。默认data/queue
queue.pagt_capacity   #启用持久队列时使用的页面数据文件的大小。默认64MB
queue.max_events      #启用持久队列时队列中未读事件的最大数量。默认0，不限制
queue.max_bytes       #队列的总容量，以字节数表示。确保磁盘容量大于此值。默认1G
#若与上一项同时配置，则Logstash会加载先配置的一项
queue.checkpoint.acks #启用持久队列时强制检查点之前的最大ACK响应事件数。默认1024，若设为0则不限制
queue.checkpoint.writes #在启用持久队列时强制检查点之前写入事件的最大数量。默认1024，若设为0则不限制
queue.drain           #启用后，Logstash将等待直到持久队列耗尽，然后才能关闭。默认false

#dead_letter_queue参数
dead_letter_queue.enable  #是否启用dead_letter_queue，简称DLQ功能。默认false不启用
dead_letter_queue.max_bytes #每个DLQ的最大大小。超出就会删除条目，默认1G
path.dead_letter_queue #DLQ目录的位置，默认为安装目录的data/dead_letter_queue

curl -XPUT 'localhost:9600/_node/logging?pretty' -H 'Content-Type: application/json' -d '
{
    "logger.logstash.outputs.elasticsearch" : "DEBUG"
}'

input {        #设置input参数
  file {
    #通过列表添加两个日志
    path => ['/var/log/httpd/access_log','/var/log/squid/access.log']
  }
}
output {        #标准输出，一定要加，否则无法输出到屏幕
  stdout {}
}

{
          "path" => "/var/log/httpd/access_log",
       "message" => "127.0.0.1 - - [02/Oct/2018:16:11:16 +0800] \"GET / HTTP/1.0\" 200 10 \"-\" \"ApacheBench/2.3\"",
    "@timestamp" => 2018-10-02T08:11:20.344Z,
      "@version" => "1",
          "host" => "VM_0_7_centos"
}
{
          "path" => "/var/log/squid/access.log",
       "message" => "1538467887.306    656 180.126.242.119 TCP_TUNNEL/200 33103 CONNECT xui.ptlogin2.qq.com:443 - HIER_DIRECT/xui.ptlogin2.qq.com -",
    "@timestamp" => 2018-10-02T08:11:27.355Z,
      "@version" => "1",
          "host" => "VM_0_7_centos"
}

input {
  file {
    path => '/usr/local/es-6.4/logs/elasticsearch.log'
    type => 'elasticsearch'
    start_position => 'beginning'   #从日志的头开始读
  }
}
output {
  elasticsearch {          #使用elasticsearch插件
    hosts => '127.0.0.1:9200'   #指定elasticsearch源
    index => 'es_message-%{+YYYY.MM.dd}'  #指定index
  }
  stdout{
    codec => rubydebug
  }
}

{
          "type" => "elasticsearch",
       "message" => "[2018-10-03T16:06:50,392][INFO ][o.e.c.m.MetaDataMappingService] [system135] [.kibana/PnIK501cQUydUEIVp0icjw] update_mapping [doc]",
    "@timestamp" => 2018-10-03T08:06:50.560Z,
      "@version" => "1",
          "host" => "system5.example.com",
          "path" => "/usr/local/es-6.4/logs/elasticsearch.log"
}

server.port: 5601          #Kibana服务端口
server.host: "localhost"   #向哪些主机开放端口，若要所有主机都能访问，即客户端能远程访问，需要设为0.0.0.0
server.name: "your-hostname" #Kibana实例名，一般为主机名
server.basePath: ""        #
server.maxPayloadBytes: 1048576
server.rewriteBasePath: false

elasticsearch.url: "http://localhost:9200"  #Elasticsearch的地址，需要设置正确
elasticsearch.preserveHost: true
kibana.index: ".kibana"
kibana.defaultAppId: "home"
#elasticsearch.username: "user"           #设置es授权用户名
#elasticsearch.password: "pass"           #设置es授权用户密码
#server.ssl.enabled: false                #是否开启ssl
#server.ssl.certificate: /path/to/your/server.crt
#server.ssl.key: /path/to/your/server.key
#elasticsearch.ssl.certificate: /path/to/your/client.crt
#elasticsearch.ssl.key: /path/to/your/client.key

curl -X PUT -H "Content-Type: application/json" localhost:9200/tech/employee/1 -d '
{
  "name": "zhangsan",
  "age": "25",
  "address": "nanjing",
  "hobby": [ "football", "tennis", "game" ]
}'

curl -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/bank/account/_bulk?pretty' --data-binary @accounts.json