为什么logstash日志中某个字段设定了geo_point的type不生效？ - 知否问答 - 万象云+社区

input { beats { port => 5044 host => "0.0.0.0" } } filter { if [fileset][module] == "nginx" { if [fileset][name] == "access" { grok { match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] } remove_field => "message" } mutate { add_field => { "read_timestamp" => "%{@timestamp}" } } date { match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ] remove_field => "[nginx][access][time]" } useragent { source => "[nginx][access][agent]" target => "[nginx][access][user_agent]" remove_field => "[nginx][access][agent]" } geoip { source => "[nginx][access][remote_ip]" database => "../GeoLite2-City.mmdb" target => "[nginx][access][geoip]" } } else if [fileset][name] == "error" { grok { match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] } remove_field => "message" } mutate { rename => { "@timestamp" => "read_timestamp" } } date { match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ] remove_field => "[nginx][error][time]" } } } } output { elasticsearch { hosts => ['127.0.0.1:9200'] manage_template => false index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" } stdout { codec => rubydebug } }

"filebeat-6.2.2": { "order": 1, "index_patterns": [ "filebeat-6.2.2-*" ], "settings": { "index": { "number_of_routing_shards": "30", "mapping": { "total_fields": { "limit": "10000" } }, "refresh_interval": "5s" } }, "mappings": { "doc": { "properties": { "auditd": { "properties": { "log": { "properties": { "new_ses": { "type": "keyword", "ignore_above": 1024 }, "pid": { "type": "keyword", "ignore_above": 1024 }, "a0": { "type": "keyword", "ignore_above": 1024 }, "record_type": { "type": "keyword", "ignore_above": 1024 }, "old_auid": { "ignore_above": 1024, "type": "keyword" }, "new_auid": { "type": "keyword", "ignore_above": 1024 }, "old_ses": { "type": "keyword", "ignore_above": 1024 }, "acct": { "type": "keyword", "ignore_above": 1024 }, "ppid": { "type": "keyword", "ignore_above": 1024 }, "items": { "type": "keyword", "ignore_above": 1024 }, "geoip": { "properties": { "continent_name": { "type": "keyword", "ignore_above": 1024 }, "city_name": { "type": "keyword", "ignore_above": 1024 }, "region_name": { "type": "keyword", "ignore_above": 1024 }, "country_iso_code": { "type": "keyword", "ignore_above": 1024 }, "location": { "type": "geo_point" } } }, "sequence": { "type": "long" }, "item": { "type": "keyword", "ignore_above": 1024 }, "res": { "type": "keyword", "ignore_above": 1024 } } } } }, "osquery": { "properties": { "result": { "properties": { "action": { "type": "keyword", "ignore_above": 1024 }, "host_identifier": { "type": "keyword", "ignore_above": 1024 }, "unix_time": { "type": "long" }, "calendar_time": { "type": "keyword", "ignore_above": 1024 }, "name": { "type": "keyword", "ignore_above": 1024 } } } } }, "redis": { "properties": { "slowlog": { "properties": { "args": { "type": "keyword", "ignore_above": 1024 }, "cmd": { "ignore_above": 1024, "type": "keyword" }, "duration": { "properties": { "us": { "type": "long" } } }, "id": { "type": "long" }, "key": { "type": "keyword", "ignore_above": 1024 } } }, "log": { "properties": { "pid": { "type": "long" }, "role": { "type": "keyword", "ignore_above": 1024 }, "level": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false } } } } }, "beat": { "properties": { "hostname": { "type": "keyword", "ignore_above": 1024 }, "timezone": { "type": "keyword", "ignore_above": 1024 }, "version": { "ignore_above": 1024, "type": "keyword" }, "name": { "type": "keyword", "ignore_above": 1024 } } }, "@timestamp": { "type": "date" }, "tags": { "type": "keyword", "ignore_above": 1024 }, "prospector": { "properties": { "type": { "ignore_above": 1024, "type": "keyword" } } }, "icinga": { "properties": { "debug": { "properties": { "facility": { "type": "keyword", "ignore_above": 1024 }, "severity": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false } } }, "main": { "properties": { "facility": { "type": "keyword", "ignore_above": 1024 }, "severity": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false } } }, "startup": { "properties": { "severity": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false }, "facility": { "ignore_above": 1024, "type": "keyword" } } } } }, "nginx": { "properties": { "access": { "properties": { "url": { "type": "keyword", "ignore_above": 1024 }, "response_code": { "type": "long" }, "body_sent": { "properties": { "bytes": { "type": "long" } } }, "referrer": { "ignore_above": 1024, "type": "keyword" }, "agent": { "type": "text", "norms": false }, "user_agent": { "properties": { "name": { "type": "keyword", "ignore_above": 1024 }, "os_minor": { "type": "long" }, "patch": { "type": "keyword", "ignore_above": 1024 }, "os": { "ignore_above": 1024, "type": "keyword" }, "os_major": { "type": "long" }, "os_name": { "type": "keyword", "ignore_above": 1024 }, "device": { "type": "keyword", "ignore_above": 1024 }, "major": { "type": "long" }, "minor": { "type": "long" } } }, "geoip": { "properties": { "continent_name": { "type": "keyword", "ignore_above": 1024 }, "country_iso_code": { "type": "keyword", "ignore_above": 1024 }, "location": { "type": "geo_point" }, "region_name": { "type": "keyword", "ignore_above": 1024 }, "city_name": { "ignore_above": 1024, "type": "keyword" } } }, "remote_ip": { "type": "keyword", "ignore_above": 1024 }, "user_name": { "type": "keyword", "ignore_above": 1024 }, "method": { "type": "keyword", "ignore_above": 1024 }, "http_version": { "type": "keyword", "ignore_above": 1024 } } }, "error": { "properties": { "level": { "type": "keyword", "ignore_above": 1024 }, "pid": { "type": "long" }, "tid": { "type": "long" }, "connection_id": { "type": "long" }, "message": { "type": "text", "norms": false } } } } }, "error": { "properties": { "message": { "type": "text", "norms": false }, "code": { "type": "long" }, "type": { "type": "keyword", "ignore_above": 1024 } } }, "meta": { "properties": { "cloud": { "properties": { "provider": { "ignore_above": 1024, "type": "keyword" }, "instance_id": { "type": "keyword", "ignore_above": 1024 }, "instance_name": { "type": "keyword", "ignore_above": 1024 }, "machine_type": { "type": "keyword", "ignore_above": 1024 }, "availability_zone": { "type": "keyword", "ignore_above": 1024 }, "project_id": { "type": "keyword", "ignore_above": 1024 }, "region": { "ignore_above": 1024, "type": "keyword" } } } } }, "message": { "type": "text", "norms": false }, "mysql": { "properties": { "error": { "properties": { "timestamp": { "type": "keyword", "ignore_above": 1024 }, "thread_id": { "type": "long" }, "level": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false } } }, "slowlog": { "properties": { "user": { "type": "keyword", "ignore_above": 1024 }, "query_time": { "properties": { "sec": { "type": "float" } } }, "rows_examined": { "type": "long" }, "timestamp": { "type": "long" }, "id": { "type": "long" }, "host": { "type": "keyword", "ignore_above": 1024 }, "ip": { "type": "keyword", "ignore_above": 1024 }, "lock_time": { "properties": { "sec": { "type": "float" } } }, "rows_sent": { "type": "long" }, "query": { "type": "keyword", "ignore_above": 1024 } } } } }, "traefik": { "properties": { "access": { "properties": { "user_agent": { "properties": { "patch": { "type": "keyword", "ignore_above": 1024 }, "name": { "type": "keyword", "ignore_above": 1024 }, "os_minor": { "type": "long" }, "os_name": { "type": "keyword", "ignore_above": 1024 }, "device": { "type": "keyword", "ignore_above": 1024 }, "minor": { "type": "long" }, "os_major": { "type": "long" }, "major": { "type": "long" }, "os": { "type": "keyword", "ignore_above": 1024 } } }, "request_count": { "type": "long" }, "response_code": { "type": "long" }, "body_sent": { "properties": { "bytes": { "type": "long" } } }, "frontend_name": { "type": "text", "norms": false }, "backend_url": { "type": "text", "norms": false }, "remote_ip": { "ignore_above": 1024, "type": "keyword" }, "method": { "type": "keyword", "ignore_above": 1024 }, "url": { "type": "keyword", "ignore_above": 1024 }, "referrer": { "ignore_above": 1024, "type": "keyword" }, "user_name": { "type": "keyword", "ignore_above": 1024 }, "http_version": { "type": "keyword", "ignore_above": 1024 }, "agent": { "type": "text", "norms": false }, "geoip": { "properties": { "continent_name": { "type": "keyword", "ignore_above": 1024 }, "country_iso_code": { "type": "keyword", "ignore_above": 1024 }, "location": { "type": "geo_point" }, "region_name": { "ignore_above": 1024, "type": "keyword" }, "city_name": { "type": "keyword", "ignore_above": 1024 } } } } } } }, "fields": { "type": "object" }, "logstash": { "properties": { "log": { "properties": { "thread": { "type": "text", "norms": false }, "log_event": { "type": "object" }, "message": { "type": "text", "norms": false }, "level": { "type": "keyword", "ignore_above": 1024 }, "module": { "type": "keyword", "ignore_above": 1024 } } }, "slowlog": { "properties": { "message": { "type": "text", "norms": false }, "level": { "type": "keyword", "ignore_above": 1024 }, "event": { "type": "text", "norms": false }, "plugin_type": { "type": "keyword", "ignore_above": 1024 }, "took_in_millis": { "type": "long" }, "took_in_nanos": { "type": "long" }, "plugin_params_object": { "type": "object" }, "module": { "type": "keyword", "ignore_above": 1024 }, "thread": { "type": "text", "norms": false }, "plugin_name": { "type": "keyword", "ignore_above": 1024 }, "plugin_params": { "type": "text", "norms": false } } } } }, "fileset": { "properties": { "module": { "type": "keyword", "ignore_above": 1024 }, "name": { "ignore_above": 1024, "type": "keyword" } } }, "apache2": { "properties": { "access": { "properties": { "url": { "type": "keyword", "ignore_above": 1024 }, "referrer": { "ignore_above": 1024, "type": "keyword" }, "geoip": { "properties": { "continent_name": { "type": "keyword", "ignore_above": 1024 }, "country_iso_code": { "type": "keyword", "ignore_above": 1024 }, "location": { "type": "geo_point" }, "region_name": { "type": "keyword", "ignore_above": 1024 }, "city_name": { "type": "keyword", "ignore_above": 1024 } } }, "remote_ip": { "type": "keyword", "ignore_above": 1024 }, "user_name": { "type": "keyword", "ignore_above": 1024 }, "method": { "type": "keyword", "ignore_above": 1024 }, "http_version": { "type": "keyword", "ignore_above": 1024 }, "response_code": { "type": "long" }, "body_sent": { "properties": { "bytes": { "type": "long" } } }, "agent": { "type": "text", "norms": false }, "user_agent": { "properties": { "minor": { "type": "long" }, "os_major": { "type": "long" }, "os_minor": { "type": "long" }, "os": { "type": "keyword", "ignore_above": 1024 }, "os_name": { "type": "keyword", "ignore_above": 1024 }, "device": { "type": "keyword", "ignore_above": 1024 }, "major": { "type": "long" }, "patch": { "ignore_above": 1024, "type": "keyword" }, "name": { "type": "keyword", "ignore_above": 1024 } } } } }, "error": { "properties": { "tid": { "type": "long" }, "module": { "type": "keyword", "ignore_above": 1024 }, "level": { "ignore_above": 1024, "type": "keyword" }, "client": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false }, "pid": { "type": "long" } } } } }, "read_timestamp": { "type": "keyword", "ignore_above": 1024 }, "kafka": { "properties": { "log": { "properties": { "message": { "type": "text", "norms": false }, "component": { "type": "keyword", "ignore_above": 1024 }, "class": { "norms": false, "type": "text" }, "trace": { "properties": { "class": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false }, "full": { "type": "text", "norms": false } } }, "timestamp": { "type": "keyword", "ignore_above": 1024 }, "level": { "type": "keyword", "ignore_above": 1024 } } } } }, "system": { "properties": { "syslog": { "properties": { "message": { "type": "keyword", "ignore_above": 1024 }, "timestamp": { "ignore_above": 1024, "type": "keyword" }, "hostname": { "type": "keyword", "ignore_above": 1024 }, "program": { "type": "keyword", "ignore_above": 1024 }, "pid": { "type": "keyword", "ignore_above": 1024 } } }, "auth": { "properties": { "groupadd": { "properties": { "name": { "type": "keyword", "ignore_above": 1024 }, "gid": { "type": "long" } } }, "program": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "keyword", "ignore_above": 1024 }, "user": { "type": "keyword", "ignore_above": 1024 }, "sudo": { "properties": { "tty": { "type": "keyword", "ignore_above": 1024 }, "pwd": { "type": "keyword", "ignore_above": 1024 }, "user": { "ignore_above": 1024, "type": "keyword" }, "command": { "type": "keyword", "ignore_above": 1024 }, "error": { "type": "keyword", "ignore_above": 1024 } } }, "useradd": { "properties": { "gid": { "type": "long" }, "home": { "ignore_above": 1024, "type": "keyword" }, "shell": { "type": "keyword", "ignore_above": 1024 }, "name": { "type": "keyword", "ignore_above": 1024 }, "uid": { "type": "long" } } }, "timestamp": { "type": "keyword", "ignore_above": 1024 }, "hostname": { "type": "keyword", "ignore_above": 1024 }, "pid": { "type": "long" }, "ssh": { "properties": { "ip": { "type": "ip" }, "dropped_ip": { "type": "ip" }, "port": { "type": "long" }, "signature": { "type": "keyword", "ignore_above": 1024 }, "geoip": { "properties": { "continent_name": { "type": "keyword", "ignore_above": 1024 }, "city_name": { "type": "keyword", "ignore_above": 1024 }, "region_name": { "type": "keyword", "ignore_above": 1024 }, "country_iso_code": { "type": "keyword", "ignore_above": 1024 }, "location": { "type": "geo_point" } } }, "event": { "type": "keyword", "ignore_above": 1024 }, "method": { "type": "keyword", "ignore_above": 1024 } } } } } } }, "docker": { "properties": { "container": { "properties": { "id": { "type": "keyword", "ignore_above": 1024 }, "image": { "type": "keyword", "ignore_above": 1024 }, "name": { "type": "keyword", "ignore_above": 1024 }, "labels": { "type": "object" } } } } }, "kubernetes": { "properties": { "annotations": { "type": "object" }, "container": { "properties": { "name": { "type": "keyword", "ignore_above": 1024 }, "image": { "type": "keyword", "ignore_above": 1024 } } }, "pod": { "properties": { "name": { "type": "keyword", "ignore_above": 1024 } } }, "namespace": { "type": "keyword", "ignore_above": 1024 }, "node": { "properties": { "name": { "type": "keyword", "ignore_above": 1024 } } }, "labels": { "type": "object" } } }, "source": { "type": "keyword", "ignore_above": 1024 }, "offset": { "type": "long" }, "stream": { "ignore_above": 1024, "type": "keyword" }, "postgresql": { "properties": { "log": { "properties": { "timezone": { "type": "keyword", "ignore_above": 1024 }, "user": { "type": "keyword", "ignore_above": 1024 }, "level": { "ignore_above": 1024, "type": "keyword" }, "duration": { "type": "float" }, "timestamp": { "type": "keyword", "ignore_above": 1024 }, "database": { "type": "keyword", "ignore_above": 1024 }, "query": { "type": "keyword", "ignore_above": 1024 }, "message": { "type": "text", "norms": false }, "thread_id": { "type": "long" } } } } } }, "_meta": { "version": "6.2.2" }, "date_detection": false, "dynamic_templates": [{ "fields": { "mapping": { "type": "keyword" }, "match_mapping_type": "string", "path_match": "fields.*" } }, { "docker.container.labels": { "mapping": { "type": "keyword" }, "match_mapping_type": "string", "path_match": "docker.container.labels.*" } }, { "strings_as_keyword": { "mapping": { "ignore_above": 1024, "type": "keyword" }, "match_mapping_type": "string" } } ] } }, "aliases": {} },

{ "_index": "filebeat-6.2.3-2018.04.02", "_type": "doc", "_id": "yHvthWIBXIJ6ALutoty5", "_version": 1, "_score": null, "_source": { "source": "/data/wwwlogs/access.log", "beat": { "version": "6.2.3", "hostname": "webserver_node1", "name": "webserver_node1" }, "nginx": { "access": { "body_sent": { "bytes": "5" }, "http_version": "1.1", "url": "/test.html", "response_code": "302", "referrer": "-", "geoip": { "latitude": 30.2936, "region_name": "Zhejiang", "country_code2": "CN", "city_name": "Hangzhou", "ip": "42.156.138.53", "country_name": "China", "continent_code": "AS", "region_code": "ZJ", "country_code3": "CN", "longitude": 120.1614, "location": { "lat": 30.2936, "lon": 120.1614 }, "timezone": "Asia/Shanghai" }, "remote_ip": "42.156.138.53", "user_name": "-", "method": "GET", "user_agent": { "os_minor": "0", "os_name": "iOS", "os": "iOS", "name": "Mobile Safari", "device": "iPhone", "major": "4", "minor": "0", "os_major": "4", "patch": "5", "build": "" } } }, "read_timestamp": "2018-04-02T10:36:06.065Z", "@version": "1", "offset": 78632335, "fileset": { "module": "nginx", "name": "access" }, "@timestamp": "2018-04-02T10:36:05.000Z", "prospector": { "type": "log" }, "tags": [ "beats_input_codec_plain_applied" ], "host": "webserver_node1" }, "fields": { "read_timestamp": [ "2018-04-02T10:36:06.065Z" ], "@timestamp": [ "2018-04-02T10:36:05.000Z" ] }, "sort": [ 1522665365000 ] }

回答动态