Update to libbeat 7.5.2

This commit is contained in:
Blerim Sheqa 2020-02-25 11:58:23 +01:00
parent f37bcf1fd9
commit cc8ca7dd1c
2308 changed files with 366444 additions and 17981 deletions

View File

@ -21,20 +21,20 @@
/x-pack/packetbeat/ @elastic/siem
# Filebeat
/filebeat/module/ @elastic/infrastructure
/filebeat/module/ @elastic/integrations
/filebeat/module/elasticsearch/ @elastic/stack-monitoring
/filebeat/module/kibana/ @elastic/stack-monitoring
/filebeat/module/logstash/ @elastic/stack-monitoring
/x-pack/filebeat/module/ @elastic/infrastructure
/x-pack/filebeat/module/ @elastic/integrations
/x-pack/filebeat/module/suricata/ @elastic/secops
# Metricbeat
/metricbeat/module/ @elastic/infrastructure
/metricbeat/module/ @elastic/integrations
/metricbeat/module/elasticsearch/ @elastic/stack-monitoring
/metricbeat/module/kibana/ @elastic/stack-monitoring
/metricbeat/module/logstash/ @elastic/stack-monitoring
/metricbeat/module/beat/ @elastic/stack-monitoring
/x-pack/metricbeat/module/ @elastic/infrastructure
/x-pack/metricbeat/module/ @elastic/integrations
# Heartbeat
/heartbeat/ @elastic/uptime

View File

@ -1 +1 @@
1.12.9
1.12.12

View File

@ -142,11 +142,11 @@ jobs:
# Generators
- os: linux
env: TARGETS="-C generator/metricbeat test"
env: TARGETS="-C generator/metricbeat test test-package"
go: $TRAVIS_GO_VERSION
stage: test
- os: linux
env: TARGETS="-C generator/beat test"
env: TARGETS="-C generator/beat test test-package"
go: $TRAVIS_GO_VERSION
stage: test
@ -166,12 +166,6 @@ jobs:
stage: test
# Kubernetes
- os: linux
install: deploy/kubernetes/.travis/setup.sh
env:
- TARGETS="-C deploy/kubernetes test"
- TRAVIS_K8S_VERSION=v1.8.0
stage: test
- os: linux
install: deploy/kubernetes/.travis/setup.sh
env:
@ -184,6 +178,24 @@ jobs:
- TARGETS="-C deploy/kubernetes test"
- TRAVIS_K8S_VERSION=v1.10.0
stage: test
- os: linux
dist: xenial
install: deploy/kubernetes/.travis/setup.sh
env:
- TARGETS="-C deploy/kubernetes test"
- TRAVIS_K8S_VERSION=v1.15.3
- TRAVIS_MINIKUBE_VERSION=v1.3.1
stage: test
addons:
apt:
update: true
packages:
- python-virtualenv
- libpcap-dev
- xsltproc
- libxml2-utils
- librpm-dev
# TODO include 1.11 once minikube supports it
#- os: linux
# install: deploy/kubernetes/.travis/setup.sh

View File

@ -12,11 +12,26 @@ other Beats should be migrated.
Note: This changelog was only started after the 6.3 release.
=== Beats version 7.5.1
https://github.com/elastic/beats/compare/v7.5.0..v7.5.1[Check the HEAD diff]
=== Beats version 7.5.0
https://github.com/elastic/beats/compare/v7.4.1..v7.5.0[Check the HEAD diff]
==== Breaking changes
- Build docker and kubernetes features only on supported platforms. {pull}13509[13509]
- Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509]
==== Added
- Compare event by event in `testadata` framework to avoid sorting problems {pull}13747[13747]
=== Beats version 7.4.1
https://github.com/elastic/beats/compare/v7.4.0..v7.4.1[Check the HEAD diff
https://github.com/elastic/beats/compare/v7.4.0..v7.4.1[Check the HEAD diff]
=== Beats version 7.4.0
https://github.com/elastic/beats/compare/v7.3.1..v7.4.0[Check the HEAD diff
https://github.com/elastic/beats/compare/v7.3.1..v7.4.0[Check the HEAD diff]
==== Breaking changes
@ -70,7 +85,6 @@ https://github.com/elastic/beats/compare/v7.1.1..v7.2.0[Check the HEAD diff]
- Add new option `IgnoreAllErrors` to `libbeat.common.schema` for skipping fields that failed while converting. {pull}12089[12089]
- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132]
=== Beats version 7.1.1
https://github.com/elastic/beats/compare/v7.1.0..v7.1.1[Check the HEAD diff]

View File

@ -21,10 +21,11 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
==== Breaking changes
- Move Fields from package libbeat/common to libbeat/mapping. {pull}11198[11198]
- For "metricbeat style" generated custom beats, the mage target `GoTestIntegration` has changed to `GoIntegTest` and `GoTestUnit` has changed to `GoUnitTest`. {pull}13341[13341]
==== Bugfixes
- Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}[]
- Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}14162[14162]
==== Added
@ -45,4 +46,10 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
- Use the go-lookslike library for testing in heartbeat. Eventually the mapval package will be replaced with it. {pull}12540[12540]
- New ReporterV2 interfaces that can receive a context on `Fetch(ctx, reporter)`, or `Run(ctx, reporter)`. {pull}11981[11981]
- Generate configuration from `mage` for all Beats. {pull}12618[12618]
- Add ClientFactory to TCP input source to add SplitFunc/NetworkFuncs per client. {pull}8543[8543]
- Introduce beat.OutputChooses publisher mode. {pull}12996[12996]
- Ensure that beat.Processor, beat.ProcessorList, and processors.ProcessorList are compatible and can be composed more easily. {pull}12996[12996]
- Add support to close beat.Client via beat.CloseRef (a subset of context.Context). {pull}13031[13031]
- Add checks for types and formats used in fields definitions in `fields.yml` files. {pull}13188[13188]
- Makefile included in generator copies files from beats repository using `git archive` instead of cp. {pull}13193[13193]
- Strip debug symbols from binaries to reduce binary sizes. {issue}12768[12768]

View File

@ -3,10 +3,220 @@
:issue: https://github.com/elastic/beats/issues/
:pull: https://github.com/elastic/beats/pull/
[[release-notes-7.5.1]]
=== Beats version 7.5.1
https://github.com/elastic/beats/compare/v7.5.0...v7.5.1[View commits]
==== Bugfixes
*Affecting all Beats*
- Fix `proxy_url` option in Elasticsearch output. {pull}14950[14950]
- Fix bug with potential concurrent reads and writes from event.Meta map by Kafka output. {issue}14542[14542] {pull}14568[14568]
*Filebeat*
- Change iis url path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225] {issue}7951[7951] {pull}13378[13378] {pull}14754[14754]
- Fix azure filesets test files. {issue}14185[14185] {pull}14235[14235]
- Update Logstash module's Grok patterns to support Logstash 7.4 logs. {pull}14743[14743]
*Metricbeat*
- Fix perfmon expanding counter path/adding counter to query when OS language is not english. {issue}14684[14684] {pull}14800[14800]
- Add extra check on `ignore_non_existent_counters` flag if the PdhExpandWildCardPathW returns no errors but does not expand the counter path successfully in windows/perfmon metricset. {pull}14797[14797]
- Fix rds metricset from reporting same values for different instances. {pull}14702[14702]
- Closing handler after verifying the registry key in diskio metricset. {issue}14683[14683] {pull}14759[14759]
- Fix docker network stats when multiple interfaces are configured. {issue}14586[14586] {pull}14825[14825]
- Fix ListMetrics pagination in aws module. {issue}14926[14926] {pull}14942[14942]
- Fix CPU count in docker/cpu in cases where no `online_cpus` are reported {pull}15070[15070]
[[release-notes-7.5.0]]
=== Beats version 7.5.0
https://github.com/elastic/beats/compare/v7.4.1...v7.5.0[View commits]
==== Breaking changes
*Affecting all Beats*
- By default, all Beats-created files and folders will have a umask of 0027 (on POSIX systems). {pull}14119[14119]
*Filebeat*
*Heartbeat*
- JSON/Regex checks against HTTP bodies will only consider the first 100MiB of the HTTP body to prevent excessive memory usage. {pull}14223[14223]
*Metricbeat*
==== Bugfixes
*Affecting all Beats*
- Disable `add_kubernetes_metadata` if no matchers found. {pull}13709[13709]
- Better wording for xpack beats when the _xpack endpoint is not reachable. {pull}13771[13771]
- Kubernetes watcher at `add_kubernetes_metadata` fails with StatefulSets {pull}13905[13905]
- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
- Fix memory leak in kubernetes autodiscover provider and add_kubernetes_metadata processor happening when pods are terminated without sending a delete event. {pull}14259[14259]
- Fix kubernetes `metaGenerator.ResourceMetadata` when parent reference controller is nil {issue}14320[14320] {pull}14329[14329]
*Auditbeat*
- Socket dataset: Fix start errors when IPv6 is disabled on the kernel. {issue}13953[13953] {pull}13966[13966]
*Filebeat*
- Fix a denial of service flaw when parsing malformed DSA public keys in Go.
If {filebeat} is configured to accept incoming TLS connections with client
authentication enabled, a remote attacker could cause the Beat to stop
processing events. (CVE-2019-17596) See https://www.elastic.co/community/security/
- Fix timezone parsing of rabbitmq module ingest pipelines. {pull}13879[13879]
- Fix conditions and error checking of date processors in ingest pipelines that use `event.timezone` to parse dates. {pull}13883[13883]
- Fix timezone parsing of Cisco module ingest pipelines. {pull}13893[13893]
- Fix timezone parsing of logstash module ingest pipelines. {pull}13890[13890]
- Fix timezone parsing of iptables, mssql and panw module ingest pipelines. {pull}13926[13926]
- Fixed increased memory usage with large files when multiline pattern does not match. {issue}14068[14068]
- Fix azure fields names. {pull}14098[14098] {pull}14132[14132]
- Fix calculation of `network.bytes` and `network.packets` for bi-directional netflow events. {pull}14111[14111]
- Accept '-' as http.response.body.bytes in apache module. {pull}14137[14137]
- Fix timezone parsing of MySQL module ingest pipelines. {pull}14130[14130]
- Improve error message in s3 input when handleSQSMessage failed. {pull}14113[14113]
- Fix race condition in S3 input plugin. {pull}14359[14359]
*Heartbeat*
- Fix storage of HTTP bodies to work when JSON/Regex body checks are enabled. {pull}14223[14223]
*Metricbeat*
- Fix a denial of service flaw when parsing malformed DSA public keys in Go.
If {metricbeat} is configured to accept incoming TLS connections with client
authentication enabled, a remote attacker could cause the Beat to stop
processing events. (CVE-2019-17596) See https://www.elastic.co/community/security/
- PdhExpandWildCardPathW will not expand counter paths in 32 bit windows systems, workaround will use a different function. {issue}12590[12590] {pull}12622[12622]
- Fix `docker.cpu.system.pct` calculation by using the reported number online cpus instead of the number of metrics per cpu. {pull}13691[13691]
- Change kubernetes.event.message to text {pull}13964[13964]
- Fix performance counter values for windows/perfmon metricset.{issue}14036[14036] {pull}14039[14039] {pull}14108[14108]
- Add FailOnRequired when applying schema and fix metric names in mongodb metrics metricset. {pull}14143[14143]
- Convert indexed ms-since-epoch timestamp fields in `elasticsearch/ml_job` metricset to ints from float64s. {issue}14220[14220] {pull}14222[14222]
- Fix ARN parsing function to work for ELB ARNs. {pull}14316[14316]
- Update azure configuration example. {issue}14224[14224]
- Limit some of the error messages to the logs only {issue}14317[14317] {pull}14327[14327]
- Fix cloudwatch metricset with names and dimensions in config. {issue}14376[14376] {pull}14391[14391]
- Fix marshaling of ms-since-epoch values in `elasticsearch/cluster_stats` metricset. {pull}14378[14378]
*Packetbeat*
- Fix parsing of the HTTP host header when it contains a port or an IPv6 address. {pull}14215[14215]
==== Added
*Affecting all Beats*
- Fail with error when autodiscover providers have no defined configs. {pull}13078[13078]
- Add autodetection mode for add_docker_metadata and enable it by default in included configuration files{pull}13374[13374]
- Add autodetection mode for add_kubernetes_metadata and enable it by default in included configuration files. {pull}13473[13473]
- Use less restrictive API to check if template exists. {pull}13847[13847]
- Do not check for alias when setup.ilm.check_exists is false. {pull}13848[13848]
- Add support for numeric time zone offsets in timestamp processor. {pull}13902[13902]
- Add condition to the config file template for add_kubernetes_metadata {pull}14056[14056]
- Marking Central Management deprecated. {pull}14018[14018]
- Add `keep_null` setting to allow Beats to publish null values in events. {issue}5522[5522] {pull}13928[13928]
- Add shared_credential_file option in aws related config for specifying credential file directory. {issue}14157[14157] {pull}14178[14178]
- Ensure that init containers are no longer tailed after they stop. {pull}14394[14394]
- Libbeat HTTP's Server can listen to a unix socket using the `unix:///tmp/hello.sock` syntax. {pull}13655[13655]
- Libbeat HTTP's Server can listen to a Windows named pipe using the `npipe:///hello` syntax. {pull}13655[13655]
- Adding new `Enterprise` license type to the licenser. {issue}14246[14246]
*Auditbeat*
- Socket: Add DNS enrichment. {pull}14004[14004]
*Filebeat*
- Add support for virtual host in Apache access logs {pull}12778[12778]
- Update CoreDNS module to populate ECS DNS fields. {issue}13320[13320] {pull}13505[13505]
- Parse query steps in PostgreSQL slowlogs. {issue}13496[13496] {pull}13701[13701]
- Add filebeat azure module with activitylogs, auditlogs, signinlogs filesets. {pull}13776[13776]
- Add support to set the document id in the json reader. {pull}5844[5844]
- Add input httpjson. {issue}13545[13545] {pull}13546[13546]
- Filebeat Netflow input: Remove beta label. {pull}13858[13858]
- Remove `event.timezone` from events that don't need it in some modules that support log formats with and without timezones. {pull}13918[13918]
- Add ExpandEventListFromField config option in the kafka input. {pull}13965[13965]
- Add ELB fileset to AWS module. {pull}14020[14020]
- Add module for MISP (Malware Information Sharing Platform). {pull}13805[13805]
- Add filebeat azure module with activitylogs, auditlogs, signinlogs filesets. {pull}13776[13776] {pull}14033[14033] {pull}14107[14107]
- Add support for all the ObjectCreated events in S3 input. {pull}14077[14077]
- Add `source.bytes` and `source.packets` for uni-directional netflow events. {pull}14111[14111]
- Add Kibana Dashboard for MISP module. {pull}14147[14147]
- Add support for gzipped files in S3 input {pull}13980[13980]
- Add Filebeat Azure Dashboards {pull}14127[14127]
*Heartbeat*
- Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498]
- Allow `hosts` to be used to configure http monitors {pull}13703[13703]
*Metricbeat*
- Add refresh list of perf counters at every fetch {issue}13091[13091]
- Add proc/vmstat data to the system/memory metricset on linux {pull}13322[13322]
- Add support for NATS version 2. {pull}13601[13601]
- Add `docker.cpu.*.norm.pct` metrics for `cpu` metricset of Docker Metricbeat module. {pull}13695[13695]
- Add `instance` label by default when using Prometheus collector. {pull}13737[13737]
- Add azure module. {pull}13196[13196] {pull}13859[13859] {pull}13988[13988]
- Add Apache Tomcat module {pull}13491[13491]
- Add ECS `container.id` and `container.runtime` to kubernetes `state_container` metricset. {pull}13884[13884]
- Add `job` label by default when using Prometheus collector. {pull}13878[13878]
- Add `state_resourcequota` metricset for Kubernetes module. {pull}13693[13693]
- Add tags filter in ec2 metricset. {pull}13872[13872] {issue}13145[13145]
- Add cloud.account.id and cloud.account.name into events from aws module. {issue}13551[13551] {pull}13558[13558]
- Add `metrics_path` as known hint for autodiscovery {pull}13996[13996]
- Leverage KUBECONFIG when creating k8s client. {pull}13916[13916]
- Add ability to filter by tags for cloudwatch metricset. {pull}13758[13758] {issue}13145[13145]
- Release cloudwatch, s3_daily_storage, s3_request, sqs and rds metricset as GA. {pull}14114[14114] {issue}14059[14059]
- Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221]
- Add new dashboards for Azure vms, vm guest metrics, vm scale sets {pull}14000[14000]
*Functionbeat*
- Make `bulk_max_size` configurable in outputs. {pull}13493[13493]
*Winlogbeat*
- Fill `event.provider`. {pull}13937[13937]
- Add support for user management events to the Security module. {pull}13530[13530]
==== Deprecated
*Metricbeat*
- `kubernetes.container.id` field for `state_container` is deprecated in favour of ECS `container.id` and `container.runtime`. {pull}13884[13884]
[[release-notes-7.4.1]]
=== Beats version 7.4.1
https://github.com/elastic/beats/compare/v7.4.0...v7.4.1[View commits]
==== Breaking changes
*Affecting all Beats*
*Auditbeat*
*Filebeat*
*Heartbeat*
*Journalbeat*
*Metricbeat*
*Packetbeat*
*Winlogbeat*
*Functionbeat*
==== Bugfixes
*Affecting all Beats*
@ -16,8 +226,6 @@ https://github.com/elastic/beats/compare/v7.4.0...v7.4.1[View commits]
*Auditbeat*
- Socket dataset: Fix start errors when IPv6 is disabled on the kernel. {issue}13953[13953] {pull}13966[13966]
*Filebeat*
- Fixed early expiration of templates (Netflow v9 and IPFIX). {pull}13821[13821]
@ -27,12 +235,63 @@ https://github.com/elastic/beats/compare/v7.4.0...v7.4.1[View commits]
- Fix delay in enforcing close_renamed and close_removed options. {issue}13488[13488] {pull}13907[13907]
- Fix missing netflow fields in index template. {issue}13768[13768] {pull}13914[13914]
- Fix cisco module's asa and ftd filesets parsing of domain names where an IP address is expected. {issue}14034[14034]
- Fixed increased memory usage with large files when multiline pattern does not match. {issue}14068[14068]
*Heartbeat*
*Journalbeat*
*Metricbeat*
- Mark Kibana usage stats as collected only if API call succeeds. {pull}13881[13881]
*Packetbeat*
*Winlogbeat*
*Functionbeat*
==== Added
*Affecting all Beats*
*Auditbeat*
*Filebeat*
*Heartbeat*
*Journalbeat*
*Metricbeat*
*Packetbeat*
*Functionbeat*
*Winlogbeat*
==== Deprecated
*Affecting all Beats*
*Filebeat*
*Heartbeat*
*Journalbeat*
*Metricbeat*
*Packetbeat*
*Winlogbeat*
*Functionbeat*
==== Known Issue
*Journalbeat*
[[release-notes-7.4.0]]
=== Beats version 7.4.0
https://github.com/elastic/beats/compare/v7.3.1...v7.4.0[View commits]

View File

@ -11,19 +11,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Affecting all Beats*
- Update to Golang 1.12.1. {pull}11330[11330]
- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
*Auditbeat*
*Filebeat*
- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
*Heartbeat*
*Journalbeat*
- Remove broken dashboard. {pull}15288[15288]
*Metricbeat*
- kubernetes.container.cpu.limit.cores and kubernetes.container.cpu.requests.cores are now floats. {issue}11975[11975]
@ -35,28 +37,46 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Functionbeat*
==== Bugfixes
*Affecting all Beats*
- Fix a race condition with the Kafka pipeline client, it is possible that `Close()` get called before `Connect()` . {issue}11945[11945]
- Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338]
*Auditbeat*
*Filebeat*
- panw module: Use geo.name instead of geo.country_iso_code for free-form location. {issue}13272[13272]
- cisco/asa fileset: Fix parsing of 302021 message code. {pull}14519[14519]
- Fix filebeat azure dashboards, event category should be `Alert`. {pull}14668[14668]
- Check content-type when creating new reader in s3 input. {pull}15252[15252] {issue}15225[15225]
- Fix session reset detection and a crash in Netflow input. {pull}14904[14904]
- netflow: Allow for options templates without scope fields. {pull}15449[15449]
- netflow: Fix bytes/packets counters on some devices (NSEL and Netstream). {pull}15449[15449]
- netflow: Fix compatibility with some Cisco devices by changing the field `class_id` from short to long. {pull}15449[15449]
- Fixed dashboard for Cisco ASA Firewall. {issue}15420[15420] {pull}15553[15553]
*Heartbeat*
- Fix recording of SSL cert metadata for Expired/Unvalidated x509 certs. {pull}13687[13687]
*Journalbeat*
*Metricbeat*
- Ignore prometheus untyped metrics with NaN value. {issue}13750[13750] {pull}13790[13790]
- Fix checking tagsFilter using length in cloudwatch metricset. {pull}14525[14525]
- Fixed bug with `elasticsearch/cluster_stats` metricset not recording license expiration date correctly. {issue}14541[14541] {pull}14591[14591]
- Log bulk failures from bulk API requests to monitoring cluster. {issue}14303[14303] {pull}14356[14356]
- Fix regular expression to detect instance name in perfmon metricset. {issue}14273[14273] {pull}14666[14666]
- Fixed bug with `elasticsearch/cluster_stats` metricset not recording license ID in the correct field. {pull}14592[14592]
- Fix `docker.container.size` fields values {issue}14979[14979] {pull}15224[15224]
- Make `kibana` module more resilient to Kibana unavailability. {issue}15258[15258] {pull}15270[15270]
- Fix panic exception with some unicode strings in perfmon metricset. {issue}15264[15264]
- Make `logstash` module more resilient to Logstash unavailability. {issue}15276[15276] {pull}15306[15306]
*Packetbeat*
@ -71,9 +91,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Affecting all Beats*
- Decouple Debug logging from fail_on_error logic for rename, copy, truncate processors {pull}12451[12451]
- Allow a beat to ship monitoring data directly to an Elasticsearch monitoring cluster. {pull}9260[9260]
- Add `providers` setting to `add_cloud_metadata` processor. {pull}13812[13812]
- Add a friendly log message when a request to docker has exceeded the deadline. {pull}15336[15336]
*Auditbeat*
@ -81,7 +99,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Filebeat*
- `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358]
- Use correct OS path separator in `add_kubernetes_metadata` to support Windows nodes. {pull}9205[9205]
- Add `index` option to all inputs to directly set a per-input index value. {pull}14010[14010]
- Include log.source.address for unparseable syslog messages. {issue}13268[13268] {pull}15453[15453]
*Heartbeat*
@ -93,6 +112,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Packetbeat*
*Functionbeat*
@ -112,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Metricbeat*
*Packetbeat*
*Winlogbeat*
@ -121,3 +142,4 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
==== Known Issue
*Journalbeat*

View File

@ -98,16 +98,12 @@ check: python-env
@git diff-index --exit-code HEAD --
.PHONY: check-headers
check-headers:
@go get -u github.com/elastic/go-licenser
@go-licenser -d -exclude x-pack
@go-licenser -d -license Elastic x-pack
check-headers: mage
@mage checkLicenseHeaders
.PHONY: add-headers
add-headers:
@go get github.com/elastic/go-licenser
@go-licenser -exclude x-pack
@go-licenser -license Elastic x-pack
add-headers: mage
@mage addLicenseHeaders
# Corrects spelling errors
.PHONY: misspell

File diff suppressed because it is too large Load Diff

View File

@ -1,4 +1,4 @@
FROM golang:1.12.9
FROM golang:1.12.12
RUN \
apt-get update \

View File

@ -14,6 +14,7 @@ auditbeat.modules:
- /etc
processors:
- add_cloud_metadata: ~
- add_docker_metadata: ~
output.elasticsearch:
hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

View File

@ -38,6 +38,10 @@ auditbeat.modules:
rate_limit: 0
include_raw_message: false
include_warnings: false
# Set to true to publish fields with null values in events.
#keep_null: false
# Load audit rules from separate files. Same format as audit.rules(7).
audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ]
audit_rules: |
@ -110,6 +114,9 @@ auditbeat.modules:
# Detect changes to files included in subdirectories. Disabled by default.
recursive: false
# Set to true to publish fields with null values in events.
#keep_null: false
#================================ General ======================================
@ -1070,7 +1077,7 @@ setup.template.settings:
#setup.ilm.enabled: auto
# Set the prefix used in the index lifecycle write alias name. The default alias
# name is 'auditbeat-%{[agent.version]}'.
# name is 'auditbeat-%{[agent.version]}'.
#setup.ilm.rollover_alias: "auditbeat"
# Set the rollover index pattern. The default is "%{now/d}-000001".
@ -1333,12 +1340,21 @@ logging.files:
# Defines if the HTTP endpoint is enabled.
#http.enabled: false
# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
# The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
# When using IP addresses, it is recommended to only use localhost.
#http.host: localhost
# Port on which the HTTP endpoint will bind. Default is 5066.
#http.port: 5066
# Define which user should be owning the named pipe.
#http.named_pipe.user:
# Define which the permissions that should be applied to the named pipe, use the Security
# Descriptor Definition Language (SDDL) to define the permission. This option cannot be used with
# `http.user`.
#http.named_pipe.security_descriptor:
#============================= Process Security ================================
# Enable or disable seccomp system call filtering on Linux. Default is enabled.

View File

@ -148,6 +148,7 @@ output.elasticsearch:
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
- add_docker_metadata: ~
#================================ Logging =====================================

View File

@ -1,6 +1,6 @@
[[filtering-and-enhancing-data]]
== Filter and enhance the exported data
include::{libbeat-dir}/docs/processors.asciidoc[]
include::{libbeat-dir}/processors.asciidoc[]
include::{libbeat-dir}/docs/processors-using.asciidoc[]
include::{libbeat-dir}/processors-using.asciidoc[]

View File

@ -4,4 +4,4 @@
You can specify settings in the +{beatname_lc}.yml+ config file to control the
general behavior of {beatname_uc}.
include::{libbeat-dir}/docs/generalconfig.asciidoc[]
include::{libbeat-dir}/generalconfig.asciidoc[]

View File

@ -0,0 +1,56 @@
//////////////////////////////////////////////////////////////////////////
//// This content is shared by all Auditbeat modules. Make sure you keep the
//// descriptions generic enough to work for all modules. To include
//// this file, use:
////
//// include::{docdir}/auditbeat-options.asciidoc[]
////
//////////////////////////////////////////////////////////////////////////
[id="module-standard-options-{modulename}"]
[float]
==== Standard configuration options
You can specify the following options for any {beatname_uc} module.
*`module`*:: The name of the module to run.
ifeval::["{modulename}"=="system"]
*`datasets`*:: A list of datasets to execute.
endif::[]
*`enabled`*:: A Boolean value that specifies whether the module is enabled.
ifeval::["{modulename}"=="system"]
*`period`*:: The frequency at which the datasets check for changes. If a system
is not reachable, {beatname_uc} returns an error for each period. This setting
is required. For most datasets, especially `process` and `socket`, a shorter
period is recommended.
endif::[]
*`fields`*:: A dictionary of fields that will be sent with the dataset event. This setting
is optional.
*`tags`*:: A list of tags that will be sent with the dataset event. This setting is
optional.
*`processors`*:: A list of processors to apply to the data generated by the dataset.
+
See <<filtering-and-enhancing-data>> for information about specifying
processors in your config.
*`index`*:: If present, this formatted string overrides the index for events from this
module (for elasticsearch outputs), or sets the `raw_index` field of the event's
metadata (for other outputs). This string can only refer to the agent name and
version and the event timestamp; for access to dynamic fields, use
`output.elasticsearch.index` or a processor.
+
Example value: `"%{[agent.name]}-myindex-%{+yyyy.MM.dd}"` might
expand to +"{beatname_lc}-myindex-2019.12.13"+.
*`keep_null`*:: If this option is set to true, fields with `null` values will be published in
the output document. By default, `keep_null` is set to `false`.
*`service.name`*:: A name given by the user to the service the data is collected from. It can be
used for example to identify information collected from nodes of different
clusters with the same `service.type`.

View File

@ -51,38 +51,38 @@ include::./auditbeat-general-options.asciidoc[]
include::./reload-configuration.asciidoc[]
include::{libbeat-dir}/docs/queueconfig.asciidoc[]
include::{libbeat-dir}/queueconfig.asciidoc[]
include::{libbeat-dir}/docs/outputconfig.asciidoc[]
include::{libbeat-dir}/outputconfig.asciidoc[]
include::{libbeat-dir}/docs/shared-ilm.asciidoc[]
include::{libbeat-dir}/shared-ilm.asciidoc[]
include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[]
include::{libbeat-dir}/shared-ssl-config.asciidoc[]
include::./auditbeat-filtering.asciidoc[]
include::{libbeat-dir}/docs/shared-config-ingest.asciidoc[]
include::{libbeat-dir}/shared-config-ingest.asciidoc[]
include::{libbeat-dir}/docs/shared-geoip.asciidoc[]
include::{libbeat-dir}/shared-geoip.asciidoc[]
include::{libbeat-dir}/docs/shared-path-config.asciidoc[]
include::{libbeat-dir}/shared-path-config.asciidoc[]
include::{libbeat-dir}/docs/shared-kibana-config.asciidoc[]
include::{libbeat-dir}/shared-kibana-config.asciidoc[]
include::{libbeat-dir}/docs/setup-config.asciidoc[]
include::{libbeat-dir}/setup-config.asciidoc[]
include::{libbeat-dir}/docs/loggingconfig.asciidoc[]
include::{libbeat-dir}/loggingconfig.asciidoc[]
:standalone:
include::{libbeat-dir}/docs/shared-env-vars.asciidoc[]
include::{libbeat-dir}/shared-env-vars.asciidoc[]
:standalone!:
:standalone:
include::{libbeat-dir}/docs/yaml.asciidoc[]
include::{libbeat-dir}/yaml.asciidoc[]
:standalone!:
include::{libbeat-dir}/docs/regexp.asciidoc[]
include::{libbeat-dir}/regexp.asciidoc[]
include::{libbeat-dir}/docs/http-endpoint.asciidoc[]
include::{libbeat-dir}/http-endpoint.asciidoc[]
include::{libbeat-dir}/docs/reference-yml.asciidoc[]
include::{libbeat-dir}/reference-yml.asciidoc[]

View File

@ -7,6 +7,6 @@ https://discuss.elastic.co/c/beats/{beatname_lc}[{beatname_uc} discussion forum]
include::./faq-ulimit.asciidoc[]
include::{libbeat-dir}/docs/faq-limit-bandwidth.asciidoc[]
include::{libbeat-dir}/faq-limit-bandwidth.asciidoc[]
include::{libbeat-dir}/docs/shared-faq.asciidoc[]
include::{libbeat-dir}/shared-faq.asciidoc[]

View File

@ -1,7 +1,7 @@
[id="{beatname_lc}-getting-started"]
== Getting started with {beatname_uc}
include::{libbeat-dir}/docs/shared-getting-started-intro.asciidoc[]
include::{libbeat-dir}/shared-getting-started-intro.asciidoc[]
* <<{beatname_lc}-installation>>
* <<{beatname_lc}-configuration>>
@ -16,14 +16,14 @@ include::{libbeat-dir}/docs/shared-getting-started-intro.asciidoc[]
Install {beatname_uc} on all the servers you want to monitor.
include::{libbeat-dir}/docs/shared-download-and-install.asciidoc[]
include::{libbeat-dir}/shared-download-and-install.asciidoc[]
[[deb]]
*deb:*
ifeval::["{release-state}"=="unreleased"]
Version {stack-version} of {beatname_uc} has not yet been released.
Version {version} of {beatname_uc} has not yet been released.
endif::[]
@ -42,7 +42,7 @@ endif::[]
ifeval::["{release-state}"=="unreleased"]
Version {stack-version} of {beatname_uc} has not yet been released.
Version {version} of {beatname_uc} has not yet been released.
endif::[]
@ -61,7 +61,7 @@ endif::[]
ifeval::["{release-state}"=="unreleased"]
Version {stack-version} of {beatname_uc} has not yet been released.
Version {version} of {beatname_uc} has not yet been released.
endif::[]
@ -75,14 +75,14 @@ tar xzvf {beatname_lc}-{version}-darwin-x86_64.tar.gz
endif::[]
include::{libbeat-dir}/docs/shared-brew-install.asciidoc[]
include::{libbeat-dir}/shared-brew-install.asciidoc[]
[[linux]]
*linux:*
ifeval::["{release-state}"=="unreleased"]
Version {stack-version} of {beatname_uc} has not yet been released.
Version {version} of {beatname_uc} has not yet been released.
endif::[]
@ -101,7 +101,7 @@ endif::[]
ifeval::["{release-state}"=="unreleased"]
Version {stack-version} of {beatname_uc} has not yet been released.
Version {version} of {beatname_uc} has not yet been released.
endif::[]
@ -122,7 +122,7 @@ See <<running-on-docker, Running on Docker>> for deploying Docker containers.
ifeval::["{release-state}"=="unreleased"]
Version {stack-version} of {beatname_uc} has not yet been released.
Version {version} of {beatname_uc} has not yet been released.
endif::[]
@ -162,7 +162,7 @@ For more information about these options, see
[id="{beatname_lc}-configuration"]
=== Step 2: Configure {beatname_uc}
include::{libbeat-dir}/docs/shared-configuring.asciidoc[]
include::{libbeat-dir}/shared-configuring.asciidoc[]
To configure {beatname_uc}:
@ -192,25 +192,25 @@ If you accept the default configuration without specifying additional modules,
+
See <<configuring-howto-{beatname_lc}>> for more details about configuring modules.
include::{libbeat-dir}/docs/step-configure-output.asciidoc[]
include::{libbeat-dir}/step-configure-output.asciidoc[]
include::{libbeat-dir}/docs/step-configure-kibana-endpoint.asciidoc[]
include::{libbeat-dir}/step-configure-kibana-endpoint.asciidoc[]
include::{libbeat-dir}/docs/step-configure-credentials.asciidoc[]
include::{libbeat-dir}/step-configure-credentials.asciidoc[]
include::{libbeat-dir}/docs/step-test-config.asciidoc[]
include::{libbeat-dir}/step-test-config.asciidoc[]
include::{libbeat-dir}/docs/step-look-at-config.asciidoc[]
include::{libbeat-dir}/step-look-at-config.asciidoc[]
[id="{beatname_lc}-template"]
=== Step 3: Load the index template in {es}
include::{libbeat-dir}/docs/shared-template-load.asciidoc[]
include::{libbeat-dir}/shared-template-load.asciidoc[]
[[load-kibana-dashboards]]
=== Step 4: Set up the {kib} dashboards
include::{libbeat-dir}/docs/dashboards.asciidoc[]
include::{libbeat-dir}/dashboards.asciidoc[]
[id="{beatname_lc}-starting"]
=== Step 5: Start {beatname_uc}
@ -246,7 +246,7 @@ in the _Beats Platform Reference_.
If you see a warning about too many open files, you need to increase the
`ulimit`. See the <<ulimit,FAQ>> for more details.
include::{libbeat-dir}/docs/shared-brew-run.asciidoc[]
include::{libbeat-dir}/shared-brew-run.asciidoc[]
*win:*
@ -280,7 +280,7 @@ To make it easier for you to start auditing the activities of users and
processes on your system, we have created example {beatname_uc} dashboards.
You loaded the dashboards earlier when you ran the `setup` command.
include::{libbeat-dir}/docs/opendashboards.asciidoc[]
include::{libbeat-dir}/opendashboards.asciidoc[]
The dashboards are provided as examples. We recommend that you
{kibana-ref}/dashboard.html[customize] them to meet your needs.

View File

@ -1,12 +1,13 @@
= Auditbeat Reference
:libbeat-dir: {docdir}/../../libbeat
:libbeat-dir: {docdir}/../../libbeat/docs
include::{libbeat-dir}/docs/version.asciidoc[]
include::{libbeat-dir}/version.asciidoc[]
include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[]
include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
:version: {stack-version}
:beatname_lc: auditbeat
:beatname_uc: Auditbeat
:beatname_pkg: {beatname_lc}
@ -19,14 +20,18 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
:docker_platform:
:win_os:
:linux_os:
:no_decode_cef_processor:
:no_decode_csv_fields_processor:
:no_script_processor:
:no_timestamp_processor:
include::{libbeat-dir}/docs/shared-beats-attributes.asciidoc[]
include::{libbeat-dir}/shared-beats-attributes.asciidoc[]
include::./overview.asciidoc[]
include::./getting-started.asciidoc[]
include::{libbeat-dir}/docs/repositories.asciidoc[]
include::{libbeat-dir}/repositories.asciidoc[]
include::./setting-up-running.asciidoc[]
@ -38,13 +43,13 @@ include::./modules.asciidoc[]
include::./fields.asciidoc[]
include::{libbeat-dir}/docs/monitoring/monitoring-beats.asciidoc[]
include::{libbeat-dir}/monitoring/monitoring-beats.asciidoc[]
include::{libbeat-dir}/docs/shared-securing-beat.asciidoc[]
include::{libbeat-dir}/shared-securing-beat.asciidoc[]
include::./troubleshooting.asciidoc[]
include::./faq.asciidoc[]
include::{libbeat-dir}/docs/contributing-to-beats.asciidoc[]
include::{libbeat-dir}/contributing-to-beats.asciidoc[]

View File

@ -218,6 +218,9 @@ time.
- `none`: No backpressure mitigation measures are enabled.
--
*`keep_null`*:: If this option is set to true, fields with `null` values will be
published in the output document. By default, `keep_null` is set to `false`.
[float]
=== Audit rules

View File

@ -122,6 +122,9 @@ of this directories are watched. If `recursive` is set to `true`, the
`file_integrity` module will watch for changes on this directories and all
their subdirectories.
*`keep_null`*:: If this option is set to true, fields with `null` values will be
published in the output document. By default, `keep_null` is set to `false`.
[float]
=== Example configuration

View File

@ -12,4 +12,4 @@ Audit Framework. You can also use {beatname_uc} to detect changes to critical
files, like binaries and configuration files, and identify potential security
policy violations.
include::{libbeat-dir}/docs/shared-libbeat-description.asciidoc[]
include::{libbeat-dir}/shared-libbeat-description.asciidoc[]

View File

@ -1,4 +1,4 @@
include::{libbeat-dir}/docs/shared-docker.asciidoc[]
include::{libbeat-dir}/shared-docker.asciidoc[]
==== Special requirements

View File

@ -6,7 +6,7 @@ check files integrity.
ifeval::["{release-state}"=="unreleased"]
However, version {stack-version} of {beatname_uc} has not yet been
However, version {version} of {beatname_uc} has not yet been
released, so no Docker image is currently available for this version.
endif::[]

View File

@ -27,16 +27,16 @@ This section includes additional information on how to set up and run
//MAINTAINERS: If you add a new file to this section, make sure you update the bulleted list ^^ too.
include::{libbeat-dir}/docs/shared-directory-layout.asciidoc[]
include::{libbeat-dir}/shared-directory-layout.asciidoc[]
include::{libbeat-dir}/docs/keystore.asciidoc[]
include::{libbeat-dir}/keystore.asciidoc[]
include::{libbeat-dir}/docs/command-reference.asciidoc[]
include::{libbeat-dir}/command-reference.asciidoc[]
include::./running-on-docker.asciidoc[]
include::./running-on-kubernetes.asciidoc[]
include::{libbeat-dir}/docs/shared-systemd.asciidoc[]
include::{libbeat-dir}/shared-systemd.asciidoc[]
include::{libbeat-dir}/docs/shared-shutdown.asciidoc[]
include::{libbeat-dir}/shared-shutdown.asciidoc[]

View File

@ -17,14 +17,14 @@ following tips:
[[getting-help]]
== Get Help
include::{libbeat-dir}/docs/getting-help.asciidoc[]
include::{libbeat-dir}/getting-help.asciidoc[]
//sets block macro for debugging.asciidoc included in next section
[id="enable-{beatname_lc}-debugging"]
== Debug
include::{libbeat-dir}/docs/debugging.asciidoc[]
include::{libbeat-dir}/debugging.asciidoc[]

View File

@ -11,6 +11,10 @@
rate_limit: 0
include_raw_message: false
include_warnings: false
# Set to true to publish fields with null values in events.
#keep_null: false
{{ end -}}
# Load audit rules from separate files. Same format as audit.rules(7).
audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ]

View File

@ -213,6 +213,9 @@ time.
- `none`: No backpressure mitigation measures are enabled.
--
*`keep_null`*:: If this option is set to true, fields with `null` values will be
published in the output document. By default, `keep_null` is set to `false`.
[float]
=== Audit rules

View File

@ -75,4 +75,7 @@
# Detect changes to files included in subdirectories. Disabled by default.
recursive: false
# Set to true to publish fields with null values in events.
#keep_null: false
{{ end }}

View File

@ -116,3 +116,6 @@ The supported hash types are `blake2b_256`, `blake2b_384`, `blake2b_512`, `md5`,
of this directories are watched. If `recursive` is set to `true`, the
`file_integrity` module will watch for changes on this directories and all
their subdirectories.
*`keep_null`*:: If this option is set to true, fields with `null` values will be
published in the output document. By default, `keep_null` is set to `false`.

View File

@ -9,6 +9,8 @@ export CHANGE_MINIKUBE_NONE_USER=true
curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$TRAVIS_K8S_VERSION/bin/linux/amd64/kubectl && \
chmod +x kubectl && sudo mv kubectl /usr/local/bin/
curl -Lo minikube https://storage.googleapis.com/minikube/releases/$TRAVIS_MINIKUBE_VERSION/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
mkdir -p $HOME/.kube $HOME/.minikube
touch $HOME/.kube/config
sudo minikube start --vm-driver=none --kubernetes-version=$TRAVIS_K8S_VERSION --logtostderr
sudo minikube update-context
JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; \

View File

@ -52,7 +52,7 @@ data:
recursive: true
---
# Deploy a auditbeat instance per node for node metrics retrieval
apiVersion: extensions/v1beta1
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: auditbeat
@ -60,6 +60,9 @@ metadata:
labels:
k8s-app: auditbeat
spec:
selector:
matchLabels:
k8s-app: auditbeat
template:
metadata:
labels:
@ -71,7 +74,7 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: auditbeat
image: docker.elastic.co/beats/auditbeat:7.4.1
image: docker.elastic.co/beats/auditbeat:7.5.1
args: [
"-c", "/etc/auditbeat.yml"
]
@ -148,7 +151,7 @@ spec:
path: /var/lib/auditbeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: auditbeat
@ -161,7 +164,7 @@ roleRef:
name: auditbeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: auditbeat

View File

@ -1,5 +1,5 @@
# Deploy a auditbeat instance per node for node metrics retrieval
apiVersion: extensions/v1beta1
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: auditbeat
@ -7,6 +7,9 @@ metadata:
labels:
k8s-app: auditbeat
spec:
selector:
matchLabels:
k8s-app: auditbeat
template:
metadata:
labels:

View File

@ -1,4 +1,4 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: auditbeat

View File

@ -1,4 +1,4 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: auditbeat

View File

@ -42,7 +42,7 @@ data:
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
---
apiVersion: extensions/v1beta1
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
@ -50,6 +50,9 @@ metadata:
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
@ -61,7 +64,7 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.4.1
image: docker.elastic.co/beats/filebeat:7.5.1
args: [
"-c", "/etc/filebeat.yml",
"-e",
@ -123,7 +126,7 @@ spec:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
@ -136,7 +139,7 @@ roleRef:
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat

View File

@ -1,4 +1,4 @@
apiVersion: extensions/v1beta1
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
@ -6,6 +6,9 @@ metadata:
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:

View File

@ -1,4 +1,4 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat

View File

@ -1,4 +1,4 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat

View File

@ -91,7 +91,7 @@ data:
hosts: ["localhost:10249"]
---
# Deploy a Metricbeat instance per node for node metrics retrieval
apiVersion: extensions/v1beta1
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: metricbeat
@ -99,6 +99,9 @@ metadata:
labels:
k8s-app: metricbeat
spec:
selector:
matchLabels:
k8s-app: metricbeat
template:
metadata:
labels:
@ -110,7 +113,7 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: metricbeat
image: docker.elastic.co/beats/metricbeat:7.4.1
image: docker.elastic.co/beats/metricbeat:7.5.1
args: [
"-c", "/etc/metricbeat.yml",
"-e",
@ -223,6 +226,8 @@ data:
- state_replicaset
- state_pod
- state_container
- state_cronjob
- state_resourcequota
# Uncomment this to get k8s events:
#- event
period: 10s
@ -230,7 +235,7 @@ data:
hosts: ["kube-state-metrics:8080"]
---
# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics
apiVersion: apps/v1beta1
apiVersion: apps/v1
kind: Deployment
metadata:
name: metricbeat
@ -238,6 +243,9 @@ metadata:
labels:
k8s-app: metricbeat
spec:
selector:
matchLabels:
k8s-app: metricbeat
template:
metadata:
labels:
@ -248,7 +256,7 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: metricbeat
image: docker.elastic.co/beats/metricbeat:7.4.1
image: docker.elastic.co/beats/metricbeat:7.5.1
args: [
"-c", "/etc/metricbeat.yml",
"-e",
@ -296,7 +304,7 @@ spec:
defaultMode: 0600
name: metricbeat-deployment-modules
---
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: metricbeat
@ -309,7 +317,7 @@ roleRef:
name: metricbeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: metricbeat

View File

@ -1,5 +1,5 @@
# Deploy a Metricbeat instance per node for node metrics retrieval
apiVersion: extensions/v1beta1
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: metricbeat
@ -7,6 +7,9 @@ metadata:
labels:
k8s-app: metricbeat
spec:
selector:
matchLabels:
k8s-app: metricbeat
template:
metadata:
labels:

View File

@ -41,6 +41,8 @@ data:
- state_replicaset
- state_pod
- state_container
- state_cronjob
- state_resourcequota
# Uncomment this to get k8s events:
#- event
period: 10s

View File

@ -1,5 +1,5 @@
# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics
apiVersion: apps/v1beta1
apiVersion: apps/v1
kind: Deployment
metadata:
name: metricbeat
@ -7,6 +7,9 @@ metadata:
labels:
k8s-app: metricbeat
spec:
selector:
matchLabels:
k8s-app: metricbeat
template:
metadata:
labels:

View File

@ -1,4 +1,4 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: metricbeat

View File

@ -1,4 +1,4 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: metricbeat

View File

@ -28,7 +28,9 @@ This script does the following:
remote
* if the --create_pr flag is used, it uses the GitHub API to create the PR
for you. Note that this requires you to have a Github token with the
public_repo scope in the `~/.elastic/github.token` file
public_repo scope in the `~/.elastic/github.token` file. This token
should be also authorized to Elastic organization so as to work with single-sign-on.
(see https://help.github.com/en/articles/authorizing-a-personal-access-token-for-use-with-saml-single-sign-on)
Note that you need to take the commit hashes from `git log` on the
from_branch, copying the IDs from Github doesn't work in case we squashed the

View File

@ -228,8 +228,10 @@ def create_notice(filename, beat, copyright, vendor_dirs, csvfile, overrides=Non
APACHE2_LICENSE_TITLES = [
"Apache License 2.0",
"Apache License Version 2.0",
"Apache License, Version 2.0",
"licensed under the Apache 2.0 license", # github.com/zmap/zcrypto
re.sub(r"\s+", " ", """Apache License
==============

View File

@ -35,6 +35,7 @@ import (
"github.com/magefile/mage/sh"
"github.com/pkg/errors"
"github.com/elastic/beats/dev-tools/mage/gotool"
"github.com/elastic/beats/libbeat/processors/dissect"
)
@ -189,6 +190,26 @@ func GoVet() error {
return errors.Wrap(err, "failed running go vet, please fix the issues reported")
}
// CheckLicenseHeaders checks license headers in .go files.
func CheckLicenseHeaders() error {
fmt.Println(">> fmt - go-licenser: Checking for missing headers")
mg.Deps(InstallGoLicenser)
var license string
switch BeatLicense {
case "ASL2", "ASL 2.0":
license = "ASL2"
case "Elastic", "Elastic License":
license = "Elastic"
default:
return errors.Errorf("unknown license type %v", BeatLicense)
}
licenser := gotool.Licenser
return licenser(licenser.Check(), licenser.License(license))
}
// CheckDashboardsFormat checks the format of dashboards
func CheckDashboardsFormat() error {
dashboardSubDir := "/_meta/kibana/"

View File

@ -105,17 +105,19 @@ func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error
}
params := map[string]interface{}{
"GOOS": EnvOr("DEV_OS", "linux"),
"GOARCH": EnvOr("DEV_ARCH", "amd64"),
"Reference": false,
"Docker": false,
"ExcludeConsole": false,
"ExcludeFileOutput": false,
"ExcludeKafka": false,
"ExcludeLogstash": false,
"ExcludeRedis": false,
"UseObserverProcessor": false,
"ExcludeDashboards": false,
"GOOS": EnvOr("DEV_OS", "linux"),
"GOARCH": EnvOr("DEV_ARCH", "amd64"),
"Reference": false,
"Docker": false,
"ExcludeConsole": false,
"ExcludeFileOutput": false,
"ExcludeKafka": false,
"ExcludeLogstash": false,
"ExcludeRedis": false,
"UseObserverProcessor": false,
"UseDockerMetadataProcessor": true,
"UseKubernetesMetadataProcessor": false,
"ExcludeDashboards": false,
}
for k, v := range args.ExtraVars {
params[k] = v

View File

@ -26,6 +26,8 @@ import (
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"
"github.com/elastic/beats/dev-tools/mage/gotool"
)
var (
@ -35,9 +37,6 @@ var (
// GoImportsLocalPrefix is a string prefix matching imports that should be
// grouped after third-party packages.
GoImportsLocalPrefix = "github.com/elastic"
// GoLicenserImportPath controls the import path used to install go-licenser.
GoLicenserImportPath = "github.com/elastic/go-licenser"
)
// Format adds license headers, formats .go files with goimports, and formats
@ -120,9 +119,7 @@ func AddLicenseHeaders() error {
fmt.Println(">> fmt - go-licenser: Adding missing headers")
if err := sh.Run("go", "get", GoLicenserImportPath); err != nil {
return err
}
mg.Deps(InstallGoLicenser)
var license string
switch BeatLicense {
@ -134,5 +131,6 @@ func AddLicenseHeaders() error {
return errors.Errorf("unknown license type %v", BeatLicense)
}
return sh.RunV("go-licenser", "-license", license)
licenser := gotool.Licenser
return licenser(licenser.License(license))
}

View File

@ -0,0 +1,31 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package gotool
type goGet func(opts ...ArgOpt) error
// Get runs `go get` and provides optionals for adding command line arguments.
var Get goGet = runGoGet
func runGoGet(opts ...ArgOpt) error {
args := buildArgs(opts)
return runVGo("get", args)
}
func (goGet) Update() ArgOpt { return flagBoolIf("-u", true) }
func (goGet) Package(pkg string) ArgOpt { return posArg(pkg) }

View File

@ -0,0 +1,257 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package gotool
import (
"os"
"strings"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)
// Args holds parameters, environment variables and flag information used to
// pass to the go tool.
type Args struct {
extra map[string]string // extra flags one can pass to the command
env map[string]string
flags map[string][]string
pos []string
}
// ArgOpt is a functional option adding info to Args once executed.
type ArgOpt func(args *Args)
type goTest func(opts ...ArgOpt) error
// Test runs `go test` and provides optionals for adding command line arguments.
var Test goTest = runGoTest
// ListProjectPackages lists all packages in the current project
func ListProjectPackages() ([]string, error) {
return ListPackages("./...")
}
// ListPackages calls `go list` for every package spec given.
func ListPackages(pkgs ...string) ([]string, error) {
return getLines(callGo(nil, "list", pkgs...))
}
// ListTestFiles lists all go and cgo test files available in a package.
func ListTestFiles(pkg string) ([]string, error) {
const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` +
`{{ range .XTestGoFiles }}{{ printf "%s\n" . }}{{ end }}`
return getLines(callGo(nil, "list", "-f", tmpl, pkg))
}
// HasTests returns true if the given package contains test files.
func HasTests(pkg string) (bool, error) {
files, err := ListTestFiles(pkg)
if err != nil {
return false, err
}
return len(files) > 0, nil
}
func (goTest) WithCoverage(to string) ArgOpt {
return combine(flagArg("-cover", ""), flagArgIf("-test.coverprofile", to))
}
func (goTest) Short(b bool) ArgOpt { return flagBoolIf("-test.short", b) }
func (goTest) Use(bin string) ArgOpt { return extraArgIf("use", bin) }
func (goTest) OS(os string) ArgOpt { return envArgIf("GOOS", os) }
func (goTest) ARCH(arch string) ArgOpt { return envArgIf("GOARCH", arch) }
func (goTest) Create() ArgOpt { return flagArg("-c", "") }
func (goTest) Out(path string) ArgOpt { return flagArg("-o", path) }
func (goTest) Package(path string) ArgOpt { return posArg(path) }
func (goTest) Verbose() ArgOpt { return flagArg("-test.v", "") }
func runGoTest(opts ...ArgOpt) error {
args := buildArgs(opts)
if bin := args.Val("use"); bin != "" {
flags := map[string][]string{}
for k, v := range args.flags {
if strings.HasPrefix(k, "-test.") {
flags[k] = v
}
}
useArgs := &Args{}
*useArgs = *args
useArgs.flags = flags
_, err := sh.Exec(useArgs.env, os.Stdout, os.Stderr, bin, useArgs.build()...)
return err
}
return runVGo("test", args)
}
func getLines(out string, err error) ([]string, error) {
if err != nil {
return nil, err
}
lines := strings.Split(out, "\n")
res := lines[:0]
for _, line := range lines {
line = strings.TrimSpace(line)
if len(line) > 0 {
res = append(res, line)
}
}
return res, nil
}
func callGo(env map[string]string, cmd string, opts ...string) (string, error) {
args := []string{cmd}
args = append(args, opts...)
return sh.OutputWith(env, mg.GoCmd(), args...)
}
func runVGo(cmd string, args *Args) error {
return execGoWith(func(env map[string]string, cmd string, args ...string) error {
_, err := sh.Exec(env, os.Stdout, os.Stderr, cmd, args...)
return err
}, cmd, args)
}
func runGo(cmd string, args *Args) error {
return execGoWith(sh.RunWith, cmd, args)
}
func execGoWith(
fn func(map[string]string, string, ...string) error,
cmd string, args *Args,
) error {
cliArgs := []string{cmd}
cliArgs = append(cliArgs, args.build()...)
return fn(args.env, mg.GoCmd(), cliArgs...)
}
func posArg(value string) ArgOpt {
return func(a *Args) { a.Add(value) }
}
func extraArg(k, v string) ArgOpt {
return func(a *Args) { a.Extra(k, v) }
}
func extraArgIf(k, v string) ArgOpt {
if v == "" {
return nil
}
return extraArg(k, v)
}
func envArg(k, v string) ArgOpt {
return func(a *Args) { a.Env(k, v) }
}
func envArgIf(k, v string) ArgOpt {
if v == "" {
return nil
}
return envArg(k, v)
}
func flagArg(flag, value string) ArgOpt {
return func(a *Args) { a.Flag(flag, value) }
}
func flagArgIf(flag, value string) ArgOpt {
if value == "" {
return nil
}
return flagArg(flag, value)
}
func flagBoolIf(flag string, b bool) ArgOpt {
if b {
return flagArg(flag, "")
}
return nil
}
func combine(opts ...ArgOpt) ArgOpt {
return func(a *Args) {
for _, opt := range opts {
if opt != nil {
opt(a)
}
}
}
}
func buildArgs(opts []ArgOpt) *Args {
a := &Args{}
combine(opts...)(a)
return a
}
// Extra sets a special k/v pair to be interpreted by the execution function.
func (a *Args) Extra(k, v string) {
if a.extra == nil {
a.extra = map[string]string{}
}
a.extra[k] = v
}
// Val returns a special functions value for a given key.
func (a *Args) Val(k string) string {
if a.extra == nil {
return ""
}
return a.extra[k]
}
// Env sets an environmant variable to be passed to the child process on exec.
func (a *Args) Env(k, v string) {
if a.env == nil {
a.env = map[string]string{}
}
a.env[k] = v
}
// Flag adds a flag to be passed to the child process on exec.
func (a *Args) Flag(flag, value string) {
if a.flags == nil {
a.flags = map[string][]string{}
}
a.flags[flag] = append(a.flags[flag], value)
}
// Add adds a positional argument to be passed to the child process on exec.
func (a *Args) Add(p string) {
a.pos = append(a.pos, p)
}
func (a *Args) build() []string {
args := make([]string, 0, 2*len(a.flags)+len(a.pos))
for k, values := range a.flags {
for _, v := range values {
args = append(args, k)
if v != "" {
args = append(args, v)
}
}
}
args = append(args, a.pos...)
return args
}

View File

@ -0,0 +1,37 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package gotool
import (
"github.com/magefile/mage/sh"
)
type goLicenser func(opts ...ArgOpt) error
// Licenser runs `go-licenser` and provides optionals for adding command line arguments.
var Licenser goLicenser = runGoLicenser
func runGoLicenser(opts ...ArgOpt) error {
args := buildArgs(opts).build()
return sh.RunV("go-licenser", args...)
}
func (goLicenser) Check() ArgOpt { return flagBoolIf("-d", true) }
func (goLicenser) License(license string) ArgOpt { return flagArgIf("-license", license) }
func (goLicenser) Exclude(path string) ArgOpt { return flagArgIf("-exclude", path) }
func (goLicenser) Path(path string) ArgOpt { return posArg(path) }

View File

@ -0,0 +1,49 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package mage
import (
"path/filepath"
"github.com/pkg/errors"
"github.com/elastic/beats/dev-tools/mage/gotool"
)
var (
// GoLicenserImportPath controls the import path used to install go-licenser.
GoLicenserImportPath = "github.com/elastic/go-licenser"
)
// InstallVendored uses go get to install a command from its vendored source
func InstallVendored(importPath string) error {
beatDir, err := ElasticBeatsDir()
if err != nil {
return errors.Wrap(err, "failed to obtain beats repository path")
}
get := gotool.Get
return get(
get.Package(filepath.Join(beatDir, "vendor", importPath)),
)
}
// InstallGoLicenser target installs go-licenser
func InstallGoLicenser() error {
return InstallVendored(GoLicenserImportPath)
}

View File

@ -0,0 +1,27 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package collectors
import (
metricbeat "github.com/elastic/beats/metricbeat/scripts/mage"
)
//CollectDocs creates the documentation under docs/
func CollectDocs() error {
return metricbeat.CollectDocs()
}

View File

@ -39,3 +39,8 @@ func Check() {
deps = append(deps, devtools.Check)
mg.SerialDeps(deps...)
}
// CheckLicenseHeaders checks license headers
func CheckLicenseHeaders() {
mg.Deps(devtools.CheckLicenseHeaders)
}

View File

@ -27,3 +27,8 @@ import (
func Fmt() {
mg.Deps(devtools.Format)
}
// AddLicenseHeaders adds license headers
func AddLicenseHeaders() {
mg.Deps(devtools.AddLicenseHeaders)
}

View File

@ -0,0 +1,25 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package update
import "github.com/magefile/mage/sh"
// Update updates the generated files (aka make update).
func Update() error {
return sh.Run("make", "update")
}

0
vendor/github.com/elastic/beats/dev-tools/promote_docs generated vendored Executable file → Normal file
View File

View File

@ -89,7 +89,7 @@ recommend that you install it.
[[update-scripts]]
=== Update scripts
The Beats use a variety of scripts based on Python to generate configuration files
The Beats use a variety of scripts based on Python, make and mage to generate configuration files
and documentation. The primary command used for this is:
[source,shell]
@ -108,6 +108,7 @@ These commands have the following dependencies:
* Python >= {python}
* https://virtualenv.pypa.io/en/latest/[virtualenv] for Python
* https://github.com/magefile/mage[Mage]
Virtualenv can be installed with the command `easy_install virtualenv` or `pip
install virtualenv`. More details can be found

View File

@ -1,9 +1,11 @@
[[beats-reference]]
= Beats Developer Guide
:libbeat-dir: {docdir}/../../libbeat
:libbeat-dir: {docdir}/../../libbeat/docs
include::{libbeat-dir}/docs/version.asciidoc[]
include::{libbeat-dir}/version.asciidoc[]
include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[]
:dev-guide: true
:beatname_lc: beatname
@ -11,13 +13,13 @@ include::{libbeat-dir}/docs/version.asciidoc[]
include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
include::{libbeat-dir}/docs/shared-beats-attributes.asciidoc[]
include::{libbeat-dir}/shared-beats-attributes.asciidoc[]
include::./pull-request-guidelines.asciidoc[]
include::./contributing.asciidoc[]
include::{libbeat-dir}/docs/communitybeats.asciidoc[]
include::{libbeat-dir}/communitybeats.asciidoc[]
include::./newbeat.asciidoc[]

View File

@ -31,13 +31,10 @@ The following topics describe how to build a new Beat:
All Beats are written in http://golang.org/[Go], so having Go installed and knowing
the basics are prerequisites for understanding this guide.
But don't worry if you aren't a Go expert. Go is a relatively new
language, and very few people are experts in it. In fact, several
people learned Go by contributing to Packetbeat and libbeat, including the
original Packetbeat authors.
*Before you begin:* Set up your Go environment as described under
<<setting-up-dev-environment>> in <<beats-contributing>>.
<<setting-up-dev-environment>> in <<beats-contributing>>. The minimum required
Go version is {go-version}.
To build your Beat on a specific version of libbeat, check out the specific
branch ({branch} in the example below):

View File

@ -1,4 +1,4 @@
FROM golang:1.12.9
FROM golang:1.12.12
RUN \
apt-get update \

View File

@ -62,6 +62,9 @@ filebeat.inputs:
# fields.
#fields_under_root: false
# Set to true to publish fields with null values in events.
#keep_null: false
# Ignore files which were modified more then the defined timespan in the past.
# ignore_older is disabled by default, so no files are ignored by setting it to 0.
# Time strings like 2h (2 hours), 5m (5 minutes) can be used.

View File

@ -326,7 +326,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
outDone := make(chan struct{}) // outDone closes down all active pipeline connections
crawler, err := crawler.New(
channel.NewOutletFactory(outDone, wgEvents).Create,
channel.NewOutletFactory(outDone, wgEvents, b.Info).Create,
config.Inputs,
b.Info.Version,
fb.done,

View File

@ -18,8 +18,11 @@
package channel
import (
"fmt"
"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/common/fmtstr"
"github.com/elastic/beats/libbeat/processors"
)
@ -31,6 +34,14 @@ type pipelineConnector struct {
pipeline beat.Pipeline
}
// addFormattedIndex is a Processor to set an event's "raw_index" metadata field
// with a given TimestampFormatString. The elasticsearch output interprets
// that field as specifying the (raw string) index the event should be sent to;
// in other outputs it is just included in the metadata.
type addFormattedIndex struct {
formatString *fmtstr.TimestampFormatString
}
// Connect passes the cfg and the zero value of beat.ClientConfig to the underlying function.
func (fn ConnectorFunc) Connect(cfg *common.Config) (Outleter, error) {
return fn(cfg, beat.ClientConfig{})
@ -51,24 +62,11 @@ func (c *pipelineConnector) ConnectWith(cfg *common.Config, clientCfg beat.Clien
return nil, err
}
var err error
var userProcessors beat.ProcessorList
userProcessors, err = processors.New(config.Processors)
procs, err := processorsForConfig(c.parent.beatInfo, config, clientCfg)
if err != nil {
return nil, err
}
if lst := clientCfg.Processing.Processor; lst != nil {
if len(userProcessors.All()) == 0 {
userProcessors = lst
} else if orig := lst.All(); len(orig) > 0 {
newLst := processors.NewList(nil)
newLst.List = append(newLst.List, lst, userProcessors)
userProcessors = newLst
}
}
setOptional := func(to common.MapStr, key string, value string) {
if value != "" {
to.Put(key, value)
@ -105,7 +103,8 @@ func (c *pipelineConnector) ConnectWith(cfg *common.Config, clientCfg beat.Clien
clientCfg.Processing.EventMetadata = config.EventMetadata
clientCfg.Processing.Meta = meta
clientCfg.Processing.Fields = fields
clientCfg.Processing.Processor = userProcessors
clientCfg.Processing.Processor = procs
clientCfg.Processing.KeepNull = config.KeepNull
client, err := c.pipeline.ConnectWith(clientCfg)
if err != nil {
return nil, err
@ -117,3 +116,64 @@ func (c *pipelineConnector) ConnectWith(cfg *common.Config, clientCfg beat.Clien
}
return outlet, nil
}
// processorsForConfig assembles the Processors for a pipelineConnector.
func processorsForConfig(
beatInfo beat.Info, config inputOutletConfig, clientCfg beat.ClientConfig,
) (*processors.Processors, error) {
procs := processors.NewList(nil)
// Processor ordering is important:
// 1. Index configuration
if !config.Index.IsEmpty() {
staticFields := fmtstr.FieldsForBeat(beatInfo.Beat, beatInfo.Version)
timestampFormat, err :=
fmtstr.NewTimestampFormatString(&config.Index, staticFields)
if err != nil {
return nil, err
}
indexProcessor := &addFormattedIndex{timestampFormat}
procs.List = append(procs.List, indexProcessor)
}
// 2. ClientConfig processors
if lst := clientCfg.Processing.Processor; lst != nil {
procs.List = append(procs.List, lst)
}
// 3. User processors
userProcessors, err := processors.New(config.Processors)
if err != nil {
return nil, err
}
// Subtlety: it is important here that we append the individual elements of
// userProcessors, rather than userProcessors itself, even though
// userProcessors implements the processors.Processor interface. This is
// because the contents of what we return are later pulled out into a
// processing.group rather than a processors.Processors, and the two have
// different error semantics: processors.Processors aborts processing on
// any error, whereas processing.group only aborts on fatal errors. The
// latter is the most common behavior, and the one we are preserving here for
// backwards compatibility.
// We are unhappy about this and have plans to fix this inconsistency at a
// higher level, but for now we need to respect the existing semantics.
procs.List = append(procs.List, userProcessors.List...)
return procs, nil
}
func (p *addFormattedIndex) Run(event *beat.Event) (*beat.Event, error) {
index, err := p.formatString.Run(event.Timestamp)
if err != nil {
return nil, err
}
if event.Meta == nil {
event.Meta = common.MapStr{}
}
event.Meta["raw_index"] = index
return event, nil
}
func (p *addFormattedIndex) String() string {
return fmt.Sprintf("add_index_pattern=%v", p.formatString)
}

View File

@ -0,0 +1,213 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package channel
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/processors"
"github.com/elastic/beats/libbeat/processors/actions"
)
func TestProcessorsForConfig(t *testing.T) {
testCases := map[string]struct {
beatInfo beat.Info
configStr string
clientCfg beat.ClientConfig
event beat.Event
expectedFields map[string]string
}{
"Simple static index": {
configStr: "index: 'test'",
expectedFields: map[string]string{
"@metadata.raw_index": "test",
},
},
"Index with agent info + timestamp": {
beatInfo: beat.Info{Beat: "TestBeat", Version: "3.9.27"},
configStr: "index: 'beat-%{[agent.name]}-%{[agent.version]}-%{+yyyy.MM.dd}'",
event: beat.Event{Timestamp: time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)},
expectedFields: map[string]string{
"@metadata.raw_index": "beat-TestBeat-3.9.27-1999.12.31",
},
},
"Set index in ClientConfig": {
clientCfg: beat.ClientConfig{
Processing: beat.ProcessingConfig{
Processor: makeProcessors(&setRawIndex{"clientCfgIndex"}),
},
},
expectedFields: map[string]string{
"@metadata.raw_index": "clientCfgIndex",
},
},
"ClientConfig processor runs after beat input Index": {
configStr: "index: 'test'",
clientCfg: beat.ClientConfig{
Processing: beat.ProcessingConfig{
Processor: makeProcessors(&setRawIndex{"clientCfgIndex"}),
},
},
expectedFields: map[string]string{
"@metadata.raw_index": "clientCfgIndex",
},
},
"Set field in input config": {
configStr: `processors: [add_fields: {fields: {testField: inputConfig}}]`,
expectedFields: map[string]string{
"fields.testField": "inputConfig",
},
},
"Set field in ClientConfig": {
clientCfg: beat.ClientConfig{
Processing: beat.ProcessingConfig{
Processor: makeProcessors(actions.NewAddFields(common.MapStr{
"fields": common.MapStr{"testField": "clientConfig"},
}, false)),
},
},
expectedFields: map[string]string{
"fields.testField": "clientConfig",
},
},
"Input config processors run after ClientConfig": {
configStr: `processors: [add_fields: {fields: {testField: inputConfig}}]`,
clientCfg: beat.ClientConfig{
Processing: beat.ProcessingConfig{
Processor: makeProcessors(actions.NewAddFields(common.MapStr{
"fields": common.MapStr{"testField": "clientConfig"},
}, false)),
},
},
expectedFields: map[string]string{
"fields.testField": "inputConfig",
},
},
}
for description, test := range testCases {
if test.event.Fields == nil {
test.event.Fields = common.MapStr{}
}
config, err := outletConfigFromString(test.configStr)
if err != nil {
t.Errorf("[%s] %v", description, err)
continue
}
processors, err := processorsForConfig(test.beatInfo, config, test.clientCfg)
if err != nil {
t.Errorf("[%s] %v", description, err)
continue
}
processedEvent, err := processors.Run(&test.event)
// We don't check if err != nil, because we are testing the final outcome
// of running the processors, including when some of them fail.
if processedEvent == nil {
t.Errorf("[%s] Unexpected fatal error running processors: %v\n",
description, err)
}
for key, value := range test.expectedFields {
field, err := processedEvent.GetValue(key)
if err != nil {
t.Errorf("[%s] Couldn't get field %s from event: %v", description, key, err)
continue
}
assert.Equal(t, field, value)
fieldStr, ok := field.(string)
if !ok {
// Note that requiring a string here is just to simplify the test setup,
// not a requirement of the underlying api.
t.Errorf("[%s] Field [%s] should be a string", description, key)
continue
}
if fieldStr != value {
t.Errorf("[%s] Event field [%s]: expected [%s], got [%s]", description, key, value, fieldStr)
}
}
}
}
func TestProcessorsForConfigIsFlat(t *testing.T) {
// This test is regrettable, and exists because of inconsistencies in
// processor handling between processors.Processors and processing.group
// (which implements beat.ProcessorList) -- see processorsForConfig for
// details. The upshot is that, for now, if the input configuration specifies
// processors, they must be returned as direct children of the resulting
// processors.Processors (rather than being collected in additional tree
// structure).
// This test should be removed once we have a more consistent mechanism for
// collecting and running processors.
configStr := `processors:
- add_fields: {fields: {testField: value}}
- add_fields: {fields: {testField2: stuff}}`
config, err := outletConfigFromString(configStr)
if err != nil {
t.Fatal(err)
}
processors, err := processorsForConfig(
beat.Info{}, config, beat.ClientConfig{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, 2, len(processors.List))
}
// setRawIndex is a bare-bones processor to set the raw_index field to a
// constant string in the event metadata. It is used to test order of operations
// for processorsForConfig.
type setRawIndex struct {
indexStr string
}
func (p *setRawIndex) Run(event *beat.Event) (*beat.Event, error) {
if event.Meta == nil {
event.Meta = common.MapStr{}
}
event.Meta["raw_index"] = p.indexStr
return event, nil
}
func (p *setRawIndex) String() string {
return fmt.Sprintf("set_raw_index=%v", p.indexStr)
}
// Helper function to convert from YML input string to an unpacked
// inputOutletConfig
func outletConfigFromString(s string) (inputOutletConfig, error) {
config := inputOutletConfig{}
cfg, err := common.NewConfigFrom(s)
if err != nil {
return config, err
}
if err := cfg.Unpack(&config); err != nil {
return config, err
}
return config, nil
}
// makeProcessors wraps one or more bare Processor objects in Processors.
func makeProcessors(procs ...processors.Processor) *processors.Processors {
procList := processors.NewList(nil)
procList.List = procs
return procList
}

View File

@ -20,6 +20,7 @@ package channel
import (
"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/common/fmtstr"
"github.com/elastic/beats/libbeat/processors"
)
@ -28,6 +29,7 @@ type OutletFactory struct {
eventer beat.ClientEventer
wgEvents eventCounter
beatInfo beat.Info
}
type eventCounter interface {
@ -46,6 +48,7 @@ type inputOutletConfig struct {
// event processing
common.EventMetadata `config:",inline"` // Fields and tags to add to events.
Processors processors.PluginConfig `config:"processors"`
KeepNull bool `config:"keep_null"`
// implicit event fields
Type string `config:"type"` // input.type
@ -56,8 +59,8 @@ type inputOutletConfig struct {
Fileset string `config:"_fileset_name"` // hidden setting
// Output meta data settings
Pipeline string `config:"pipeline"` // ES Ingest pipeline name
Pipeline string `config:"pipeline"` // ES Ingest pipeline name
Index fmtstr.EventFormatString `config:"index"` // ES output index pattern
}
// NewOutletFactory creates a new outlet factory for
@ -65,10 +68,12 @@ type inputOutletConfig struct {
func NewOutletFactory(
done <-chan struct{},
wgEvents eventCounter,
beatInfo beat.Info,
) *OutletFactory {
o := &OutletFactory{
done: done,
wgEvents: wgEvents,
beatInfo: beatInfo,
}
if wgEvents != nil {

View File

@ -17,8 +17,6 @@ The {beatname_uc} configuration file uses http://yaml.org/[YAML] for its syntax.
See the {beats-ref}/config-file-format.html[Config File Format] section of the
_Beats Platform Reference_ for more about the structure of the config file.
include::../../libbeat/docs/shared-cm-tip.asciidoc[]
The following topics describe how to configure Filebeat:
* <<configuration-filebeat-modules>>
@ -58,44 +56,44 @@ include::./filebeat-general-options.asciidoc[]
include::./reload-configuration.asciidoc[]
include::{libbeat-dir}/docs/queueconfig.asciidoc[]
include::{libbeat-dir}/queueconfig.asciidoc[]
include::{libbeat-dir}/docs/outputconfig.asciidoc[]
include::{libbeat-dir}/outputconfig.asciidoc[]
include::../../libbeat/docs/shared-ilm.asciidoc[]
include::./load-balancing.asciidoc[]
include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[]
include::{libbeat-dir}/shared-ssl-config.asciidoc[]
include::./filebeat-filtering.asciidoc[]
include::{libbeat-dir}/docs/shared-config-ingest.asciidoc[]
include::{libbeat-dir}/shared-config-ingest.asciidoc[]
include::{libbeat-dir}/docs/shared-geoip.asciidoc[]
include::{libbeat-dir}/shared-geoip.asciidoc[]
include::{libbeat-dir}/docs/shared-path-config.asciidoc[]
include::{libbeat-dir}/shared-path-config.asciidoc[]
include::{libbeat-dir}/docs/shared-kibana-config.asciidoc[]
include::{libbeat-dir}/shared-kibana-config.asciidoc[]
include::{libbeat-dir}/docs/setup-config.asciidoc[]
include::{libbeat-dir}/setup-config.asciidoc[]
include::{libbeat-dir}/docs/loggingconfig.asciidoc[]
include::{libbeat-dir}/loggingconfig.asciidoc[]
:standalone:
include::{libbeat-dir}/docs/shared-env-vars.asciidoc[]
include::{libbeat-dir}/shared-env-vars.asciidoc[]
:standalone!:
:autodiscoverJolokia:
:autodiscoverHints:
include::{libbeat-dir}/docs/shared-autodiscover.asciidoc[]
include::{libbeat-dir}/shared-autodiscover.asciidoc[]
:standalone:
include::{libbeat-dir}/docs/yaml.asciidoc[]
include::{libbeat-dir}/yaml.asciidoc[]
:standalone!:
include::{libbeat-dir}/docs/regexp.asciidoc[]
include::{libbeat-dir}/regexp.asciidoc[]
include::{libbeat-dir}/docs/http-endpoint.asciidoc[]
include::{libbeat-dir}/http-endpoint.asciidoc[]
include::{libbeat-dir}/docs/reference-yml.asciidoc[]
include::{libbeat-dir}/reference-yml.asciidoc[]

View File

@ -122,6 +122,6 @@ reached EOF or not. Note that this option can lead to data loss if the file is
deleted before {beatname_uc} reaches the end of the file.
include::{libbeat-dir}/docs/faq-limit-bandwidth.asciidoc[]
include::{libbeat-dir}/faq-limit-bandwidth.asciidoc[]
include::{libbeat-dir}/docs/shared-faq.asciidoc[]
include::{libbeat-dir}/shared-faq.asciidoc[]

File diff suppressed because it is too large Load Diff

View File

@ -22,7 +22,7 @@ global processing across all data exported by {beatname_uc}.
[[using-processors]]
=== Processors
include::{libbeat-dir}/docs/processors.asciidoc[]
include::{libbeat-dir}/processors.asciidoc[]
[float]
[[drop-event-example]]
@ -103,4 +103,4 @@ The resulting output looks something like this:
}
-----------------------------------------------------
include::{libbeat-dir}/docs/processors-using.asciidoc[]
include::{libbeat-dir}/processors-using.asciidoc[]

View File

@ -42,13 +42,15 @@ NOTE: The content stored in filebeat/data.json is compatible to the old registry
The permissions mask to apply on registry data file. The default value is 0600. The permissions option must be a valid Unix-style file permissions mask expressed in octal notation. In Go, numbers in octal notation must start with 0.
The most permissive mask allowed is 0640. If a higher permissions mask is
specified via this setting, it will be subject to a umask of 0027.
This option is not supported on Windows.
Examples:
0644: give read and write access to the file owner, and read access to all others.
0640: give read and write access to the file owner, and read access to members of the group associated with the file.
0600: give read and write access to the file owner, and no access to all others.
0664: give read and write access to the file owner and members of the group associated with the file, as well as read access to all other users.
[source,yaml]
-------------------------------------------------------------------------------------
@ -141,4 +143,4 @@ Example configuration:
filebeat.shutdown_timeout: 5s
-------------------------------------------------------------------------------------
include::{libbeat-dir}/docs/generalconfig.asciidoc[]
include::{libbeat-dir}/generalconfig.asciidoc[]

View File

@ -18,7 +18,7 @@ implement and deploy a log monitoring solution.
* <<enable-modules-cli>>
* <<enable-modules-config-file>>
include::{libbeat-dir}/docs/shared-note-file-permissions.asciidoc[]
include::{libbeat-dir}/shared-note-file-permissions.asciidoc[]
When you enable modules, you can also
<<specify-variable-settings,specify variable settings>> to change the default

View File

@ -1,7 +1,7 @@
[[filebeat-getting-started]]
== Getting Started With Filebeat
include::{libbeat-dir}/docs/shared-getting-started-intro.asciidoc[]
include::{libbeat-dir}/shared-getting-started-intro.asciidoc[]
* <<filebeat-installation>>
* <<filebeat-configuration>>
@ -15,7 +15,7 @@ include::{libbeat-dir}/docs/shared-getting-started-intro.asciidoc[]
[[filebeat-installation]]
=== Step 1: Install Filebeat
include::{libbeat-dir}/docs/shared-download-and-install.asciidoc[]
include::{libbeat-dir}/shared-download-and-install.asciidoc[]
[[deb]]
*deb:*
@ -74,7 +74,7 @@ tar xzvf filebeat-{version}-darwin-x86_64.tar.gz
endif::[]
include::{libbeat-dir}/docs/shared-brew-install.asciidoc[]
include::{libbeat-dir}/shared-brew-install.asciidoc[]
[[linux]]
*linux:*
@ -146,7 +146,7 @@ started experience for common log formats. If you are using Filebeat modules,
skip this section, including the remaining getting started steps, and go
directly to <<filebeat-modules-quickstart>>.
include::{libbeat-dir}/docs/shared-configuring.asciidoc[]
include::{libbeat-dir}/shared-configuring.asciidoc[]
Here is a sample of the `filebeat` section of the `filebeat.yml` file. Filebeat uses predefined
default values for most configuration options.
@ -185,27 +185,25 @@ To fetch all files from a predefined level of subdirectories, the following patt
fetch log files from the `/var/log` folder itself. Currently it is not possible to recursively
fetch all files in all subdirectories of a directory.
include::{libbeat-dir}/docs/step-configure-output.asciidoc[]
include::{libbeat-dir}/step-configure-output.asciidoc[]
include::{libbeat-dir}/docs/step-configure-kibana-endpoint.asciidoc[]
include::{libbeat-dir}/step-configure-kibana-endpoint.asciidoc[]
include::{libbeat-dir}/docs/step-configure-credentials.asciidoc[]
include::{libbeat-dir}/step-configure-credentials.asciidoc[]
include::{libbeat-dir}/docs/step-test-config.asciidoc[]
include::{libbeat-dir}/step-test-config.asciidoc[]
include::{libbeat-dir}/docs/step-look-at-config.asciidoc[]
include::../../libbeat/docs/shared-cm-tip.asciidoc[]
include::{libbeat-dir}/step-look-at-config.asciidoc[]
[[filebeat-template]]
=== Step 3: Load the index template in Elasticsearch
include::{libbeat-dir}/docs/shared-template-load.asciidoc[]
include::{libbeat-dir}/shared-template-load.asciidoc[]
[[load-kibana-dashboards]]
=== Step 4: Set up the Kibana dashboards
include::{libbeat-dir}/docs/dashboards.asciidoc[]
include::{libbeat-dir}/dashboards.asciidoc[]
[[filebeat-starting]]
=== Step 5: Start Filebeat
@ -242,7 +240,7 @@ specified. See
{beats-ref}/config-file-permissions.html[Config File Ownership and Permissions]
in the _Beats Platform Reference_.
include::{libbeat-dir}/docs/shared-brew-run.asciidoc[]
include::{libbeat-dir}/shared-brew-run.asciidoc[]
*win:*
@ -263,7 +261,7 @@ To make it easier for you to explore Filebeat data in Kibana, we've created
example {beatname_uc} dashboards. You loaded the dashboards earlier when you
ran the `setup` command.
include::{libbeat-dir}/docs/opendashboards.asciidoc[]
include::{libbeat-dir}/opendashboards.asciidoc[]
These dashboards are designed to work out-of-the box when you use
<<filebeat-modules-overview,Filebeat modules>>. However, you can also use them

Binary file not shown.

After

Width:  |  Height:  |  Size: 786 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 629 KiB

View File

@ -1,12 +1,13 @@
= Filebeat Reference
:libbeat-dir: {docdir}/../../libbeat
:libbeat-dir: {docdir}/../../libbeat/docs
include::{libbeat-dir}/docs/version.asciidoc[]
include::{libbeat-dir}/version.asciidoc[]
include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[]
include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
:version: {stack-version}
:beatname_lc: filebeat
:beatname_uc: Filebeat
:beatname_pkg: {beatname_lc}
@ -18,10 +19,6 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
:has_solutions:
:ignores_max_retries:
:has_docker_label_ex:
:has_decode_cef_processor:
:has_decode_csv_fields_processor:
:has_script_processor:
:has_timestamp_processor:
:has_modules_command:
:has_registry:
:deb_os:
@ -31,7 +28,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
:docker_platform:
:win_os:
include::{libbeat-dir}/docs/shared-beats-attributes.asciidoc[]
include::{libbeat-dir}/shared-beats-attributes.asciidoc[]
include::./overview.asciidoc[]
@ -39,7 +36,7 @@ include::./getting-started.asciidoc[]
include::./modules-getting-started.asciidoc[]
include::{libbeat-dir}/docs/repositories.asciidoc[]
include::{libbeat-dir}/repositories.asciidoc[]
include::./setting-up-running.asciidoc[]
@ -49,19 +46,19 @@ include::./how-filebeat-works.asciidoc[]
include::./configuring-howto.asciidoc[]
include::{libbeat-dir}/docs/shared-central-management.asciidoc[]
include::{libbeat-dir}/shared-central-management.asciidoc[]
include::./modules.asciidoc[]
include::./fields.asciidoc[]
include::{libbeat-dir}/docs/monitoring/monitoring-beats.asciidoc[]
include::{libbeat-dir}/monitoring/monitoring-beats.asciidoc[]
include::{libbeat-dir}/docs/shared-securing-beat.asciidoc[]
include::{libbeat-dir}/shared-securing-beat.asciidoc[]
include::./troubleshooting.asciidoc[]
include::./faq.asciidoc[]
include::{libbeat-dir}/docs/contributing-to-beats.asciidoc[]
include::{libbeat-dir}/contributing-to-beats.asciidoc[]

View File

@ -191,6 +191,10 @@ must be at the top level in the JSON object and the value associated with the
key must be a string, otherwise no filtering or multiline aggregation will
occur.
*`document_id`*:: Option configuration setting that specifies the JSON key to
set the document id. If configured, the field will be removed from the original
json document and stored in `@metadata.id`
*`ignore_decoding_error`*:: An optional configuration setting that specifies if
JSON decoding errors should be logged or not. If set to true, errors will not
be logged. The default is false.

View File

@ -64,7 +64,7 @@ If this option is set to true, the custom
<<{beatname_lc}-input-{type}-fields,fields>> are stored as top-level fields in
the output document instead of being grouped under a `fields` sub-dictionary. If
the custom field names conflict with other field names added by {beatname_uc},
then the custom fields overwrite the other fields.
then the custom fields overwrite the other fields.
[float]
===== `processors`
@ -84,3 +84,20 @@ this option usually results in simpler configuration files. If the pipeline is
configured both in the input and output, the option from the
input is used.
[float]
===== `keep_null`
If this option is set to true, fields with `null` values will be published in
the output document. By default, `keep_null` is set to `false`.
[float]
===== `index`
If present, this formatted string overrides the index for events from this input
(for elasticsearch outputs), or sets the `raw_index` field of the event's
metadata (for other outputs). This string can only refer to the agent name and
version and the event timestamp; for access to dynamic fields, use
`output.elasticsearch.index` or a processor.
Example value: `"%{[agent.name]}-myindex-%{+yyyy.MM.dd}"` might
expand to `"filebeat-myindex-2019.11.01"`.

View File

@ -9,7 +9,7 @@
Use the `kafka` input to read from topics in a Kafka cluster.
To configure this input, specify a list of one or more <<kafka-hosts,`hosts`>> in the
To configure this input, specify a list of one or more <<hosts,`hosts`>> in the
cluster to bootstrap the connection with, a list of <<topics,`topics`>> to
track, and a <<groupid,`group_id`>> for the connection.
@ -27,6 +27,26 @@ Example configuration:
----
The following example shows how to use the `kafka` input to ingest data from
Microsoft Azure Event Hubs that have Kafka compatibility enabled:
["source","yaml",subs="attributes"]
----
{beatname_lc}.inputs:
- type: kafka
hosts: ["<your event hub namespace>.servicebus.windows.net:9093"]
topics: ["<your event hub instance>"]
group_id: "<your consumer group>"
username: "$ConnectionString"
password: "<your connection string>"
ssl.enabled: true
----
For more details on the mapping between Kafka and Event Hubs configuration
parameters, see the
link:https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-for-kafka-ecosystem-overview[Azure documentation].
[id="{beatname_lc}-input-{type}-options"]
==== Configuration options
@ -107,6 +127,17 @@ Kafka fetch settings:
*`max`*:: The maximum number of bytes to read per request. Defaults to 0
(no limit).
===== `expand_event_list_from_field`
If the fileset using this input expects to receive multiple messages bundled under a specific field then the config option `expand_event_list_from_field` value can be assigned the name of the field.
For example in the case of azure filesets the events are found under the json object "records".
```
{
"records": [ {event1}, {event2}]
}
```
This setting will be able to split the messages under the group value ('records') into separate events.
===== `rebalance`
Kafka rebalance settings:

View File

@ -40,9 +40,9 @@ To set up and run {beatname_uc} modules:
installation. By default, {beatname_uc} assumes {es} is running locally on port
9200.
+
include::{libbeat-dir}/docs/step-configure-output.asciidoc[]
include::{libbeat-dir}/step-configure-output.asciidoc[]
include::{libbeat-dir}/docs/step-configure-credentials.asciidoc[]
include::{libbeat-dir}/step-configure-credentials.asciidoc[]
. Enable the modules you want to run. For example, the following command enables
the system, nginx, and mysql modules:

View File

@ -73,6 +73,19 @@ include::../include/var-paths.asciidoc[]
include::../include/timezone-support.asciidoc[]
[float]
=== Virtual Host
See customlog documentation https://httpd.apache.org/docs/2.4/en/mod/mod_log_config.html
Add %v config in httpd.conf in log section
["source","sh",subs="attributes"]
-----
# Replace
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
# By
LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
-----
:has-dashboards!:
:fileset_ex!:

View File

@ -14,8 +14,9 @@ beta[]
This is a module for aws logs. It uses filebeat s3 input to get log files from
AWS S3 buckets with SQS notification. This module supports reading s3 server
access logs with `s3access` fileset. Server access logging provides detailed
records for the requests that are made to a bucket.
access logs with `s3access` fileset and ELB access logs with `elb` fileset.
Access logs contain detailed information about the requests made to these
services.
[float]
=== Example dashboard

View File

@ -0,0 +1,117 @@
////
This file is generated! See scripts/docs_collector.py
////
[[filebeat-module-azure]]
[role="xpack"]
:modulename: azure
:has-dashboards: false
== azure module
beta[]
This is the azure module.
The azure module will concentrate on retrieving different types of log data from Azure.
There are several requirements before using the module since the logs will actually be read from azure event hubs.
- the event hubs the azure module will read logs from must have the kafka option enabled .
- the logs have to be exported first to the event hubs https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create-kafka-enabled
- to export activity logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-export
- to export audit and sign-in logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/active-directory/reports-monitoring/tutorial-azure-monitor-stream-logs-to-event-hub
The module will contain the following filesets:
`activitylogs` ::
Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription.
`signinlogs` ::
Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities.
`auditlogs` ::
Will retrieve azure Active Directory audit logs. The audit logs provide traceability through logs for all changes done by various features within Azure AD. Examples of audit logs include changes made to any resources within Azure AD like adding or removing users, apps, groups, roles and policies.
[float]
=== Dashboards
The azure module comes with several predefined dashboards for general cloud overview, user activity and alerts. For example:
image::./images/filebeat-azure-overview.png[]
[float]
=== Module configuration
```
- module: azure
activitylogs:
enabled: true
var:
namespace: "obseventhubs.servicebus.windows.net:9093"
eventhub: ["insights-operational-logs"]
consumer_group: "$Default"
connection_string: ""
auditlogs:
enabled: true
var:
namespace: ""
eventhub: ["insights-logs-auditlogs"]
consumer_group: "$Default"
connection_string: ""
signinlogs:
enabled: true
var:
namespace: ""
eventhub: ["insights-logs-signinlogs"]
consumer_group: "$Default"
connection_string: ""
```
A side by side kafka/event hubs notation, we will follow Azure notations in this case.
`namespace` ::
_string_
An Event Hubs namespace provides a unique scoping container, referenced by its fully qualified domain name, in which users can create one or more event hubs or Kafka topics.
`eventhub` ::
_[]string_
Or kafka topic, is a fully managed, real-time data ingestion service.
Default value `insights-operational-logs`
`consumer_group` ::
_string_
The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets.
Default value: `$Default`
`connection_string` ::
_string_
The connection string required to communicate with Event Hubs, steps here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string.
include::../include/what-happens.asciidoc[]
[float]
=== Compatibility
TODO: document with what versions of the software is this tested
[float]
=== Fields
For a description of each field in the module, see the
<<exported-fields-azure,exported fields>> section.

View File

@ -33,7 +33,7 @@ include::../include/configuring-intro.asciidoc[]
The following example shows how to set paths in the +modules.d/{modulename}.yml+
file to override the default paths for logs:
["source","yaml",subs="attributes"]
[source,yaml]
-----
- module: kafka
log:
@ -48,7 +48,7 @@ file to override the default paths for logs:
To specify the same settings at the command line, you use:
["source","sh",subs="attributes"]
[source,yaml]
-----
-M "kafka.log.var.paths=[/path/to/logs/controller.log*, /path/to/logs/server.log*, /path/to/logs/state-change.log*, /path/to/logs/kafka-*.log*]"
-----
@ -62,6 +62,19 @@ include::../include/config-option-intro.asciidoc[]
[float]
==== `log` fileset settings
*`var.kafka_home`*::
The path to your Kafka installation. The default is `/opt`. For example:
+
[source,yaml]
----
- module: kafka
log:
enabled: true
var.kafka_home: /usr/share/kafka_2.12-2.4.0
...
----
include::../include/var-paths.asciidoc[]
include::../include/timezone-support.asciidoc[]

View File

@ -0,0 +1,41 @@
////
This file is generated! See scripts/docs_collector.py
////
[[filebeat-module-misp]]
[role="xpack"]
:modulename: misp
:has-dashboards: false
== MISP module
beta[]
This is a filebeat module for reading threat intel information from the MISP platform (https://www.circl.lu/doc/misp/). It uses the httpjson input to access the MISP REST API interface.
The configuration in the config.yml file uses the following format:
* var.api_key: specifies the API key to access MISP.
* var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute".
* var.url: URL of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch"
[float]
=== Example dashboard
This module comes with a sample dashboard. For example:
[role="screenshot"]
image::./images/kibana-misp.png[]
:has-dashboards!:
:modulename!:
[float]
=== Fields
For a description of each field in the module, see the
<<exported-fields-misp,exported fields>> section.

View File

@ -59,6 +59,10 @@ expiration.
processing. Use this setting to avoid packet-loss when dealing with occasional
bursts of traffic.
`var.detect_sequence_reset`:: Flag controlling whether {beatname_uc} should
monitor sequence numbers in the Netflow packets to detect an Exporting Process
reset. See <<filebeat-input-netflow,netflow input>> for details.
:has-dashboards!:
:fileset_ex!:

View File

@ -109,8 +109,8 @@ in ECS that are added under the `panw.panos` prefix:
| Category | | panw.panos.url.category
| Severity | log.level |
| Direction | network.direction |
| Source Location | source.geo.name |
| Destination Location | destination.geo.name |
| Source Location | source.geo.country_iso_code |
| Destination Location | destination.geo.country_iso_code |
| PCAP_id | | panw.panos.network.pcap_id
| Filedigest | | panw.panos.file.hash
| User Agent | user_agent.original |

View File

@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py
* <<filebeat-module-apache>>
* <<filebeat-module-auditd>>
* <<filebeat-module-aws>>
* <<filebeat-module-azure>>
* <<filebeat-module-cef>>
* <<filebeat-module-cisco>>
* <<filebeat-module-coredns>>
@ -20,6 +21,7 @@ This file is generated! See scripts/docs_collector.py
* <<filebeat-module-kafka>>
* <<filebeat-module-kibana>>
* <<filebeat-module-logstash>>
* <<filebeat-module-misp>>
* <<filebeat-module-mongodb>>
* <<filebeat-module-mssql>>
* <<filebeat-module-mysql>>
@ -44,6 +46,7 @@ include::modules-overview.asciidoc[]
include::modules/apache.asciidoc[]
include::modules/auditd.asciidoc[]
include::modules/aws.asciidoc[]
include::modules/azure.asciidoc[]
include::modules/cef.asciidoc[]
include::modules/cisco.asciidoc[]
include::modules/coredns.asciidoc[]
@ -58,6 +61,7 @@ include::modules/iptables.asciidoc[]
include::modules/kafka.asciidoc[]
include::modules/kibana.asciidoc[]
include::modules/logstash.asciidoc[]
include::modules/misp.asciidoc[]
include::modules/mongodb.asciidoc[]
include::modules/mssql.asciidoc[]
include::modules/mysql.asciidoc[]

View File

@ -23,4 +23,4 @@ image::./images/filebeat.png[Beats design]
For more information about inputs and harvesters, see
<<how-filebeat-works>>.
include::{libbeat-dir}/docs/shared-libbeat-description.asciidoc[]
include::{libbeat-dir}/shared-libbeat-description.asciidoc[]

View File

@ -6,7 +6,7 @@ allowing you to separate your configuration into multiple smaller
configuration files. See the <<load-input-config>> and the
<<load-module-config>> sections for details.
include::{libbeat-dir}/docs/shared-note-file-permissions.asciidoc[]
include::{libbeat-dir}/shared-note-file-permissions.asciidoc[]
[float]
[[load-input-config]]
@ -129,4 +129,4 @@ set the `period` to less than 1s because the modification time of files is often
stored in seconds. Setting the `period` to less than 1s will result in
unnecessary overhead.
include::{libbeat-dir}/docs/shared-note-file-permissions.asciidoc[]
include::{libbeat-dir}/shared-note-file-permissions.asciidoc[]

View File

@ -1 +1 @@
include::{libbeat-dir}/docs/shared-docker.asciidoc[]
include::{libbeat-dir}/shared-docker.asciidoc[]

View File

@ -6,7 +6,7 @@ retrieve and ship container logs.
ifeval::["{release-state}"=="unreleased"]
However, version {stack-version} of {beatname_uc} has not yet been
However, version {version} of {beatname_uc} has not yet been
released, so no Docker image is currently available for this version.
endif::[]

View File

@ -27,16 +27,16 @@ This section includes additional information on how to set up and run
//MAINTAINERS: If you add a new file to this section, make sure you update the bulleted list ^^ too.
include::{libbeat-dir}/docs/shared-directory-layout.asciidoc[]
include::{libbeat-dir}/shared-directory-layout.asciidoc[]
include::{libbeat-dir}/docs/keystore.asciidoc[]
include::{libbeat-dir}/keystore.asciidoc[]
include::{libbeat-dir}/docs/command-reference.asciidoc[]
include::{libbeat-dir}/command-reference.asciidoc[]
include::./running-on-docker.asciidoc[]
include::./running-on-kubernetes.asciidoc[]
include::{libbeat-dir}/docs/shared-systemd.asciidoc[]
include::{libbeat-dir}/shared-systemd.asciidoc[]
include::{libbeat-dir}/docs/shared-shutdown.asciidoc[]
include::{libbeat-dir}/shared-shutdown.asciidoc[]

View File

@ -17,14 +17,14 @@ following tips:
[[getting-help]]
== Get help
include::{libbeat-dir}/docs/getting-help.asciidoc[]
include::{libbeat-dir}/getting-help.asciidoc[]
//sets block macro for debugging.asciidoc included in next section
[[enable-filebeat-debugging]]
== Debug
include::{libbeat-dir}/docs/debugging.asciidoc[]
include::{libbeat-dir}/debugging.asciidoc[]

View File

@ -5,6 +5,7 @@ filebeat.config:
processors:
- add_cloud_metadata: ~
- add_docker_metadata: ~
output.elasticsearch:
hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'

View File

@ -439,6 +439,9 @@ filebeat.inputs:
# fields.
#fields_under_root: false
# Set to true to publish fields with null values in events.
#keep_null: false
# Ignore files which were modified more then the defined timespan in the past.
# ignore_older is disabled by default, so no files are ignored by setting it to 0.
# Time strings like 2h (2 hours), 5m (5 minutes) can be used.
@ -1771,7 +1774,7 @@ setup.template.settings:
#setup.ilm.enabled: auto
# Set the prefix used in the index lifecycle write alias name. The default alias
# name is 'filebeat-%{[agent.version]}'.
# name is 'filebeat-%{[agent.version]}'.
#setup.ilm.rollover_alias: "filebeat"
# Set the rollover index pattern. The default is "%{now/d}-000001".
@ -2034,12 +2037,21 @@ logging.files:
# Defines if the HTTP endpoint is enabled.
#http.enabled: false
# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
# The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
# When using IP addresses, it is recommended to only use localhost.
#http.host: localhost
# Port on which the HTTP endpoint will bind. Default is 5066.
#http.port: 5066
# Define which user should be owning the named pipe.
#http.named_pipe.user:
# Define which the permissions that should be applied to the named pipe, use the Security
# Descriptor Definition Language (SDDL) to define the permission. This option cannot be used with
# `http.user`.
#http.named_pipe.security_descriptor:
#============================= Process Security ================================
# Enable or disable seccomp system call filtering on Linux. Default is enabled.

View File

@ -176,6 +176,8 @@ output.elasticsearch:
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
#================================ Logging =====================================

Some files were not shown because too many files have changed in this diff Show More