Process events to prevent unintended datatypes and field names

fixes #1
This commit is contained in:
Blerim Sheqa 2017-01-27 13:58:49 +01:00
parent 5fda97545f
commit 1ea21190d2
9 changed files with 974 additions and 130 deletions

View File

@ -31,12 +31,12 @@ icingabeat:
# * Notification
# * AcknowledgementSet
# * AcknowledgementCleared
# * CommentAdded (See https://github.com/Icinga/icingabeat/issues/1)
# * CommentRemoved (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeAdded (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeRemoved (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeStarted (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeTriggered (See https://github.com/Icinga/icingabeat/issues/1)
# * CommentAdded
# * CommentRemoved
# * DowntimeAdded
# * DowntimeRemoved
# * DowntimeStarted
# * DowntimeTriggered
#
# To disable eventstream, leave the types empty or comment out the option
types:

View File

@ -2,11 +2,15 @@
title: icingabeat
description: Data received from the Icinga 2 API
fields:
- name: timestamp
type: date
description: >
Timestamp of event occurrence
- name: type
type: keyword
required: true
description: >
Type of the document.
Type of the document
- name: host
type: keyword
@ -51,25 +55,20 @@
- name: acknowledgement_type
type: integer
description: >
Type of the acknowledgement
- name: comment
type: text
description: >
Content of a comment message
Type of an acknowledgement
- name: expiry
type: integer
type: date
description: >
If an acknowledgement or not
Expiry of an acknowledgement
- name: notify
type: keyword
description: >
If users are notified
If has been sent out
- name: check_result.active
type: keyword
type: boolean
description: >
If check was active or passive
@ -84,12 +83,12 @@
Command that was executed
- name: check_result.execution_end
type: text
type: date
description: >
Time when execution of check ended
- name: check_result.execution_start
type: text
type: date
description: >
Time when execution of check started
@ -109,14 +108,15 @@
Performance data in text format
- name: check_result.schedule_end
type: text
type: date
description: >
Time when scheduling of the check ended
- name: check_result.schedule_start
type: text
type: date
description: >
Time when check was scheduled
- name: check_result.state
type: integer
description: >
@ -133,7 +133,7 @@
Check attempt after check execution
- name: check_result.vars_after.reachable
type: keyword
type: boolean
description: >
Reachable state after check execution
@ -153,7 +153,7 @@
Check attempt before check execution
- name: check_result.vars_before.reachable
type: keyword
type: boolean
description: >
Reachable state before check execution
@ -166,3 +166,193 @@
type: integer
description: >
State type before check execution
- name: comment.__name
type: text
description: >
Unique identifier of a comment
- name: comment.author
type: keyword
description: >
Author of a comment
- name: comment.entry_time
type: date
description: >
Entry time of a comment
- name: comment.entry_type
type: integer
description: >
Entry type of a comment
- name: comment.expire_time
type: date
description: >
Expire time of a comment
- name: comment.host_name
type: keyword
description: >
Host name of a comment
- name: comment.legacy_id
type: integer
description: >
Legacy ID of a comment
- name: comment.name
type: keyword
description: >
Identifier of a comment
- name: comment.package
type: keyword
description: >
Config package of a comment
- name: comment.service_name
type: keyword
description: >
Service name of a comment
- name: comment.templates
type: text
description: >
Templates used by a comment
- name: comment.text
type: text
description: >
Text of a comment
- name: comment.type
type: keyword
description: >
Comment type
- name: comment.version
type: keyword
description: >
Config version of comment object
- name: comment.zone
type: keyword
description: >
Zone where comment was generated
- name: downtime.__name
type: text
description: >
Unique identifier of a downtime
- name: downtime.author
type: keyword
description: >
Author of a downtime
- name: downtime.comment
type: text
description: >
Text of a downtime
- name: downtime.config_owner
type: text
description: >
Config owner
- name: downtime.duration
type: integer
description: >
Duration of a downtime
- name: downtime.end_time
type: date
description: >
Timestamp of downtime end
- name: downtime.entry_time
type: date
description: >
Timestamp when downtime was created
- name: downtime.fixed
type: boolean
description: >
If downtime is fixed or flexible
- name: downtime.host_name
type: keyword
description: >
Hostname of a downtime
- name: downtime.legacy_id
type: integer
description: >
The integer ID of a downtime
- name: downtime.name
type: keyword
description: >
Downtime config identifier
- name: downtime.package
type: keyword
description: >
Configuration package of downtime
- name: downtime.scheduled_by
type: text
description: >
By whom downtime was scheduled
- name: downtime.service_name
type: keyword
description: >
Service name of a downtime
- name: downtime.start_time
type: date
description: >
Timestamp when downtime starts
- name: downtime.templates
type: text
description: >
Templates used by this downtime
- name: downtime.trigger_time
type: date
description: >
Timestamp when downtime was triggered
- name: downtime.triggered_by
type: text
description: >
By whom downtime was triggered
- name: downtime.triggers
type: text
description: >
Downtime triggers
- name: downtime.type
type: keyword
description: >
Downtime type
- name: downtime.version
type: keyword
description: >
Config version of downtime
- name: downtime.was_cancelled
type: boolean
description: >
If downtime was cancelled
- name: downtime.zone
type: keyword
description: >
Zone of downtime

File diff suppressed because one or more lines are too long

View File

@ -36,6 +36,67 @@ func NewEventstream(bt *Icingabeat, cfg config.Config) *Eventstream {
return eventstream
}
// BuildEvent ...
func BuildEvent(e []byte) common.MapStr {
var event common.MapStr
var icingaEvent map[string]interface{}
if err := json.Unmarshal(e, &icingaEvent); err != nil {
logp.Warn("Error decoding event: %v", err)
}
event = common.MapStr{
"@timestamp": common.Time(time.Now()),
}
for key, value := range icingaEvent {
event.Put(key, value)
}
logp.Debug("icingabeat", "Type: %v", icingaEvent["type"])
switch icingaEvent["type"] {
case "CheckResult", "StateChange", "Notification":
checkResult := icingaEvent["check_result"].(map[string]interface{})
event.Put("check_result.execution_start", FloatToTimestamp(checkResult["execution_start"].(float64)))
event.Put("check_result.execution_end", FloatToTimestamp(checkResult["execution_end"].(float64)))
event.Put("check_result.schedule_start", FloatToTimestamp(checkResult["schedule_start"].(float64)))
event.Put("check_result.schedule_end", FloatToTimestamp(checkResult["schedule_end"].(float64)))
event.Delete("check_result.performance_data")
case "AcknowledgementSet":
event.Delete("comment")
event.Put("comment.text", icingaEvent["comment"])
event.Put("expiry", FloatToTimestamp(icingaEvent["expiry"].(float64)))
case "CommentAdded", "CommentRemoved":
comment := icingaEvent["comment"].(map[string]interface{})
event.Put("comment.entry_time", FloatToTimestamp(comment["entry_time"].(float64)))
event.Put("comment.expire_time", FloatToTimestamp(comment["expire_time"].(float64)))
case "DowntimeAdded", "DowntimeRemoved", "DowntimeStarted", "DowntimeTriggered":
downtime := icingaEvent["downtime"].(map[string]interface{})
event.Put("downtime.end_time", FloatToTimestamp(downtime["end_time"].(float64)))
event.Put("downtime.entry_time", FloatToTimestamp(downtime["entry_time"].(float64)))
event.Put("downtime.start_time", FloatToTimestamp(downtime["start_time"].(float64)))
event.Put("downtime.trigger_time", FloatToTimestamp(downtime["trigger_time"].(float64)))
}
event.Put("type", "icingabeat.event."+strings.ToLower(icingaEvent["type"].(string)))
event.Put("timestamp", FloatToTimestamp(icingaEvent["timestamp"].(float64)))
return event
}
// FloatToTimestamp ...
func FloatToTimestamp(stamp float64) time.Time {
sec := int64(stamp)
nsec := int64((stamp - float64(int64(stamp))) * 1e9)
t := time.Unix(sec, nsec)
return t
}
// Run evenstream receiver
func (es *Eventstream) Run() error {
queue := "icingabeat"
@ -87,24 +148,7 @@ func (es *Eventstream) Run() error {
logp.Err("Error reading line %#v", err)
}
var event common.MapStr
if err := json.Unmarshal(line, &event); err != nil {
logp.Info("Unmarshal problem %v", err)
es.mutex.Lock()
tst := es.closer == nil
es.mutex.Unlock()
if tst || err == io.ErrUnexpectedEOF || err == io.EOF {
break
}
continue
}
event["@timestamp"] = common.Time(time.Now())
documentType := strings.ToLower(event["type"].(string))
event["type"] = "icingabeat.event." + documentType
es.icingabeat.client.PublishEvent(event)
es.icingabeat.client.PublishEvent(BuildEvent(line))
logp.Info("Event sent")
}
@ -114,7 +158,7 @@ func (es *Eventstream) Run() error {
default:
}
} else {
logp.Info("Error connecting to API: %v", responseErr)
logp.Err("Error connecting to API: %v", responseErr)
}
select {

View File

@ -127,14 +127,20 @@ Region in which this host is running.
Data received from the Icinga 2 API
[float]
=== timestamp
type: date
Timestamp of event occurrence
[float]
=== type
type: keyword
required: True
Type of the document.
Type of the document
[float]
@ -206,23 +212,15 @@ Affected users of a notification
type: integer
Type of the acknowledgement
[float]
=== comment
type: text
Content of a comment message
Type of an acknowledgement
[float]
=== expiry
type: integer
type: date
If an acknowledgement or not
Expiry of an acknowledgement
[float]
@ -230,13 +228,13 @@ If an acknowledgement or not
type: keyword
If users are notified
If has been sent out
[float]
=== check_result.active
type: keyword
type: boolean
If check was active or passive
@ -260,7 +258,7 @@ Command that was executed
[float]
=== check_result.execution_end
type: text
type: date
Time when execution of check ended
@ -268,7 +266,7 @@ Time when execution of check ended
[float]
=== check_result.execution_start
type: text
type: date
Time when execution of check started
@ -300,7 +298,7 @@ Performance data in text format
[float]
=== check_result.schedule_end
type: text
type: date
Time when scheduling of the check ended
@ -308,7 +306,7 @@ Time when scheduling of the check ended
[float]
=== check_result.schedule_start
type: text
type: date
Time when check was scheduled
@ -340,7 +338,7 @@ Check attempt after check execution
[float]
=== check_result.vars_after.reachable
type: keyword
type: boolean
Reachable state after check execution
@ -372,7 +370,7 @@ Check attempt before check execution
[float]
=== check_result.vars_before.reachable
type: keyword
type: boolean
Reachable state before check execution
@ -393,3 +391,307 @@ type: integer
State type before check execution
[float]
=== comment.__name
type: text
Unique identifier of a comment
[float]
=== comment.author
type: keyword
Author of a comment
[float]
=== comment.entry_time
type: date
Entry time of a comment
[float]
=== comment.entry_type
type: integer
Entry type of a comment
[float]
=== comment.expire_time
type: date
Expire time of a comment
[float]
=== comment.host_name
type: keyword
Host name of a comment
[float]
=== comment.legacy_id
type: integer
Legacy ID of a comment
[float]
=== comment.name
type: keyword
Identifier of a comment
[float]
=== comment.package
type: keyword
Config package of a comment
[float]
=== comment.service_name
type: keyword
Service name of a comment
[float]
=== comment.templates
type: text
Templates used by a comment
[float]
=== comment.text
type: text
Text of a comment
[float]
=== comment.type
type: keyword
Comment type
[float]
=== comment.version
type: keyword
Config version of comment object
[float]
=== comment.zone
type: keyword
Zone where comment was generated
[float]
=== downtime.__name
type: text
Unique identifier of a downtime
[float]
=== downtime.author
type: keyword
Author of a downtime
[float]
=== downtime.comment
type: text
Text of a downtime
[float]
=== downtime.config_owner
type: text
Config owner
[float]
=== downtime.duration
type: integer
Duration of a downtime
[float]
=== downtime.end_time
type: date
Timestamp of downtime end
[float]
=== downtime.entry_time
type: date
Timestamp when downtime was created
[float]
=== downtime.fixed
type: boolean
If downtime is fixed or flexible
[float]
=== downtime.host_name
type: keyword
Hostname of a downtime
[float]
=== downtime.legacy_id
type: integer
The integer ID of a downtime
[float]
=== downtime.name
type: keyword
Downtime config identifier
[float]
=== downtime.package
type: keyword
Configuration package of downtime
[float]
=== downtime.scheduled_by
type: text
By whom downtime was scheduled
[float]
=== downtime.service_name
type: keyword
Service name of a downtime
[float]
=== downtime.start_time
type: date
Timestamp when downtime starts
[float]
=== downtime.templates
type: text
Templates used by this downtime
[float]
=== downtime.trigger_time
type: date
Timestamp when downtime was triggered
[float]
=== downtime.triggered_by
type: text
By whom downtime was triggered
[float]
=== downtime.triggers
type: text
Downtime triggers
[float]
=== downtime.type
type: keyword
Downtime type
[float]
=== downtime.version
type: keyword
Config version of downtime
[float]
=== downtime.was_cancelled
type: boolean
If downtime was cancelled
[float]
=== downtime.zone
type: keyword
Zone of downtime

View File

@ -31,12 +31,12 @@ icingabeat:
# * Notification
# * AcknowledgementSet
# * AcknowledgementCleared
# * CommentAdded (See https://github.com/Icinga/icingabeat/issues/1)
# * CommentRemoved (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeAdded (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeRemoved (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeStarted (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeTriggered (See https://github.com/Icinga/icingabeat/issues/1)
# * CommentAdded
# * CommentRemoved
# * DowntimeAdded
# * DowntimeRemoved
# * DowntimeStarted
# * DowntimeTriggered
#
# To disable eventstream, leave the types empty or comment out the option
types:

View File

@ -55,9 +55,7 @@
"check_result": {
"properties": {
"active": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
"type": "boolean"
},
"check_source": {
"ignore_above": 1024,
@ -72,18 +70,10 @@
"type": "string"
},
"execution_end": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
"type": "date"
},
"execution_start": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
"type": "date"
},
"exit_status": {
"type": "long"
@ -103,18 +93,10 @@
"type": "string"
},
"schedule_end": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
"type": "date"
},
"schedule_start": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
"type": "date"
},
"state": {
"type": "long"
@ -130,9 +112,7 @@
"type": "long"
},
"reachable": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
"type": "boolean"
},
"state": {
"type": "long"
@ -148,9 +128,7 @@
"type": "long"
},
"reachable": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
"type": "boolean"
},
"state": {
"type": "long"
@ -163,14 +141,201 @@
}
},
"comment": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
"properties": {
"__name": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"author": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"entry_time": {
"type": "date"
},
"entry_type": {
"type": "long"
},
"expire_time": {
"type": "date"
},
"host_name": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"legacy_id": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"package": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"service_name": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"templates": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"text": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"type": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"version": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"zone": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
}
}
},
"downtime": {
"properties": {
"__name": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"author": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"comment": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"config_owner": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"duration": {
"type": "long"
},
"end_time": {
"type": "date"
},
"entry_time": {
"type": "date"
},
"fixed": {
"type": "boolean"
},
"host_name": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"legacy_id": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"package": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"scheduled_by": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"service_name": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"start_time": {
"type": "date"
},
"templates": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"trigger_time": {
"type": "date"
},
"triggered_by": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"triggers": {
"index": "analyzed",
"norms": {
"enabled": false
},
"type": "string"
},
"type": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"version": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"was_cancelled": {
"type": "boolean"
},
"zone": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
}
}
},
"expiry": {
"type": "long"
"type": "date"
},
"host": {
"ignore_above": 1024,
@ -248,6 +413,9 @@
},
"type": "string"
},
"timestamp": {
"type": "date"
},
"type": {
"ignore_above": 1024,
"index": "not_analyzed",

View File

@ -48,8 +48,7 @@
"check_result": {
"properties": {
"active": {
"ignore_above": 1024,
"type": "keyword"
"type": "boolean"
},
"check_source": {
"ignore_above": 1024,
@ -60,12 +59,10 @@
"type": "text"
},
"execution_end": {
"norms": false,
"type": "text"
"type": "date"
},
"execution_start": {
"norms": false,
"type": "text"
"type": "date"
},
"exit_status": {
"type": "long"
@ -79,12 +76,10 @@
"type": "text"
},
"schedule_end": {
"norms": false,
"type": "text"
"type": "date"
},
"schedule_start": {
"norms": false,
"type": "text"
"type": "date"
},
"state": {
"type": "long"
@ -99,8 +94,7 @@
"type": "long"
},
"reachable": {
"ignore_above": 1024,
"type": "keyword"
"type": "boolean"
},
"state": {
"type": "long"
@ -116,8 +110,7 @@
"type": "long"
},
"reachable": {
"ignore_above": 1024,
"type": "keyword"
"type": "boolean"
},
"state": {
"type": "long"
@ -130,11 +123,155 @@
}
},
"comment": {
"norms": false,
"type": "text"
"properties": {
"__name": {
"norms": false,
"type": "text"
},
"author": {
"ignore_above": 1024,
"type": "keyword"
},
"entry_time": {
"type": "date"
},
"entry_type": {
"type": "long"
},
"expire_time": {
"type": "date"
},
"host_name": {
"ignore_above": 1024,
"type": "keyword"
},
"legacy_id": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"package": {
"ignore_above": 1024,
"type": "keyword"
},
"service_name": {
"ignore_above": 1024,
"type": "keyword"
},
"templates": {
"norms": false,
"type": "text"
},
"text": {
"norms": false,
"type": "text"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
},
"zone": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"downtime": {
"properties": {
"__name": {
"norms": false,
"type": "text"
},
"author": {
"ignore_above": 1024,
"type": "keyword"
},
"comment": {
"norms": false,
"type": "text"
},
"config_owner": {
"norms": false,
"type": "text"
},
"duration": {
"type": "long"
},
"end_time": {
"type": "date"
},
"entry_time": {
"type": "date"
},
"fixed": {
"type": "boolean"
},
"host_name": {
"ignore_above": 1024,
"type": "keyword"
},
"legacy_id": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"package": {
"ignore_above": 1024,
"type": "keyword"
},
"scheduled_by": {
"norms": false,
"type": "text"
},
"service_name": {
"ignore_above": 1024,
"type": "keyword"
},
"start_time": {
"type": "date"
},
"templates": {
"norms": false,
"type": "text"
},
"trigger_time": {
"type": "date"
},
"triggered_by": {
"norms": false,
"type": "text"
},
"triggers": {
"norms": false,
"type": "text"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
},
"was_cancelled": {
"type": "boolean"
},
"zone": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"expiry": {
"type": "long"
"type": "date"
},
"host": {
"ignore_above": 1024,
@ -198,6 +335,9 @@
"norms": false,
"type": "text"
},
"timestamp": {
"type": "date"
},
"type": {
"ignore_above": 1024,
"type": "keyword"

View File

@ -31,12 +31,12 @@ icingabeat:
# * Notification
# * AcknowledgementSet
# * AcknowledgementCleared
# * CommentAdded (See https://github.com/Icinga/icingabeat/issues/1)
# * CommentRemoved (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeAdded (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeRemoved (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeStarted (See https://github.com/Icinga/icingabeat/issues/1)
# * DowntimeTriggered (See https://github.com/Icinga/icingabeat/issues/1)
# * CommentAdded
# * CommentRemoved
# * DowntimeAdded
# * DowntimeRemoved
# * DowntimeStarted
# * DowntimeTriggered
#
# To disable eventstream, leave the types empty or comment out the option
types: