Skip to content

Commit

Permalink
Don't send duplicate notifications for fixed downtimes
Browse files Browse the repository at this point in the history
  • Loading branch information
@yhabteab @julianbrost
yhabteab authored and julianbrost committed May 13, 2024
1 parent 7b93e1f commit 0ebcc4b
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 0 deletions.
6 changes: 6 additions & 0 deletions internal/icinga2/api_responses.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,12 @@ type Downtime struct {
// RemoveTime is used to indicate whether a downtime was ended automatically or cancelled prematurely by a user.
// It is set to zero time for the former case, otherwise to the timestamp at which time has been cancelled.
RemoveTime UnixFloat `json:"remove_time"`

// IsFixed is used to differentiate between fixed and flexible downtimes.
// Fixed downtimes always emits a start and triggered event and cause two notifications being sent
// for the very (same) event. Flexible downtimes, on the other hand, only emits a trigger event, and
// don't produce duplicates for the same event.
IsFixed bool `json:"fixed"`
}

// HostServiceRuntimeAttributes are common attributes of both Host and Service objects.
Expand Down
71 changes: 71 additions & 0 deletions internal/icinga2/api_responses_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,40 @@ func TestObjectQueriesResult_UnmarshalJSON(t *testing.T) {
Host: "dummy-11",
Author: "icingaadmin",
Comment: "turn down for what",
IsFixed: true,
},
},
},
{
// $ curl -k -s -u root:icinga 'https://localhost:5665/v1/objects/downtimes' | jq -c '[.results[] | select(.attrs.fixed == false)][1]'
name: "flexible-downtime-host",
jsonData: `{"attrs":{"__name":"dummy-7!691d508b-c93f-4565-819c-3e46ffef1555","active":true,"author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714043658,"entry_time":1714040073.241627,"fixed":false,"ha_mode":0,"host_name":"dummy-7","legacy_id":4,"name":"691d508b-c93f-4565-819c-3e46ffef1555","original_attributes":null,"package":"_api","parent":"","paused":false,"remove_time":0,"scheduled_by":"","service_name":"","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/dummy-7!691d508b-c93f-4565-819c-3e46ffef1555.conf"},"start_time":1714040058,"templates":["691d508b-c93f-4565-819c-3e46ffef1555"],"trigger_time":1714040073.241627,"triggered_by":"","triggers":[],"type":"Downtime","version":1714040073.241642,"was_cancelled":false,"zone":"master"},"joins":{},"meta":{},"name":"dummy-7!691d508b-c93f-4565-819c-3e46ffef1555","type":"Downtime"}`,
resp: &ObjectQueriesResult[Downtime]{},
expected: &ObjectQueriesResult[Downtime]{
Name: "dummy-7!691d508b-c93f-4565-819c-3e46ffef1555",
Type: "Downtime",
Attrs: Downtime{
Host: "dummy-7",
Author: "icingaadmin",
Comment: "Flexible",
IsFixed: false,
},
},
},
{
// $ curl -k -s -u root:icinga 'https://localhost:5665/v1/objects/downtimes' | jq -c '[.results[] | select(.attrs.fixed == false)][0]'
name: "flexible-downtime-service",
jsonData: `{"attrs":{"__name":"docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d","active":true,"author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714042731,"entry_time":1714039143.459298,"fixed":false,"ha_mode":0,"host_name":"docker-master","legacy_id":3,"name":"97078a44-8902-495a-9f2a-c1f6802bc63d","original_attributes":null,"package":"_api","parent":"","paused":false,"remove_time":0,"scheduled_by":"","service_name":"disk /","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/docker-master!disk %2F!97078a44-8902-495a-9f2a-c1f6802bc63d.conf"},"start_time":1714039131,"templates":["97078a44-8902-495a-9f2a-c1f6802bc63d"],"trigger_time":1714039143.459298,"triggered_by":"","triggers":[],"type":"Downtime","version":1714039143.459324,"was_cancelled":false,"zone":""},"joins":{},"meta":{},"name":"docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d","type":"Downtime"}`,
resp: &ObjectQueriesResult[Downtime]{},
expected: &ObjectQueriesResult[Downtime]{
Name: "docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d",
Type: "Downtime",
Attrs: Downtime{
Host: "docker-master",
Service: "disk /",
Author: "icingaadmin",
Comment: "Flexible",
IsFixed: false,
},
},
},
Expand All @@ -147,6 +181,7 @@ func TestObjectQueriesResult_UnmarshalJSON(t *testing.T) {
Service: "load",
Author: "icingaadmin",
Comment: "Scheduled downtime for backup",
IsFixed: true,
},
},
},
Expand Down Expand Up @@ -419,6 +454,7 @@ func TestApiResponseUnmarshal(t *testing.T) {
Host: "dummy-157",
Author: "icingaadmin",
Comment: "updates",
IsFixed: true,
},
},
},
Expand All @@ -432,6 +468,7 @@ func TestApiResponseUnmarshal(t *testing.T) {
Service: "http",
Author: "icingaadmin",
Comment: "broken until Monday",
IsFixed: true,
},
},
},
Expand All @@ -444,6 +481,7 @@ func TestApiResponseUnmarshal(t *testing.T) {
Host: "dummy-157",
Author: "icingaadmin",
Comment: "updates",
IsFixed: true,
},
},
},
Expand All @@ -457,6 +495,7 @@ func TestApiResponseUnmarshal(t *testing.T) {
Service: "http",
Author: "icingaadmin",
Comment: "broken until Monday",
IsFixed: true,
},
},
},
Expand All @@ -469,6 +508,20 @@ func TestApiResponseUnmarshal(t *testing.T) {
Host: "dummy-157",
Author: "icingaadmin",
Comment: "updates",
IsFixed: true,
},
},
},
{
name: "flexible-downtimetriggered-host",
jsonData: `{"downtime":{"__name":"dummy-7!691d508b-c93f-4565-819c-3e46ffef1555","author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714043658,"entry_time":1714040073.241627,"fixed":false,"host_name":"dummy-7","legacy_id":4,"name":"691d508b-c93f-4565-819c-3e46ffef1555","package":"_api","parent":"","remove_time":0,"scheduled_by":"","service_name":"","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/dummy-7!691d508b-c93f-4565-819c-3e46ffef1555.conf"},"start_time":1714040058,"templates":["691d508b-c93f-4565-819c-3e46ffef1555"],"trigger_time":0,"triggered_by":"","triggers":[],"type":"Downtime","version":1714040073.241642,"zone":"master"},"timestamp":1714040073.242575,"type":"DowntimeAdded"}`,
expected: &DowntimeTriggered{
Timestamp: UnixFloat(time.UnixMicro(1714040073242575)),
Downtime: Downtime{
Host: "dummy-7",
Author: "icingaadmin",
Comment: "Flexible",
IsFixed: false,
},
},
},
Expand All @@ -482,6 +535,21 @@ func TestApiResponseUnmarshal(t *testing.T) {
Service: "http",
Author: "icingaadmin",
Comment: "broken until Monday",
IsFixed: true,
},
},
},
{
name: "flexible-downtimetriggered-service",
jsonData: `{"downtime":{"__name":"docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d","author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714042731,"entry_time":1714039143.459298,"fixed":false,"host_name":"docker-master","legacy_id":3,"name":"97078a44-8902-495a-9f2a-c1f6802bc63d","package":"_api","parent":"","remove_time":0,"scheduled_by":"","service_name":"disk /","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/docker-master!disk %2F!97078a44-8902-495a-9f2a-c1f6802bc63d.conf"},"start_time":1714039131,"templates":["97078a44-8902-495a-9f2a-c1f6802bc63d"],"trigger_time":1714039143.459298,"triggered_by":"","triggers":[],"type":"Downtime","version":1714039143.459324,"zone":""},"timestamp":1714039143.460918,"type":"DowntimeTriggered"}`,
expected: &DowntimeTriggered{
Timestamp: UnixFloat(time.UnixMicro(1714039143460918)),
Downtime: Downtime{
Host: "docker-master",
Service: "disk /",
Author: "icingaadmin",
Comment: "Flexible",
IsFixed: false,
},
},
},
Expand All @@ -495,6 +563,7 @@ func TestApiResponseUnmarshal(t *testing.T) {
Author: "icingaadmin",
Comment: "updates",
RemoveTime: UnixFloat(time.Time{}),
IsFixed: true,
},
},
},
Expand All @@ -508,6 +577,7 @@ func TestApiResponseUnmarshal(t *testing.T) {
Author: "icingaadmin",
Comment: "updates",
RemoveTime: UnixFloat(time.UnixMicro(1697207096187718)),
IsFixed: true,
},
},
},
Expand All @@ -522,6 +592,7 @@ func TestApiResponseUnmarshal(t *testing.T) {
Author: "icingaadmin",
Comment: "broken until Monday",
RemoveTime: UnixFloat(time.UnixMicro(1697207144746117)),
IsFixed: true,
},
},
},
Expand Down
14 changes: 14 additions & 0 deletions internal/icinga2/client_api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,9 +449,23 @@ func (client *Client) listenEventStream() error {
ev, err = client.buildDowntimeEvent(client.Ctx, respT.Downtime, false)
evTime = respT.Timestamp.Time()
case *DowntimeStarted:
if !respT.Downtime.IsFixed {
// This may never happen, but Icinga 2 does the same thing, and we need to ignore the start
// event for flexible downtime, as there will definitely be a triggered event for it.
client.Logger.Debugf("Skipping flexible downtime start event, %#v", respT)
continue
}

ev, err = client.buildDowntimeEvent(client.Ctx, respT.Downtime, true)
evTime = respT.Timestamp.Time()
case *DowntimeTriggered:
if respT.Downtime.IsFixed {
// Fixed downtimes generate two events (start, triggered), the latter applies here and must
// be ignored, since we're going to process its start event to avoid duplicated notifications.
client.Logger.Debugf("Skipping fixed downtime triggered event, %#v", respT)
continue
}

ev, err = client.buildDowntimeEvent(client.Ctx, respT.Downtime, true)
evTime = respT.Timestamp.Time()
case *Flapping:
Expand Down

0 comments on commit 0ebcc4b

Please sign in to comment.