icinga2: Configurable Icinga 2 API HTTP Timeout

First, raise the default HTTP timeout for Icinga 2 API connections to
one minute. This became necessary on one development system for high
load during startup.

As, however, one minute is kinda arbitrary, it was made configurable in
the daemon's YAML configuration file.

internal/daemon/config.go

@@ -7,12 +7,14 @@ import (
 	"github.com/icinga/icinga-go-library/database"
 	"github.com/icinga/icinga-go-library/logging"
 	"os"
+	"time"
 )
 
 type ConfigFile struct {
 	Listen           string          `yaml:"listen" default:"localhost:5680"`
 	DebugPassword    string          `yaml:"debug-password"`
 	ChannelPluginDir string          `yaml:"channel-plugin-dir" default:"/usr/libexec/icinga-notifications/channel"`
+	ApiTimeout       time.Duration   `yaml:"api-timeout" default:"1m"`
 	Icingaweb2URL    string          `yaml:"icingaweb2-url"`
 	Database         database.Config `yaml:"database"`
 	Logging          logging.Config  `yaml:"logging"`

internal/icinga2/client.go

@@ -32,8 +32,7 @@ type catchupEventMsg struct {
 // Client for the Icinga 2 Event Stream API with support for other Icinga 2 APIs to gather additional information and
 // perform a catch-up of unknown events either when starting up to or in case of a connection loss.
 //
-// Within the icinga-notifications scope, one or multiple Client instances can be generated from the configuration by
-// calling NewClientsFromConfig.
+// Within the icinga-notifications scope, one or multiple Client instances can be generated by the Launcher.
 //
 // A Client must be started by calling its Process method, which blocks until Ctx is marked as done. Reconnections and
 // the necessary state replaying in an internal catch-up-phase from the Icinga 2 API will be taken care off. Internally,
@@ -44,8 +43,19 @@ type Client struct {
 	ApiBaseURL       string
 	ApiBasicAuthUser string
 	ApiBasicAuthPass string
+
+	// ApiHttpTransport is a shared http.Transport and http.RoundTripper for all API connections, set by the Launcher.
 	ApiHttpTransport http.RoundTripper
 
+	// ApiTimeout specifies the timeout for API connections outside the Event Stream API.
+	//
+	// Defaults to 1m if unset within Client.Process.
+	//
+	// Depending on both the Icinga 2 server power and the amount of objects to be queried, the initial flood of
+	// request might take a bit longer. However, one minute is still a very huge time which shouldn't be reached
+	// for most setups unless the system is under immense stress or other issues are also present.
+	ApiTimeout time.Duration
+
 	// EventSourceId to be reflected in generated event.Events.
 	EventSourceId int64
 	// IcingaWebRoot points to the Icinga Web 2 endpoint for generated URLs.
@@ -438,6 +448,10 @@ func (client *Client) worker() {
 // takes care of reconnections, messages are being logged while generated event.Event will be dispatched to the
 // CallbackFn function.
 func (client *Client) Process() {
+	if client.ApiTimeout == 0 {
+		client.ApiTimeout = time.Minute
+	}
+
 	client.eventDispatcherEventStream = make(chan *eventMsg)
 	client.catchupPhaseRequest = make(chan struct{})
 

internal/icinga2/client_api.go

@@ -103,7 +103,7 @@ func (client *Client) queryObjectsApi(
 	// The underlying network connection is reused by using client.ApiHttpTransport.
 	httpClient := &http.Client{
 		Transport: client.ApiHttpTransport,
-		Timeout:   3 * time.Second,
+		Timeout:   client.ApiTimeout,
 	}
 	res, err := httpClient.Do(req)
 	if err != nil {

internal/icinga2/launcher.go

@@ -118,8 +118,11 @@ func (launcher *Launcher) launch(src *config.Source) {
 		ApiBaseURL:       src.Icinga2BaseURL.String,
 		ApiBasicAuthUser: src.Icinga2AuthUser.String,
 		ApiBasicAuthPass: src.Icinga2AuthPass.String,
+
 		ApiHttpTransport: trans,
 
+		ApiTimeout: daemon.Config().ApiTimeout,
+
 		EventSourceId: src.ID,
 		IcingaWebRoot: daemon.Config().Icingaweb2URL,