Apply shutdown timeout to http server to limit reload delay

Part of elastic#14337

internal/beater/http.go

@@ -106,9 +106,9 @@ func (h *httpServer) start() error {
 	return h.Serve(h.httpListener)
 }
 
-func (h *httpServer) stop() {
+func (h *httpServer) stop(ctx context.Context) {
 	h.logger.Infof("Stop listening on: %s", h.Server.Addr)
-	if err := h.Shutdown(context.Background()); err != nil {
+	if err := h.Shutdown(ctx); err != nil {
 		h.logger.Errorf("error stopping http server: %s", err.Error())
 		if err := h.Close(); err != nil {
 			h.logger.Errorf("error closing http server: %s", err.Error())

internal/beater/server.go

@@ -222,10 +222,11 @@ func (s server) run(ctx context.Context) error {
 	})
 	g.Go(func() error {
 		<-ctx.Done()
-		// httpServer should stop before grpcServer to avoid a panic caused by placing a new connection into
-		// a closed grpc connection channel during shutdown.
-		// See https://github.com/elastic/gmux/issues/13
-		s.httpServer.stop()
+		stopctx, cancel := context.WithTimeout(context.Background(), s.cfg.ShutdownTimeout)
+		defer cancel()
+		s.httpServer.stop(stopctx)
+		// FIXME: will httpServer.stop terminate the grpc connections, making grpcServer.GracefulStop a no-op?
+		// Will it be better to grpcServer.GracefulStop first as the gmux panic is fixed, so that GOAWAY is sent?
 		s.grpcServer.GracefulStop()
 		return nil
 	})