PMM-11341 Auto discovery limit. (#2568)

* PMM-11341 Auto discovery control. * PMM-11341 Fix. * PMM-11341 Same comments on all places. * PMM-11341 Remove live service info broker scrape. * PMM-11341 Lint. * PMM-11341 Connection checker for PG. * PMM-11341 Fix comment. * PMM-11341 Discover limit field even for QAN agents. * PMM-11341 Remove version check from connection checker. * PMM-11341 Remove whole PG connection check block on managed side. * Revert "PMM-11341 Discover limit field even for QAN agents." This reverts commit cec000c. * PMM-11341 Remove now unused field. * PMM-11341 Fix bad proto numbering after revert. * PMM-11341 Agreed changes. * PMM-11341 agent, admin identify as 2.41.0 version. * PMM-11341 Add check for auto discovery limit value. * PMM-11341 Count without excluded DBs. * PMM-11341 Fix to scan. * PMM-11341 Remove forced version. * PMM-11341 Gen. * PMM-11341 Bad naming. * PMM-11341 Close to rows. * PMM-11341 Fix. * PMM-11341 Status error instead common one. * PMM-11341 Naming. * PMM-11341 Fix typo. * PMM-11341 RDS field. * PMM-11341 RDS. * PMM-11341Align with table stats limit. * PMM-11341 Align with MySQL. * PMM-11341 Format. * PMM-11341 Lint. * PMM-11341 Lower case. * PMM-11341 FIx MySQL, PG pmm-admin description. * PMM-11341 DB list query moved above. * PMM-11341 Lint. * PMM-11341 MySQL, PG comments fix for limit. * PMM-11341 Lint. * PMM-11341 Remove nolint. * PMM-11341 Ifs refactor. * PMM-12341 Refactor. * PMM-11341 Change default limit to 10. * PMM-11341 Small description change. * PMM-11341 Fix RDS test after change of default value. * PMM-11341 Fix description for MySQL. * PMM-11341 Fix mariaDB 10.4 expected queryID. * PMM-11341 Fix API tests for add PG.
percona · Nov 16, 2023 · 6c37873 · 6c37873
1 parent 21d018b
commit 6c37873
Show file tree

Hide file tree

Showing 53 changed files with 3,825 additions and 3,049 deletions.
diff --git a/admin/commands/inventory/add_agent_mysqld_exporter.go b/admin/commands/inventory/add_agent_mysqld_exporter.go
@@ -62,7 +62,7 @@ func (res *addAgentMysqldExporterResult) TablestatStatus() string {
 	}
 
 	switch {
-	case res.Agent.TablestatsGroupTableLimit == 0: // no limit
+	case res.Agent.TablestatsGroupTableLimit == 0: // server defined
 		s += " (the table count limit is not set)."
 	case res.Agent.TablestatsGroupTableLimit < 0: // always disabled
 		s += " (always)."
@@ -94,7 +94,7 @@ type AddAgentMysqldExporterCommand struct {
 	TLSCAFile                 string            `name:"tls-ca" help:"Path to certificate authority certificate file"`
 	TLSCertFile               string            `name:"tls-cert" help:"Path to client certificate file"`
 	TLSKeyFile                string            `name:"tls-key" help:"Path to client key file"`
-	TablestatsGroupTableLimit int32             `placeholder:"number" help:"Tablestats group collectors will be disabled if there are more than that number of tables (default: 0 - always enabled; negative value - always disabled)"`
+	TablestatsGroupTableLimit int32             `placeholder:"number" help:"Tablestats group collectors will be disabled if there are more than that number of tables (default: server-defined, -1: always disabled)"`
 	PushMetrics               bool              `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
 	DisableCollectors         []string          `help:"Comma-separated list of collector names to exclude from exporter"`
 	LogLevel                  string            `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`

diff --git a/admin/commands/inventory/add_agent_postgres_exporter.go b/admin/commands/inventory/add_agent_postgres_exporter.go
@@ -46,6 +46,8 @@ func (res *addAgentPostgresExporterResult) String() string {
 }
 
 // AddAgentPostgresExporterCommand is used by Kong for CLI flags and commands.
+//
+//nolint:lll
 type AddAgentPostgresExporterCommand struct {
 	PMMAgentID          string            `arg:"" help:"The pmm-agent identifier which runs this instance"`
 	ServiceID           string            `arg:"" help:"Service identifier"`
@@ -62,6 +64,7 @@ type AddAgentPostgresExporterCommand struct {
 	TLSCertFile         string            `help:"TLS certificate file"`
 	TLSKeyFile          string            `help:"TLS certificate key file"`
 	LogLevel            string            `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`
+	AutoDiscoveryLimit  int32             `default:"0" placeholder:"NUMBER" help:"Auto-discovery will be disabled if there are more than that number of databases (default: server-defined, -1: always disabled)"`
 }
 
 func (cmd *AddAgentPostgresExporterCommand) RunCmd() (commands.Result, error) {
@@ -99,6 +102,7 @@ func (cmd *AddAgentPostgresExporterCommand) RunCmd() (commands.Result, error) {
 			SkipConnectionCheck: cmd.SkipConnectionCheck,
 			PushMetrics:         cmd.PushMetrics,
 			DisableCollectors:   commands.ParseDisableCollectors(cmd.DisableCollectors),
+			AutoDiscoveryLimit:  cmd.AutoDiscoveryLimit,
 
 			TLS:           cmd.TLS,
 			TLSSkipVerify: cmd.TLSSkipVerify,

diff --git a/admin/commands/management/add_mysql.go b/admin/commands/management/add_mysql.go
@@ -71,7 +71,7 @@ func (res *addMySQLResult) TablestatStatus() string {
 	s := "Table statistics collection " + status
 
 	switch {
-	case res.MysqldExporter.TablestatsGroupTableLimit == 0: // no limit
+	case res.MysqldExporter.TablestatsGroupTableLimit == 0: // server defined
 		s += " (the table count limit is not set)."
 	case res.MysqldExporter.TablestatsGroupTableLimit < 0: // always disabled
 		s += " (always)."

diff --git a/admin/commands/management/add_postgresql.go b/admin/commands/management/add_postgresql.go
@@ -73,6 +73,7 @@ type AddPostgreSQLCommand struct {
 	DisableQueryExamples bool              `name:"disable-queryexamples" help:"Disable collection of query examples"`
 	MetricsMode          string            `enum:"${metricsModesEnum}" default:"auto" help:"Metrics flow mode, can be push - agent will push metrics, pull - server scrape metrics from agent or auto - chosen by server"`
 	DisableCollectors    []string          `help:"Comma-separated list of collector names to exclude from exporter"`
+	AutoDiscoveryLimit   int32             `default:"0" placeholder:"NUMBER" help:"Auto-discovery will be disabled if there are more than that number of databases (default: server-defined, -1: always disabled)"`
 
 	AddCommonFlags
 	AddLogLevelNoFatalFlags
@@ -203,6 +204,7 @@ func (cmd *AddPostgreSQLCommand) RunCmd() (commands.Result, error) {
 			DisableQueryExamples: cmd.DisableQueryExamples,
 			MetricsMode:          pointer.ToString(strings.ToUpper(cmd.MetricsMode)),
 			DisableCollectors:    commands.ParseDisableCollectors(cmd.DisableCollectors),
+			AutoDiscoveryLimit:   cmd.AutoDiscoveryLimit,
 			LogLevel:             &cmd.AddLogLevel,
 		},
 		Context: commands.Ctx,

diff --git a/agent/agents/mysql/perfschema/perfschema_test.go b/agent/agents/mysql/perfschema/perfschema_test.go
@@ -304,8 +304,8 @@ func TestPerfSchema(t *testing.T) {
 
 	case "10.4-mariadb":
 		digests = map[string]string{
-			"SELECT `sleep` (?)":   "ce5b40e78030bb319c84965637255c18",
-			"SELECT * FROM `city`": "978a3813c9f566d7a72d65b88a9149d9",
+			"SELECT `sleep` (?)":   "0a01e0e8325cdd1db9a0746270ab8ce9",
+			"SELECT * FROM `city`": "a65e76b1643273fa3206b11c4f4d8739",
 		}
 
 	default:

diff --git a/agent/serviceinfobroker/service_info_broker.go b/agent/serviceinfobroker/service_info_broker.go
@@ -219,12 +219,32 @@ func (sib *ServiceInfoBroker) getPostgreSQLInfo(ctx context.Context, dsn string,
 	db := sql.OpenDB(c)
 	defer db.Close() //nolint:errcheck
 
+	var databaseList []string
+	databaseListQuery := "SELECT /* agent='serviceinfobroker' */ datname FROM pg_database WHERE datallowconn = true AND datistemplate = false AND has_database_privilege(current_user, datname, 'connect')" //nolint:lll
+	rows, err := db.QueryContext(ctx, databaseListQuery)
+	if err != nil {
+		res.Error = err.Error()
+		return &res
+	}
+	defer rows.Close() //nolint:errcheck
+	for rows.Next() {
+		var databaseName string
+		err := rows.Scan(&databaseName)
+		if err != nil {
+			res.Error = err.Error()
+			return &res
+		}
+
+		databaseList = append(databaseList, databaseName)
+	}
+	res.DatabaseList = databaseList
+
 	var version string
 	if err = db.QueryRowContext(ctx, "SHOW /* agent='serviceinfobroker' */ SERVER_VERSION").Scan(&version); err != nil {
 		res.Error = err.Error()
 	}
-
 	res.Version = version
+
 	return &res
 }
 

diff --git a/api-tests/management/postgresql_test.go b/api-tests/management/postgresql_test.go
@@ -59,6 +59,7 @@ func TestAddPostgreSQL(t *testing.T) {
 
 				SkipConnectionCheck: true,
 				DisableCollectors:   []string{"custom_query.ml", "custom_query.mr.directory"},
+				AutoDiscoveryLimit:  0,
 			},
 		}
 		addPostgreSQLOK, err := client.Default.PostgreSQL.AddPostgreSQL(params)
@@ -106,6 +107,7 @@ func TestAddPostgreSQL(t *testing.T) {
 					DisabledCollectors: []string{"custom_query.ml", "custom_query.mr.directory"},
 					PushMetricsEnabled: true,
 					Status:             &AgentStatusUnknown,
+					AutoDiscoveryLimit: 10,
 				},
 			},
 		}, *listAgents.Payload)
@@ -138,6 +140,7 @@ func TestAddPostgreSQL(t *testing.T) {
 				DisableQueryExamples:            true,
 
 				SkipConnectionCheck: true,
+				AutoDiscoveryLimit:  15,
 			},
 		}
 		addPostgreSQLOK, err := client.Default.PostgreSQL.AddPostgreSQL(params)
@@ -189,6 +192,7 @@ func TestAddPostgreSQL(t *testing.T) {
 					Username:           "username",
 					PushMetricsEnabled: true,
 					Status:             &AgentStatusUnknown,
+					AutoDiscoveryLimit: 15,
 				},
 			},
 			QANPostgresqlPgstatementsAgent: []*agents.ListAgentsOKBodyQANPostgresqlPgstatementsAgentItems0{
@@ -292,6 +296,7 @@ func TestAddPostgreSQL(t *testing.T) {
 				Port:        5432,
 
 				SkipConnectionCheck: true,
+				AutoDiscoveryLimit:  -2,
 			},
 		}
 		addPostgreSQLOK, err := client.Default.PostgreSQL.AddPostgreSQL(params)
@@ -344,6 +349,7 @@ func TestAddPostgreSQL(t *testing.T) {
 				Username:    "username",
 
 				SkipConnectionCheck: true,
+				AutoDiscoveryLimit:  -1,
 			},
 		}
 		_, err := client.Default.PostgreSQL.AddPostgreSQL(params)
@@ -382,6 +388,7 @@ func TestAddPostgreSQL(t *testing.T) {
 				Username:    "username",
 
 				SkipConnectionCheck: true,
+				AutoDiscoveryLimit:  5,
 			},
 		}
 		addPostgreSQLOK, err := client.Default.PostgreSQL.AddPostgreSQL(params)
@@ -433,6 +440,7 @@ func TestAddPostgreSQL(t *testing.T) {
 					Username:           "username",
 					PushMetricsEnabled: true,
 					Status:             &AgentStatusUnknown,
+					AutoDiscoveryLimit: 5,
 				},
 			},
 		}, *listAgents.Payload)
@@ -659,6 +667,7 @@ func TestAddPostgreSQL(t *testing.T) {
 					Username:           "username",
 					PushMetricsEnabled: true,
 					Status:             &AgentStatusUnknown,
+					AutoDiscoveryLimit: 10,
 				},
 			},
 		}, *listAgents.Payload)
@@ -728,11 +737,12 @@ func TestAddPostgreSQL(t *testing.T) {
 		assert.Equal(t, agents.ListAgentsOKBody{
 			PostgresExporter: []*agents.ListAgentsOKBodyPostgresExporterItems0{
 				{
-					AgentID:    listAgents.Payload.PostgresExporter[0].AgentID,
-					ServiceID:  serviceID,
-					PMMAgentID: pmmAgentID,
-					Username:   "username",
-					Status:     &AgentStatusUnknown,
+					AgentID:            listAgents.Payload.PostgresExporter[0].AgentID,
+					ServiceID:          serviceID,
+					PMMAgentID:         pmmAgentID,
+					Username:           "username",
+					Status:             &AgentStatusUnknown,
+					AutoDiscoveryLimit: 10,
 				},
 			},
 		}, *listAgents.Payload)
@@ -808,6 +818,7 @@ func TestAddPostgreSQL(t *testing.T) {
 					Username:           "username",
 					PushMetricsEnabled: true,
 					Status:             &AgentStatusUnknown,
+					AutoDiscoveryLimit: 10,
 				},
 			},
 		}, *listAgents.Payload)