diff --git a/README.md b/README.md index a060483..0be9b01 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ virt-install --name=test_vm --arch=x86_64 --vcpus=1 --ram=512 --os-type=linux -- ``` 4. Load Snap libvirt collector plugin and create task: ``` -wget https://raw.githubusercontent.com/intelsdi-x/snap-plugin-collector-libvirt/master/example/task-example.yaml +wget https://raw.githubusercontent.com/intelsdi-x/snap-plugin-collector-libvirt/master/example/tasks/task-example.yaml snaptel plugin load snap-plugin-collector-libvirt snaptel task create -t task-example.yaml ``` diff --git a/example/snapteld-config-sample.json b/example/config/snapteld-config-sample.json similarity index 100% rename from example/snapteld-config-sample.json rename to example/config/snapteld-config-sample.json diff --git a/example/task-example.yaml b/example/tasks/task-example.yaml similarity index 100% rename from example/task-example.yaml rename to example/tasks/task-example.yaml diff --git a/example/tasks/task-static-example.yaml b/example/tasks/task-static-example.yaml new file mode 100644 index 0000000..56ea6a6 --- /dev/null +++ b/example/tasks/task-static-example.yaml @@ -0,0 +1,22 @@ + +--- + version: 1 + schedule: + type: "simple" + interval: "1s" + max-failures: 10 + workflow: + collect: + metrics: + /intel/libvirt/test/cpu/cputime: {} + /intel/libvirt/test/disk/hd0/rdbytes: {} + /intel/libvirt/test/disk/hd0/rdreq: {} + /intel/libvirt/test/disk/hd0/wrbytes: {} + /intel/libvirt/test/disk/hd0/wrreq: {} + /intel/libvirt/test/network/vir0/rxbytes: {} + /intel/libvirt/test/network/vir0/txbytes: {} + publish: + - plugin_name: "file" + config: + file: "/tmp/libvirt_metrics.log" + diff --git a/libvirt/libvirt.go b/libvirt/libvirt.go index f63821a..87c674f 100644 --- a/libvirt/libvirt.go +++ b/libvirt/libvirt.go @@ -31,6 +31,8 @@ type Libvirt interface { GetDomainInterfaces(domain libvirt.VirDomain) ([]string, error) GetDomainDisks(domain libvirt.VirDomain) ([]string, error) GetVCPUStatistics(domain libvirt.VirDomain) (map[string]int64, error) + GetRequestedInstances(conn libvirt.VirConnection, domainNames []string) ([]libvirt.VirDomain, error) + GetInstanceByDomainName(conn libvirt.VirConnection, domainName string) (libvirt.VirDomain, error) } // GetInstanceIds return all names of active VirDomains @@ -67,3 +69,22 @@ func GetInstances(conn libvirt.VirConnection) ([]libvirt.VirDomain, error) { } return libvirtDomains, nil } + +// GetRequestedInstances return all instances from domainNames slice +func GetRequestedInstances(conn libvirt.VirConnection, domainNames []string) ([]libvirt.VirDomain, error) { + var libvirtDomains []libvirt.VirDomain + for _, domainName := range domainNames { + domain, err := GetInstanceByDomainName(conn, domainName) + if err != nil { + return libvirtDomains, err + } + libvirtDomains = append(libvirtDomains, domain) + } + return libvirtDomains, nil + +} + +// GetInstanceByDomainName return instance by DomainName +func GetInstanceByDomainName(conn libvirt.VirConnection, domainName string) (libvirt.VirDomain, error) { + return conn.LookupDomainByName(domainName) +} diff --git a/libvirt/libvirt_test.go b/libvirt/libvirt_test.go index 87e89fe..8952e28 100644 --- a/libvirt/libvirt_test.go +++ b/libvirt/libvirt_test.go @@ -38,6 +38,13 @@ func TestLibirt(t *testing.T) { So(err, ShouldNotBeNil) So(len(interf), ShouldResemble, 0) }) + Convey("Get DomainNames", t, func() { + conn, err := libvirt.NewVirConnection("test:///default") + So(err, ShouldBeNil) + domains, err := GetInstanceIds(conn) + So(err, ShouldBeNil) + So(len(domains), ShouldResemble, 1) + }) Convey("Get Interface Name when Interface exist", t, func() { buf, err := ioutil.ReadFile("./test_domain.xml") if err != nil { @@ -133,4 +140,20 @@ func TestLibirt(t *testing.T) { expectedData := map[string]string{"nova_uuid": "5a26891c-efb0-4c6a-8bef-b54c45296136"} So(data, ShouldResemble, expectedData) }) + Convey("Get Instances from a slice", t, func() { + conn, err := libvirt.NewVirConnection("test:///default") + instances := []string{"test"} + So(err, ShouldBeNil) + domains, err := GetRequestedInstances(conn, instances) + So(err, ShouldBeNil) + So(len(domains), ShouldResemble, 1) + }) + Convey("Get Instances from a slice, when instance doesn't exist", t, func() { + conn, err := libvirt.NewVirConnection("test:///default") + instances := []string{"testi1"} + So(err, ShouldBeNil) + domains, err := GetRequestedInstances(conn, instances) + So(err, ShouldNotBeNil) + So(len(domains), ShouldResemble, 0) + }) } diff --git a/libvirtcollector/common.go b/libvirtcollector/common.go index 2f2210b..03be073 100644 --- a/libvirtcollector/common.go +++ b/libvirtcollector/common.go @@ -1,11 +1,23 @@ package libvirtcollector import ( + "bytes" + "strings" "time" + wrapper "github.com/intelsdi-x/snap-plugin-collector-libvirt/libvirt" "github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin" + libvirt "github.com/sandlbn/libvirt-go" ) +func getLibvirtURI(cfg plugin.Config) string { + uri, err := cfg.GetString("uri") + if err != nil { + return defaultURI + } + return uri +} + func createMetric(ns plugin.Namespace) plugin.Metric { metricType := plugin.Metric{ @@ -22,6 +34,7 @@ func filterNamespace(metricType string, mts []plugin.Metric) (int, []plugin.Metr filteredMetrics = append(filteredMetrics, m) } } + filteredMetrics = removeDuplicates(filteredMetrics) return len(filteredMetrics), filteredMetrics } @@ -65,3 +78,64 @@ func copyNamespaceElements(ns []plugin.NamespaceElement) []plugin.NamespaceEleme copy(newNs, ns) return newNs } + +func removeDuplicates(elements []plugin.Metric) []plugin.Metric { + // Use map to record duplicates as we find them. + encountered := map[string]bool{} + result := []plugin.Metric{} + + for _, v := range elements { + ns := strings.Join(v.Namespace.Strings(), "/") + if !encountered[ns] { + encountered[ns] = true + result = append(result, v) + } + } + // Return the new slice. + return result +} + +func metricStored(elements []plugin.Metric, newNamespace []plugin.NamespaceElement) bool { + for _, v := range elements { + ns := strings.Join(v.Namespace.Strings(), "") + if ns == joinNamespaceElements(newNamespace) { + return true + } + } + return false +} + +func joinNamespaceElements(ns []plugin.NamespaceElement) string { + var buffer bytes.Buffer + for _, v := range ns { + buffer.WriteString(v.Value) + } + return buffer.String() +} + +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +func getInstances(conn libvirt.VirConnection, elements []plugin.Metric) ([]libvirt.VirDomain, error) { + + instances := []string{} + + for _, v := range elements { + domain := v.Namespace.Strings()[nsDomainPosition] + + if domain == "*" { + return wrapper.GetInstances(conn) + } + if !contains(instances, domain) { + instances = append(instances, domain) + } + } + return wrapper.GetRequestedInstances(conn, instances) + +} diff --git a/libvirtcollector/libvirtcollector.go b/libvirtcollector/libvirtcollector.go index e4d9d55..760f480 100644 --- a/libvirtcollector/libvirtcollector.go +++ b/libvirtcollector/libvirtcollector.go @@ -33,7 +33,7 @@ const ( // Plugin plugin name Plugin = "libvirt" // Version of plugin - Version = 12 + Version = 13 nsDomainPosition = 2 nsMetricPostion = 3 nsDevicePosition = 4 @@ -79,7 +79,7 @@ func (LibvirtCollector) CollectMetrics(mts []plugin.Metric) ([]plugin.Metric, er } defer conn.UnrefAndCloseConnection() - ids, err := wrapper.GetInstances(conn) + ids, err := getInstances(conn, mts) if err != nil { return metrics, err } @@ -107,39 +107,49 @@ func (LibvirtCollector) CollectMetrics(mts []plugin.Metric) ([]plugin.Metric, er } for _, mt := range netMetrics { ns := copyNamespace(mt) - ns[nsDomainPosition].Value, err = id.GetName() + if ns[nsDomainPosition].IsDynamic() { + ns[nsDomainPosition].Value, err = id.GetName() + } for k, v := range netCounters { newNamespace := copyNamespaceElements(ns) - newNamespace[nsDevicePosition].Value = k - var value int64 - switch ns[nsSubMetricPostion].Value { - case "rxbytes": - value = v.RxBytes - case "rxpackets": - value = v.RxPackets - case "rxerrs": - value = v.RxErrs - case "rxdrop": - value = v.RxDrop - case "txbytes": - value = v.TxBytes - case "txpackets": - value = v.TxPackets - case "txerrs": - value = v.TxErrs - case "txdrop": - value = v.TxDrop + if newNamespace[nsDevicePosition].IsDynamic() { + newNamespace[nsDevicePosition].Value = k + } + if newNamespace[nsDevicePosition].Value == k { + var value int64 + switch ns[nsSubMetricPostion].Value { + case "rxbytes": + value = v.RxBytes + case "rxpackets": + value = v.RxPackets + case "rxerrs": + value = v.RxErrs + case "rxdrop": + value = v.RxDrop + case "txbytes": + value = v.TxBytes + case "txpackets": + value = v.TxPackets + case "txerrs": + value = v.TxErrs + case "txdrop": + value = v.TxDrop + } + if !metricStored(metrics, newNamespace) { + metrics = append(metrics, createNamespace(mt, value, newNamespace, meta)) + } } - metrics = append(metrics, createNamespace(mt, value, newNamespace, meta)) } } for _, mt := range diskMetrics { ns := copyNamespace(mt) - ns[nsDomainPosition].Value, err = id.GetName() - if err != nil { - return metrics, err + if ns[nsDomainPosition].IsDynamic() { + ns[nsDomainPosition].Value, err = id.GetName() + if err != nil { + return metrics, err + } } if diskCount > 0 { diskCounters, err = wrapper.GetBlockStatistics(id) @@ -149,41 +159,53 @@ func (LibvirtCollector) CollectMetrics(mts []plugin.Metric) ([]plugin.Metric, er } for k, v := range diskCounters { newNamespace := copyNamespaceElements(ns) - newNamespace[nsDevicePosition].Value = k - var value int64 - switch ns[nsSubMetricPostion].Value { - case "wrreq": - value = v.RdReq - case "rdreq": - value = v.RdReq - case "wrbytes": - value = v.WrBytes - case "rdbytes": - value = v.RdBytes + if newNamespace[nsDevicePosition].IsDynamic() { + newNamespace[nsDevicePosition].Value = k + } + if newNamespace[nsDevicePosition].Value == k { + var value int64 + switch ns[nsSubMetricPostion].Value { + case "wrreq": + value = v.RdReq + case "rdreq": + value = v.RdReq + case "wrbytes": + value = v.WrBytes + case "rdbytes": + value = v.RdBytes + } + if !metricStored(metrics, newNamespace) { + metrics = append(metrics, createNamespace(mt, value, newNamespace, meta)) + } } - metrics = append(metrics, createNamespace(mt, value, newNamespace, meta)) } } for _, mt := range memoryMetrics { ns := copyNamespace(mt) - ns[nsDomainPosition].Value, err = id.GetName() + if ns[nsDomainPosition].IsDynamic() { + ns[nsDomainPosition].Value, err = id.GetName() - if err != nil { - return metrics, err + if err != nil { + return metrics, err + } } memKey := ns[nsDevicePosition].Value memoryStat, err := wrapper.GetMemoryStatistics(id, memKey) if err != nil { return metrics, err } - metrics = append(metrics, createNamespace(mt, memoryStat[memKey], ns, meta)) + if !metricStored(metrics, ns) { + metrics = append(metrics, createNamespace(mt, memoryStat[memKey], ns, meta)) + } } for _, mt := range cpuMetrics { ns := copyNamespace(mt) - ns[nsDomainPosition].Value, err = id.GetName() - if err != nil { - return metrics, err + if ns[nsDomainPosition].IsDynamic() { + ns[nsDomainPosition].Value, err = id.GetName() + if err != nil { + return metrics, err + } } secondlastElement := len(ns) - 2 if ns[secondlastElement].IsDynamic() { @@ -201,7 +223,9 @@ func (LibvirtCollector) CollectMetrics(mts []plugin.Metric) ([]plugin.Metric, er if err != nil { return metrics, err } - metrics = append(metrics, createNamespace(mt, cpuTime, ns, meta)) + if !metricStored(metrics, ns) { + metrics = append(metrics, createNamespace(mt, cpuTime, ns, meta)) + } } @@ -227,6 +251,51 @@ func (LibvirtCollector) GetMetricTypes(cfg plugin.Config) ([]plugin.Metric, erro var metrics []plugin.Metric + uri := getLibvirtURI(cfg) + + conn, err := libvirt.NewVirConnection(uri) + defer conn.CloseConnection() + + ids, err := wrapper.GetInstances(conn) + if err != nil { + return metrics, err + } + + for _, domain := range ids { + + domainName, err := domain.GetName() + if err != nil { + return metrics, err + } + + ns := plugin.NewNamespace(Vendor, Plugin, domainName) + for _, value := range nsTypes.cpu { + metrics = append(metrics, createMetric(ns.AddStaticElements("cpu", value))) + metrics = append(metrics, createMetric(ns.AddStaticElement("cpu").AddDynamicElement("cpu_id", "id of vcpu").AddStaticElement(value))) + + } + for _, value := range nsTypes.disk { + //ignoring errors, domain don't need to have disk attached + disks, _ := wrapper.GetDomainDisks(domain) + for _, disk := range disks { + metrics = append(metrics, createMetric( + ns.AddStaticElements("disk", disk, value))) + } + } + for _, value := range nsTypes.network { + //ignoring errors, domain don't need to have network interface + interfaces, _ := wrapper.GetDomainInterfaces(domain) + for _, netInterface := range interfaces { + metrics = append(metrics, createMetric( + ns.AddStaticElements("network", netInterface, value))) + } + } + for _, value := range nsTypes.memory { + metrics = append(metrics, createMetric(ns.AddStaticElements("memory", value))) + } + + } + ns := plugin.NewNamespace(Vendor, Plugin). AddDynamicElement("domain_id", "an id of libvirt domain") diff --git a/libvirtcollector/libvirtcollector_test.go b/libvirtcollector/libvirtcollector_test.go index 6d0c9c1..4841d9a 100644 --- a/libvirtcollector/libvirtcollector_test.go +++ b/libvirtcollector/libvirtcollector_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin" + libvirt "github.com/sandlbn/libvirt-go" . "github.com/smartystreets/goconvey/convey" ) @@ -51,10 +52,13 @@ func TestLibirtPlugin(t *testing.T) { }) Convey("Get Metric Types", t, func() { libvirtCol := LibvirtCollector{} - var cfg = plugin.Config{} + cfg := plugin.Config{ + "uri": "test:///default", + "nova": false, + } metrics, err := libvirtCol.GetMetricTypes(cfg) So(err, ShouldBeNil) - So(len(metrics), ShouldResemble, 25) + So(len(metrics), ShouldResemble, 38) }) Convey("Collect Metrics", t, func() { libvirtCol := LibvirtCollector{} @@ -84,7 +88,6 @@ func TestLibirtPlugin(t *testing.T) { mts := []plugin.Metric{} mts = append(mts, plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddDynamicElement("domain_id", "an id of libvirt domain").AddStaticElement("cpu").AddDynamicElement("cpu_id", "id of vcpu").AddStaticElement("cputime"), Config: config}) - metrics, err := libvirtCol.CollectMetrics(mts) So(err, ShouldBeNil) So(len(metrics), ShouldResemble, 2) @@ -95,6 +98,24 @@ func TestLibirtPlugin(t *testing.T) { So(metrics[1].Namespace.Strings()[secondLastElement], ShouldBeIn, []string{"0", "1"}) So(metrics[1].Namespace.Strings()[2], ShouldResemble, "test") + }) + Convey("Check if metric is not collected twice", t, func() { + libvirtCol := LibvirtCollector{} + + config := plugin.Config{ + "uri": "test:///default", + "nova": false, + } + mts := []plugin.Metric{} + + mts = append(mts, plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddDynamicElement("domain_id", "an id of libvirt domain").AddStaticElement("cpu").AddDynamicElement("cpu_id", "id of vcpu").AddStaticElement("cputime"), Config: config}) + mts = append(mts, plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddDynamicElement("domain_id", "an id of libvirt domain").AddStaticElement("cpu").AddDynamicElement("cpu_id", "id of vcpu").AddStaticElement("cputime"), Config: config}) + mts = append(mts, plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddDynamicElement("domain_id", "an id of libvirt domain").AddStaticElements("cpu", "cputime"), Config: config}) + mts = append(mts, plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddDynamicElement("domain_id", "an id of libvirt domain").AddStaticElements("cpu", "cputime"), Config: config}) + metrics, err := libvirtCol.CollectMetrics(mts) + So(err, ShouldBeNil) + So(len(metrics), ShouldResemble, 3) + }) Convey("Merge two map[string]string", t, func() { @@ -138,5 +159,40 @@ func TestLibirtPlugin(t *testing.T) { So(countResult, ShouldResemble, 0) }) + Convey("metric Stored", t, func() { + mts := []plugin.Metric{ + plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin, "test", "cpu", "cputime")}, + } + + ns := plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddDynamicElement("domain_id", "an id of libvirt domain").AddStaticElements("cpu", "cputime")} + ns.Namespace[2].Value = "test" + + stored := metricStored(mts, ns.Namespace) + So(stored, ShouldResemble, true) + + }) + Convey("GetInstances", t, func() { + mts := []plugin.Metric{} + conn, err := libvirt.NewVirConnection("test:///default") + config := plugin.Config{ + "uri": "test:///default", + "nova": false, + } + So(err, ShouldBeNil) + + mts = append(mts, plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddDynamicElement("domain_id", "an id of libvirt domain").AddStaticElement("cpu").AddDynamicElement("cpu_id", "id of vcpu").AddStaticElement("cputime"), Config: config}) + mts = append(mts, plugin.Metric{Namespace: plugin.NewNamespace(Vendor, Plugin).AddStaticElement("test").AddStaticElement("cpu").AddDynamicElement("cpu_id", "id of vcpu").AddStaticElement("cputime"), Config: config}) + result, err := getInstances(conn, mts) + So(len(result), ShouldResemble, 1) + So(err, ShouldBeNil) + + }) + Convey("Contains", t, func() { + source := []string{"one", "two"} + + So(true, ShouldResemble, contains(source, "one")) + So(false, ShouldResemble, contains(source, "three")) + + }) }