Add some runtime metrics of go and nodejs application.

.env

@@ -28,7 +28,7 @@ OTEL_COLLECTOR_HOST=otelcol
 OTEL_COLLECTOR_PORT_GRPC=4317
 OTEL_COLLECTOR_PORT_HTTP=4318
 OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-config.yml
-OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-config-extras.yml
+OTEL_COLLECTOR_CONFIG_EXTRAS=./instana/otelcollector/otelcol-config-extras.yml
 OTEL_EXPORTER_OTLP_ENDPOINT=http://${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT_GRPC}
 PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:8080/otlp-http/v1/traces
 

instana/otelcollector/otelcol-config-extras.yml

@@ -0,0 +1,39 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+# extra settings to be merged into OpenTelemetry Collector configuration
+# do not delete this file
+
+## Example configuration for sending data to your own OTLP HTTP backend
+## Note: the spanmetrics exporter must be included in the exporters array
+## if overriding the traces pipeline.
+##
+processors:
+  resource:
+    attributes:
+      - key: service.instance.id
+        value: otel-demo
+        action: upsert
+
+# Replace the INSTANA_ENDPOINT, more information:
+# https://www.ibm.com/docs/en/instana-observability/current?topic=opentelemetry-sending-data-instana
+exporters:
+  otlp/instana:
+    endpoint: INSTANA_ENDPOINT
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [transform, batch, resource]
+      exporters: [otlp, debug, spanmetrics, otlp/instana]
+    metrics:
+      receivers: [otlp, spanmetrics]
+      processors: [batch, resource]
+      exporters: [otlphttp/prometheus, debug, otlp/instana]
+    logs:
+      receivers: [otlp]
+      processors: [batch, resource]
+      exporters: [opensearch, debug]

src/checkoutservice/main.go

@@ -155,6 +155,11 @@ func main() {
 		}
 	}()
 
+	meter := mp.Meter("checkoutservice")
+	if err := recordRuntimeMetrics(meter); err != nil {
+		log.Fatal(err)
+	}
+
 	err := runtime.Start(runtime.WithMinimumReadMemStatsInterval(time.Second))
 	if err != nil {
 		log.Fatal(err)

src/checkoutservice/runtime_metrics.go

@@ -0,0 +1,61 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+package main
+
+import (
+	"context"
+	"math"
+	"runtime"
+
+	"go.opentelemetry.io/otel/metric"
+)
+
+func recordRuntimeMetrics(meter metric.Meter) error {
+	// Create metric instruments
+
+	var (
+		err error
+
+		memSys       metric.Int64ObservableUpDownCounter
+		pauseTotalMs metric.Int64ObservableCounter
+	)
+
+	if pauseTotalMs, err = meter.Int64ObservableCounter(
+		"process.runtime.go.gc.pause_total_ms",
+		metric.WithDescription("Cumulative nanoseconds in GC stop-the-world pauses since the program started"),
+	); err != nil {
+		return err
+	}
+
+	if memSys, err = meter.Int64ObservableUpDownCounter(
+		"process.runtime.go.mem.sys",
+		metric.WithUnit("By"),
+		metric.WithDescription("Bytes of memory obtained from the OS"),
+	); err != nil {
+		return err
+	}
+
+	// Record the runtime stats periodically
+	if _, err := meter.RegisterCallback(
+		func(ctx context.Context, o metric.Observer) error {
+			var memStats runtime.MemStats
+			runtime.ReadMemStats(&memStats)
+
+			o.ObserveInt64(pauseTotalMs, clampUint64(memStats.PauseTotalNs)/1e6) // GC Pause in ms
+			o.ObserveInt64(memSys, clampUint64(memStats.Sys))
+			return nil
+		},
+		pauseTotalMs, memSys,
+	); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func clampUint64(v uint64) int64 {
+	if v > math.MaxInt64 {
+		return math.MaxInt64
+	}
+	return int64(v)
+}

src/paymentservice/charge.js

@@ -9,6 +9,7 @@ const { FlagdProvider} = require('@openfeature/flagd-provider');
 const flagProvider = new FlagdProvider();
 
 const logger = require('./logger');
+const runtimeMetrics = require('./runtime-metrics');
 const tracer = trace.getTracer('paymentservice');
 const meter = metrics.getMeter('paymentservice');
 const transactionsCounter = meter.createCounter('app.payment.transactions')
@@ -64,5 +65,8 @@ module.exports.charge = async request => {
   const { units, nanos, currencyCode } = request.amount;
   logger.info({transactionId, cardType, lastFourDigits, amount: { units, nanos, currencyCode }}, "Transaction complete.");
   transactionsCounter.add(1, {"app.payment.currency": currencyCode})
+
+  runtimeMetrics.setupRuntimeMetrics();
+
   return { transactionId }
 }

src/paymentservice/opentelemetry.js

@@ -11,6 +11,7 @@ const {awsEc2Detector, awsEksDetector} = require('@opentelemetry/resource-detect
 const {containerDetector} = require('@opentelemetry/resource-detector-container')
 const {gcpDetector} = require('@opentelemetry/resource-detector-gcp')
 const {envDetector, hostDetector, osDetector, processDetector} = require('@opentelemetry/resources')
+const {RuntimeNodeInstrumentation} = require('@opentelemetry/instrumentation-runtime-node')
 
 const sdk = new opentelemetry.NodeSDK({
   traceExporter: new OTLPTraceExporter(),
@@ -20,6 +21,9 @@ const sdk = new opentelemetry.NodeSDK({
       '@opentelemetry/instrumentation-fs': {
         requireParentSpan: true,
       },
+    }),
+    new RuntimeNodeInstrumentation({
+      monitoringPrecision: 5000,
     })
   ],
   metricReader: new PeriodicExportingMetricReader({
@@ -38,4 +42,4 @@ const sdk = new opentelemetry.NodeSDK({
   ],
 })
 
-sdk.start();
+sdk.start();

src/paymentservice/package.json

@@ -16,18 +16,19 @@
     "@grpc/proto-loader": "0.7.13",
     "@openfeature/flagd-provider": "0.13.0",
     "@openfeature/server-sdk": "1.14.0",
+    "@opentelemetry/api": "1.8.0",
     "@opentelemetry/auto-instrumentations-node": "0.46.1",
     "@opentelemetry/core": "1.24.1",
-    "@opentelemetry/resources": "1.24.1",
-    "@opentelemetry/api": "1.8.0",
-    "@opentelemetry/sdk-metrics": "1.24.1",
-    "@opentelemetry/exporter-trace-otlp-grpc": "0.51.1",
     "@opentelemetry/exporter-metrics-otlp-grpc": "0.51.1",
-    "@opentelemetry/sdk-node": "0.51.1",
+    "@opentelemetry/exporter-trace-otlp-grpc": "0.51.1",
+    "@opentelemetry/instrumentation-runtime-node": "^0.10.0",
     "@opentelemetry/resource-detector-alibaba-cloud": "0.28.9",
     "@opentelemetry/resource-detector-aws": "1.5.0",
     "@opentelemetry/resource-detector-container": "0.3.9",
     "@opentelemetry/resource-detector-gcp": "0.29.9",
+    "@opentelemetry/resources": "1.24.1",
+    "@opentelemetry/sdk-metrics": "1.24.1",
+    "@opentelemetry/sdk-node": "0.51.1",
     "grpc-js-health-check": "1.1.0",
     "pino": "8.16.1",
     "simple-card-validator": "1.1.0",