OpenTelemetry + APM Metrics issue

go.mod

module otel-example

go 1.17

require (
	github.com/rs/zerolog v1.25.0
	go.opentelemetry.io/otel v1.0.1
	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.24.0
	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.24.0
	go.opentelemetry.io/otel/metric v0.24.0
	go.opentelemetry.io/otel/sdk v1.0.1
	go.opentelemetry.io/otel/sdk/metric v0.24.0
	google.golang.org/grpc v1.41.0
)

require (
	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
	github.com/golang/protobuf v1.5.2 // indirect
	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
	go.opentelemetry.io/otel/internal/metric v0.24.0 // indirect
	go.opentelemetry.io/otel/sdk/export/metric v0.24.0 // indirect
	go.opentelemetry.io/otel/trace v1.0.1 // indirect
	go.opentelemetry.io/proto/otlp v0.9.0 // indirect
	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
	golang.org/x/text v0.3.3 // indirect
	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
	google.golang.org/protobuf v1.27.1 // indirect
)

main.go

package main

import (
	"context"
	"github.com/rs/zerolog/log"
	"time"

	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/metric"
	"go.opentelemetry.io/otel/metric/global"
)

func main() {
	ctx := context.Background()

	telemetryShutdown, err := NewTelemetry(TelConfig{
		Endpoint: "localhost",
	}, log.Logger).Start(ctx)
	if err != nil {
		panic(err)
	}

	defer telemetryShutdown()

	meter := global.Meter("my meter")

	numberOfExecutions := metric.Must(meter).
		NewFloat64Counter(
			"integer.counter",
			metric.WithDescription("I count one integer but I'm actually a float"),
		).Bind(
		[]attribute.KeyValue{
			attribute.String(
				"text",
				"description")}...)
	numberOfExecutions.Add(ctx, 1)

	counter, err := meter.NewFloat64Counter("seconds.counter")
	if err != nil {
		otel.Handle(err)
	}

	log.Info().Msg("Starting counter")

	countSeconds(ctx, counter)
}

func countSeconds(ctx context.Context, counter metric.Float64Counter) {
	ticker := time.NewTicker(1 * time.Second)

	for {
		select {
		case <-ticker.C:
			counter.Add(
				ctx,
				1,
				attribute.Int("code", 299),
				attribute.String("text", "my text"),
				attribute.String("server", "localhost"),
			)
		}
	}
}

telemetry.go

package main

import (
	"context"
	"crypto/tls"
	"github.com/rs/zerolog"
	"go.opentelemetry.io/otel/sdk/metric/selector/simple"
	"google.golang.org/grpc/credentials"

	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric"
	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
	"go.opentelemetry.io/otel/metric/global"
	"go.opentelemetry.io/otel/propagation"
	controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
	processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
	"strings"
	"time"
)

const (
	telemetryConnectTimeout = 10 * time.Second
	telemetryCollectPeriod  = 5 * time.Second
)

type TelConfig struct {
	Endpoint    string
}

type Telemetry struct {
	config TelConfig
	log    zerolog.Logger
}

func NewTelemetry(config TelConfig, log zerolog.Logger) Telemetry {
	return Telemetry{
		config: config,
		log:    log.With().Str("context", "telemetry").Logger(),
	}
}

func (t Telemetry) Start(ctx context.Context) (func(), error) {
	metricExporter, err := t.initMetricExporter(ctx)
	if err != nil {
		return nil, err
	}

	stopMetrics, err := t.initMetrics(ctx, metricExporter)
	if err != nil {
		return nil, err
	}

	return func() {
		// Running from `defer`, so `recover()` should return an error (if panicked)
		panicErr := recover()

		defer func() {
			if panicErr != nil {
				// Make sure the panic still emits
				panic(err)
			}
		}()

		// Stop the pusher (so it will push the last exports to the receiver)
		stopMetrics(ctx)
	}, nil
}

func (t Telemetry) initMetricExporter(ctx context.Context) (*otlpmetric.Exporter, error) {
	timeoutCtx, cancel := context.WithTimeout(ctx, telemetryConnectTimeout)
	defer cancel()

	securityDialOption := otlpmetricgrpc.WithInsecure()
	if strings.HasSuffix(t.config.Endpoint, ":443") {
		securityDialOption = otlpmetricgrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{}))
	}

	exporter, err := otlpmetricgrpc.New(timeoutCtx,
		securityDialOption,
		otlpmetricgrpc.WithEndpoint(t.config.Endpoint),
		otlpmetricgrpc.WithDialOption(), // Non-blocking, as we don't want the user to wait
	)
	if err != nil {
		return nil, err
	}

	return exporter, nil
}

func (t Telemetry) initMetrics(
	ctx context.Context,
	exporter *otlpmetric.Exporter,
) (func(context.Context), error) {
	exporterController := controller.New(
		processor.NewFactory(
			simple.NewWithExactDistribution(),
			exporter,
		),
		controller.WithExporter(exporter),
		controller.WithCollectPeriod(telemetryCollectPeriod),
	)

	err := exporterController.Start(ctx)
	if err != nil {
		return nil, err
	}

	global.SetMeterProvider(exporterController)

	propagator := propagation.NewCompositeTextMapPropagator(
		propagation.Baggage{},
		propagation.TraceContext{},
	)
	otel.SetTextMapPropagator(propagator)

	return func(ctx context.Context) {
		if err = exporterController.Stop(ctx); err != nil {
			t.log.Warn().Err(err).Msg("unable to stop the exporter")
		}
	}, nil
}