Golang HTTP Server Context Propagation Issue

main.go

package main

import (
	"context"
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"time"

	"github.com/gorilla/mux"

	"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
	"go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace"
	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
	"go.opentelemetry.io/contrib/propagators/aws/xray"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp"
	"go.opentelemetry.io/otel/propagation"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

var tracer = otel.Tracer("work-service")

var statusCodesCommonWeighted = [35]int{
	200, 200, 200, 200, 200, 200, 200, 200, 200, 200,
	300, 301, 302, 304, 307, 400, 401, 403, 404, 404,
	404, 404, 404, 410, 500, 500, 500, 500, 500, 501,
	503, 503, 503, 503, 550}

var client = http.Client{
	Transport: otelhttp.NewTransport(http.DefaultTransport),
}

func initTracer() {
	// Create new OTLP Exporter struct
	exporter, err := otlp.NewExporter(
		context.Background(),
		otlp.WithInsecure(),
		otlp.WithAddress("localhost:55680"),
	)
	if err != nil {
		log.Fatal("failed to create otlp exporter", err)
	}

	// Instantiate a new ECS Resource detector
	// eksResourceDetector := eks.ResourceDetector{}
	// resource, err := eksResourceDetector.Detect(context.Background())
	// if err != nil {
	// 	log.Fatal("failed to start eks resource detector")
	// }

	// AlwaysSample() returns a Sampler that samples every trace.
	// Be careful about using this sampler in a production application with
	// significant traffic: a new trace will be started and exported for every request.
	cfg := sdktrace.Config{
		DefaultSampler: sdktrace.AlwaysSample(),
	}

	// A custom ID Generator to generate traceIDs that conform to
	// AWS X-Ray traceID format
	idg := xray.NewIDGenerator()

	// Create a new TraceProvider object passing in the config, the exporter
	// and the ID Generator we want to use for our tracing
	tp := sdktrace.NewTracerProvider(
		sdktrace.WithConfig(cfg),
		sdktrace.WithSyncer(exporter),
		sdktrace.WithIDGenerator(idg),
		// sdktrace.WithResource(resource),
	)

	// Set the traceprovider and the propagator we want to use
	otel.SetTracerProvider(tp)
	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
		propagation.TraceContext{},
		propagation.Baggage{},
		xray.Propagator{},
	))
}

func init() {
	rand.Seed(time.Now().Unix())
}

func main() {
	initTracer()
	r := mux.NewRouter()
	r.Use(otelmux.Middleware("service-a"))

	r.HandleFunc("/work", handler).Methods(http.MethodGet)

	go func() {
		log.Fatal(http.ListenAndServe(":8080", r))
	}()

	// call yourself to generate some traffic
	go func() {
		for {
			<-time.After(5 * time.Second)
			resp, err := http.Get("http://service-a.aws-otel-eks.svc.cluster.local/work")
			if err != nil {
				log.Println("failed to make /work request", err)
				continue
			}
			log.Printf("work request status code: %d", resp.StatusCode)
		}
	}()

	select {}
}

func handler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")

	// call service-b, passing request context
	req, _ := http.NewRequest(http.MethodGet, "http://service-b.aws-otel-eks.svc.cluster.local/work", nil)
	ctx, req := otelhttptrace.W3C(r.Context(), req)
	otelhttptrace.Inject(ctx, req)

	resp, err := client.Do(req)
	if err != nil {
		log.Println("failed to make /work request", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	log.Printf("service-b request status code: %d", resp.StatusCode)

	_, span := tracer.Start(r.Context(), "handle workB response")
	defer span.End()

	// do some work
	randInt := rand.Intn(10)
	time.Sleep(time.Duration(randInt) * time.Millisecond)

	w.WriteHeader(statusCodesCommonWeighted[rand.Intn(len(statusCodesCommonWeighted))])

	json.NewEncoder(w).Encode(time.Now().Local().String())
}