Most of memory is allocated for labels slice and actual labels (strings).

pprof

Active filters:
   ignore=lines
Showing nodes accounting for 144.91GB, 59.15% of 245GB total
Dropped 1435 nodes (cum <= 1.22GB)
Showing top 25 nodes out of 124
      flat  flat%   sum%        cum   cum%
         0     0%     0%   151.11GB 61.68%  net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2007
         0     0%     0%   151.10GB 61.67%  github.com/prometheus/prometheus/web.(*Handler).testReady.func1 /app/web/web.go:483
         0     0%     0%   151.09GB 61.67%  github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1 /app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
         0     0%     0%   151.09GB 61.67%  github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP /app/util/httputil/compression.go:90
         0     0%     0%   151.09GB 61.67%  github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2 /app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
         0     0%     0%   151.08GB 61.67%  github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1 /app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
         0     0%     0%   151.08GB 61.66%  github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1 /app/web/web.go:1120
         0     0%     0%   151.07GB 61.66%  github.com/prometheus/common/route.(*Router).handle.func1 /app/vendor/github.com/prometheus/common/route/route.go:83
         0     0%     0%   151.06GB 61.66%  github.com/julienschmidt/httprouter.(*Router).ServeHTTP /app/vendor/github.com/julienschmidt/httprouter/router.go:334
         0     0%     0%   151.06GB 61.66%  github.com/prometheus/common/route.(*Router).ServeHTTP /app/vendor/github.com/prometheus/common/route/route.go:121
         0     0%     0%   151.05GB 61.65%  net/http.StripPrefix.func1 /usr/local/go/src/net/http/server.go:2046
         0     0%     0%   151.04GB 61.65%  net/http.(*ServeMux).ServeHTTP /usr/local/go/src/net/http/server.go:2387
         0     0%     0%   151.03GB 61.65%  github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5 /app/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:140
         0     0%     0%   150.65GB 61.49%  github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1 /app/web/api/v1/api.go:243
         0     0%     0%   148.93GB 60.79%  github.com/prometheus/prometheus/storage/tsdb.seriesSet.Next /app/storage/tsdb/tsdb.go:275
         0     0%     0%   147.73GB 60.30%  github.com/prometheus/prometheus/tsdb.(*blockSeriesSet).Next /app/tsdb/querier.go:883
         0     0%     0%   144.23GB 58.87%  github.com/prometheus/prometheus/tsdb.(*mergedSeriesSet).Next /app/tsdb/querier.go:599
         0     0%     0%   144.23GB 58.87%  github.com/prometheus/prometheus/tsdb.(*mergedSeriesSet).nextWithID /app/tsdb/querier.go:576
         0     0%     0%   143.33GB 58.50%  github.com/prometheus/prometheus/web/api/v1.(*API).series /app/web/api/v1/api.go:563
         0     0%     0%   142.65GB 58.22%  github.com/prometheus/prometheus/tsdb.(*populatedChunkSeries).Next /app/tsdb/querier.go:824
         0     0%     0%    96.19GB 39.26%  github.com/prometheus/prometheus/web.withStackTracer.func1 /app/web/web.go:107
         0     0%     0%    96.16GB 39.25%  net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:2802
         0     0%     0%    94.53GB 38.59%  net/http.(*conn).serve /usr/local/go/src/net/http/server.go:1890
   84.81GB 34.62% 34.62%    84.81GB 34.62%  github.com/prometheus/prometheus/tsdb.(*baseChunkSeries).Next /app/tsdb/querier.go:759
   60.10GB 24.53% 59.15%    60.10GB 24.53%  github.com/prometheus/prometheus/tsdb/encoding.(*Decbuf).UvarintStr /app/tsdb/encoding/encoding.go:187