Created
May 26, 2020 12:49
-
-
Save nabam/6f028b2dfedaf68c6ebee4842e5b78e2 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <chrono> | |
#include <thread> | |
#include <malloc.h> | |
#include <stdio.h> | |
#include <stdlib.h> | |
#include "opencensus/stats/stats.h" | |
#include "absl/strings/string_view.h" | |
#define MALLOC_INFO_BUFFER 1024 * 1024 | |
namespace webinterface { | |
const size_t TRIM_MEMORY_AFTER_RSS_B = 1L * 1024 * 1024 * 1024; | |
const size_t TRIM_MEMORY_AFTER_RSS_AGGRESSIVE_B = 1L * 3 * 1024 * 1024 * 1024; | |
const unsigned int RELEASE_MEMORY_CHECK_INTERVAL_MS = 30 * 1000; | |
const unsigned int TRY_RELEASE_MEMORY_EVERY_N_CHECKS = 10; | |
ABSL_CONST_INIT const absl::string_view kMallinfoArenaMeasureName = "mallinfo/arena"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoOrdblksMeasureName = "mallinfo/ordblks"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoSmblksMeasureName = "mallinfo/smblks"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoHblksMeasureName = "mallinfo/hblks"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoHblkhdMeasureName = "mallinfo/hblkhd"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoUsmblksMeasureName = "mallinfo/usmblks"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoFsmblksMeasureName = "mallinfo/fsmblks"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoUordblksMeasureName = "mallinfo/uordblks"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoFordblksMeasureName = "mallinfo/fordblks"; | |
ABSL_CONST_INIT const absl::string_view kMallinfoKeepcostMeasureName = "mallinfo/keepcost"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoFastCountMeasureName = "malloc_info/fast_count"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoFastSizeMeasureName = "malloc_info/fast_size"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoRestCountMeasureName = "malloc_info/rest_count"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoRestSizeMeasureName = "malloc_info/rest_size"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoMmapCountMeasureName = "malloc_info/mmap_count"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoMmapSizeMeasureName = "malloc_info/mmap_size"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoCurrentSizeMeasureName = "malloc_info/current_size"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoMaxSizeMeasureName = "malloc_info/max_size"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoTotalSizeMeasureName = "malloc_info/total_size"; | |
ABSL_CONST_INIT const absl::string_view kMallocInfoMprotectSizeMeasureName = "malloc_info/mprotect_size"; | |
opencensus::stats::MeasureInt64 MallocInfoFastCountMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoFastCountMeasureName, "fast_count", "n"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoFastSizeMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoFastSizeMeasureName, "fast_size", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoRestCountMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoRestCountMeasureName, "rest_count", "n"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoRestSizeMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoRestSizeMeasureName, "rest_size", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoMmapCountMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoMmapCountMeasureName, "mmap_count", "n"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoMmapSizeMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoMmapSizeMeasureName, "mmap_size", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoCurrentSizeMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoCurrentSizeMeasureName, "current_size", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoMaxSizeMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoMaxSizeMeasureName, "max_size", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoTotalSizeMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoTotalSizeMeasureName, "total_size", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallocInfoMprotectSizeMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallocInfoMprotectSizeMeasureName, "mprotect_size", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoArenaMeasure() { | |
static const opencensus::stats::MeasureInt64 m = | |
opencensus::stats::MeasureInt64::Register(kMallinfoArenaMeasureName, "Non-mmapped space allocated", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoOrdblksMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallinfoOrdblksMeasureName, "Number of free chunks", "chunks"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoSmblksMeasure() { | |
static const opencensus::stats::MeasureInt64 m = | |
opencensus::stats::MeasureInt64::Register(kMallinfoSmblksMeasureName, "Number of free fastbin blocks", "blocks"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoHblksMeasure() { | |
static const opencensus::stats::MeasureInt64 m = | |
opencensus::stats::MeasureInt64::Register(kMallinfoHblksMeasureName, "Number of mmapped regions", "regions"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoHblkhdMeasure() { | |
static const opencensus::stats::MeasureInt64 m = | |
opencensus::stats::MeasureInt64::Register(kMallinfoHblkhdMeasureName, "Space allocated in mmapped regions", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoUsmblksMeasure() { | |
static const opencensus::stats::MeasureInt64 m = | |
opencensus::stats::MeasureInt64::Register(kMallinfoUsmblksMeasureName, "Maximum total allocated space", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoFsmblksMeasure() { | |
static const opencensus::stats::MeasureInt64 m = | |
opencensus::stats::MeasureInt64::Register(kMallinfoFsmblksMeasureName, "Space in freed fastbin blocks", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoUordblksMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallinfoUordblksMeasureName, "Total allocated space", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoFordblksMeasure() { | |
static const opencensus::stats::MeasureInt64 m = opencensus::stats::MeasureInt64::Register(kMallinfoFordblksMeasureName, "Total free space", "bytes"); | |
return m; | |
} | |
opencensus::stats::MeasureInt64 MallinfoKeepcostMeasure() { | |
static const opencensus::stats::MeasureInt64 m = | |
opencensus::stats::MeasureInt64::Register(kMallinfoKeepcostMeasureName, "Top-most, releasable space", "bytes"); | |
return m; | |
} | |
void register_malloc_metrics_views() { | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/arena") | |
.set_description("Non-mmapped space allocated") | |
.set_measure(kMallinfoArenaMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/ordblks") | |
.set_description("Number of free chunks") | |
.set_measure(kMallinfoOrdblksMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/smblks") | |
.set_description("Number of free fastbin blocks") | |
.set_measure(kMallinfoSmblksMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/hblks") | |
.set_description("Number of mmapped regions") | |
.set_measure(kMallinfoHblksMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/hblkhd") | |
.set_description("Space allocated in mmapped regions") | |
.set_measure(kMallinfoHblkhdMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/usmblks") | |
.set_description("Maximum total allocated space") | |
.set_measure(kMallinfoUsmblksMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/fsmblks") | |
.set_description("Space in freed fastbin blocks") | |
.set_measure(kMallinfoFsmblksMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/uordblks") | |
.set_description("Total allocated space") | |
.set_measure(kMallinfoUordblksMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/fordblks") | |
.set_description("Total free space") | |
.set_measure(kMallinfoFordblksMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/mallinfo/keepcost") | |
.set_description("Top-most, releasable space") | |
.set_measure(kMallinfoKeepcostMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/fast_count") | |
.set_description("fast_count") | |
.set_measure(kMallocInfoFastCountMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/fast_size") | |
.set_description("fast_size") | |
.set_measure(kMallocInfoFastSizeMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/rest_count") | |
.set_description("rest_count") | |
.set_measure(kMallocInfoRestCountMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/rest_size") | |
.set_description("rest_size") | |
.set_measure(kMallocInfoRestSizeMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/mmap_count") | |
.set_description("mmap_count") | |
.set_measure(kMallocInfoMmapCountMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/mmap_size") | |
.set_description("mmap_size") | |
.set_measure(kMallocInfoMmapSizeMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/current_size") | |
.set_description("current_size") | |
.set_measure(kMallocInfoCurrentSizeMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/max_size") | |
.set_description("max_size") | |
.set_measure(kMallocInfoMaxSizeMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/total_size") | |
.set_description("total_size") | |
.set_measure(kMallocInfoTotalSizeMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
opencensus::stats::ViewDescriptor() | |
.set_name("com.mwapi/malloc_info/mprotect_size") | |
.set_description("mprotect_size") | |
.set_measure(kMallocInfoMprotectSizeMeasureName) | |
.set_aggregation(opencensus::stats::Aggregation::LastValue()) | |
.RegisterForExport(); | |
} | |
void update_malloc_metrics() { | |
// Silly parser for anal xml output of malloc_info | |
static char buffer[MALLOC_INFO_BUFFER]; | |
FILE* f = fmemopen(buffer, sizeof(buffer), "w"); | |
malloc_info(0, f); | |
fclose(f); | |
size_t fast_count = 0, fast_size = 0, rest_count = 0, rest_size = 0, mmap_count = 0, mmap_size = 0, current = 0, max = 0, total = 0, mprotect = 0; | |
char* c = buffer; | |
while ((c - buffer) < sizeof(buffer) && *c != '\0') { | |
if (*c++ != '\n') { | |
continue; | |
} | |
if (sscanf(c, | |
"</heap>\n" | |
"<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n" | |
"<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n" | |
"<total type=\"mmap\" count=\"%zu\" size=\"%zu\"/>\n" | |
"<system type=\"current\" size=\"%zu\"/>\n" | |
"<system type=\"max\" size=\"%zu\"/>\n" | |
"<aspace type=\"total\" size=\"%zu\"/>\n" | |
"<aspace type=\"mprotect\" size=\"%zu\"/>\n" | |
"</malloc>\n", | |
&fast_count, &fast_size, &rest_count, &rest_size, &mmap_count, &mmap_size, ¤t, &max, &total, &mprotect) > 0) { | |
break; | |
} | |
} | |
struct mallinfo mi; | |
mi = mallinfo(); | |
opencensus::stats::Record({{MallinfoArenaMeasure(), (unsigned int)mi.arena}, | |
{MallinfoOrdblksMeasure(), (unsigned int)mi.ordblks}, | |
{MallinfoSmblksMeasure(), (unsigned int)mi.smblks}, | |
{MallinfoHblksMeasure(), (unsigned int)mi.hblks}, | |
{MallinfoHblkhdMeasure(), (unsigned int)mi.hblkhd}, | |
{MallinfoUsmblksMeasure(), (unsigned int)mi.usmblks}, | |
{MallinfoFsmblksMeasure(), (unsigned int)mi.fsmblks}, | |
{MallinfoUordblksMeasure(), (unsigned int)mi.uordblks}, | |
{MallinfoFordblksMeasure(), (unsigned int)mi.fordblks}, | |
{MallinfoKeepcostMeasure(), (unsigned int)mi.keepcost}, | |
{MallocInfoFastCountMeasure(), fast_count}, | |
{MallocInfoFastSizeMeasure(), fast_size}, | |
{MallocInfoRestCountMeasure(), rest_count}, | |
{MallocInfoRestSizeMeasure(), rest_size}, | |
{MallocInfoMmapCountMeasure(), mmap_count}, | |
{MallocInfoMmapSizeMeasure(), mmap_size}, | |
{MallocInfoCurrentSizeMeasure(), current}, | |
{MallocInfoMaxSizeMeasure(), max}, | |
{MallocInfoTotalSizeMeasure(), total}, | |
{MallocInfoMprotectSizeMeasure(), mprotect}}); | |
} | |
size_t get_current_rss() { | |
long rss = 0L; | |
FILE* fp = NULL; | |
if ((fp = fopen("/proc/self/statm", "r")) == NULL) return (size_t)0L; | |
if (fscanf(fp, "%*s%ld", &rss) != 1) { | |
fclose(fp); | |
return (size_t)0L; | |
} | |
fclose(fp); | |
return (size_t)rss * (size_t)sysconf(_SC_PAGESIZE); | |
} | |
void release_memory_loop() { | |
for (int i = 0;; i++) { | |
auto x = std::chrono::steady_clock::now() + std::chrono::milliseconds(RELEASE_MEMORY_CHECK_INTERVAL_MS); | |
size_t rss = get_current_rss(); | |
size_t new_rss = rss; | |
if (rss > TRIM_MEMORY_AFTER_RSS_AGGRESSIVE_B) { | |
malloc_trim(0); | |
new_rss = get_current_rss(); | |
std::cout << "Memory release attempted. RSS: " + std::to_string(rss / 1024 / 1024.0) + "MB -> " + std::to_string(new_rss / 1024 / 1024.0) + "MB" | |
<< std::endl; | |
} else if (i % TRY_RELEASE_MEMORY_EVERY_N_CHECKS == 0 && rss > TRIM_MEMORY_AFTER_RSS_B) { | |
malloc_trim(0); | |
new_rss = get_current_rss(); | |
std::cout << "Memory release attempted. RSS: " + std::to_string(rss / 1024 / 1024.0) + "MB -> " + std::to_string(new_rss / 1024 / 1024.0) + "MB" | |
<< std::endl; | |
} | |
// Also update malloc metrics | |
update_malloc_metrics(); | |
std::this_thread::sleep_until(x); | |
} | |
} | |
void initialize_linux_memory_tools() { | |
// Register malloc metrics | |
update_malloc_metrics(); | |
register_malloc_metrics_views(); | |
// Try to release memory to OS periodically | |
// mwCore seems to perform a lot of small allocations which make it hard for glibc allocator | |
// to trigger return of available heap space to the OS on free() using madvice system call. | |
// Instead we invoke malloc_trim(0) glibc function to merge fastbins and call madvice periodically. | |
// See: https://sources.debian.org/src/glibc/2.31-0experimental0/malloc/malloc.c/#L4385-L4416 | |
auto rmt = std::thread(release_memory_loop); | |
pthread_setname_np(rmt.native_handle(), "memory_keeper"); | |
rmt.detach(); | |
} | |
} // namespace webinterface |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment