M1 buffer size measuring tool
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// WARNING: must be run as root on an M1 device | |
// WARNING: fragile, uses private apple APIs | |
// currently no command line interface, see variables at top of main | |
/* | |
no warranty; use at your own risk - i believe this code needs | |
some minor changes to work on some later hardware and/or software revisions, | |
which is unsurprising given the use of undocumented, private APIs. | |
------------------------------------------------------------------------------ | |
This code is available under 2 licenses -- choose whichever you prefer. | |
------------------------------------------------------------------------------ | |
ALTERNATIVE A - MIT License | |
Copyright (c) 2020 Dougall Johnson | |
Permission is hereby granted, free of charge, to any person obtaining a copy of | |
this software and associated documentation files (the "Software"), to deal in | |
the Software without restriction, including without limitation the rights to | |
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies | |
of the Software, and to permit persons to whom the Software is furnished to do | |
so, subject to the following conditions: | |
The above copyright notice and this permission notice shall be included in all | |
copies or substantial portions of the Software. | |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
SOFTWARE. | |
------------------------------------------------------------------------------ | |
ALTERNATIVE B - Public Domain (www.unlicense.org) | |
This is free and unencumbered software released into the public domain. | |
Anyone is free to copy, modify, publish, use, compile, sell, or distribute this | |
software, either in source code form or as a compiled binary, for any purpose, | |
commercial or non-commercial, and by any means. | |
In jurisdictions that recognize copyright laws, the author or authors of this | |
software dedicate any and all copyright interest in the software to the public | |
domain. We make this dedication for the benefit of the public at large and to | |
the detriment of our heirs and successors. We intend this dedication to be an | |
overt act of relinquishment in perpetuity of all present and future rights to | |
this software under copyright law. | |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION | |
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | |
------------------------------------------------------------------------------ | |
*/ | |
/* | |
Based on https://github.com/travisdowns/robsize | |
Henry Wong <henry@stuffedcow.net> | |
http://blog.stuffedcow.net/2013/05/measuring-rob-capacity/ | |
2014-10-14 | |
*/ | |
#include <assert.h> | |
#include <dlfcn.h> | |
#include <pthread.h> | |
#include <stdio.h> | |
#include <stdlib.h> | |
#include <sys/mman.h> | |
#include <libkern/OSCacheControl.h> | |
static int its = 8192; | |
static int outer_its = 64; | |
static int unroll = 1; // TODO | |
const char *delim = "\t"; | |
#define KPERF_LIST \ | |
/* ret, name, params */ \ | |
F(int, kpc_get_counting, void) \ | |
F(int, kpc_force_all_ctrs_set, int) \ | |
F(int, kpc_set_counting, uint32_t) \ | |
F(int, kpc_set_thread_counting, uint32_t) \ | |
F(int, kpc_set_config, uint32_t, void *) \ | |
F(int, kpc_get_config, uint32_t, void *) \ | |
F(int, kpc_set_period, uint32_t, void *) \ | |
F(int, kpc_get_period, uint32_t, void *) \ | |
F(uint32_t, kpc_get_counter_count, uint32_t) \ | |
F(uint32_t, kpc_get_config_count, uint32_t) \ | |
F(int, kperf_sample_get, int *) \ | |
F(int, kpc_get_thread_counters, int, unsigned int, void *) | |
#define F(ret, name, ...) \ | |
typedef ret name##proc(__VA_ARGS__); \ | |
static name##proc *name; | |
KPERF_LIST | |
#undef F | |
#define CFGWORD_EL0A32EN_MASK (0x10000) | |
#define CFGWORD_EL0A64EN_MASK (0x20000) | |
#define CFGWORD_EL1EN_MASK (0x40000) | |
#define CFGWORD_EL3EN_MASK (0x80000) | |
#define CFGWORD_ALLMODES_MASK (0xf0000) | |
#define CPMU_NONE 0 | |
#define CPMU_CORE_CYCLE 0x02 | |
#define CPMU_INST_A64 0x8c | |
#define CPMU_INST_BRANCH 0x8d | |
#define CPMU_SYNC_DC_LOAD_MISS 0xbf | |
#define CPMU_SYNC_DC_STORE_MISS 0xc0 | |
#define CPMU_SYNC_DTLB_MISS 0xc1 | |
#define CPMU_SYNC_ST_HIT_YNGR_LD 0xc4 | |
#define CPMU_SYNC_BR_ANY_MISP 0xcb | |
#define CPMU_FED_IC_MISS_DEM 0xd3 | |
#define CPMU_FED_ITLB_MISS 0xd4 | |
#define KPC_CLASS_FIXED (0) | |
#define KPC_CLASS_CONFIGURABLE (1) | |
#define KPC_CLASS_POWER (2) | |
#define KPC_CLASS_RAWPMU (3) | |
#define KPC_CLASS_FIXED_MASK (1u << KPC_CLASS_FIXED) | |
#define KPC_CLASS_CONFIGURABLE_MASK (1u << KPC_CLASS_CONFIGURABLE) | |
#define KPC_CLASS_POWER_MASK (1u << KPC_CLASS_POWER) | |
#define KPC_CLASS_RAWPMU_MASK (1u << KPC_CLASS_RAWPMU) | |
#define COUNTERS_COUNT 10 | |
#define CONFIG_COUNT 8 | |
#define KPC_MASK (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_FIXED_MASK) | |
uint64_t g_counters[COUNTERS_COUNT]; | |
uint64_t g_config[COUNTERS_COUNT]; | |
static void configure_rdtsc() { | |
if (kpc_set_config(KPC_MASK, g_config)) { | |
printf("kpc_set_config failed\n"); | |
return; | |
} | |
if (kpc_force_all_ctrs_set(1)) { | |
printf("kpc_force_all_ctrs_set failed\n"); | |
return; | |
} | |
if (kpc_set_counting(KPC_MASK)) { | |
printf("kpc_set_counting failed\n"); | |
return; | |
} | |
if (kpc_set_thread_counting(KPC_MASK)) { | |
printf("kpc_set_thread_counting failed\n"); | |
return; | |
} | |
} | |
static void init_rdtsc() { | |
void *kperf = dlopen( | |
"/System/Library/PrivateFrameworks/kperf.framework/Versions/A/kperf", | |
RTLD_LAZY); | |
if (!kperf) { | |
printf("kperf = %p\n", kperf); | |
return; | |
} | |
#define F(ret, name, ...) \ | |
name = (name##proc *)(dlsym(kperf, #name)); \ | |
if (!name) { \ | |
printf("%s = %p\n", #name, (void *)name); \ | |
return; \ | |
} | |
KPERF_LIST | |
#undef F | |
// TODO: KPC_CLASS_RAWPMU_MASK | |
if (kpc_get_counter_count(KPC_MASK) != COUNTERS_COUNT) { | |
printf("wrong fixed counters count\n"); | |
return; | |
} | |
if (kpc_get_config_count(KPC_MASK) != CONFIG_COUNT) { | |
printf("wrong fixed config count\n"); | |
return; | |
} | |
// Not all counters can count all things: | |
// CPMU_CORE_CYCLE {0-7} | |
// CPMU_FED_IC_MISS_DEM {0-7} | |
// CPMU_FED_ITLB_MISS {0-7} | |
// CPMU_INST_BRANCH {3, 4, 5} | |
// CPMU_SYNC_DC_LOAD_MISS {3, 4, 5} | |
// CPMU_SYNC_DC_STORE_MISS {3, 4, 5} | |
// CPMU_SYNC_DTLB_MISS {3, 4, 5} | |
// CPMU_SYNC_BR_ANY_MISP {3, 4, 5} | |
// CPMU_SYNC_ST_HIT_YNGR_LD {3, 4, 5} | |
// CPMU_INST_A64 {5} | |
// using "CFGWORD_ALLMODES_MASK" is much noisier | |
g_config[0] = CPMU_CORE_CYCLE | CFGWORD_EL0A64EN_MASK; | |
// configs[3] = CPMU_SYNC_DC_LOAD_MISS | CFGWORD_EL0A64EN_MASK; | |
// configs[4] = CPMU_SYNC_DTLB_MISS | CFGWORD_EL0A64EN_MASK; | |
// configs[5] = CPMU_INST_A64 | CFGWORD_EL0A64EN_MASK; | |
configure_rdtsc(); | |
} | |
static unsigned long long int rdtsc() { | |
if (kpc_get_thread_counters(0, COUNTERS_COUNT, g_counters)) { | |
printf("kpc_get_thread_counters failed\n"); | |
return 1; | |
} | |
return g_counters[2]; | |
} | |
static void shuffle(int *array, size_t n) { | |
if (n > 1) { | |
size_t i; | |
for (i = 0; i < n - 1; i++) { | |
size_t j = i + rand() / (RAND_MAX / (n - i) + 1); | |
int t = array[j]; | |
array[j] = array[i]; | |
array[i] = t; | |
} | |
} | |
} | |
static void init_dbufs(uint64_t **out_data1, uint64_t **out_data2) { | |
// Initialize two 256MB data buffers, with the same linked-list | |
// of offsets. | |
size_t size = 256 * 1024 * 1024; | |
size_t cache_line_size = 64; | |
size_t count = size / cache_line_size; | |
size_t stride = cache_line_size / sizeof(void *); | |
int *numbers = malloc(count * sizeof(int)); | |
for (int i = 0; i < count; i++) { | |
numbers[i] = i; | |
} | |
shuffle(numbers, count); | |
uint64_t *data1 = calloc(size, 1); | |
uint64_t *data2 = (uint64_t *)((char *)calloc(size + 64, 1) + 64); | |
int next = numbers[count - 1]; | |
for (int i = 0; i < count; i++) { | |
int n = numbers[i]; | |
data1[stride * n] = next * stride; | |
data2[stride * n] = next * stride; | |
next = n; | |
} | |
*out_data1 = data1; | |
*out_data2 = data2; | |
free(numbers); | |
} | |
static int add_prep(uint32_t *ibuf, int instr_type) { | |
int o = 0; | |
// free as much of the prf as possible | |
switch (instr_type) { | |
case 4: // gpr prf size | |
for (int i = 5; i < 31; i++) | |
ibuf[o++] = 0xd2800000 | i; // mov xi, #0 | |
break; | |
case 5: // simd/fp prf size | |
for (int i = 0; i < 32; i++) | |
ibuf[o++] = 0x4ea11c20 | i; // mov.16b vi, v1 | |
break; | |
} | |
return o; | |
} | |
static int add_filler(uint32_t *ibuf, int instr_type, int j) { | |
int o = 0; | |
// "spike" is used to mean the first icount where the minimum time had clearly | |
// jumped up | |
switch (instr_type) { | |
case 0: // OOO window maximum size (firestorm spike at 2295, icestorm spike at 415) | |
ibuf[o++] = 0xd503201f; // nop | |
break; | |
case 1: // maximum in flight renames (firestorm spike at 623, icestorm spike at 111) | |
ibuf[o++] = 0xd2800005; // mov x5, 0 | |
break; | |
case 2: // load buffer size (firestorm spike at 129, icestorm spike at 29 (?)) | |
ibuf[o++] = 0xf9400045; // ldr x5, [x2] | |
break; | |
case 3: // store buffer size (firestorm spike at 108, icestorm spike at 36 (?)) | |
ibuf[o++] = 0xf9000445; // str x5, [x2, #8] | |
break; | |
case 4: // gpr prf size (firestorm spike at 380, icestorm spike at 79) | |
ibuf[o++] = 0x8b0b0165; // add x5, x11, x11 | |
break; | |
case 5: // simd/fp prf size (firestorm spike at 434, icestorm spike at 87) | |
ibuf[o++] = 0x4e228420; // add v0.16b, v1.16b, v2.16b | |
break; | |
case 6: // scheduler (rs) size (firestorm spike at 158, icestorm spike at 34) | |
ibuf[o++] = 0x8b010005; // add x5, x0, x1 (depends on pending load) | |
break; | |
case 7: // untaken branches (firestorm spike at 144, icestorm spike at 32) | |
if (j == 0) | |
ibuf[o++] = 0xeb0500bf; // cmp x5, x5 | |
ibuf[o++] = 0x54000781; // b.ne .+0xF0 | |
break; | |
case 8: // confused attempt to get a reoder buffer size (firestorm spike at 853) | |
if (j == 0) { | |
ibuf[o++] = 0xeb0500bf; // cmp x5, x5 | |
} else if (j - 1 < 100) { | |
ibuf[o++] = 0xf9000445; // str x5, [x2, #8] | |
} else if (j - 1 - 100 < 130) { | |
ibuf[o++] = 0x54000781; // b.ne .+0xF0 | |
} else { | |
ibuf[o++] = 0xd2800005; // mov x5, 0 | |
} | |
break; | |
case 9: // calls in flight (firestorm spike at 30, icestorm spike at 11) | |
ibuf[o++] = 0x94000002; // bl +8 | |
ibuf[o++] = 0x14000002; // b +8 | |
ibuf[o++] = 0xd65f03c0; // ret | |
break; | |
case 10: // uncond branch (firestorm spike at 88, icestorm spike at 32) | |
ibuf[o++] = 0x14000001; // b +4 | |
break; | |
case 11: // taken branch (firestorm spike at 88, icestorm spike at 32) | |
if (j == 0) | |
ibuf[o++] = 0xeb0500bf; // cmp x5, x5 | |
ibuf[o++] = 0x54000020; // b.eq .+4 | |
break; | |
case 12: // not-taken compare+branch (firestorm spike at 129) | |
ibuf[o++] = 0xeb0500bf; // cmp x5, x5 | |
ibuf[o++] = 0x54000021; // b.ne .+4 | |
break; | |
case 13: // taken compare+branch (firestorm spike at 88) | |
ibuf[o++] = 0xeb0500bf; // cmp x5, x5 | |
ibuf[o++] = 0x54000020; // b.eq .+4 | |
break; | |
} | |
return o; | |
} | |
void make_routine(uint32_t *ibuf, int icount, int instr_type) { | |
pthread_jit_write_protect_np(0); | |
int o = 0; | |
// prologue | |
if (instr_type == 5) { | |
ibuf[o++] = 0x6dbb3bef; // stp d15, d14, [sp, #-80]! | |
ibuf[o++] = 0x6d0133ed; // stp d13, d12, [sp, #16] | |
ibuf[o++] = 0x6d022beb; // stp d11, d10, [sp, #32] | |
ibuf[o++] = 0x6d0323e9; // stp d9, d8, [sp, #48] | |
ibuf[o++] = 0xa9047bfd; // stp x29, x30, [sp, #64] | |
} else { | |
ibuf[o++] = 0xa9b87bfd; // stp x29, x30, [sp, #-128]! | |
ibuf[o++] = 0xa9016ffc; // stp x28, x27, [sp, #16] | |
ibuf[o++] = 0xa90267fa; // stp x26, x25, [sp, #32] | |
ibuf[o++] = 0xa9035ff8; // stp x24, x23, [sp, #48] | |
ibuf[o++] = 0xa90457f6; // stp x22, x21, [sp, #64] | |
ibuf[o++] = 0xa9054ff4; // stp x20, x19, [sp, #80] | |
ibuf[o++] = 0xa90647f2; // stp x18, x17, [sp, #96] | |
ibuf[o++] = 0xa9073ff0; // stp x16, x15, [sp, #112] | |
} | |
// next, next, data1, data2, its | |
// x0 = offset into data1 | |
// x1 = offset into data2 | |
// x2 = data1 | |
// x3 = data2 | |
// x4 = its | |
o += add_prep(ibuf + o, instr_type); | |
int start = o; | |
int load_count = 1; | |
for (int i = 0; i < load_count; i++) | |
ibuf[o++] = 0xf8607840; // ldr x0, [x2, x0, lsl #3] | |
for (int j = 0; j < icount; j++) { | |
o += add_filler(ibuf + o, instr_type, j); | |
} | |
for (int i = 0; i < load_count; i++) | |
ibuf[o++] = 0xf8617861; // ldr x1, [x3, x1, lsl #3] | |
for (int j = 0; j < icount; j++) { | |
// o += add_filler(ibuf+o, instr_type, j); | |
} | |
// lfence mode? | |
ibuf[o++] = 0xD5033B9F; // DSB ISH | |
ibuf[o++] = 0xD5033FDF; // ISB | |
// loop back to top | |
ibuf[o++] = 0x71000484; // subs w4, w4, #1 | |
int off = start - o; | |
assert(off < 0 && off > -0x40000); | |
ibuf[o++] = 0x54000001 | ((off & 0x7ffff) << 5); // b.ne | |
// epilogue | |
if (instr_type == 5) { | |
ibuf[o++] = 0xa9447bfd; // ldp x29, x30, [sp, #64] | |
ibuf[o++] = 0x6d4323e9; // ldp d9, d8, [sp, #48] | |
ibuf[o++] = 0x6d422beb; // ldp d11, d10, [sp, #32] | |
ibuf[o++] = 0x6d4133ed; // ldp d13, d12, [sp, #16] | |
ibuf[o++] = 0x6cc53bef; // ldp d15, d14, [sp], #80 | |
ibuf[o++] = 0xd65f03c0; // ret | |
} else { | |
ibuf[o++] = 0xa9473ff0; // ldp x16, x15, [sp, #112] | |
ibuf[o++] = 0xa94647f2; // ldp x18, x17, [sp, #96] | |
ibuf[o++] = 0xa9454ff4; // ldp x20, x19, [sp, #80] | |
ibuf[o++] = 0xa94457f6; // ldp x22, x21, [sp, #64] | |
ibuf[o++] = 0xa9435ff8; // ldp x24, x23, [sp, #48] | |
ibuf[o++] = 0xa94267fa; // ldp x26, x25, [sp, #32] | |
ibuf[o++] = 0xa9416ffc; // ldp x28, x27, [sp, #16] | |
ibuf[o++] = 0xa8c87bfd; // ldp x29, x30, [sp], #128 | |
ibuf[o++] = 0xd65f03c0; // ret | |
} | |
pthread_jit_write_protect_np(1); | |
sys_icache_invalidate(ibuf, o * 4); | |
} | |
int main(int argc, char **argv) { | |
int test_high_perf_cores = 1; | |
int instr_type = 1; | |
int start_icount = 600; | |
int stop_icount = 700; | |
int stride_icount = 1; | |
// TODO: can we force this to run on the fast cores? | |
// counters seemingly fail to update if we initialise | |
// them, then switch cores, although the fixed thread | |
// counters don't have this problem. | |
// QOS_CLASS_BACKGROUND does seem to pin it to the slow | |
// cores though. | |
if (test_high_perf_cores) { | |
pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0); | |
} else { | |
pthread_set_qos_class_self_np(QOS_CLASS_BACKGROUND, 0); | |
} | |
init_rdtsc(); | |
uint64_t *data1, *data2; | |
init_dbufs(&data1, &data2); | |
void *mapping = mmap(NULL, 0x400000, PROT_READ | PROT_WRITE | PROT_EXEC, | |
MAP_ANON | MAP_PRIVATE | MAP_JIT, -1, 0); | |
uint32_t *ibuf = (uint32_t *)mapping; | |
uint64_t next = 0; | |
for (int icount = start_icount; icount <= stop_icount; icount += stride_icount) { | |
make_routine(ibuf, icount, instr_type); | |
uint64_t (*routine)(uint64_t, uint64_t, uint64_t *, uint64_t *, uint64_t) = (void *)ibuf; | |
uint64_t min_diff = 0x7fffffffffffffffLL; | |
uint64_t max_diff = 0x0; | |
uint64_t sum_diff = 0; | |
next = routine(next, next, data1, data2, its); | |
for (int i = 0; i < outer_its; i++) { | |
// in case we were on the wrong core earlier | |
configure_rdtsc(); | |
long long start = rdtsc(); | |
next = routine(next, next, data1, data2, its); | |
long long stop = rdtsc(); | |
uint64_t cycles = stop - start; | |
sum_diff += cycles; | |
if (min_diff > cycles) { | |
min_diff = cycles; | |
} | |
if (max_diff < cycles) { | |
max_diff = cycles; | |
} | |
} | |
printf("%d%s%.2f%s%.2f%s%.2f\n", icount, delim, | |
0.5 * min_diff / its / unroll, delim, | |
0.5 * sum_diff / its / unroll / outer_its, delim, | |
0.5 * max_diff / its / unroll); | |
} | |
return 0; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment