Skip to content

Instantly share code, notes, and snippets.

@rygorous
Last active November 4, 2023 23:47
Show Gist options
  • Save rygorous/1323ab0812f21bc2b588cf5903402a6b to your computer and use it in GitHub Desktop.
Save rygorous/1323ab0812f21bc2b588cf5903402a6b to your computer and use it in GitHub Desktop.
// advance
for (int i = 0; i < num_streams; ++i)
{
std::string desc = formatf("advance %d", i);
bool is_reverse_stream = (i % 3) == 1;
if (EARLY_CLZ != 2)
{
if (EARLY_CLZ == 0)
bb->append(CLZ(bits[i], bits[i]).set_comment(desc)); // figure out how many bits we consumed
if (is_reverse_stream)
bb->append(SUB(inptr[i], inptr[i], LSR(bits[i], 3)).set_comment(desc));
else
bb->append(ADD(inptr[i], inptr[i], LSR(bits[i], 3)).set_comment(desc));
}
else
{
bb->append(UBFX(temp[i], bits[i], 3, 3).set_comment(desc));
if (is_reverse_stream)
bb->append(SUB(inptr[i], inptr[i], temp[i]).set_comment(desc));
else
bb->append(ADD(inptr[i], inptr[i], temp[i]).set_comment(desc));
}
}
// refill
for (int i = 0; i < num_streams; ++i)
{
std::string desc = formatf("refill %d", i);
bool is_reverse = (i % 3) == 1;
// load
bb->append(AND(bits[i], bits[i], 7).set_comment(desc + " leftover bits"));
bb->append(LDR(temp[i], Addr::base(inptr[i])).set_comment(desc + " load"));
// byte-reverse middle streams
if (is_reverse)
bb->append(REV(temp[i], temp[i]).set_comment(desc + " LE->BE"));
bb->append(ORR(temp[i], temp[i], 1ull<<63).set_comment(desc + " mark"));
bb->append(LSR(bits[i], temp[i], bits[i]).set_comment(desc + " consume leftover"));
// Prefetch if requested
if (PREFETCH_DIST)
{
// We use 8-byte loads that access bytes [ptr,ptr+7]. So make forward
// prefetch distances relative to ptr+7 and backwards prefetch distances
// relative to ptr. That way, 0==no prefetch (=only prefetching bytes we
// already accessed) naturally.
int dist = is_reverse ? -PREFETCH_DIST : PREFETCH_DIST + 7;
bb->append(PRFUM(PRF::PLD | PRF::L1 | PRF::STRM, Addr::base(inptr[i], dist)).set_comment(desc + " prefetch"));
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment