tcg: Group pages into "super-pages" for jumps and invalidation

gistfile1.txt

From 24547b9cf3c10b2e14c54ffadd462e58d7a3e8ce Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 17 Oct 2023 16:15:55 +0100
Subject: [PATCH] tcg: Group pages into "super-pages" for jumps and
 invalidation

---
 accel/tcg/tb-maint.c   | 95 ++++++++++++++++++++++++++----------------
 accel/tcg/translator.c |  7 +++-
 2 files changed, 64 insertions(+), 38 deletions(-)

diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index e678d20dc2..44502ba8a1 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -32,6 +32,14 @@
 #include "internal-common.h"
 #include "internal-target.h"
 
+/* "Page" in this file refers to a group of 1 << SUPER_PAGE_SHIFT
+ * target pages.  Fast jumps are allowed between all target pages in
+ * one of these super-pages.  Writing to code in any target page
+ * invalidates the whole super-page.
+ */
+#define SUPER_PAGE_SHIFT 3 /* 8 target pages in one super-page */
+#define SUPER_PAGE_BITS (TARGET_PAGE_BITS + SUPER_PAGE_SHIFT)
+#define SUPER_PAGE_MASK ((target_long)-1 << SUPER_PAGE_BITS)
 
 /* List iterators for lists of tagged pointers in TranslationBlock. */
 #define TB_FOR_EACH_TAGGED(head, tb, n, field)                          \
@@ -188,15 +196,15 @@ void page_table_config_init(void)
 {
     uint32_t v_l1_bits;
 
-    assert(TARGET_PAGE_BITS);
+    assert(SUPER_PAGE_BITS);
     /* The bits remaining after N lower levels of page tables.  */
-    v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS;
+    v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - SUPER_PAGE_BITS) % V_L2_BITS;
     if (v_l1_bits < V_L1_MIN_BITS) {
         v_l1_bits += V_L2_BITS;
     }
 
     v_l1_size = 1 << v_l1_bits;
-    v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits;
+    v_l1_shift = L1_MAP_ADDR_SPACE_BITS - SUPER_PAGE_BITS - v_l1_bits;
     v_l2_levels = v_l1_shift / V_L2_BITS - 1;
 
     assert(v_l1_bits <= V_L1_MAX_BITS);
@@ -399,13 +407,13 @@ static void page_unlock(PageDesc *pd)
 
 void tb_lock_page0(tb_page_addr_t paddr)
 {
-    page_lock(page_find_alloc(paddr >> TARGET_PAGE_BITS, true));
+    page_lock(page_find_alloc(paddr >> SUPER_PAGE_BITS, true));
 }
 
 void tb_lock_page1(tb_page_addr_t paddr0, tb_page_addr_t paddr1)
 {
-    tb_page_addr_t pindex0 = paddr0 >> TARGET_PAGE_BITS;
-    tb_page_addr_t pindex1 = paddr1 >> TARGET_PAGE_BITS;
+    tb_page_addr_t pindex0 = paddr0 >> SUPER_PAGE_BITS;
+    tb_page_addr_t pindex1 = paddr1 >> SUPER_PAGE_BITS;
     PageDesc *pd0, *pd1;
 
     if (pindex0 == pindex1) {
@@ -438,8 +446,8 @@ void tb_lock_page1(tb_page_addr_t paddr0, tb_page_addr_t paddr1)
 
 void tb_unlock_page1(tb_page_addr_t paddr0, tb_page_addr_t paddr1)
 {
-    tb_page_addr_t pindex0 = paddr0 >> TARGET_PAGE_BITS;
-    tb_page_addr_t pindex1 = paddr1 >> TARGET_PAGE_BITS;
+    tb_page_addr_t pindex0 = paddr0 >> SUPER_PAGE_BITS;
+    tb_page_addr_t pindex1 = paddr1 >> SUPER_PAGE_BITS;
 
     if (pindex0 != pindex1) {
         page_unlock(page_find_alloc(pindex1, false));
@@ -450,8 +458,8 @@ static void tb_lock_pages(TranslationBlock *tb)
 {
     tb_page_addr_t paddr0 = tb_page_addr0(tb);
     tb_page_addr_t paddr1 = tb_page_addr1(tb);
-    tb_page_addr_t pindex0 = paddr0 >> TARGET_PAGE_BITS;
-    tb_page_addr_t pindex1 = paddr1 >> TARGET_PAGE_BITS;
+    tb_page_addr_t pindex0 = paddr0 >> SUPER_PAGE_BITS;
+    tb_page_addr_t pindex1 = paddr1 >> SUPER_PAGE_BITS;
 
     if (unlikely(paddr0 == -1)) {
         return;
@@ -471,8 +479,8 @@ void tb_unlock_pages(TranslationBlock *tb)
 {
     tb_page_addr_t paddr0 = tb_page_addr0(tb);
     tb_page_addr_t paddr1 = tb_page_addr1(tb);
-    tb_page_addr_t pindex0 = paddr0 >> TARGET_PAGE_BITS;
-    tb_page_addr_t pindex1 = paddr1 >> TARGET_PAGE_BITS;
+    tb_page_addr_t pindex0 = paddr0 >> SUPER_PAGE_BITS;
+    tb_page_addr_t pindex1 = paddr1 >> SUPER_PAGE_BITS;
 
     if (unlikely(paddr0 == -1)) {
         return;
@@ -546,7 +554,7 @@ static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data)
  */
 static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
 {
-    tb_page_addr_t index = addr >> TARGET_PAGE_BITS;
+    tb_page_addr_t index = addr >> SUPER_PAGE_BITS;
     struct page_entry *pe;
     PageDesc *pd;
 
@@ -604,8 +612,8 @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
     tb_page_addr_t index;
     PageDesc *pd;
 
-    start >>= TARGET_PAGE_BITS;
-    last >>= TARGET_PAGE_BITS;
+    start >>= SUPER_PAGE_BITS;
+    last >>= SUPER_PAGE_BITS;
     g_assert(start <= last);
 
     set->tree = q_tree_new_full(tb_page_addr_cmp, NULL, NULL,
@@ -624,7 +632,7 @@ static struct page_collection *page_collection_lock(tb_page_addr_t start,
         if (pd == NULL) {
             continue;
         }
-        if (page_trylock_add(set, index << TARGET_PAGE_BITS)) {
+        if (page_trylock_add(set, index << SUPER_PAGE_BITS)) {
             q_tree_foreach(set->tree, page_entry_unlock, NULL);
             goto retry;
         }
@@ -703,7 +711,13 @@ static void tb_page_add(PageDesc *p, TranslationBlock *tb, unsigned int n)
      * allocated in a physical page.
      */
     if (!page_already_protected) {
-        tlb_protect_code(tb->page_addr[n] & TARGET_PAGE_MASK);
+        size_t i;
+        ram_addr_t ra = tb->page_addr[n] & SUPER_PAGE_MASK;
+
+        for (i = 0; i < 1 << SUPER_PAGE_SHIFT; ++i) {
+            tlb_protect_code(ra);
+            ra += TARGET_PAGE_SIZE;
+        }
     }
 }
 
@@ -711,8 +725,8 @@ static void tb_record(TranslationBlock *tb)
 {
     tb_page_addr_t paddr0 = tb_page_addr0(tb);
     tb_page_addr_t paddr1 = tb_page_addr1(tb);
-    tb_page_addr_t pindex0 = paddr0 >> TARGET_PAGE_BITS;
-    tb_page_addr_t pindex1 = paddr0 >> TARGET_PAGE_BITS;
+    tb_page_addr_t pindex0 = paddr0 >> SUPER_PAGE_BITS;
+    tb_page_addr_t pindex1 = paddr0 >> SUPER_PAGE_BITS;
 
     assert(paddr0 != -1);
     if (unlikely(paddr1 != -1) && pindex0 != pindex1) {
@@ -743,8 +757,8 @@ static void tb_remove(TranslationBlock *tb)
 {
     tb_page_addr_t paddr0 = tb_page_addr0(tb);
     tb_page_addr_t paddr1 = tb_page_addr1(tb);
-    tb_page_addr_t pindex0 = paddr0 >> TARGET_PAGE_BITS;
-    tb_page_addr_t pindex1 = paddr0 >> TARGET_PAGE_BITS;
+    tb_page_addr_t pindex0 = paddr0 >> SUPER_PAGE_BITS;
+    tb_page_addr_t pindex1 = paddr0 >> SUPER_PAGE_BITS;
 
     assert(paddr0 != -1);
     if (unlikely(paddr1 != -1) && pindex0 != pindex1) {
@@ -1025,8 +1039,8 @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
 {
     tb_page_addr_t start, last;
 
-    start = addr & TARGET_PAGE_MASK;
-    last = addr | ~TARGET_PAGE_MASK;
+    start = addr & SUPER_PAGE_MASK;
+    last = addr | ~SUPER_PAGE_MASK;
     tb_invalidate_phys_range(start, last);
 }
 
@@ -1060,8 +1074,8 @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
     assert_memory_lock();
     current_tb = tcg_tb_lookup(pc);
 
-    last = addr | ~TARGET_PAGE_MASK;
-    addr &= TARGET_PAGE_MASK;
+    last = addr | ~SUPER_PAGE_MASK;
+    addr &= SUPER_PAGE_MASK;
     current_tb_modified = false;
 
     PAGE_FOR_EACH_TB(addr, last, unused, tb, n) {
@@ -1108,7 +1122,7 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
 #endif /* TARGET_HAS_PRECISE_SMC */
 
     /* Range may not cross a page. */
-    tcg_debug_assert(((start ^ last) & TARGET_PAGE_MASK) == 0);
+    tcg_debug_assert(((start ^ last) & SUPER_PAGE_MASK) == 0);
 
     /*
      * We remove all the TBs in the range [start, last].
@@ -1121,10 +1135,10 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
         tb_start = tb_page_addr0(tb);
         tb_last = tb_start + tb->size - 1;
         if (n == 0) {
-            tb_last = MIN(tb_last, tb_start | ~TARGET_PAGE_MASK);
+            tb_last = MIN(tb_last, tb_start | ~SUPER_PAGE_MASK);
         } else {
             tb_start = tb_page_addr1(tb);
-            tb_last = tb_start + (tb_last & ~TARGET_PAGE_MASK);
+            tb_last = tb_start + (tb_last & ~SUPER_PAGE_MASK);
         }
         if (!(tb_last < start || tb_start > last)) {
 #ifdef TARGET_HAS_PRECISE_SMC
@@ -1147,7 +1161,14 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
 
     /* if no code remaining, no need to continue to use slow writes */
     if (!p->first_tb) {
-        tlb_unprotect_code(start);
+        size_t i;
+        ram_addr_t ra;
+
+        ra = start + TARGET_PAGE_SIZE * (1 << SUPER_PAGE_SHIFT);
+        for (i = 0; i < 1 << SUPER_PAGE_SHIFT; ++i) {
+            ra -= TARGET_PAGE_SIZE;
+            tlb_unprotect_code(ra);
+        }
     }
 
 #ifdef TARGET_HAS_PRECISE_SMC
@@ -1172,13 +1193,13 @@ void tb_invalidate_phys_page(tb_page_addr_t addr)
     tb_page_addr_t start, last;
     PageDesc *p;
 
-    p = page_find(addr >> TARGET_PAGE_BITS);
+    p = page_find(addr >> SUPER_PAGE_BITS);
     if (p == NULL) {
         return;
     }
 
-    start = addr & TARGET_PAGE_MASK;
-    last = addr | ~TARGET_PAGE_MASK;
+    start = addr & SUPER_PAGE_MASK;
+    last = addr | ~SUPER_PAGE_MASK;
     pages = page_collection_lock(start, last);
     tb_invalidate_phys_page_range__locked(pages, p, start, last, 0);
     page_collection_unlock(pages);
@@ -1198,8 +1219,8 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
 
     pages = page_collection_lock(start, last);
 
-    index_last = last >> TARGET_PAGE_BITS;
-    for (index = start >> TARGET_PAGE_BITS; index <= index_last; index++) {
+    index_last = last >> SUPER_PAGE_BITS;
+    for (index = start >> SUPER_PAGE_BITS; index <= index_last; index++) {
         PageDesc *pd = page_find(index);
         tb_page_addr_t page_start, page_last;
 
@@ -1207,8 +1228,8 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
             continue;
         }
         assert_page_locked(pd);
-        page_start = index << TARGET_PAGE_BITS;
-        page_last = page_start | ~TARGET_PAGE_MASK;
+        page_start = index << SUPER_PAGE_BITS;
+        page_last = page_start | ~SUPER_PAGE_MASK;
         page_last = MIN(page_last, last);
         tb_invalidate_phys_page_range__locked(pages, pd,
                                               page_start, page_last, 0);
@@ -1225,7 +1246,7 @@ static void tb_invalidate_phys_page_fast__locked(struct page_collection *pages,
 {
     PageDesc *p;
 
-    p = page_find(start >> TARGET_PAGE_BITS);
+    p = page_find(start >> SUPER_PAGE_BITS);
     if (!p) {
         return;
     }
diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c
index 575b9812ad..47ccf6cb21 100644
--- a/accel/tcg/translator.c
+++ b/accel/tcg/translator.c
@@ -112,6 +112,11 @@ static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
     }
 }
 
+/* XXXXXX put it in a header file */
+#define SUPER_PAGE_SHIFT 3 /* 8 target pages in one super-page */
+#define SUPER_PAGE_BITS (TARGET_PAGE_BITS + SUPER_PAGE_SHIFT)
+#define SUPER_PAGE_MASK ((target_long)-1 << SUPER_PAGE_BITS)
+
 bool translator_use_goto_tb(DisasContextBase *db, vaddr dest)
 {
     /* Suppress goto_tb if requested. */
@@ -120,7 +125,7 @@ bool translator_use_goto_tb(DisasContextBase *db, vaddr dest)
     }
 
     /* Check for the dest on the same page as the start of the TB.  */
-    return ((db->pc_first ^ dest) & TARGET_PAGE_MASK) == 0;
+    return ((db->pc_first ^ dest) & SUPER_PAGE_MASK) == 0;
 }
 
 void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
-- 
2.41.0