Pop open either the L large or S small dialog, and see how the buttons are 'compressed' into a single row, since they fail to get allocated a height of 3 needed for layout. The dialogs height is set dynamically depending on the number of rows required. However the distribution of those rows to the contents is the issue.

attempt2.py

from textual.app import App, ComposeResult
from textual.widgets import Label, Button, Footer
from typing import Dict, Optional, List
from textual.screen import ModalScreen
from textual.binding import Binding
from textual.widget import Widget
from textual.containers import Grid, Horizontal, Vertical



class EditTraitScreen(ModalScreen[Optional[int]]):
    DEFAULT_CSS = """
    EditTraitScreen > Vertical {
        width: 90;
        border: heavy yellow;
        background: $surface;
        padding: 0 0 0 0;
    }
    EditTraitScreen > Vertical > Horizontal > Button {
        margin: 1;
    }
    EditTraitScreen > Vertical > Grid {
        margin: 0 1 0 1;
        grid-size: 2;
        grid-columns: 1fr 3fr;
        padding: 0 0 0 0;
    }
    """

    BINDINGS = [
        Binding("escape", "app.pop_screen", "", show=False),
    ]

    def __init__(self, count: int):
        super(EditTraitScreen, self).__init__()

        self.controls: List[Widget] = []
        for j in range(count):
            self.controls.append(Label(str(j)))
            self.controls.append(Label(str(j)))

    def compose(self) -> ComposeResult:
        g = Grid(
            Label("Setting"),
            Label("hello"),
            *self.controls,
        )
        v = Vertical(
            g,
            Horizontal(
                Button("Set", variant="primary", id="quit"),
                Button("Cancel", variant="default", id="cancel"),
        ))
        yield v
        v.border_title = "Set value"
        dh = len(self.controls)//2
        g.styles.height = 1 + dh
        v.styles.height = 3 + 4 + dh

    def on_button_pressed(self, event: Button.Pressed) -> None:
        self.dismiss(None)

class TableApp(App):

    DEFAULT_CSS = """
    Screen {
        align: center middle;
    }
    """

    BINDINGS = [
        ("l", "large_popup", "Large popup"),
        ("s", "small_popup", "Small popup"),
    ]

    def __init__(self):
        super().__init__()

    def compose(self) -> ComposeResult:
        yield Label("Hello world")
        yield Footer()

    def action_large_popup(self) -> None:
        self.push_screen(EditTraitScreen(12))

    def action_small_popup(self) -> None:
        self.push_screen(EditTraitScreen(3))

app = TableApp()
if __name__ == "__main__":
    app.run()

    

dialog_grid_test.py

from textual.app import App, ComposeResult
from textual.widgets import Label, Button, Footer
from typing import Dict, Optional, List
from textual.screen import ModalScreen
from textual.binding import Binding
from textual.widget import Widget
from textual.containers import Grid



class EditTraitScreen(ModalScreen[Optional[int]]):

    DEFAULT_CSS = """
    #dialog3 {
        width: 90;
        grid-size: 2;
        grid-columns: 1fr 3fr;
        padding: 0 1 0 1;
        border: heavy yellow;
        background: $surface;
    }
    """

    BINDINGS = [
        Binding("escape", "app.pop_screen", "", show=False),
    ]

    def __init__(self, count: int):
        super(EditTraitScreen, self).__init__()

        self.controls: List[Widget] = []
        for j in range(count):
            self.controls.append(Label(str(j)))
            self.controls.append(Label(str(j)))

    def compose(self) -> ComposeResult:
        g = Grid(
            Label("Setting"),
            Label("hello"),
            *self.controls,
            Button("Set", variant="primary", id="quit"),
            Button("Cancel", variant="default", id="cancel"),
            id="dialog3",
        )
        yield g
        g.border_title = "Set value"
        g.styles.height = 3 + 2 + 3 + (len(self.controls)/2)

    def on_button_pressed(self, event: Button.Pressed) -> None:
        self.dismiss(None)

class TableApp(App):

    DEFAULT_CSS = """
    Screen {
        align: center middle;
    }
    """

    BINDINGS = [
        ("l", "large_popup", "Large popup"),
        ("s", "small_popup", "Small popup"),
    ]

    def __init__(self):
        super().__init__()

    def compose(self) -> ComposeResult:
        yield Label("Hello world")
        yield Footer()

    def action_large_popup(self) -> None:
        self.push_screen(EditTraitScreen(12))

    def action_small_popup(self) -> None:
        self.push_screen(EditTraitScreen(3))

app = TableApp()
if __name__ == "__main__":
    app.run()