Qwen3.6 35B A3B NVFP4 SGLang Inference Engine Bootstrap Script

qwen36_nvfp4_bootstrap_rtx5090.sh

#!/usr/bin/env bash
set -euo pipefail

mkdir -p /workspace /var/log/portal /root/.cache/huggingface
cd /workspace

gpu_count_from_list() {
  local devices="$1"
  case "$devices" in
    "" | all | none | void)
      return 1
      ;;
  esac

  local count=0
  local device
  local IFS=,
  for device in $devices; do
    if [ -n "$device" ]; then
      count=$((count + 1))
    fi
  done

  if [ "$count" -gt 0 ]; then
    printf '%s\n' "$count"
    return 0
  fi

  return 1
}

detect_gpu_count() {
  gpu_count_from_list "${CUDA_VISIBLE_DEVICES:-}" && return
  gpu_count_from_list "${NVIDIA_VISIBLE_DEVICES:-}" && return

  local count
  if command -v nvidia-smi >/dev/null 2>&1 &&
    count=$(nvidia-smi --query-gpu=name --format=csv,noheader,nounits | sed '/^[[:space:]]*$/d' | wc -l | tr -d ' '); then
    if [ "$count" -gt 0 ]; then
      printf '%s\n' "$count"
      return 0
    fi
  fi

  echo "could not detect GPU count; set SGLANG_DATA_PARALLEL_SIZE explicitly" >&2
  return 1
}

export HF_HOME="${HF_HOME:-/root/.cache/huggingface}"
export MODEL_NAME="${MODEL_NAME:-Qwen/Qwen3.6-35B-A3B}"
export MODEL_PATH="${MODEL_PATH:-/workspace/qwen36_nvfp4_modelopt_mixed}"
export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-$MODEL_NAME}"
export NVFP4_REPO="${NVFP4_REPO:-nvidia/Qwen3.6-35B-A3B-NVFP4}"
export BASE_REPO="${BASE_REPO:-Qwen/Qwen3.6-35B-A3B}"

export SGLANG_CONTEXT_LENGTH="${SGLANG_CONTEXT_LENGTH:-32768}"
export SGLANG_MEM_FRACTION_STATIC="${SGLANG_MEM_FRACTION_STATIC:-0.90}"
export SGLANG_CHUNKED_PREFILL_SIZE="${SGLANG_CHUNKED_PREFILL_SIZE:-8192}"
export SGLANG_MAX_PREFILL_TOKENS="${SGLANG_MAX_PREFILL_TOKENS:-16384}"
export SGLANG_KV_CACHE_DTYPE="${SGLANG_KV_CACHE_DTYPE:-fp8_e4m3}"
export SGLANG_PORT="${SGLANG_PORT:-18000}"
export SGLANG_HOST="${SGLANG_HOST:-127.0.0.1}"
export SGLANG_RANDOM_SEED="${SGLANG_RANDOM_SEED:-12345}"
export SGLANG_FP4_GEMM_BACKEND="${SGLANG_FP4_GEMM_BACKEND:-auto}"
export SGLANG_TP_SIZE="${SGLANG_TP_SIZE:-1}"
export SGLANG_MAX_RUNNING_REQUESTS="${SGLANG_MAX_RUNNING_REQUESTS:-10}"
export SGLANG_PREFILL_MAX_REQUESTS="${SGLANG_PREFILL_MAX_REQUESTS:-$SGLANG_MAX_RUNNING_REQUESTS}"
export SGLANG_SCHEDULE_CONSERVATIVENESS="${SGLANG_SCHEDULE_CONSERVATIVENESS:-1.0}"
export SGLANG_LOAD_BALANCE_METHOD="${SGLANG_LOAD_BALANCE_METHOD:-total_requests}"
if [ -z "${SGLANG_DATA_PARALLEL_SIZE:-}" ]; then
  SGLANG_DATA_PARALLEL_SIZE="$(detect_gpu_count)"
fi
export SGLANG_DATA_PARALLEL_SIZE
export SGLANG_LOG="${SGLANG_LOG:-/var/log/portal/sglang_tp${SGLANG_TP_SIZE}_dp${SGLANG_DATA_PARALLEL_SIZE}_final.log}"
export SGLANG_BASE_GPU_ID="${SGLANG_BASE_GPU_ID:-0}"
export SGLANG_GPU_ID_STEP="${SGLANG_GPU_ID_STEP:-1}"
export SGLANG_DIST_INIT_ADDR="${SGLANG_DIST_INIT_ADDR:-}"
export SGLANG_NCCL_PORT="${SGLANG_NCCL_PORT:-}"

SITE=$(python3 - <<'PY_SITE'
import site

print(site.getsitepackages()[0])
PY_SITE
)
if [ -d "$SITE/nvidia" ]; then
  NVIDIA_LIBS=$(find "$SITE/nvidia" -type d -name lib | paste -sd: -)
  export LD_LIBRARY_PATH="${NVIDIA_LIBS}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
fi

python3 - <<'PY_PATCH_SGLANG'
from pathlib import Path
import importlib.util
import site

paths = [
    Path(root) / "sglang/srt/layers/quantization/modelopt_quant.py"
    for root in site.getsitepackages()
]
config_paths = [
    Path(root) / "sglang/srt/configs/model_config.py"
    for root in site.getsitepackages()
]
spec = importlib.util.find_spec("sglang")
if spec and spec.submodule_search_locations:
    paths.extend(
        Path(root) / "srt/layers/quantization/modelopt_quant.py"
        for root in spec.submodule_search_locations
    )
    config_paths.extend(
        Path(root) / "srt/configs/model_config.py"
        for root in spec.submodule_search_locations
    )

for path in paths:
    if not path.exists():
        continue

    text = path.read_text()
    if 'if quant_algo == "W4A16_NVFP4":' in text:
        break

    needle = "        quant_algo = self._resolve_quant_algo(prefix)\n"
    patch = (
        needle
        + '        if quant_algo == "W4A16_NVFP4":\n'
        + '            quant_algo = "NVFP4"\n'
    )
    if needle not in text:
        raise SystemExit(f"could not patch {path}: missing quant_algo line")

    path.write_text(text.replace(needle, patch, 1))
    print(f"patched {path}")
    break
else:
    raise SystemExit("could not find SGLang modelopt_quant.py")

for path in config_paths:
    if not path.exists():
        continue

    text = path.read_text()
    needle = '            return {"quant_method": "w4afp8", "quant_algo": quant_algo}\n'
    patch = '            return {"quant_method": "modelopt_mixed", "quant_algo": quant_algo}\n'
    if needle not in text:
        if patch in text:
            break
        raise SystemExit(f"could not patch {path}: missing MIXED_PRECISION w4afp8 line")

    path.write_text(text.replace(needle, patch, 1))
    print(f"patched {path}")
    break
else:
    raise SystemExit("could not find SGLang model_config.py")
PY_PATCH_SGLANG

python3 - <<'PY_PATCH_QWEN3_STREAMING'
from pathlib import Path
import importlib.util
import re
import site

# HARNESS_STREAM_PATCH: stream string-parameter chunks to avoid read-stall
# aborts on jumbo heredocs; fragments match stock output byte-for-byte.
METHOD_SOURCE = '''
    # HARNESS_STREAM_PATCH: stream string parameter values incrementally.
    # The stock parser emits nothing until a parameter's terminator arrives, so
    # a jumbo string param (a heredoc writing a whole file) is silent for its
    # entire decode and clients abort healthy streams on read timeouts. String
    # params now stream as JSON fragments as tokens arrive; a short holdback
    # covers partial terminator tags and the trimmable trailing newline, and
    # values that might be the literal `null` are held until disambiguated.
    # Non-string params keep the buffered path (they need whole-value type
    # conversion). Fragment concatenation is byte-identical to the buffered
    # parser's output for any increment split.

    _HP_VALUE_TERMINATORS = ("</parameter>", "<parameter=", "</function>")
    _HP_STRING_TYPES = ("string", "str", "text", "varchar", "char", "enum")

    def _hp_reset_param_state(self):
        self._hp_in_value = False
        self._hp_param_name = None
        self._hp_streamable = False
        self._hp_prefix_sent = False
        self._hp_lead_newline_done = False
        self._hp_pending = ""

    def _hp_param_is_string(self, param_name, tools):
        cfg = self._get_arguments_config(self.current_func_name, tools)
        if param_name not in cfg:
            # The buffered path returns unknown params as raw strings.
            return True
        entry = cfg[param_name]
        if isinstance(entry, dict) and "type" in entry:
            ptype = str(entry["type"]).strip().lower()
        else:
            ptype = "string"
        return ptype in self._HP_STRING_TYPES

    def _hp_holdback_len(self, avail):
        """Chars at the end of `avail` that cannot be emitted yet: the longest
        suffix that is a prefix of a value terminator, plus the immediately
        preceding newline (trimmed if the terminator follows it)."""
        max_len = min(len(avail), max(len(t) for t in self._HP_VALUE_TERMINATORS))
        hold = 0
        for length in range(max_len, 0, -1):
            suffix = avail[len(avail) - length :]
            if any(t.startswith(suffix) for t in self._HP_VALUE_TERMINATORS):
                hold = length
                break
        if len(avail) > hold and avail[len(avail) - hold - 1] == "\\n":
            hold += 1
        return hold

    def _hp_stream_value_chunk(self, raw_chunk, tools):
        """Emit fragments for a mid-value chunk. The caller advances the cursor
        by len(raw_chunk) regardless; trims only affect emitted text."""
        out = []
        chunk = raw_chunk
        if not self._hp_lead_newline_done:
            self._hp_lead_newline_done = True
            if chunk.startswith("\\n"):
                chunk = chunk[1:]
        if not self._hp_prefix_sent:
            self._hp_pending += chunk
            low = self._hp_pending.lower()
            if len(self._hp_pending) <= 4 and "null".startswith(low):
                return out  # may still be the literal null
            if not self._hp_streamable:
                return out  # buffered until the terminator
            sep = ", " if self.current_tool_param_count > 0 else ""
            out.append(
                ToolCallItem(
                    tool_index=self.current_tool_id,
                    parameters=f'{sep}{json.dumps(self._hp_param_name)}: "',
                )
            )
            self._hp_prefix_sent = True
            chunk = self._hp_pending
            self._hp_pending = ""
        elif not self._hp_streamable:
            self._hp_pending += chunk
            return out
        if chunk:
            fragment = json.dumps(chunk, ensure_ascii=False)[1:-1]
            if fragment:
                out.append(
                    ToolCallItem(
                        tool_index=self.current_tool_id, parameters=fragment
                    )
                )
        return out

    def _hp_finish_value(self, raw_tail, tools):
        """Close the parameter whose terminator begins right after raw_tail."""
        out = []
        tail = raw_tail
        if not self._hp_lead_newline_done:
            self._hp_lead_newline_done = True
            if tail.startswith("\\n"):
                tail = tail[1:]
        if self._hp_prefix_sent:
            # Streaming already began; the value's final newline (if any) was
            # held back by _hp_holdback_len, so the whole-value trailing trim
            # reduces to trimming this tail.
            if tail.endswith("\\n"):
                tail = tail[:-1]
            if tail:
                fragment = json.dumps(tail, ensure_ascii=False)[1:-1]
                if fragment:
                    out.append(
                        ToolCallItem(
                            tool_index=self.current_tool_id, parameters=fragment
                        )
                    )
            out.append(
                ToolCallItem(tool_index=self.current_tool_id, parameters='"')
            )
        else:
            value = self._hp_pending + tail
            if value.endswith("\\n"):
                value = value[:-1]
            param_config = self._get_arguments_config(self.current_func_name, tools)
            converted = self._convert_param_value(
                value, self._hp_param_name, param_config, self.current_func_name
            )
            json_key_val = (
                f"{json.dumps(self._hp_param_name)}: "
                f"{json.dumps(converted, ensure_ascii=False)}"
            )
            sep = ", " if self.current_tool_param_count > 0 else ""
            out.append(
                ToolCallItem(
                    tool_index=self.current_tool_id,
                    parameters=f"{sep}{json_key_val}",
                )
            )
        return out

    def parse_streaming_increment(
        self, new_text: str, tools: List[Tool]
    ) -> StreamingParseResult:
        """
        Robust cursor-based streaming parser with incremental string-parameter
        streaming (HARNESS_STREAM_PATCH).
        """
        if not hasattr(self, "_hp_in_value"):
            self._hp_reset_param_state()

        self._buffer += new_text

        if not self._buffer:
            return StreamingParseResult()

        calls = []
        normal_text_chunks = []

        while True:
            current_slice = self._buffer[self.parsed_pos :]

            if not current_slice:
                break

            # Must precede every tag check: value payload may contain
            # tag-lookalike text, which only the terminator search may end.
            if self._hp_in_value:
                best_pos = -1
                best_token_len = 0
                for token in self._HP_VALUE_TERMINATORS:
                    idx = current_slice.find(token)
                    if idx != -1 and (best_pos == -1 or idx < best_pos):
                        best_pos = idx
                        # Only </parameter> is consumed with the value; the
                        # other terminators are re-parsed by the main loop.
                        best_token_len = (
                            len(token) if token == self.parameter_end_token else 0
                        )
                if best_pos != -1:
                    calls.extend(
                        self._hp_finish_value(current_slice[:best_pos], tools)
                    )
                    self.parsed_pos += best_pos + best_token_len
                    self.current_tool_param_count += 1
                    self._hp_reset_param_state()
                    continue
                hold = self._hp_holdback_len(current_slice)
                emit = current_slice[: len(current_slice) - hold]
                if emit:
                    calls.extend(self._hp_stream_value_chunk(emit, tools))
                    self.parsed_pos += len(emit)
                break

            if current_slice.startswith(self.tool_call_start_token):
                self.parsed_pos += len(self.tool_call_start_token)
                self.is_inside_tool_call = True
                continue

            if current_slice.startswith(self.tool_call_prefix):
                end_angle = current_slice.find(">")
                if end_angle != -1:
                    func_name = current_slice[len(self.tool_call_prefix) : end_angle]

                    self.current_tool_id += 1
                    self.current_tool_name_sent = True
                    self.current_tool_param_count = 0
                    self.json_started = False
                    self.current_func_name = func_name

                    calls.append(
                        ToolCallItem(
                            tool_index=self.current_tool_id,
                            name=func_name,
                            parameters="",
                        )
                    )

                    self.parsed_pos += end_angle + 1
                    continue
                else:
                    break

            if current_slice.startswith(self.parameter_prefix):
                name_end = current_slice.find(">")
                if name_end == -1:
                    break
                param_name = current_slice[len(self.parameter_prefix) : name_end]

                if not self.json_started:
                    calls.append(
                        ToolCallItem(
                            tool_index=self.current_tool_id, parameters="{"
                        )
                    )
                    self.json_started = True

                self._hp_reset_param_state()
                self._hp_in_value = True
                self._hp_param_name = param_name
                self._hp_streamable = self._hp_param_is_string(param_name, tools)
                self.parsed_pos += name_end + 1
                continue

            if current_slice.startswith(self.function_end_token):
                if not self.json_started:
                    calls.append(
                        ToolCallItem(tool_index=self.current_tool_id, parameters="{")
                    )
                    self.json_started = True

                calls.append(
                    ToolCallItem(tool_index=self.current_tool_id, parameters="}")
                )
                self.parsed_pos += len(self.function_end_token)
                self.current_func_name = None
                continue

            if current_slice.startswith(self.tool_call_end_token):
                self.parsed_pos += len(self.tool_call_end_token)
                self.is_inside_tool_call = False
                continue

            next_open_angle = current_slice.find("<")

            if next_open_angle == -1:
                if not self.is_inside_tool_call:
                    normal_text_chunks.append(current_slice)
                self.parsed_pos += len(current_slice)
                continue

            elif next_open_angle == 0:
                possible_tags = [
                    self.tool_call_start_token,
                    self.tool_call_end_token,
                    self.tool_call_prefix,
                    self.function_end_token,
                    self.parameter_prefix,
                    self.parameter_end_token,
                ]

                is_potential_tag = False
                for tag in possible_tags:
                    if tag.startswith(current_slice):
                        is_potential_tag = True
                        break

                # Hold a partial tag prefix until later bytes disambiguate it.
                if is_potential_tag:
                    break
                else:
                    if not self.is_inside_tool_call:
                        normal_text_chunks.append("<")
                    self.parsed_pos += 1
                    continue

            else:
                text_segment = current_slice[:next_open_angle]
                if not self.is_inside_tool_call:
                    normal_text_chunks.append(text_segment)
                self.parsed_pos += next_open_angle
                continue

        if self.parsed_pos > 0:
            self._buffer = self._buffer[self.parsed_pos :]
            self.parsed_pos = 0

        normal_text = "".join(normal_text_chunks) if normal_text_chunks else ""
        return StreamingParseResult(calls=calls, normal_text=normal_text)
'''

paths = [
    Path(root) / "sglang/srt/function_call/qwen3_coder_detector.py"
    for root in site.getsitepackages()
]
spec = importlib.util.find_spec("sglang")
if spec and spec.submodule_search_locations:
    paths.extend(
        Path(root) / "srt/function_call/qwen3_coder_detector.py"
        for root in spec.submodule_search_locations
    )

for path in paths:
    if not path.exists():
        continue
    text = path.read_text()
    if "HARNESS_STREAM_PATCH" in text:
        print(f"already patched {path}")
        break
    span = re.compile(
        r"\n    def parse_streaming_increment\(.*?"
        r"(?=\n    def supports_structural_tag)",
        re.DOTALL,
    )
    if not span.search(text):
        raise SystemExit(
            f"could not patch {path}: parse_streaming_increment span not found"
        )
    # Callable replacement: a plain string would have its backslash escapes
    # (the method source's "\n" literals) expanded by re.sub.
    patched = span.sub(lambda _m: "\n" + METHOD_SOURCE.strip("\n"), text, count=1)
    compile(patched, str(path), "exec")  # fail fast on syntax damage
    backup = path.with_suffix(path.suffix + ".orig")
    if not backup.exists():
        backup.write_text(text)
    path.write_text(patched)
    print(f"patched {path}")
    break
else:
    raise SystemExit("could not find SGLang qwen3_coder_detector.py")
PY_PATCH_QWEN3_STREAMING

if [ ! -f "$MODEL_PATH/.lm_head_bf16_patched" ]; then
  python3 - <<'PY_BUILD_MODEL'
from pathlib import Path
import json
import os
import shutil
import subprocess
import sys

for module, package in [
    ("huggingface_hub", "huggingface_hub"),
    ("safetensors", "safetensors"),
]:
    try:
        __import__(module)
    except ImportError:
        subprocess.check_call(
            [sys.executable, "-m", "pip", "install", "--no-cache-dir", package]
        )

from huggingface_hub import snapshot_download
from safetensors import safe_open
from safetensors.torch import load_file, save_file

nv_repo = os.environ["NVFP4_REPO"]
base_repo = os.environ["BASE_REPO"]
out = Path(os.environ["MODEL_PATH"])
base = Path("/workspace/qwen36_base_lm_head")
out.mkdir(parents=True, exist_ok=True)
base.mkdir(parents=True, exist_ok=True)

snapshot_download(nv_repo, local_dir=str(out))
snapshot_download(
    base_repo,
    local_dir=str(base),
    allow_patterns=["model.safetensors.index.json"],
)
base_index = json.loads((base / "model.safetensors.index.json").read_text())
base_shard_name = base_index["weight_map"]["lm_head.weight"]
snapshot_download(
    base_repo,
    local_dir=str(base),
    allow_patterns=["model.safetensors.index.json", base_shard_name],
)

nv_index_path = out / "model.safetensors.index.json"
nv_index = json.loads(nv_index_path.read_text())
nv_shard = out / nv_index["weight_map"]["lm_head.weight"]
base_shard = base / base_shard_name

tensors = load_file(nv_shard)
for key in (
    "lm_head.input_scale",
    "lm_head.weight",
    "lm_head.weight_scale",
    "lm_head.weight_scale_2",
):
    tensors.pop(key, None)
with safe_open(base_shard, framework="pt", device="cpu") as base_file:
    tensors["lm_head.weight"] = base_file.get_tensor("lm_head.weight")
with safe_open(nv_shard, framework="pt", device="cpu") as old_file:
    metadata = old_file.metadata()

tmp = nv_shard.with_suffix(nv_shard.suffix + ".tmp")
save_file(tensors, tmp, metadata=metadata)
os.replace(tmp, nv_shard)

for key in (
    "lm_head.input_scale",
    "lm_head.weight_scale",
    "lm_head.weight_scale_2",
):
    nv_index["weight_map"].pop(key, None)
nv_index["weight_map"]["lm_head.weight"] = nv_shard.name
nv_index_path.write_text(json.dumps(nv_index, indent=2) + "\n")


def remove_lm_head_quant(path: Path) -> None:
    data = json.loads(path.read_text())
    quant = data.get("quantization_config") or data.get("quantization") or data
    layers = quant.get("quantized_layers")
    if isinstance(layers, list):
        quant["quantized_layers"] = [x for x in layers if x != "lm_head"]
    elif isinstance(layers, dict):
        layers.pop("lm_head", None)

    for group_key in ("config_groups", "quant_cfg"):
        for group in quant.get(group_key, {}).values():
            targets = group.get("targets")
            if isinstance(targets, list):
                group["targets"] = [x for x in targets if x != "lm_head"]

    path.write_text(json.dumps(data, indent=2) + "\n")


for rel in ("config.json", "hf_quant_config.json"):
    path = out / rel
    if path.exists():
        remove_lm_head_quant(path)

with safe_open(nv_shard, framework="pt", device="cpu") as patched:
    keys = [key for key in patched.keys() if key.startswith("lm_head")]
    print("lm_head keys", keys)
    print("lm_head shape", tuple(patched.get_tensor("lm_head.weight").shape))

(out / ".lm_head_bf16_patched").write_text("ok\n")
shutil.rmtree(base, ignore_errors=True)
PY_BUILD_MODEL
fi

python3 - <<'PY_STOP_SGLANG'
import os
import signal
import time


def read_cmdline(pid: int) -> str:
    try:
        return (
            open(f"/proc/{pid}/cmdline", "rb")
            .read()
            .replace(b"\0", b" ")
            .decode("utf-8", "ignore")
        )
    except OSError:
        return ""


def read_comm(pid: int) -> str:
    try:
        return open(f"/proc/{pid}/comm").read()
    except OSError:
        return ""


def child_map() -> dict[int, list[int]]:
    children: dict[int, list[int]] = {}
    for name in os.listdir("/proc"):
        if not name.isdigit():
            continue
        pid = int(name)
        try:
            stat = open(f"/proc/{pid}/stat").read().split()
        except OSError:
            continue
        if len(stat) > 3:
            children.setdefault(int(stat[3]), []).append(pid)
    return children


roots = [
    int(name)
    for name in os.listdir("/proc")
    if name.isdigit()
    and int(name) != os.getpid()
    and (
        "sglang.launch_server" in read_cmdline(int(name))
        or read_comm(int(name)).startswith("sglang::")
    )
]
children = child_map()
targets: set[int] = set()


def collect(pid: int) -> None:
    if pid in targets:
        return
    targets.add(pid)
    for child in children.get(pid, []):
        collect(child)


for root in roots:
    collect(root)

for sig, delay in ((signal.SIGTERM, 3), (signal.SIGKILL, 1)):
    for pid in sorted(targets, reverse=True):
        try:
            os.kill(pid, sig)
        except ProcessLookupError:
            pass
    if targets:
        time.sleep(delay)
PY_STOP_SGLANG

: > "$SGLANG_LOG"

DIST_INIT_ARGS=()
if [ -n "$SGLANG_DIST_INIT_ADDR" ]; then
  DIST_INIT_ARGS=(--dist-init-addr "$SGLANG_DIST_INIT_ADDR")
fi

NCCL_ARGS=()
if [ -n "$SGLANG_NCCL_PORT" ]; then
  NCCL_ARGS=(--nccl-port "$SGLANG_NCCL_PORT")
fi

setsid python3 -u -m sglang.launch_server \
  --model-path "$MODEL_PATH" \
  --served-model-name "$SERVED_MODEL_NAME" \
  --host "$SGLANG_HOST" \
  --port "$SGLANG_PORT" \
  --tp-size "$SGLANG_TP_SIZE" \
  --data-parallel-size "$SGLANG_DATA_PARALLEL_SIZE" \
  --base-gpu-id "$SGLANG_BASE_GPU_ID" \
  --gpu-id-step "$SGLANG_GPU_ID_STEP" \
  "${DIST_INIT_ARGS[@]}" \
  "${NCCL_ARGS[@]}" \
  --context-length "$SGLANG_CONTEXT_LENGTH" \
  --mem-fraction-static "$SGLANG_MEM_FRACTION_STATIC" \
  --chunked-prefill-size "$SGLANG_CHUNKED_PREFILL_SIZE" \
  --max-prefill-tokens "$SGLANG_MAX_PREFILL_TOKENS" \
  --schedule-conservativeness "$SGLANG_SCHEDULE_CONSERVATIVENESS" \
  --attention-backend triton \
  --kv-cache-dtype "$SGLANG_KV_CACHE_DTYPE" \
  --disable-radix-cache \
  --enable-deterministic-inference \
  --random-seed "$SGLANG_RANDOM_SEED" \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen3_coder \
  --grammar-backend xgrammar \
  --enable-strict-thinking \
  --language-only \
  --enable-tokenizer-batch-encode \
  --quantization modelopt_mixed \
  --fp4-gemm-backend "$SGLANG_FP4_GEMM_BACKEND" \
  --enable-metrics \
  --enable-metrics-for-all-schedulers \
  --enable-request-time-stats-logging \
  --show-time-cost \
  --max-running-requests "$SGLANG_MAX_RUNNING_REQUESTS" \
  --prefill-max-requests "$SGLANG_PREFILL_MAX_REQUESTS" \
  --load-balance-method "$SGLANG_LOAD_BALANCE_METHOD" \
  > "$SGLANG_LOG" 2>&1 &

echo "$!" >/tmp/sglang_qwen36_nvfp4_rtx5090.pid
echo "started SGLang pid $(cat /tmp/sglang_qwen36_nvfp4_rtx5090.pid)"
echo "target: RTX 5090 32GB"
echo "health: http://${SGLANG_HOST}:${SGLANG_PORT}/health"
echo "log: $SGLANG_LOG"