frank-wei

import tensorrt as trt
import torch

# https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#python_topics

"""
TensorRT Initialization
"""
TRT_LOGGER = trt.Logger(trt.Logger.VERBOSE)

frank-wei

0
True
True
True
1
True
True
True
2
True

frank-wei

from openai import OpenAI

# Point to your vLLM server (default: localhost:8000)
client = OpenAI(
    base_url="http://localhost:8000/v1",  # adjust port if different
    api_key="EMPTY"  # vLLM doesn’t require authentication
)

# Choose your GPT-OSS model (must match the one you served via vllm serve)
MODEL_NAME = "/home/wwei6/local/checkpoints/gpt-oss-120b"   # or "openai/gpt-oss-120b"

frank-wei

P1977951167 - copy
FBID: 2225683307937911
(An Untitled Masterwork)
Visible to All Users
Author
wwei6
Created
Sat Oct 4, 2025 10:29pm
Forks

frank-wei

P1977390868 - copy
FBID: 751895307897003
(An Untitled Masterwork)
Visible to All Users
Author
wwei6
Created
Sat Oct 4, 2025 10:39am
Forks

frank-wei

{
             "type": "structural_tag",
               "format": {
                   "type":
                   "triggered_tags",
                   "stop_after_first":
                   False,
                   "tags": [{
                       "begin": "container.exec <|constrain|>json",
                       "content": {

frank-wei

server:
vllm serve  /data/local/model/Qwen2.5-3B-Instruct/   --port 8081 

Client:
```
from openai import OpenAI
import json
client = OpenAI(
    base_url="http://localhost:8081/v1",
    api_key="-",