My custom High Quality XBOX 360 Compatible Handbrake profile. Video: AVC, 3000kbps Constant, High@L4.1 - Audio: AC3, 640kbps Constant, 6ch. This does make an MKV, which the 360 doesn't support. You will need to use My MP4Box GUI or similar to change the container to MP4. There are some issues getting Handrbake to go MKV to MP4 directly.

XBOX 360 HQ.plist

<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist>
  <array>
    <dict>
      <key>AudioList</key>
      <array>
        <dict>
          <key>AudioBitrate</key>
          <string>640</string>
          <key>AudioEncoder</key>
          <string>AC3</string>
          <key>AudioMixdown</key>
          <string>5.1 Channels</string>
          <key>AudioSamplerate</key>
          <string>48</string>
          <key>AudioTrack</key>
          <integer>1</integer>
          <key>AudioTrackDRCSlider</key>
          <real>0</real>
          <key>AudioTrackDescription</key>
          <string>Unknown</string>
          <key>AudioTrackGainSlider</key>
          <real>0</real>
        </dict>
      </array>
      <key>AudioAllowAACPass</key>
      <true />
      <key>AudioAllowAC3Pass</key>
      <true />
      <key>AudioAllowDTSHDPass</key>
      <true />
      <key>AudioAllowDTSPass</key>
      <true />
      <key>AudioAllowMP3Pass</key>
      <true />
      <key>AudioEncoderFallback</key>
      <string>AC3</string>
      <key>ChapterMarkers</key>
      <false />
      <key>Default</key>
      <integer>0</integer>
      <key>FileFormat</key>
      <string>MKV file</string>
      <key>Folder</key>
      <false />
      <key>Mp4HttpOptimize</key>
      <integer>0</integer>
      <key>Mp4iPodCompatible</key>
      <integer>0</integer>
      <key>PictureAutoCrop</key>
      <integer>1</integer>
      <key>PictureDeblock</key>
      <integer>4</integer>
      <key>PictureDecombDeinterlace</key>
      <false />
      <key>PictureDecombCustom</key>
      <string />
      <key>PictureDecomb</key>
      <integer>0</integer>
      <key>PictureDeinterlaceCustom</key>
      <string />
      <key>PictureDeinterlace</key>
      <integer>0</integer>
      <key>PictureDenoiseFilter</key>
      <string>off</string>
      <key>PictureDenoiseCustom</key>
      <string />
      <key>PictureDenoisePreset</key>
      <string>weak</string>
      <key>PictureDetelecine</key>
      <integer>0</integer>
      <key>PictureDetelecineCustom</key>
      <string />
      <key>PictureHeight</key>
      <integer>0</integer>
      <key>PictureKeepRatio</key>
      <integer>1</integer>
      <key>PictureModulus</key>
      <integer>16</integer>
      <key>PicturePAR</key>
      <integer>1</integer>
      <key>PictureLeftCrop</key>
      <integer>0</integer>
      <key>PictureRightCrop</key>
      <integer>0</integer>
      <key>PictureTopCrop</key>
      <integer>0</integer>
      <key>PictureBottomCrop</key>
      <integer>0</integer>
      <key>PictureWidth</key>
      <integer>0</integer>
      <key>PresetBuildNumber</key>
      <string>2014112200</string>
      <key>PresetDescription</key>
      <string>No Description</string>
      <key>PresetName</key>
      <string>Cyber HQ</string>
      <key>Type</key>
      <integer>1</integer>
      <key>UsesMaxPictureSettings</key>
      <integer>1</integer>
      <key>UsesPictureFilters</key>
      <integer>1</integer>
      <key>UsesPictureSettings</key>
      <integer>1</integer>
      <key>VideoAvgBitrate</key>
      <string>3000</string>
      <key>VideoEncoder</key>
      <string>H.264 (x264)</string>
      <key>VideoFramerate</key>
      <string>Same as source</string>
      <key>VideoFramerateMode</key>
      <string>cfr</string>
      <key>VideoGrayScale</key>
      <false />
      <key>VideoQualitySlider</key>
      <real>20</real>
      <key>h264Level</key>
      <string>4.1</string>
      <key>x264OptionExtra</key>
      <string />
      <key>x264Preset</key>
      <string>medium</string>
      <key>h264Profile</key>
      <string>high</string>
      <key>x264Tune</key>
      <string>none</string>
      <key>x264UseAdvancedOptions</key>
      <integer>0</integer>
      <key>h265Profile</key>
      <string>none</string>
      <key>x265Preset</key>
      <string>veryfast</string>
      <key>x265Tune</key>
      <string>none</string>
      <key>VideoQualityType</key>
      <integer>1</integer>
      <key>VideoTargetSize</key>
      <string />
      <key>VideoTurboTwoPass</key>
      <integer>0</integer>
      <key>VideoTwoPass</key>
      <integer>0</integer>
      <key>x264Option</key>
      <string />
    </dict>
  </array>
</plist>