My pulseaudio configs

client.conf

# This file is part of PulseAudio.
#
# PulseAudio is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# PulseAudio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PulseAudio; if not, see <http://www.gnu.org/licenses/>.

## Configuration file for PulseAudio clients. See pulse-client.conf(5) for
## more information. Default values are commented out.  Use either ; or # for
## commenting.

; default-sink =
; default-source =
; default-server =
; default-dbus-server =

; autospawn = yes
; daemon-binary = /usr/bin/pulseaudio
; extra-arguments = --log-target=syslog

; cookie-file =

; enable-shm = yes
; shm-size-bytes = 0 # setting this 0 will use the system-default, usually 64 MiB

; auto-connect-localhost = no
; auto-connect-display = no

daemon.conf

# This file is part of PulseAudio.
#
# PulseAudio is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# PulseAudio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PulseAudio; if not, see <http://www.gnu.org/licenses/>.

## Configuration file for the PulseAudio daemon. See pulse-daemon.conf(5) for
## more information. Default values are commented out.  Use either ; or # for
## commenting.

; daemonize = no
; fail = yes
; allow-module-loading = yes
; allow-exit = yes
; use-pid-file = yes
; system-instance = no
; local-server-type = user
; enable-shm = yes
; shm-size-bytes = 0 # setting this 0 will use the system-default, usually 64 MiB
; lock-memory = no
; cpu-limit = no

####################
# I'm not using PA as default, because sometimes it "hangs" at 100% CPU
# So I do not want it in high priority or as realtime task
####################
high-priority = no
nice-level = 10

realtime-scheduling = no
; realtime-priority = 5

####################
# Pulseaudio will quit after 20 sec after I close game.
# Also, I needed to remove /etc/xdg/autostart/pulseaudio.desktop installed with pulseaudio package (Gentoo specific?),
# because otherwise PA is started when I'm logging in. We do not want it.
####################
exit-idle-time = 20
; scache-idle-time = 20

; dl-search-path = (depends on architecture)

; load-default-script-file = yes
; default-script-file = /etc/pulse/default.pa

; log-target = auto
; log-level = notice
; log-meta = no
; log-time = no
; log-backtrace = 0

; resample-method = speex-float-1
resample-method = speex-fixed-0
; enable-remixing = yes
; enable-lfe-remixing = yes
; lfe-crossover-freq = 120

####################
# I always hated that part, it is probably not used in current configuration
# When I set system volume on 50%, do not EVER set it higher for me.
####################
flat-volumes = no

; rlimit-fsize = -1
; rlimit-data = -1
; rlimit-stack = -1
; rlimit-core = -1
; rlimit-as = -1
; rlimit-rss = -1
; rlimit-nproc = -1
; rlimit-nofile = 256
; rlimit-memlock = -1
; rlimit-locks = -1
; rlimit-sigpending = -1
; rlimit-msgqueue = -1
; rlimit-nice = 31
; rlimit-rtprio = 9
; rlimit-rttime = 200000

####################
# This are my settings in alsa
# defaults were different here
###################
default-sample-format = s16le
default-sample-rate = 48000
alternate-sample-rate = 44100
; default-sample-channels = 2
; default-channel-map = front-left,front-right

; default-fragments = 4
; default-fragment-size-msec = 25

enable-deferred-volume = no
; deferred-volume-safety-margin-usec = 8000
; deferred-volume-extra-delay-usec = 0

default.pa

#!/usr/bin/pulseaudio -nF
#
# This file is part of PulseAudio.
#
# PulseAudio is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# PulseAudio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PulseAudio; if not, see <http://www.gnu.org/licenses/>.

# This startup script is used only if PulseAudio is started per-user
# (i.e. not in system mode)

.nofail

### Load something into the sample cache
#load-sample-lazy x11-bell /usr/share/sounds/freedesktop/stereo/bell.oga
#load-sample-lazy pulse-hotplug /usr/share/sounds/freedesktop/stereo/device-added.oga
#load-sample-lazy pulse-coldplug /usr/share/sounds/freedesktop/stereo/device-added.oga
#load-sample-lazy pulse-access /usr/share/sounds/freedesktop/stereo/message.oga

.fail

### Automatically restore the volume of streams and devices
# load-module module-device-restore
# load-module module-stream-restore
# load-module module-card-restore

### Automatically augment property information from .desktop files
### stored in /usr/share/application
# load-module module-augment-properties

### Should be after module-*-restore but before module-*-detect
# load-module module-switch-on-port-available

### Load audio drivers statically
### (it's probably better to not load these drivers manually, but instead
### use module-udev-detect -- see below -- for doing this automatically)

####################
# This part we need
# Do not use device=default when alsa default is set to pulse!
####################

load-module module-alsa-sink device=default
load-module module-alsa-source device=default

#load-module module-null-sink
#load-module module-pipe-sink

### Automatically load driver modules depending on the hardware available
# .ifexists module-udev-detect.so
# load-module module-udev-detect
# .else
### Use the static hardware detection module (for systems that lack udev support)
#load-module module-detect
# .endif

### Automatically connect sink and source if JACK server is present
# .ifexists module-jackdbus-detect.so
# .nofail
# load-module module-jackdbus-detect channels=2
# .fail
# .endif

### Automatically load driver modules for Bluetooth hardware
# .ifexists module-bluetooth-policy.so
# load-module module-bluetooth-policy
# .endif

# .ifexists module-bluetooth-discover.so
# load-module module-bluetooth-discover
# .endif

### Load several protocols
# .ifexists module-esound-protocol-unix.so
# load-module module-esound-protocol-unix
# .endif

####################
# This part is used in clients linking to libpulse*.so
# so we need this
####################

load-module module-native-protocol-unix

### Network access (may be configured with paprefs, so leave this commented
### here if you plan to use paprefs)
#load-module module-esound-protocol-tcp
#load-module module-native-protocol-tcp

### Load the RTP receiver module (also configured via paprefs, see above)
#load-module module-rtp-recv

### Load the RTP sender module (also configured via paprefs, see above)
#load-module module-null-sink sink_name=rtp format=s16be channels=2 rate=44100 sink_properties="device.description='RTP Multicast Sink'"
#load-module module-rtp-send source=rtp.monitor

### Load additional modules from GConf settings. This can be configured with the paprefs tool.
### Please keep in mind that the modules configured by paprefs might conflict with manually
### loaded modules.
# .ifexists module-gconf.so
# .nofail
# load-module module-gconf
# .fail
# .endif

### Automatically restore the default sink/source when changed by the user
### during runtime
### NOTE: This should be loaded as early as possible so that subsequent modules
### that look up the default sink/source get the right value
# load-module module-default-device-restore

### Automatically move streams to the default sink if the sink they are
### connected to dies, similar for sources
# load-module module-rescue-streams

### Make sure we always have a sink around, even if it is a null sink.
# load-module module-always-sink

### Honour intended role device property
# load-module module-intended-roles

### Automatically suspend sinks/sources that become idle for too long
# load-module module-suspend-on-idle

### If autoexit on idle is enabled we want to make sure we only quit
### when no local session needs us anymore.
# .ifexists module-console-kit.so
# load-module module-console-kit
# .endif
# .ifexists module-systemd-login.so
# load-module module-systemd-login
# .endif

### Enable positioned event sounds
# load-module module-position-event-sounds

### Cork music/video streams when a phone stream is active
# load-module module-role-cork

### Modules to allow autoloading of filters (such as echo cancellation)
### on demand. module-filter-heuristics tries to determine what filters
### make sense, and module-filter-apply does the heavy-lifting of
### loading modules and rerouting streams.
# load-module module-filter-heuristics
# load-module module-filter-apply

# X11 modules should not be started from default.pa so that one daemon
# can be shared by multiple sessions.

### Load X11 bell module
#load-module module-x11-bell sample=x11-bell

### Register ourselves in the X11 session manager
#load-module module-x11-xsmp

### Publish connection data in the X11 root window
#.ifexists module-x11-publish.so
#.nofail
#load-module module-x11-publish
#.fail
#.endif

### Make some devices default
#set-default-sink output
#set-default-source input

I will write it again here:
I needed to remove /etc/xdg/autostart/pulseaudio.desktop installed with pulseaudio package (Gentoo specific?), otherwise PA is started when I'm logging in. We do not want it that way.