Try to compare rate distortion factor between h.264 and vp8 encoding.

video RDF check script

#!/bin/sh
set -e
IN="$1"
TMP=$(mktemp -d)
mkdir "$TMP/src"
#create our sample videos
avconv -f image2 -i "$IN" "$TMP/src/frame-%04d.png"
#H.264 profiles
avconv -f image2 -i "$IN" -c:v h264 -c:a none -pix_fmt yuv420p -profile:v main -level 31 "$TMP/out.mp4"
avconv -f image2 -i "$IN" -c:v h264 -c:a none -pix_fmt yuv420p -profile:v high -level 41 "$TMP/out_h.mp4"

#VP8
avconv -f image2 -i "$IN" -c:v libvpx -c:a libvorbis -qmin 20 -qmax 35 -threads auto "$TMP/out.webm"
avconv -f image2 -i "$IN" -pass 1 -y -passlogfile $TMP/vp8.fpf -c:v libvpx -c:a libvorbis -qmin 20 -qmax 35 -threads auto "$TMP/out_2.webm"
avconv -f image2 -i "$IN" -pass 2 -y -passlogfile $TMP/vp8.fpf -c:v libvpx -c:a libvorbis -qmin 20 -qmax 35 -threads auto "$TMP/out_2.webm"

#VP9
avconv -f image2 -i "$IN" -c:v libvpx-vp9 -c:a none -qmin 20 -qmax 35 -threads auto "$TMP/out_9.webm"

avconv -f image2 -i "$IN" -pass 1 -y -passlogfile $TMP/vp9.fpf -c:v libvpx-vp9 -c:a none -qmin 20 -qmax 35 -threads auto "$TMP/out_92.webm"
avconv -f image2 -i "$IN" -pass 2 -y -passlogfile $TMP/vp9.fpf -c:v libvpx-vp9 -c:a none -qmin 20 -qmax 35 -threads auto "$TMP/out_92.webm"


#extract png frames from them
extract(){
	mkdir "$TMP/$2"
	avconv -i "$TMP/$1"  "$TMP/$2/frame-%04d.png"
}
extract "out.mp4"  "mp4"
extract "out_h.mp4" "mp4h"
extract "out.webm" "vp8"
extract "out_2.webm" "vp82"
extract "out_9.webm" "vp9"
extract "out_92.webm" "vp92"
#Compare
get_size(){
	file_size=`wc -c "$TMP/$1"|cut -d " " -f 1`
	echo "$((file_size/1000))"
}

frames_count="$(ls -1 $TMP/src/ |wc -l)"

get_noise(){
	set +e #because compare "fails"
	local noise=0
	for i in $(seq -f "%04g" 1 $frames_count) ;do
		noise=$(echo "$noise + $(compare -metric PSNR "$TMP/$1/frame-$i.png" "$TMP/src/frame-$i.png" null: 2>&1)"|bc)
	done
	set -e
	#noise=$(echo "$noise / $frames_count" |bc)
	echo "$noise"
}

metrics(){
	echo "$1 : $(get_size "$2")kB . PSNR : $(get_noise "$3")"
}

metrics "MP4(main)" "out.mp4" "mp4"
metrics "MP4(high)" "out_h.mp4" "mp4h"
metrics "VP8 (1 p)" "out.webm" "vp8"
metrics "VP8 (2 p)" "out_2.webm" "vp82"
metrics "VP9 (1 p)" "out_9.webm" "vp9"
metrics "VP9 (2 p)" "out_92.webm" "vp92"
rm -rf "$TMP"