data57b.rb

require 'statsample'


def f2(n)
	return n.odd? ? (n * 3 + 1) / 2 : n / 2

end

def f1(n)
	n = (n * 3 + 1) / 2 if (n.odd?)
	n /= 2 if (n.even?)
	return n

end

def dense(w, d)
	w2 = w - 1
	
	a = (0...w2).to_a
	s = '0' * w2
	(1..(d * w - 1)).map { a.delete_at(rand(a.size)) }.each { |x| s[x, 1] = '1' }
	return ('1' + s).to_i(2) 
	
end

def stat(l)
	l = [0] if (l.empty?)
        t = t2 = 0
        l.each \
        {
		|x|
		t += x
		t2 += x ** 2
        }
        c = l.size
        a = t.to_f / c
        z = t2.to_f / c - a ** 2
        sd = Math.sqrt(z < 0 ? 0 : z)

	return a, sd, l.max.to_f, l.min.to_f
end

def stat2(l, t, n)
	return Hash[[["a#{n}", "sd#{n}", "mx#{n}"], stat(l)[0..2].map { |x| x / t }].transpose]
end

def d(s)
        c = s.split('').select { |x| x == '1' }.size
	d = c.to_f / s.length
        return d
        
end

def data(n)
	ns = n.to_s(2)
	nl = ns.length
	m = nl / 2
	
	nsh = ns[0..m]
	nsl = ns[m..-1]
	
	asdm1 = stat2(ns.split(/0+/).map { |x| x.length }, nl, 1)
	
	l1 = ns.split(/1+/)
	
	l1.shift
	asdm0 = stat2(l1.map { |x| x.length }, nl, 0)
	
	return {'n' => n, 'ns' => ns, 'nl' => nl, 'd' => d(ns), 'dh' => d(nsh), 'dl' => d(nsl)}.merge(asdm1).merge(asdm0)
end

def pair2(n, m)
	n1 = n
	
	m.times { n = f2(n) }
	x = {0 => data(n1), 1 => data(n)}

	x[1]['wr'] = x[1]['nl'].to_f / x[0]['nl'].to_f
	return x
end

def pair(d, m)
	w = 100
	return(pair2(dense(w, d), m))
end

def dist(m, c)
		
	w = 100
	d = 0.0
	l = []
	c.times \
	{
		|i|
		
		l << pair(i.to_f / (c - 1), m)
	}
#	$stderr.puts("#{l.size} pts")
	return l
end

def adv(n, m)

	n1 = n
	l = [n]

	begin
	
		n2 = n
		m.times { n2 = f2(n2) }
		l << n2
		n = n2
	end while (n2 >= n1)
	return l
	
end

def pair2b(n, m)

	l = adv(n, m)
	l = l.map { |x| data(x) }
	
	l1 = []
	(l.size - 1).times \
	{
		|i|
		x = {0 => l[i], 1 => l[i + 1]}
		x[1]['wr'] = x[1]['nl'].to_f / x[0]['nl'].to_f
		x[0]['x'] = adv(l[i]['n'], m).size
		x[0]['y'] = [i, x[0]['d']]
		l1 << x
	}
	
	return l1
end

def pairb(d, m)
	w = 100
	return(pair2b(dense(w, d), m))
end

def distb(m)
		
	w = 100
	d = 0.0
	c = 250
	l = []
	c.times \
	{
		|i|
		
		l += pairb(i.to_f / (c - 1), m)
	}
#	$stderr.puts("#{l.size} pts")
	return l
end

def sum(l)

	t = 0.0
	l.each { |x| t += x }
	return t
end

def av(l)
	return nil if (l.empty?)
	return sum(l) / l.size
end

def corr(l, y1)
	
	yp = "#{y1}_p"
	ye = "#{y1}_e"
	xav = av(l.map { |x| x[y1] })
	yav = av(l.map { |x| x[yp] })

	tx = ty = txy = e = 0.0
	m = nil
	l.each \
	{
		|z|
		
		x = z[y1]
		y = z[yp]
		
		txy += (x - xav) * (y - yav)
		tx += (x - xav) ** 2
		ty += (y - yav) ** 2
		
		z[ye] = (x - y)
		e1 = z[ye]
		e += e1
		m = [m, e1.abs].compact.max
	}
	r = txy / (Math.sqrt(tx) * Math.sqrt(ty))
	e /= l.size
	return {'r' => r, 'e_a' => e, 'e_m' => m}
	
end

def dot(x, z)

	t = z['c']
	
	(z.keys - ['c']).each { |v| t += z[v] * x[v] }
	return t
		
end

def predict(l, vy, z)
	l.each \
	{
		|x|
		
		x[1]["#{vy}_p"] = dot(x[0], z)
	}
end

def fit(l, vx, vy)

	a = {}

	vx.each { |v| a[v] = l.map { |x| x[0][v] }.to_vector() }

	a[vy] = l.map { |x| x[1][vy] }.to_vector()
	
	ds = a.to_dataset()

	r = Statsample::Regression.multiple(ds, vy)
#	$stderr.puts(r.summary)

	z = r.coeffs.merge({'c' => r.constant})
	
	predict(l, vy, z)
	
	a = corr(l.map { |x| x[1] }, vy)
	
	return z.merge!(a)

end

def fmt(a)
	a2 = {}
	a.each \
	{
		|k, v|
		a2[k] = v.is_a?(Numeric) ? sprintf("%.3g", v).to_f : v
	}
	return a2
	
end

def fit1(v, x, l)
	a = fit(l, v, x)
	l1 = l.map { |y| y[1][x] }
	
	a.merge!({'mn' => l1.min, 'mx' => l1.max})

#	$stderr.puts(fmt({'x' => x}.merge(a.select { |k, v| ['r', 'e_m', 'e_a', 'mn', 'mx'].member?(k) })).inspect)
	return a
end

def model(a)

	(a['v'] + ['wr']).each \
	{
		|x|
		a[x] = fit1(a['v'], x, a['l'])
	}

#	$stderr.puts
	return a
	
end

def out(fn, a)
	
	return if (fn.nil?)
	f = File.open(fn, 'a')
	f.puts(a.keys.join("\t")) if (f.size == 0)
	if (a.nil?)
		f.puts
	else
		f.puts(a.values.join("\t"))
	end
	f.close
end

def count(m)
	a = {'l' => dist(m, 100), 'v' => ['a1', 'a0', 'dh', 'dl', 'sd0', 'sd1', 'mx1']}

	a = model(a)
	return a
end

def opt(m, a)
	l = []

	a['l'].size.times \
	{
		|i|
		l << {'ns' => a['l'][i][0]['ns'], 'wr_e' => a['l'][i][1]['wr_e']}
	}
	
	c = 20
	w = 100
	1000.times \
	{
		|n|
		
		c = l.size / 4
		l.sort_by! { |x| x['wr_e'] }
#		l.reverse!
		
		x = rand(c)
		y = rand(c)
		y += 1 if (y == x)
		ns = ns1 = ns2 = nil
		case rand(3)
			when 0
				ns1 = l[x]['ns']
				ns = ns1.dup
				i = rand(w - 1) + 1
				ns[i, 1] = (ns1[i, 1].to_i ^ 1).to_s
			when 1
				ns1 = l[x]['ns']
				ns2 = l[y]['ns']
				ns = ''
				w.times \
				{
					|i|
					ns[i, 1] = [ns1[i, 1], ns2[i, 1]][rand(2)]
				}
				
			when 2
				ns1 = l[x]['ns']
				ns2 = l[y]['ns']
				i = rand(w)
				ns = ns1[0...i] + ns2[i..-1]
		end

		w2 = ns.to_i(2).to_s(2).length
		raise if (w2 != w)
		
		next if (l.map { |x| x['ns'] }.member?(ns))
		
		x = pair2(ns.to_i(2), m)


		(a['v'] + ['wr']).each \
		{
			|vy|
			z = a[vy].select { |k, v| (a['v'] + ['c']).member?(k) }
			
			predict([x], vy, z)
			corr([x[1]], vy)
		}

		l.pop if (l.size > 2000)
		l << {'ns'=> ns, 'wr_e' => x[1]['wr_e']}
		
#		puts([l[0]['wr_e'], l[-1]['wr_e'], d(ns), ns].join("\t")) if (n % 40 == 0)
		$stdout.flush
	}
	return l.first
end

def detect(x1, a, j, l)
	
	x1 = x1.dup
	i = 0
	c = $c
	x1['nl'] = 1
	loop \
	{
		x2 = {}
		(a['v'] + ['wr']).each \
		{
			|vy|
			
			x2[vy] = dot(x1, a[vy].select { |k, x| (a['v'] + ['c']).member?(k) }) 
			
			x2[vy] += l[j][1]["#{vy}_e"] if (!j.nil?)
			
			return c if (x2[vy].abs > 1e10)
		}
		x2['nl'] = x1['nl'] * x2['wr']
		x1 = x2
		i += 1
		break if (x1['nl'] < 1 || i == c)
	}
	return i

end

def count2(l, a, y, l3, l4 = (0...l.size).to_a)
	
	t2 = 0
	l.size.times \
	{
		|i|
		t2 += 1 if (detect(l[i][0], a, nil, nil) == $c)
	}

	t = 0
	l2 = []
	l.size.times \
	{
		|i|
		l1 = []
		l0 = []
		l4.each \
		{
			|j|
			x = detect(l[i][0], a, j, l)
			l1 << j if (x == $c)
			l0 << x
		}
		t += l1.size
		l2 << l1
		l3 << l0
		$stderr.print("#{i} ") if (i % 10 == 0)
	}
	$stderr.puts
	return {"t#{y}" => t, "t#{y}2" => t2, "l#{y}" => l2}
end

def stat1(l)
	return Hash[[['a', 'sd', 'mx', 'mn'], stat(l)].transpose]
end

def test(l, a)

	(a['v'] + ['wr']).each \
	{
		|vy|
		z = a[vy].select { |k, v| (a['v'] + ['c']).member?(k) }
			
		predict(l, vy, z)
		corr(l.map { |x| x[1] }, vy)
	}
	a.merge!({'l2' => l})
	
end

def plot(z, l, li, lo, y, t, nc, f2)
	f = File.open("out#{z}1.txt", 'w')
	l.each_with_index \
	{
		|l1, i|
		l1.each { |x| f.puts([i, x, 0].join("\t")) }
	}
	f.close
	
	f = File.open("out#{z}2.txt", 'w')
	l.each_with_index { |l1, i| f.puts([i, l1.max, l1.min, li.nil? ? nil : li[i], lo.nil? ? nil : lo[i]].join("\t")) }
	f.close
		
	f2.print("set y2tics; plot 'out#{z}1.txt' using 1:2:3 linecolor variable lt 5 ps 0.10 title '#{t}',")
	f2.print("'out#{z}2.txt' using 1:4 with line lc 3 lw 3 title '#{t}i',") if (!li.nil?)
	f2.print("'out#{z}2.txt' using 1:5 with line lc 4 lw 3 title '#{t}o',") if (!lo.nil?)
	f2.print("'out#{z}2.txt' using 1:2 with line lc 2 lw 3 title 'mx',")
	f2.print("'out#{z}2.txt' using 1:3 with line lc 1 lw 3 title 'mn',")
	
	f2.print("#{y} lc 6 lw 3,#{nc}")
	f2.puts
	f2.puts("pause -1; unset y2tics;")
	
	if (t == 'wr_e') then
		f = File.open("out#{z}3.txt", 'w')
		l.each_with_index \
		{
			|l1, i|
			mn = l1.min
			mx = l1.max
			mx += (mx * 0.01)
			c = 10
			l2 = [0] * c
			l1.each { |x| l2[(((x - mn) / (mx - mn)) * c).to_i] += 1 }
			
			(0...c).each { |j| f.puts([j, l2[j], i].join("\t")) }
			f.puts
		}
		f.close
		f2.puts("plot 'out#{z}3.txt' using 1:2:3 with line linecolor palette lw 2")
		f2.puts("pause -1")
	end
	
end

def retest(d, n, f2)

	i = n - 1
	a = Kernel.eval(File.open("#{d}/out#{i}.txt").readlines[0])
		
	l = a['lb'].map { |x| x.size }

	raise if (sum(l) != 0)
	
	
	test(dist($m, 100), a)
	
	l2 = []
	p(count2(a['l2'], a, 'b', l2))
	f = File.open('out.txt', 'w')
	l2.each { |x| f.puts(x.join("\t")) }
	f.close
	f2.puts("plot 'out.txt' matrix with image; pause -1;")

end

def stats(d)
	m = $m
	$stderr.puts(d)

	n = Dir.glob("#{d}/*").select { |x| x =~ /out\d+.txt$/}.size
	
	z = [{'l' => [], 'li' => [], 'lo' => [], 'z' => lambda { |x| x[0]['d'] }, 'b' => 0.5, 't' => 'd', 'f' => 'x'},
		{'l' => [], 'li' => [], 'lo' => [], 'z' => lambda { |x| x[1]['wr_e'] }, 'b' => 0.0, 't' => 'wr_e', 'f' => 'y'}]
		
	f = File.open(fn = 'outx.txt', 'w')
	f2 = File.open(fn2 = 'outy.txt' , 'w')
	
	l2 = []
	n.times \
	{
		|i|
		a = Kernel.eval(File.open("#{d}/out#{i}.txt").readlines[0])
		
		if (a.member?('lb')) then
			
			l = a['lb'].map { |x| x.size }
			
			l.each_with_index { |x, j| f.puts([j, x, i].join("\t")) }
			f.puts
			f2.puts([i, sum(l)].join("\t"))
		end

		z.each \
		{
			|b|
			
			b['l'] << a['l'].map { |x| b['z'].call(x) }
			
			b['lo'] << b['z'].call(a['x1'])
			b['li'] << b['z'].call(a['l'][a['j']])
		}
		l2 << a['l2'].map { |x| x[1]['wr_e'] }
	}
	f.close
	f2.close
	
	f = File.open('gnuplot.cmd', 'w')
	f.puts("set colors classic; set ytics nomirror;")
	
	nc = "'#{fn2}' using 1:2 with line lc 5 lw 3 title 'ncc' axes x1y2"
	z.each { |b| plot(b['f'], b['l'], b['li'], b['lo'], b['b'], b['t'], nc, f) }
	
	plot('z', l2, nil, nil, 0.0, 'wr_e', nc, f)
	
	f.puts("plot '#{fn}' using 1:2:3 with line linecolor palette; pause -1;")
	
	retest(d, n, f)
	f.close
	exit
end

def run(d, n)

	m = $m
	Dir.mkdir(d)
	Dir.chdir(d)
	$stderr.puts(d)
	
	srand(n)

	a = count(m)

	File.open(fn = 'out.txt', 'w').close

	i = 0
	z = false

	while (!z) 
		x = opt(m, a)
		
		j = (0...a['l'].size).max_by { |x| a['l'][x][1]['wr_e'] }
		
		a['x1'] = a['l'][j]
		a['j'] = j
		a['l'][j] = pair2(x['ns'].to_i(2), m)

		a = model(a)
		test(dist(m, 100), a)
		
		if (i % 1 == 0) then
			x = stat1(a['l'].map { |x| x[1]['wr_e'] }).merge(count2(a['l'], a, 'a', [])).merge(count2(a['l2'], a, 'b', []))
			out(fn, x.select { |k, v| !['la', 'lb'].member?(k) })
			z = (x['tb'] == 0)
			a.merge!(x.select { |k, v| ['la', 'lb'].member?(k) })
		end
		
		f = File.open("out#{i}.txt", 'w')
		f.puts(a.inspect)
		f.close
		a.delete_if { |k, | ['la', 'lb'].member?(k) }
		
		i += 1
	end

end

def grid(d, k, l1, a, f2)
	
	l1.sort_by! { |x| x[0]['d'] }
	
	l4 = (0...l1.size).sort_by { |x| l1[x][1]['wr_e'] }
	l2 = []
	count2(l1, a, 'b', l2, l4)
	f = File.open("#{d}/out#{k}.txt", 'w')
	l2.each { |x| f.puts(x.join("\t")) }
	f.close
	f2.puts("set title '#{d} #{k}'; plot '#{d}/out#{k}.txt' matrix with image; pause -1;")
	f2.flush
	$stderr.puts([d, k, l1.size].inspect)
end

def retest2(d)
	$stderr.puts(d)
	f2 = File.open("#{d}/gnuplot.cmd", 'w')

	n = Dir.glob("#{d}/*").select { |x| x =~ /out\d+.txt$/}.size

	f = File.open("#{d}/outc.txt", 'w')
	n.times \
	{
		|i|
		a = Kernel.eval(File.open("#{d}/out#{i}.txt").readlines[0])
		
		a['l2'].size.times \
		{
			|j|
			x = a['l2'][j]
			x1 = adv(x[0]['n'], $m).size
			x2 = detect(x[0], a, nil, nil)
			f.puts([j, i, x1, x2].join("\t"))
		}
		f.puts
	}
	f.close
	f2.puts("set title '#{d} c'; plot '#{d}/outc.txt' using 1:3:2 with line linecolor palette,'' using 1:4:2 with line linecolor palette; pause -1;")

	i = n - 1
	
	a = Kernel.eval(File.open("#{d}/out#{i}.txt").readlines[0])
	l = a['lb'].map { |x| x.size }

	raise if (sum(l) != 0)
	
	c = 250
	
	l = dist($m, c)
	test(l, a)
	grid(d, 'z', l, a, f2)
	
	
	l = distb($m)
	test(l, a)
		
	['d', 'n', 'x', 'nl', 'y'].each \
	{
		|k|
		l.sort_by! { |x| x[0][k] }
		
		l1 = []
		
		d1 = (l.size - 1).to_f / c
		c.times \
		{
			|i|
			l1 << l[(i * d1).to_i]
		}
		grid(d, k, l1, a, f2)
	}
	
	f2.close

end

def stats2(l)
	
	l1 = Dir.glob("out/*/gnuplot.cmd")
	File.delete(*l1)
	$stderr.puts("deleted #{l1.size}") if (!l1.empty?)
	
	l.sort_by! { |x| x.split('/')[1].to_i }
#	l = l[10..-1]
	l[0..-2].each { |d| retest2(d) }
	exit
	
end

d = 'out'
l = Dir.glob("#{d}/*")
n = l.size

d = Dir.pwd + "/#{d}"
$m = 1
$c = 500

stats2(l) if (ARGV[0] == 'x')
	
stats("#{d}/#{ARGV[0]}") if (!ARGV[0].nil?)

loop \
{
	n += 1
	run("#{d}/#{n}", n)
}