fit3b.rb

require 'statsample'



def f2(n)

	n = (n * 3 + 1) / 2 while (n.odd?)
	n /= 2 while (n.even?)
	return n

end

def adv(x)

	n1 = n = x['n']
	l = [n]
	while (n >= n1 && n != 1)
		
		n = f2(n)
		l << n
	end
	

	x['l'] = l
	x['ls'] = l.size
	
	x['ns'] = x['n'].to_s(2).length
	
	return x
	
end

def next2(z)
	
	l = [z]
	p = z['p'] + 1
	
	l << adv({'n'=>z['n'] + 2**p, 'p'=>p})
	l << z.merge({'p'=>p})
	return l
end

def insert(l, x)
	l << x
end

def delete(l, j)
	z = l.delete_at(j)
	return z
end

def sum(l)

	t = 0
	l.each { |x| t += x }
	return t
end

def stat(l)
	l = [0] if (l.empty?)
        t = t2 = 0
        l.each \
        {
		|x|
		t += x
		t2 += x ** 2
        }
        c = l.size
        a = t.to_f / c
        z = t2.to_f / c - a ** 2
        sd = Math.sqrt(z < 0 ? 0 : z)

	return a, sd, l.max.to_f
end

def dist(l)
	
	l2 = []
	l.each_with_index \
	{
		|x, i|
		ls = x[1]['ls']
		l2[ls] = [] if (l2[ls].nil?)
		l2[ls] << i
	}
	l1 = (0...l2.size).sort_by { |i| l2[i].nil? ? 0 : l2[i].size }
	
	return l2, l1
end

def rank(l1, l2)
	
	l1h, l1s = dist(l1)
	l2h, l2s = dist(l2)
	
	j = l1s.find { |x| !l2h[x].nil? && (l1h[x].nil? || l1h[x].size < $n) }


	j = l2h.size - 1 if (j.nil?)
	k = l2s.find { |x| !l2h[x].nil? }
	
	l = (0...l2.size).to_a
	l.sort_by! { |x| l2[x][1]['ls'] }
	k = l.find { |x| x != j }
	
	
	return l2h[j][rand(l2h[j].size)], k

end

def opt(c)
	
	l = []
	l1 = []
	
	insert(l, next2({'n'=>1, 'p'=>0}))


	puts('# ' + Time.now.to_s)
	t = Time.now.to_i

	c.times \
	{
		|i|
		$stderr.puts([i, sprintf('%.1fm', (Time.now.to_i - t) / 60.0), Time.now.to_s].join("\t")) if (i % 100 == 0)
		
		j, k = rank(l1, l)

		if (l.size > 1000) then
			
			z2 = delete(l, [j, k].max)
			z1 = delete(l, [j, k].min)
			
			l1 += [z1, z2]
			z = j < k ? z1 : z2
		else 
			z = delete(l, j)
			l1 += [z]
		end

		insert(l, next2(z[1]))
		insert(l, next2(z[2]))
		
		$stdout.flush
	}

	puts('# ' + Time.now.to_s)
	return l1.map { |x| x[1] }
end

def stat2(l, t)
	return stat(l).map { |x| x / t }
end

def d(s)
        c = s.split('').select { |x| x == '1' }.size
	d = c.to_f / s.length
        return d
        
end

def data(l, ls)
	l2 = []
	l.each \
	{
		|x|
		
		ns = x.to_s(2)
		nl = ns.length
		m = nl / 2
		
		nsh = ns[0..m]
		nsl = ns[m..-1]
		
		asdm1 = stat2(ns.split(/0+/).map { |x| x.length }, nl)
		
		l1 = ns.split(/1+/)
		l1.shift
		asdm0 = stat2(l1.map { |x| x.length }, nl)
		
		z = ls.to_f / nl
		
		l2 << [nl, ls, z, d(ns), d(nsh), d(nsl), asdm1].flatten
	}
	return l2
end

def fit(l1, c)

	l1 = l1.transpose

	a = {}
	a['y'] = l1[c].to_vector()

	($c...l1.size).each \
	{
		|i|
		a["d#{i}"] = l1[i].to_vector()
	}

	ds = a.to_dataset()


	r = Statsample::Regression.multiple(ds, 'y')
#	$stderr.puts(r.summary)

	return [r.constant] + r.coeffs.values_at(*(a.keys - ['y']))

end

def err(z, l1, c, fn)
	
	e1 = e2 = 0
	z = z.dup
	c1 = z.shift
	i2 = 0
	l = []
	l1.each_with_index \
	{
		|l2, j|
		nl, ls = l2
		
		t = c1
		z.size.times { |i| t += z[i] * l2[i + $c] }
		
		a = fn.call(l2[c], nl)
		b = fn.call(t, nl)
		
		e1 += (l2[c] - t).abs
		e2 += (a - b).abs
		
		l << [nl, ls, l2[c], t, a, b, e1 / (j + 1), e2 / (j + 1)]
	}
	return l
end

def samplefit(c)
	
	l = opt(c)

	h = {}
	l.each \
	{ 
		|x| 
		x['l'].reverse.each_with_index { |y, i| h[i + 1] = h.fetch(i + 1, []) | [y] }
	}
	
#	h.sort.each { |k, v| p([k, v.size]) }; exit
	
	l2 = []
	l3 = []
	l3b = []
	c1 = 10
	
	h1 = h.select { |k, v| v.size >= c1 }
	
	h1.each_with_index \
	{
		|kv, n|
		
		k, v = kv
		l1 = v.sort

#		if (n < h1.size / 2) then
			j = l1.size / 2
			l0 = l1.values_at(j - 1, j, j)
			
			l3 << [k, l0]
			l2 += data(l0, k)
#		end

		l0 = []
		c1.times \
		{
			|i|
			l0 << l1[(i.to_f / (c1 - 1) * (l1.size - 1)).to_i]
		}
		l3b << [k, l0]
	}
	
	f = File.open('out2.txt', 'w')
	l2.each { |x| f.puts(x.join("\t")) }
	f.close

	$stderr.puts("#{l2.size} pts")
	
	$c = 3
	z = fit(l2, $c - 1)


	l4 = err(z, l2, $c - 1, lambda { |x, nl| x * nl })
	f = File.open('out3.txt', 'w')
	l4.each { |x| f.puts(x.join("\t")) }
	f.close
	
	return z, l3, l3b
	
end

def adv2(n, c)

	l = [n]
	(c - 1).times \
	{
		n = f2(n)
		l << n
	}
	return l
end

def avg(l)
	return sum(l) / l.size
end

def improve(z, l3)
	
	l = l3.select { |x| x[0] >= $n }
	$stderr.puts("#{l.size} pts2")
	
	l.each \
	{
		|ls, l|
		l.each \
		{
			|n|
			l2 = data(adv2(n, $n), ls)
			l1 = err(z, l2, $c - 1, lambda { |x, nl| x * nl })
			
			
			l0 = (0...$n).map { |i| l1[i][5] + i }
			puts([ls, avg(l0)].join("\t"))
		}
		puts
	}
end

def compare()
	
	$n = 20
	z, l3, l3b = samplefit(1500)
	
	improve(z, l3)
	improve(z, l3b)
end


l2 = compare()