outline9b.rb

def outin(fn)
        l = (f = File.open(fn)).readlines
	f.close
	raise if (l.shift.chop != '$dat << eof')
        k = l.shift.split
	i = l.index("eof\n")
	raise if (i.nil?)
	l[i..-1] = []
        l2 = l.map { |x| Hash[[k, x.split.map { |x| Kernel.eval(x) }].transpose] }
	
	l2.each { |x| x['ns'] = x['ns'].to_s }
	$stderr.puts([fn, l2.size].inspect)
	return l2
end

def load(l1 = [1])

	l = []
	l1.each \
	{
		|i|
		l.concat(outin("tmp/gnuplot#{i}-1.cmd"))
	}
	return l
end

def sample2(l, n)
	
	d = (l.size - 1).to_f / (n - 1)
	l2 = []
	n.times \
	{
		|i|
		j = (i * d).to_i
		l2 << l[j]
	}
	return l2
end


def len(ns, p)
	l = ns.split(p)
	l = [] if (l.nil?)
	l.shift if (l[0] == '')
	return l.map { |x| x.length }
end

def len1(ns)
	return len(ns, /0+/)
end

def len0(ns)
	return len(ns, /1+/)
end

def len01(ns)
	return len1(ns), len0(ns)
end

def e(ns)
	return 0 if (ns.empty?)
	return len01(ns).flatten.size.to_f / ns.length
end

def d(s)
	return 0 if (s.empty?)
        c = s.split('').select { |x| x == '1' }.size
	d = c.to_f / s.length
        return d
        
end

def midpt(ns)
	w2 = ns.length / 4
	l = ns.split('')
	i = j = 0
	while (i < w2 && j < l.size)
		i += l[j].to_i
		j += 1
	end
	return j.to_f / ns.length
end

def d1(s)
        c = s.split('').select { |x| x == '1' }.size
        return c
        
end

def midpt2(ns)
	w2 = d1(ns) / 2
	l = ns.split('')
	i = j = 0
	while (i < w2 && j < l.size)
		i += l[j].to_i
		j += 1
	end
	return j.to_f / ns.length
end

def log2(x)
	return Math.log(x) / Math.log(2.0)
end

def features(ns)
	d = d(ns)
	e = e(ns)
	
	nw = ns.length
	nw2 = nw / 2
	nshi = ns[0...nw2]
	nslo = ns[nw2..-1]
	
	dlo = d(nslo) - 0.5
	dhi = d(nshi) - 0.5
	elo = e(nslo) - 0.5
	ehi = e(nshi) - 0.5
	mp1 = midpt(ns) - 0.5
	mp2 = midpt2(ns) - 0.5
	
	mx0 = len0(ns).max
	mx0 = 0 if (mx0.nil?)
	mx1 = len1(ns).max
	mx01 = [mx0, mx1].max
	mx = log2(log2(ns.to_i(2).to_f))
	
	a0 = avg(len0(ns))
	a1 = avg(len1(ns))
	a01 = avg(len01(ns).flatten)
		
	return {
		"d" => d,
		"e" => e,
		"ea" => (0.5 - e).abs,
		"da" => (0.5 -d).abs,
		"dlo" => dlo,
		"dhi" => dhi,
		"elo" => elo,
		"ehi" => ehi,
		'mp1' => mp1,
		"mp2" => mp2,
			'mx0' => mx0,
			'mx1' => mx1,
			'mx01' => mx01,
			'mx' => mx,
			'a0' => a0,
			'a1' => a1,
			'a01' => a01,
			'a1m' => a1 / mx,
		"nw" => ns.length
	}
end

def dataset(l)

	return l.map { |x| features(x['ns']).merge({'cg' => x['cg']}) }

end

def near(x, l, k2 = '', l1 = [])

	l.each_with_index \
	{
		|x1, j|
		next if (x == x1)
		
		z = 0
		['d', 'e', 'da', 'ea', 'dlo', 'dhi', 'elo', 'ehi', 'mp1', 'mp2',
		'mx0', 'mx1', 'mx01', 'mx', 'a0', 'a01', 'a1m'].each \
		{
			|k1|
			
			k = "#{k1}#{k2}"
			z += (x[k] - x1[k1]) ** 2
		}
		l1 << [z, j]
	}
	l1.sort_by! { |x| x[0] }
	
	i = 0
	i += 1 while (i + 1 < l1.size && l1[i][0] == l1[i + 1][0])
	
	return (0..i).map { |x| l[l1[x][1]].merge({'z' => l1[x][0]}) }
end

def predict(l2, l1)
	e1 = 0
	l1.each { |x| x['nn'] = []; x.delete('er2') }
	
	ky = 'cg'
	
	l2.each_with_index \
	{
		|x, j|
		l = near(x, l1)
		
		et = yt = 0
		l.each \
		{
			|x1|
			yt += x1[ky]
			e = (x1[ky] - x[ky]).abs
			et += e
			x1['nn'] << { 
				'er' => e, 
				'j' => j, 
				'z' => x1['z']}
		}
		
		x['y'] = yt / l.size
		x['er1'] = et / l.size
		e1 += x['er1']
	}
	et = e1 / l2.size
	return et
end

def sum(l)
	return l.inject(0) { |t, x| t + x }
end

def avg(l)
	return sum(l).to_f / l.size
end

def opt(l2, l1)
	
	et = predict(l2, l1)
	
	l1.each { |x| x['er2'] = x['nn'].empty? ? 0 : avg(x['nn'].map { |x1| x1['er'] }) }
	j = (0...l1.size).max_by { |x| l1[x].fetch('er2', 0) }
	x = l1[j]
	
	a = {'et' => et, 'er2max' => x['er2'], 'j' => j}
	return a
end

def outd(f, l)
	f.puts('$dat << eof')
	k = l[0].keys
	
	f.puts(k.join("\t"))
	l.each { |x| f.puts(x.values.join("\t")) }

	f.puts('eof')
	return k
end

def hide(a, x)
	return (a.member?(x) && a[x].nil?)
end

def outa(f, l, a = {}, t = '')
	
	k = outd(f, l)
	f.puts("set colors classic; set title '#{t}'; ")
#	f.puts("set key top right opaque; ")
#	f.puts("set ytics nomirror; set y2tics;")
	f.puts("plot \\")
	ct = ''
	k, ct = [k - ['t'], "(column('t')):"] if (k.member?('t') && !hide(a, 't'))
	k.each \
	{ 
		|x| 
		next if (hide(a, x))
		opt = a.fetch(x, '')
		opt = ' with line lw 2 ' + opt if (!opt.include?('with'))
		f.puts("'$dat' using #{ct}(column('#{x}')) #{opt} title '#{x}',\\") 
	}
	f.puts
#	f.puts("reset; pause -1;")

end

def outafn(l, a = {}, fn = nil, t = '')
	fn = 'gnuplot.cmd' if (fn.nil?)
	outa(f = File.open(fn, 'w'), l, a, t)
	f.close
	$stderr.puts([fn, l.size, t].inspect)
end

def resample(l, c)

	mn = l[0]['cg']
	mx = l[-1]['cg']
	p([mn, mx])
	l2 = []
	c.times \
	{
		|i|
		y = mn + i.to_f / (c - 1) * (mx - mn)
		x = l.min_by { |x1| (x1['cg'] - y).abs }
		l2 << x if (!l2.member?(x))
	}
	return l2
	
end


l = dataset(load())
c1 = 250
l2 = resample(l, c1)
l2.each { |x| x['y'] = nil }

c2 = c1
(l3 = (0...l.size).select { |x| !l[x].member?('y') }).shuffle[0...c2].each { |x| l[x]['x'] = nil }

f = File.open('out.txt', 'w')


t = 0
emn = emx = et2 = nil
loop \
{
	
	l1 = l.select { |x| x.member?('x') }

	a = opt(l2, l1)
	a['et2'] = et2.nil? ? '-' : et2
	emx = [emx, a['et']].compact.max
	emn = [emn, a['et']].compact.min
	
	a['er'] = a['et'].to_f / emx
	a['er2'] = et2.nil? ? '-' : et2.to_f / emx
	
	if (t == 0) then
		f.puts(a.keys.join("\t"))
		f1 = File.open('gnuplot1.cmd', 'w')
		f1.puts("set colors classic; ")
		f1.puts("set ytics nomirror; set y2tics; set y2range [0:];")
		f1.puts("plot \\")
		a1 = {'j' => 'pt 5', 
			'er' => 'with line lw 2 axes x1y2',
			'er2' => 'with line lt 7 lw 2 axes x1y2'
			}
		a.keys.each \
		{
			|k|
			opt = a1.fetch(k, 'with line lw 2')
			f1.puts("'out.txt' using (column('#{k}')) #{opt},\\")
		}
		f1.puts
		f1.close
	end
	
	f.puts(a.values.join("\t"))
	f.flush
	
	break if ((er3 = a['et'].to_f / emn) > 1.10)
		
	l1[a['j']].delete('x')
	
	
	l4 = (0...l.size).select { |x| l[x].member?('x') }

	l5 = l3 - l4
	
	l[i = l5[rand(l5.size)]]['x'] = nil
	l4 += [i]
	l4.sort!
	
	if ((t + 1) % 5 == 0) then

		l6 = (0...l2.size).map \
			{ 
				|x| 
				{'y' => l2[x]['y'], 
				'e' => l2[x]['er1'], 
				'cg' => l2[x]['cg'],
				'i' => l4[x]}				
			}
		
		l2b = l.values_at(*(0...l.size).to_a.shuffle[0...l2.size].sort)
		
		et2 = predict(l2b, l1)
		
		l2b.size.times { |x| l6[x]['y2'] = l2b[x]['y'] }
		
		outafn(l6)
	end
	t += 1
	$stderr.puts([t, er3.round(3)].inspect)
}