Steam account age estimator

accountage.lua

local inspect = require "inspect"
local posix = require"posix"
local lfs = require'lfs'
local cjson = require'cjson.safe'
local lmfit = require 'lmfit'
function PrintTable(...)
        print(inspect{...})
end
math.randomseed(os.time())


local bn = require'bc'
function sid64_to_accountid(sid64)
        sid64 = bn.number(sid64)
        local aid = sid64 - (sid64/2^32)*(2^32)
        return tonumber(tostring(aid))
end
local sidto64= bn.number"76561197960265728"
function aid_to_sid64(accountid)
        accountid = bn.number(accountid)
        local sid64 = accountid + sidto64
        return tostring(sid64)
end

local function sort_accountid(a,b)
        return a.accountid<b.accountid
end

local function sort_age(a,b)
        return a.age<b.age
end

function table.shuffle(tbl)
  size = #tbl
  for i = size, 1, -1 do
    local rand = math.random(size)
    tbl[i], tbl[rand] = tbl[rand], tbl[i]
  end
  return tbl
end

local path="/home/python/srcds/garrysmod/data/steamplinfo/"
lfs.chdir(path)
local t={}

for file in lfs.dir(path) do
        if lfs.attributes(file,"mode") == "file" then
                t[#t+1]=file
        end
end
table.shuffle(t)


local agepairs      = {{},{}}

local function parse(file)
        local accountid = sid64_to_accountid(file:match("^(%d+)%.txt$"))
        assert(accountid and accountid>0)

        local fd = assert(io.open(file,'rb'))
        local data = assert(fd:read'*all')
        if data=="" then return end
        local data,err = cjson.decode(data)
        if not data then
                print("fail",file,lfs.attributes (file, "size"),data)
                return
        end
        fd:close()
        --io.stdout:write"."
        io.stdout:flush()
        local age = data[23]
        if type(age)~='number' then return end
        local agepair = {accountid=accountid,age=age}

        local bucket = accountid%2 + 1
        --if bucket==1 then
        --      print(accountid,age)
        --end
        table.insert(agepairs[bucket],agepair)
end

for n,file in next,t do
        parse(file)
        if n>1500 then break end
end

local rules = {
        {-math.huge,257654515},
        {257654515,271888889},
        {271888889,math.huge}
}

local res=rules
for rulenum,rules in next,rules do

        local agepairs_orig = agepairs
        local agepairs = {{},{}}
        local rule_min,rule_max = rules[1],rules[2]
        print("\n\n------------------------------\nRANGE",rule_min,rule_max)
        for k,v in next,agepairs_orig do
                for kk,vv in next,v do
                        if rule_min<vv.accountid and rule_max>vv.accountid then
                                table.insert(agepairs[k],vv)
                        end
                end
        end

        local function my_fit_function( t, p )
          return (p[1]*t +
                                        (1 - p[1] + p[2] + p[3])
                          * t*t)

                                /
                         (1 + p[2]*t + p[3] * t*t)
        end


        local function account_creation_time_approximator(x, p)
                        return p[1] +
                        p[2] * x
                 +  p[3] * x^2
                 +   p[4] * x^3
        end
        --my_fit_function = account_creation_time_approximator

        for bucketid,agepairs_bucket in next,agepairs do
                print( "\n  BUCKET",bucketid,"\n" )
                assert(#agepairs_bucket>0)

                table.sort(agepairs_bucket,sort_age)
                local age_min,age_max = agepairs_bucket[1].age,agepairs_bucket[#agepairs_bucket].age

                table.sort(agepairs_bucket,sort_accountid)
                local aid_min,aid_max = agepairs_bucket[1].accountid,agepairs_bucket[#agepairs_bucket].accountid

                -- find the discontinuations
                local page,paccid=0,0
                for i=1,#agepairs_bucket do
                        local data = agepairs_bucket[i]
                        local aid,age = data.accountid,data.age
                        if age<page and math.abs(age-page)/60/60/24>10 then
                                print("      discontinuation at",page,age,"DAYS:"..(age-page)/60/60/24,"AID:"..aid,"AID2:"..paccid,"D:"..(aid-paccid))
                        end
                        page=age
                        paccid=aid
                end

                local function normalize(accountid,age)
                        assert(accountid>0 and accountid<2^32,"bad accountid")
                        local accountid_norm = (accountid-aid_min)/(aid_max-aid_min)
                        assert(accountid_norm>=0 and accountid_norm<=1,("Normalized %s to %s"):format(tostring(accountid),tostring(accountid_norm)))

                        if not age then return accountid_norm end
                        assert(age>0 and age<math.huge,"bad age")

                        local age_norm = (age-age_min)/(age_max-age_min)
                        assert(age_norm>=0 and age_norm<=1)
                        return accountid_norm,age_norm
                end


                local function denormalize(accountid,age)
                        local accountid_norm = (aid_min)+accountid*(aid_max-aid_min)
                        if not age then return accountid_norm end
                        local age_norm = (age_min)+age*(age_max-age_min)
                        return accountid_norm,age_norm
                end

                local fitx,fity={},{}
                local p = { 10,10,10 }
                for _,data in next,agepairs_bucket do

                        local accountid_norm,age_norm = normalize(data.accountid,data.age)

                        table.insert(fitx,accountid_norm)
                        table.insert(fity,age_norm)

                end

                -- test normalization (there might be small differences though so this might fail)
                local test1,test2 = normalize(agepairs_bucket[4].accountid,agepairs_bucket[4].age)
                test1,test2 = denormalize(test1,test2)
                assert(math.abs(test1-agepairs_bucket[4].accountid)<0.0001)
                assert(math.abs(test2-agepairs_bucket[4].age)<0.0001)

                local p, status, evals = lmfit.minimize( fitx, fity, p, my_fit_function )

                local r = res[rulenum]

                r.buckets=r.buckets or {}
                r.buckets[bucketid] = r.buckets[bucketid] or {}
                r.buckets[bucketid].p=p
                r.buckets[bucketid].ranges={age_min=age_min,age_max=age_max,aid_min=aid_min,aid_max=aid_max}


                print('  par:',table.concat(p,', '))
                print('  status:' ,status,lmfit.message(status))
                print('  evals:',evals)

                local function predict(accountid)
                        assert(accountid>0 and accountid<2^32)
                        local accountid_norm = normalize(accountid)
                        local age = my_fit_function(accountid_norm,p)
                        local _,age = denormalize(accountid_norm,age)
                        return math.floor(age+0.5)
                end



                print"   TESTING: \n"
                local testpairs={}
                for i=1,10 do
                        local pair = agepairs_bucket[math.random(1,#agepairs_bucket)]
                        table.insert(testpairs,pair)
                end

                table.sort(testpairs,sort_accountid)

                for k,pair in next,testpairs do
                        local accountid,age = pair.accountid,pair.age
                        print("    ",accountid,predict(accountid),age,math.ceil(math.abs(age-predict(accountid))/60/60/24),"days")
                end


                print"\n\n"
        end

end

PrintTable(res)