A few days ago, I posted about a use of hash that allowed learning much faster than normal language models. I'm busy with other projects and I didn't have time to finish what I was doing, but yesterday by mistake I ran a test scratcb that didn't work, where the input was 20 hashes, and output 1000 tokens, I had previously tried without success, …

gistfile1.txt

const _ = require('lodash');

const fs = require('fs');
let tf=require('@tensorflow/tfjs-node-gpu')
global.tf=tf


let cl={}
let a1 = new TextEncoder()
let b1 = new TextDecoder()
function cluster_keys(tokens) {
    let v=tokens.slice().reverse()
    let c1=v.map(s=>s).slice()
    let res=new Array(10).fill(0)
    res.push(c1[0])
    res.shift()
    let mod=2**16
    function reduction(vec,res,n=1) {
        let a=0;
        let b=0
        let cn = _.chunk(vec, 2).filter(s => s.length == 2).map(s =>{
            a=(a+s[0])%mod
            b=(a+b)%mod
            a=(a+s[1])%mod
            b=(a+b)%mod
            return s[0]+s[1]
        })
        res.push(cn[0]||0)
        res.shift()
        if(cn.length>1){
               
             
                n=n+1
                return reduction(cn,res,n)
            }
  
            return res
          

    }

return     reduction(c1,res,1)

    
}

let d=_.chunk(fs.readFileSync('./valid.txt').toString().split(''),10000).map(s=>s.join(''))

function inference(text) {

    let vec2 = Array.from(a1.encode(text))

    function nextToken(vec, n = 0) {
        let res
        tf.tidy(()=>{
        let keys1 = cluster_keys(vec)
        let keys2 = cluster_keys(vec.reverse())
        let keys=keys1.concat(keys2)
        let xs=tf.tensor([keys])
        
      //  let xs2=tf.tensor([_.takeRight(vec,64)])
        let resv = model.predict(xs)
        res=resv.argMax(-1).flatten().abs().round().arraySync()
    
        vec=res
        n++
    })
        if (n < 1) {
            return nextToken(vec, n)
        }
        
        return vec.map(s => String.fromCharCode(s)).join('')
    }

    return nextToken(vec2)

}
let train=[]
let tree=new Array(7).fill(0).map(s=>({}))
let treeh=new Array(7).fill(0).map(s=>({}))
let n=0
for(let k=0;k<1;k++){
    let temp=[]

    let b=a1.encode(d[k])
    if(b.length>2000){

    for(let i=1000;i<b.length-1000;i+=1){
        temp=b.slice(i-1000,i)
        let i2=i+1000
        let temp2=b.slice(i2-1000,i2)
       n++
      
        
      
        let r1=cluster_keys(temp.slice())
        let r2=cluster_keys(temp.slice().reverse())
        let r=r1.concat(r2)
        if(r.find(s=>s==undefined))console.log(r)

        train.push([r,[],_.chunk(temp2.slice().map(s=>(s)),100)])
      
    }      
    }
    if(k%10==0)console.log(k,'row',d.length,n,'train steps')
}

let input = tf.layers.input({
    shape: [20]
})

let x=input
let k=[]
x=tf.layers.embedding({inputDim:2**16,outputDim:128}).apply(x)
//x = tf.layers.dense({ units:1000, activation: 'swish' }).apply(x)
x=tf.layers.flatten().apply(x)
//x=tf.layers.lstm({units:2048,recurrentInitializer:'glorotNormal',recurrentActivation:'sigmoid'}).apply(x)
x=tf.layers.dense({units:10000,activation:'swish'}).apply(x)

x = tf.layers.reshape({ targetShape:[10,100,10] }).apply(x)
x=tf.layers.batchNormalization().apply(x)
x = tf.layers.dense({ units:256, activation: 'softmax' }).apply(x)
let model = tf.model({
    inputs: [input],
    outputs: x
})

this.op = tf.train.adam(.1)

model.compile({
    optimizer: this.op,
    loss: tf.losses.huberLoss,
    metrics: ['acc']
})

model.summary()
let bestLoss = Infinity
let bestWeights = null;
let a=train.splice(0, 400)
let xxs = tf.tensor(a.map(s=>s[0]))
let yys = tf.tensor(a.map(s=>s[2]))
const onEpochEnd = async (epoch, logs) => {
    let loss = model.evaluate(xxs, yys).dataSync()[0]

    if (loss < bestLoss) {
        bestLoss = loss;
        console.log(loss)
        bestWeights = model.getWeights().map(w => w.clone());
    } else {
        // Reverter para os melhores pesos se a loss não melhorou
        model.setWeights(bestWeights);
    }
};

model.compile({
    optimizer: tf.train.adam(.001),
    loss: 'sparseCategoricalCrossentropy',
    metrics: []
})
let idx=_.shuffle(new Array(train.length).fill(0).map((s,i)=>i))
function makeIterator() {
    const numElements = train.length
    let index = 0;
    let n = 0
    const iterator = {
        next: x => tf.tidy(() => {
            let result;
 
            setx = []
            n++

            for (let k = 0; k < 64; k++) {
                let index=idx.shift()
                if(idx.length<100)idx=_.shuffle(new Array(train.length).fill(0).map((s,i)=>i))
                setx.push([train[index][0],train[index][2]])
             


            }
            let tx1 = tf.tensor(setx.map(s => s[0]))

            let ty1 = tf.tensor(setx.map(s => s[1])).expandDims(-1)
       
            result = {
                value: {
                    xs: tx1,
                    ys: ty1
                },
                done: false
            };

            return result;
        })
    }

    return iterator;
}



const ds = tf.data.generator(makeIterator);
 model.fitDataset(ds, {
  verbose:1,
    batchesPerEpoch: Math.round(train.length/64),
    epochs: 1000,
    callbacks: {
       
        async onEpochEnd() {
                await model.save('file://./llm2')
            tf.tidy(() => {
                    console.log(inference(`
Once upon a time there was a little girl named Lucy. She loved to go to the store to buy sweets with her mom and dad. On this special day, Lucy entered the store with her mom and dad, feeling so excited.
As they were looking around, Lucy noticed a little girl playing with a toy in the corner of the store. She gasped in excitement and ran towards her. Lucy asked if she could play too but the little girl said no. She was rather grumpy and was not in the mood to play.
Lucy's mom saw what was going on and told Lucy, "Let's try to be peaceful and kind to her. Have patience and understanding. Together, you can both be happy!"
So, Lucy smiled at the girl and said, "Can we play together?" The little girl softened and smiled back. She agreed to share the toy and even let Lucy have a turn first.
Lucy and the little girl played together happily. In the end, they both learnt an important lesson: be peaceful, kind, and understanding when faced with a conflict. And that is why Lucy and the little girl became great friends.
`.substring(0,1000),512))
console.log(`
Once upon a time there was a little girl named Lucy. She loved to go to the store to buy sweets with her mom and dad. On this special day, Lucy entered the store with her mom and dad, feeling so excited.
As they were looking around, Lucy noticed a little girl playing with a toy in the corner of the store. She gasped in excitement and ran towards her. Lucy asked if she could play too but the little girl said no. She was rather grumpy and was not in the mood to play.
Lucy's mom saw what was going on and told Lucy, "Let's try to be peaceful and kind to her. Have patience and understanding. Together, you can both be happy!"
So, Lucy smiled at the girl and said, "Can we play together?" The little girl softened and smiled back. She agreed to share the toy and even let Lucy have a turn first.
Lucy and the little girl played together happily. In the end, they both learnt an important lesson: be peaceful, kind, and understanding when faced with a conflict. And that is why Lucy and the little girl became great friends.
`.toLowerCase().substring(0,1000))
               
            })
        }
    }
})