chunk_split.go

chunk_split.go

func chunk_split(chunks []Chunk, lim int) [][]chunk_slices {
	var chunk []Chunk
	chunk_slices := make([][]Chunk, 0, len(chunks)/lim+1)
	for len(chunks) >= lim {
		chunk, chunks = chunks[:lim], chunks[lim:]
		chunk_slices = append(chunk_slices, chunk)
	}
	if len(buf) > 0 {
		chunk_slices = append(chunk_slices, chunks[:len(chunks)])
	}
	return chunk_slices
}

partition.go

func partition(chunks []Chunk, chunksPerTask int) []taskEntry {
	// generate uniq id across job using nanosecond + randint + counter
	// FIXME(typhoonzero): this is a workaround, use uuid
	randStart := rand.Int()
	timestamp := time.Now().Nanosecond()
	
	if chunksPerTask <= 0 {
		chunksPerTask = 1
	}

	var result []taskEntry
    chunk_slices := chunk_split(chunks, chunksPerTask)
    
    for i,c := range chunk_slices {
        var cur taskEntry
        cur.Task.Meta.ID = timestamp + randStart + (i + 1)
        cur.Task.Chunks = chunk_slices[i]
        result = append(result, cur)
    }
    
	return result
}