Skip to content

Instantly share code, notes, and snippets.

@5kg
Created April 27, 2015 07:14
Show Gist options
  • Save 5kg/a2eb0da501668042b6b6 to your computer and use it in GitHub Desktop.
Save 5kg/a2eb0da501668042b6b6 to your computer and use it in GitHub Desktop.
package main
import (
"fmt"
"github.com/golang/protobuf/proto"
pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
"io/ioutil"
"os"
"strconv"
)
const nRow = 2000
const nColumn = 1000
const nTask = 4
func main() {
in, _ := os.Open(os.Args[1])
mat := make([][]float32, nRow)
for i := 0; i < nRow; i++ {
mat[i] = make([]float32, nColumn)
for j := 0; j < nColumn; j++ {
var z float32
fmt.Fscanf(in, "%f", &z)
mat[i][j] = z
}
}
for iter := 0; iter < nTask; iter++ {
v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, nRow)}
for i := 0; i < nRow; i++ {
m := nColumn / nTask
fmt.Println(m)
v.Row[i] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
for j := 0; j < m; j++ {
z := mat[i][iter*m + j]
if z == 0 {
continue
}
v.Row[i].At[int32(j)] = z
}
}
buf, err := proto.Marshal(v)
if err != nil {
fmt.Println("marshaling error: ", err)
}
ioutil.WriteFile("20ng_column_shard.dat."+strconv.Itoa(iter), buf, 0644)
}
}
package main
import (
"fmt"
"github.com/golang/protobuf/proto"
pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
"io/ioutil"
"os"
"strconv"
)
const nRow = 2000
const nColumn = 1000
const nTask = 4
func main() {
in, _ := os.Open(os.Args[1])
mat := make([][]float32, nRow)
for i := 0; i < nRow; i++ {
mat[i] = make([]float32, nColumn)
for j := 0; j < nColumn; j++ {
var z float32
fmt.Fscanf(in, "%f", &z)
mat[i][j] = z
}
}
for iter := 0; iter < nTask; iter++ {
m := nColumn / nTask
v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, m)}
for c := 0; c < m; c++ {
v.Row[c] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
for r := 0; r < nRow; r++ {
z := mat[r][iter*m+c]
if z == 0 {
continue
}
v.Row[c].At[int32(r)] = z
}
}
buf, err := proto.Marshal(v)
if err != nil {
fmt.Println("marshaling error: ", err)
}
ioutil.WriteFile("20ng_column_shard.t.dat."+strconv.Itoa(iter), buf, 0644)
}
}
package main
import (
"fmt"
"os"
// "testing"
"github.com/golang/protobuf/proto"
pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
"io/ioutil"
"strconv"
)
const nRow = 2000
const nColumn = 1000
const nTask = 4
func main() {
in, _ := os.Open(os.Args[1])
for iter := 0; iter < nTask; iter++ {
n := nRow/nTask
v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, n)}
for i := 0; i < n; i++ {
v.Row[i] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
for j := 0; j < nColumn; j++ {
var z float32
fmt.Fscanf(in, "%f", &z)
if z == 0 {
continue
}
v.Row[i].At[int32(j)] = z
}
}
buf, err := proto.Marshal(v)
if err != nil {
fmt.Println("marshaling error: ", err)
}
ioutil.WriteFile("20ng_row_shard.dat."+strconv.Itoa(iter), buf, 0644)
}
}
package main
import (
"fmt"
"github.com/golang/protobuf/proto"
pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
"io/ioutil"
"os"
"strconv"
)
const nColumn = 1000
const nTask = 1
const nTopic = 10
func main() {
for t := 0; t < nTask; t++ {
in, _ := os.Open(os.Args[1] + "." + strconv.Itoa(t))
buf, _ := ioutil.ReadAll(in)
message := &pb.DenseMatrixShard{}
unmarshErr := proto.Unmarshal(buf, message)
if unmarshErr != nil {
panic(unmarshErr)
}
for r := 0; r < nColumn / nTask; r++ {
for c := 0; c < nTopic; c++ {
fmt.Printf("%f ", message.Row[r].At[int32(c)])
}
fmt.Println()
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment