read_nodebb_database.go

package main

import (
	"database/sql"
	"database/sql/driver"
	"fmt"
	"log"
	"math"
	"net"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/lib/pq"
	"github.com/pkg/errors"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var objects *mgo.Collection

func main() {
	ip, err := exec.Command("docker", "inspect", "-f", "{{.NetworkSettings.Networks.wtdwtf.IPAddress}}", "wtdwtf-mongo").Output()
	if err != nil {
		log.Fatalf("%+v", err)
	}

	mongo, err := mgo.Dial(string(ip[:len(ip)-1]))
	if err != nil {
		log.Fatalf("%+v", err)
	}
	defer mongo.Close()

	mongo.SetCursorTimeout(0)

	objects = mongo.DB("0").C("objects")

	if err := ForSortedSet("users:joindate", HandleUser); err != nil {
		log.Fatalf("%+v", err)
	}
	if err := ForSortedSet("users:banned:expire", UpdateBanExpirations); err != nil {
		log.Fatalf("%+v", err)
	}
}

func HandleUser(uidString string, joinTS float64) error {
	var err error
	var user struct {
		ID              int64
		Name            sql.NullString
		FullName        sql.NullString
		Password        []byte
		Email           sql.NullString
		EmailConfirmed  bool
		Banned          bool
		LastOnline      pq.NullTime
		JoinDate        pq.NullTime
		Picture         sql.NullString
		UploadedPicture sql.NullString
		CoverURL        sql.NullString
		CoverPosition   NullPosition
		GroupTitle      sql.NullString
		Website         sql.NullString
		Location        sql.NullString
		Signature       sql.NullString
		AboutMe         sql.NullString
		Birthday        pq.NullTime
		ProfileViews    sql.NullInt64
		Settings        struct {
			TopicsPerPage             sql.NullInt64
			PostsPerPage              sql.NullInt64
			ShowEmail                 bool
			ShowFullName              bool
			ScrollToMyPost            bool
			UpvoteNotificationLevel   sql.NullString
			UsePagination             bool
			NotificationSounds        bool
			SendChatNotifications     bool
			SendPostNotifications     bool
			FollowTopicsOnCreate      bool
			FollowTopicsOnReply       bool
			UserLang                  sql.NullString
			OpenOutgoingLinksInNewTab bool
			HomePageRoute             sql.NullString
			DailyDigestFreq           sql.NullString
			RestrictChat              bool
		}

		IPs []struct {
			IP       net.IP
			LastSeen time.Time
		}
		IgnoredCategories []int64
		IgnoredTopics     []int64
		FollowedTopics    []int64
		FavouritePosts    []int64
		FollowedUsers     []int64
		UpvotedPosts      []int64
		DownvotedPosts    []int64
	}
	user.ID, err = strconv.ParseInt(uidString, 10, 64)
	if err != nil {
		return errors.Wrap(err, "parsing user ID")
	}

	var data bson.M
	err = ByKey("user:"+uidString, &data)
	if err != nil {
		return err
	}

	for k, v := range data {
		switch k {
		case "_id", "_key", "uid", "status", "passwordExpiry", "showemail":
			// ignore
		case "followerCount", "followingCount", "postcount", "topiccount", "userslug", "lastposttime", "reputation":
			// ignore, derived field
		case "username":
			user.Name, err = String(v)
		case "fullname":
			user.FullName, err = String(v)
		case "password":
			user.Password, err = Bytes(v)
		case "email":
			user.Email, err = String(v)
		case "email:confirmed":
			user.EmailConfirmed, err = Bool(v)
		case "banned":
			user.Banned, err = Bool(v)
		case "gplusid":
			// TODO
		case "githubid":
			// TODO
		case "fbid":
			// TODO
		case "fbaccesstoken":
			// TODO
		case "fbrefreshtoken":
			// TODO
		case "twid":
			// TODO
		case "lastonline":
			user.LastOnline, err = Time(v)
		case "joindate":
			user.JoinDate, err = Time(v)
		case "picture":
			user.Picture, err = String(v)
		case "uploadedpicture":
			user.UploadedPicture, err = String(v)
		case "cover:url":
			user.CoverURL, err = String(v)
		case "cover:position":
			user.CoverPosition, err = Position(v)
		case "groupTitle":
			user.GroupTitle, err = String(v)
		case "website":
			user.Website, err = String(v)
		case "location":
			user.Location, err = String(v)
		case "signature":
			user.Signature, err = String(v)
		case "aboutme":
			user.AboutMe, err = String(v)
		case "birthday":
			user.Birthday, err = Date(v)
		case "profileviews":
			user.ProfileViews, err = Int64(v)
		default:
			if !strings.HasPrefix(k, "_imported_") {
				err = errors.Errorf("unknown user field: %q %T %#v", k, v, v)
			}
		}
		if err != nil {
			return errors.Wrapf(err, "for field %q", k)
		}
	}

	data = nil
	err = ByKey("user:"+uidString+":settings", &data)
	if errors.Cause(err) != mgo.ErrNotFound {
		if err != nil {
			return err
		}
		for k, v := range data {
			switch k {
			case "_id", "_key", "topicSearchEnabled", "delayImageLoading", "groupTitle", "topicPostSort", "bootswatchSkin", "categoryTopicSort":
				// ignore
			case "topicsPerPage":
				user.Settings.TopicsPerPage, err = Int64(v)
			case "postsPerPage":
				user.Settings.PostsPerPage, err = Int64(v)
			case "showemail":
				user.Settings.ShowEmail, err = Bool(v)
			case "showfullname":
				user.Settings.ShowFullName, err = Bool(v)
			case "scrollToMyPost":
				user.Settings.ScrollToMyPost, err = Bool(v)
			case "upvoteNotificationLevel":
				user.Settings.UpvoteNotificationLevel, err = String(v)
			case "usePagination":
				user.Settings.UsePagination, err = Bool(v)
			case "notificationSounds":
				user.Settings.NotificationSounds, err = Bool(v)
			case "sendChatNotifications":
				user.Settings.SendChatNotifications, err = Bool(v)
			case "sendPostNotifications":
				user.Settings.SendPostNotifications, err = Bool(v)
			case "followTopicsOnCreate":
				user.Settings.FollowTopicsOnCreate, err = Bool(v)
			case "followTopicsOnReply":
				user.Settings.FollowTopicsOnReply, err = Bool(v)
			case "userLang":
				user.Settings.UserLang, err = String(v)
			case "openOutgoingLinksInNewTab":
				user.Settings.OpenOutgoingLinksInNewTab, err = Bool(v)
			case "homePageRoute":
				user.Settings.HomePageRoute, err = String(v)
			case "dailyDigestFreq":
				user.Settings.DailyDigestFreq, err = String(v)
			case "restrictChat":
				user.Settings.RestrictChat, err = Bool(v)
			default:
				if !strings.HasPrefix(k, "_imported_") {
					err = errors.Errorf("unknown user settings field: %q %T %#v", k, v, v)
				}
			}
			if err != nil {
				return errors.Wrapf(err, "for settings field %q", k)
			}
		}
	}

	if err := ForSortedSet("uid:"+uidString+":ip", func(ipString string, lastSeenTS float64) error {
		if ipString == "Unknown" {
			return nil
		}

		ip := net.ParseIP(ipString)
		if ip == nil {
			return errors.New("invalid IP format")
		}
		lastSeen, err := Time(lastSeenTS)
		if err == nil && !lastSeen.Valid {
			err = errors.New("invalid timestamp")
		}
		if err != nil {
			return errors.Wrap(err, "cannot parse last seen time")
		}

		user.IPs = append(user.IPs, struct {
			IP       net.IP
			LastSeen time.Time
		}{ip, lastSeen.Time})

		return nil
	}); err != nil {
		return errors.Wrap(err, "getting IPs")
	}
	if err := SortedSetIDs("uid:"+uidString+":ignored:cids", &user.IgnoredCategories); err != nil {
		return errors.Wrap(err, "getting ignored categories")
	}
	if err := SortedSetIDs("uid:"+uidString+":ignored_tids", &user.IgnoredTopics); err != nil {
		return errors.Wrap(err, "getting ignored topics")
	}
	if err := SortedSetIDs("uid:"+uidString+":followed_tids", &user.FollowedTopics); err != nil {
		return errors.Wrap(err, "getting followed topics")
	}
	if err := SortedSetIDs("uid:"+uidString+":favourites", &user.FavouritePosts); err != nil {
		return errors.Wrap(err, "getting favourited posts")
	}
	if err := SortedSetIDs("following:"+uidString, &user.FollowedUsers); err != nil {
		return errors.Wrap(err, "getting followed users")
	}
	if err := SortedSetIDs("uid:"+uidString+":upvote", &user.UpvotedPosts); err != nil {
		return errors.Wrap(err, "getting upvoted posts")
	}
	if err := SortedSetIDs("uid:"+uidString+":downvote", &user.DownvotedPosts); err != nil {
		return errors.Wrap(err, "getting downvoted posts")
	}

	log.Printf("user:%d:%q", user.ID, user.Name.String)

	return nil
}

func UpdateBanExpirations(uidString string, expirationTS float64) error {
	uid, err := strconv.ParseInt(uidString, 10, 64)
	if err != nil {
		return errors.Wrap(err, "invalid user ID")
	}

	expiration, err := Time(expirationTS)
	if err == nil && !expiration.Valid {
		err = errors.New("invalid expiration timestamp")
	}
	if err != nil {
		return errors.Wrap(err, "invalid expiration")
	}

	_, _ = uid, expiration.Time

	return nil
}

func String(v interface{}) (sql.NullString, error) {
	if v == nil {
		return sql.NullString{}, nil
	}
	if s, ok := v.(string); ok {
		return sql.NullString{String: s, Valid: s != ""}, nil
	}
	return sql.NullString{}, errors.Errorf("expected string, but got %T: %#v", v, v)
}

func Bytes(v interface{}) ([]byte, error) {
	s, err := String(v)
	if err != nil {
		return nil, errors.Wrap(err, "cannot convert to []byte")
	}
	if s.Valid {
		return []byte(s.String), nil
	}
	return nil, nil
}

type NullPosition struct {
	Valid bool
	X, Y  float64
}

func (pos *NullPosition) Scan(v interface{}) error {
	if v == nil {
		return nil
	}
	b, ok := v.([]byte)
	if !ok {
		return errors.Errorf("unexpected SQL type for NullPosition: %T", v)
	}
	_, err := fmt.Sscanf(string(b), "{%f,%f}", &pos.X, &pos.Y)
	if err != nil {
		pos.Valid = true
	}
	return errors.Wrapf(err, "for NullPosition %q", b)
}

func (pos NullPosition) Value() (driver.Value, error) {
	if !pos.Valid {
		return nil, nil
	}
	return []byte(fmt.Sprintf("{%g,%g}", pos.X, pos.Y)), nil
}

func Position(v interface{}) (NullPosition, error) {
	var f NullPosition
	s, err := String(v)
	if err != nil || !s.Valid {
		return f, errors.Wrap(err, "cannot convert to position")
	}
	_, err = fmt.Sscanf(s.String, "%f%% %f%%", &f.X, &f.Y)
	if err == nil {
		f.Valid = true
	}
	return f, errors.Wrapf(err, "invalid format for position: %q", s)
}

func Int64(v interface{}) (sql.NullInt64, error) {
	if v == nil {
		return sql.NullInt64{}, nil
	}
	if i, ok := v.(int64); ok {
		return sql.NullInt64{Int64: i, Valid: true}, nil
	}
	if i, ok := v.(int); ok {
		return sql.NullInt64{Int64: int64(i), Valid: true}, nil
	}
	if f, ok := v.(float64); ok {
		if i, frac := math.Modf(f); frac != 0 || i > 2<<53 || i < -2<<53 {
			return sql.NullInt64{}, errors.Errorf("float64 cannot be converted to int64 without losing precision: %v", f)
		}
		return sql.NullInt64{Int64: int64(f), Valid: true}, nil
	}
	return sql.NullInt64{}, errors.Errorf("expected int64, but got %T: %#v", v, v)
}

func Bool(v interface{}) (bool, error) {
	i, err := Int64(v)
	if err != nil || !i.Valid {
		return false, errors.Wrap(err, "cannot convert to bool")
	}
	switch i.Int64 {
	case 0:
		return false, nil
	case 1:
		return true, nil
	default:
		return false, errors.Errorf("unexpected value for bool: %d", i.Int64)
	}
}

func Time(v interface{}) (pq.NullTime, error) {
	i, err := Int64(v)
	if err != nil || !i.Valid {
		return pq.NullTime{}, errors.Wrap(err, "cannot convert to time.Time")
	}
	if i.Int64 < 0 {
		return pq.NullTime{}, errors.Errorf("unsupported negative timestamp %d", i.Int64)
	}

	return pq.NullTime{Time: time.Unix(i.Int64/1000, (i.Int64%1000)*int64(time.Millisecond)), Valid: true}, nil
}

func Date(v interface{}) (pq.NullTime, error) {
	s, err := String(v)
	if err != nil || !s.Valid {
		return pq.NullTime{}, errors.Wrap(err, "cannot parse date")
	}
	t, err := time.Parse("1/2/2006", s.String)
	return pq.NullTime{Time: t, Valid: true}, errors.Wrapf(err, "for date %q", s.String)
}

func ByKey(key string, data interface{}) error {
	return errors.Wrapf(objects.Find(bson.M{"_key": key}).One(data), "ByKey(%q)", key)
}

func ForSortedSet(key string, f func(string, float64) error) error {
	var el struct {
		Value string  `bson:"value"`
		Score float64 `bson:"score"`
	}
	var err error

	it := objects.
		Find(bson.M{"_key": key}).
		Sort("score", "value").
		Select(bson.M{"_id": 0, "value": 1, "score": 1}).
		Iter()

	for err == nil && it.Next(&el) {
		err = errors.Wrapf(f(el.Value, el.Score), "ForSortedSet(%q, (%q, %v))", key, el.Value, el.Score)
	}

	if e := errors.Wrapf(it.Close(), "closing ForSortedSet(%q)", key); err == nil {
		err = e
	}

	return err
}

func SortedSetIDs(key string, ids *[]int64) error {
	return ForSortedSet(key, func(idString string, ts float64) error {
		id, err := strconv.ParseInt(idString, 10, 64)
		if err != nil {
			return err
		}

		*ids = append(*ids, id)

		return nil
	})
}