Merge jsonl with db users and org users.

go.mod

module warpcore_fix

go 1.22

require github.com/mattermost/mattermost/server/public v0.1.0

require (
	github.com/blang/semver/v4 v4.0.0 // indirect
	github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect
	github.com/francoispqt/gojay v1.2.13 // indirect
	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
	github.com/google/uuid v1.6.0 // indirect
	github.com/gorilla/websocket v1.5.1 // indirect
	github.com/kr/text v0.2.0 // indirect
	github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect
	github.com/mattermost/ldap v0.0.0-20231116144001-0f480c025956 // indirect
	github.com/mattermost/logr/v2 v2.0.21 // indirect
	github.com/pborman/uuid v1.2.1 // indirect
	github.com/pelletier/go-toml v1.9.5 // indirect
	github.com/philhofer/fwd v1.1.2 // indirect
	github.com/pkg/errors v0.9.1 // indirect
	github.com/tinylib/msgp v1.1.9 // indirect
	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
	github.com/wiggin77/merror v1.0.5 // indirect
	github.com/wiggin77/srslog v1.0.1 // indirect
	golang.org/x/crypto v0.20.0 // indirect
	golang.org/x/net v0.21.0 // indirect
	golang.org/x/text v0.14.0 // indirect
	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
	gopkg.in/yaml.v2 v2.4.0 // indirect
)

import_types.go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.

package main

import (
	"archive/zip"

	"github.com/mattermost/mattermost/server/public/model"
)

// Import Data Models

type LineImportData struct {
	Type          string                   `json:"type"`
	Scheme        *SchemeImportData        `json:"scheme,omitempty"`
	Team          *TeamImportData          `json:"team,omitempty"`
	Channel       *ChannelImportData       `json:"channel,omitempty"`
	User          *UserImportData          `json:"user,omitempty"`
	Post          *PostImportData          `json:"post,omitempty"`
	DirectChannel *DirectChannelImportData `json:"direct_channel,omitempty"`
	DirectPost    *DirectPostImportData    `json:"direct_post,omitempty"`
	Emoji         *EmojiImportData         `json:"emoji,omitempty"`
	Version       *int                     `json:"version,omitempty"`
}

type TeamImportData struct {
	Name            *string `json:"name"`
	DisplayName     *string `json:"display_name"`
	Type            *string `json:"type"`
	Description     *string `json:"description,omitempty"`
	AllowOpenInvite *bool   `json:"allow_open_invite,omitempty"`
	Scheme          *string `json:"scheme,omitempty"`
}

type ChannelImportData struct {
	Team        *string            `json:"team"`
	Name        *string            `json:"name"`
	DisplayName *string            `json:"display_name"`
	Type        *model.ChannelType `json:"type"`
	Header      *string            `json:"header,omitempty"`
	Purpose     *string            `json:"purpose,omitempty"`
	Scheme      *string            `json:"scheme,omitempty"`
}

type UserImportData struct {
	ProfileImage       *string   `json:"profile_image,omitempty"`
	ProfileImageData   *zip.File `json:"-"`
	Username           *string   `json:"username"`
	Email              *string   `json:"email"`
	AuthService        *string   `json:"auth_service"`
	AuthData           *string   `json:"auth_data,omitempty"`
	Password           *string   `json:"password,omitempty"`
	Nickname           *string   `json:"nickname"`
	FirstName          *string   `json:"first_name"`
	LastName           *string   `json:"last_name"`
	Position           *string   `json:"position"`
	Roles              *string   `json:"roles"`
	Locale             *string   `json:"locale"`
	UseMarkdownPreview *string   `json:"feature_enabled_markdown_preview,omitempty"`
	UseFormatting      *string   `json:"formatting,omitempty"`
	ShowUnreadSection  *string   `json:"show_unread_section,omitempty"`
	DeleteAt           *int64    `json:"delete_at,omitempty"`

	Teams *[]UserTeamImportData `json:"teams,omitempty"`

	Theme               *string `json:"theme,omitempty"`
	UseMilitaryTime     *string `json:"military_time,omitempty"`
	CollapsePreviews    *string `json:"link_previews,omitempty"`
	MessageDisplay      *string `json:"message_display,omitempty"`
	CollapseConsecutive *string `json:"collapse_consecutive_messages,omitempty"`
	ColorizeUsernames   *string `json:"colorize_usernames,omitempty"`
	ChannelDisplayMode  *string `json:"channel_display_mode,omitempty"`
	TutorialStep        *string `json:"tutorial_step,omitempty"`
	EmailInterval       *string `json:"email_interval,omitempty"`

	NotifyProps *UserNotifyPropsImportData `json:"notify_props,omitempty"`
}

type UserNotifyPropsImportData struct {
	Desktop      *string `json:"desktop"`
	DesktopSound *string `json:"desktop_sound"`

	Email *string `json:"email"`

	Mobile           *string `json:"mobile"`
	MobilePushStatus *string `json:"mobile_push_status"`

	ChannelTrigger  *string `json:"channel"`
	CommentsTrigger *string `json:"comments"`
	MentionKeys     *string `json:"mention_keys"`
}

type UserTeamImportData struct {
	Name     *string                  `json:"name"`
	Roles    *string                  `json:"roles"`
	Theme    *string                  `json:"theme,omitempty"`
	Channels *[]UserChannelImportData `json:"channels,omitempty"`
}

type UserChannelImportData struct {
	Name        *string                           `json:"name"`
	Roles       *string                           `json:"roles"`
	NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props,omitempty"`
	Favorite    *bool                             `json:"favorite,omitempty"`
}

type UserChannelNotifyPropsImportData struct {
	Desktop    *string `json:"desktop"`
	Mobile     *string `json:"mobile"`
	MarkUnread *string `json:"mark_unread"`
}

type EmojiImportData struct {
	Name  *string   `json:"name"`
	Image *string   `json:"image"`
	Data  *zip.File `json:"-"`
}

type ReactionImportData struct {
	User      *string `json:"user"`
	CreateAt  *int64  `json:"create_at"`
	EmojiName *string `json:"emoji_name"`
}

type ReplyImportData struct {
	User *string `json:"user"`

	Type     *string `json:"type"`
	Message  *string `json:"message"`
	CreateAt *int64  `json:"create_at"`
	EditAt   *int64  `json:"edit_at"`

	FlaggedBy   *[]string               `json:"flagged_by,omitempty"`
	Reactions   *[]ReactionImportData   `json:"reactions,omitempty"`
	Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
}

type PostImportData struct {
	Team    *string `json:"team"`
	Channel *string `json:"channel"`
	User    *string `json:"user"`

	Type     *string                `json:"type"`
	Message  *string                `json:"message"`
	Props    *model.StringInterface `json:"props"`
	CreateAt *int64                 `json:"create_at"`
	EditAt   *int64                 `json:"edit_at"`

	FlaggedBy   *[]string               `json:"flagged_by,omitempty"`
	Reactions   *[]ReactionImportData   `json:"reactions,omitempty"`
	Replies     *[]ReplyImportData      `json:"replies,omitempty"`
	Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
	IsPinned    *bool                   `json:"is_pinned,omitempty"`
}

type DirectChannelImportData struct {
	Members     *[]string `json:"members"`
	FavoritedBy *[]string `json:"favorited_by"`

	Header *string `json:"header"`
}

type DirectPostImportData struct {
	ChannelMembers *[]string `json:"channel_members"`
	User           *string   `json:"user"`

	Type     *string                `json:"type"`
	Message  *string                `json:"message"`
	Props    *model.StringInterface `json:"props"`
	CreateAt *int64                 `json:"create_at"`
	EditAt   *int64                 `json:"edit_at"`

	FlaggedBy   *[]string               `json:"flagged_by"`
	Reactions   *[]ReactionImportData   `json:"reactions"`
	Replies     *[]ReplyImportData      `json:"replies"`
	Attachments *[]AttachmentImportData `json:"attachments"`
	IsPinned    *bool                   `json:"is_pinned,omitempty"`
}

type SchemeImportData struct {
	Name                    *string         `json:"name"`
	DisplayName             *string         `json:"display_name"`
	Description             *string         `json:"description"`
	Scope                   *string         `json:"scope"`
	DefaultTeamAdminRole    *RoleImportData `json:"default_team_admin_role"`
	DefaultTeamUserRole     *RoleImportData `json:"default_team_user_role"`
	DefaultChannelAdminRole *RoleImportData `json:"default_channel_admin_role"`
	DefaultChannelUserRole  *RoleImportData `json:"default_channel_user_role"`
	DefaultTeamGuestRole    *RoleImportData `json:"default_team_guest_role"`
	DefaultChannelGuestRole *RoleImportData `json:"default_channel_guest_role"`
}

type RoleImportData struct {
	Name        *string   `json:"name"`
	DisplayName *string   `json:"display_name"`
	Description *string   `json:"description"`
	Permissions *[]string `json:"permissions"`
}

type LineImportWorkerData struct {
	LineImportData
	LineNumber int
}

type LineImportWorkerError struct {
	Error      *model.AppError
	LineNumber int
}

type AttachmentImportData struct {
	Path *string   `json:"path"`
	Data *zip.File `json:"-"`
}

type ComparablePreference struct {
	Category string
	Name     string
}

main.go

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"strings"
)

type OrgUser struct {
	ID      string `json:"id"`
	Name    string `json:"name"`
	Deleted bool   `json:"deleted"`
	Profile struct {
		Title     string `json:"title"`
		Email     string `json:"email"`
		FirstName string `json:"first_name"`
		LastName  string `json:"last_name"`
		RealName  string `json:"real_name_normalized"`
	} `json:"profile"`
}

type User struct {
	ID        string `json:"id"`
	Username  string `json:"username"`
	Email     string `json:"email"`
	FirstName string `json:"firstname"` // Assuming these fields are already present in the users struct
	LastName  string `json:"lastname"`
	Title     string `json:"title"`
}

func main() {
	// Reading JSONL File
	importFile, err := os.Open("mattermost_import.jsonl")
	if err != nil {
		fmt.Println("Error opening mattermost_import.jsonl file:", err)
		return
	}
	defer closeFile(importFile)

	scanner := bufio.NewScanner(importFile)

	var usersToUpdate []LineImportData

	// finding the users in the jsonl file
	for scanner.Scan() {
		var line LineImportData
		err := json.Unmarshal([]byte(scanner.Text()), &line)
		if err != nil {
			fmt.Println("Error unmarshalling JSON line:", err)
			return
		}
		if line.Type == "user" {
			usersToUpdate = append(usersToUpdate, line)
		}
	}

	fmt.Printf("Found %d users to update\n", len(usersToUpdate))

	// Read "usersDump.json"
	usersDumpData, err := os.ReadFile("usersDump.json")
	if err != nil {
		panic(err)
	}
	var usersDump []User
	err = json.Unmarshal(usersDumpData, &usersDump)
	if err != nil {
		panic(err)
	}

	// Read "org_users.json"
	orgUsersData, err := os.ReadFile("org_users.json")
	if err != nil {
		panic(err)
	}
	var orgUsers []OrgUser
	err = json.Unmarshal(orgUsersData, &orgUsers)
	if err != nil {
		panic(err)
	}

	updatedUsers := processUsers(usersToUpdate, usersDump, orgUsers)

	fmt.Println("Updated users:", len(updatedUsers))
	// Writing the updated data back to the JSONL file
	writeOutputJsonlFile("updatedUsers.jsonl", updatedUsers)
}

// This function will process each user and update their information based on the usersDump and orgUsers
func processUsers(usersToUpdate []LineImportData, usersDump []User, orgUsers []OrgUser) (userLines []LineImportData) {
	// Your code for updating the users according to the requirements
	// Remember, first check in usersDump, then in orgUsers if not already updated

	// loop over each user in the usersToUpdate slice
	for _, line := range usersToUpdate {
		dbUser := findDatabaseUser(line.User, usersDump)
		if dbUser != nil {
			fmt.Println("")
			line.User.Email = &dbUser.Email
			line.User.Username = &dbUser.Username
			line.User.FirstName = &dbUser.FirstName
			line.User.LastName = &dbUser.LastName
			line.User.Position = &dbUser.Title
			userLines = append(userLines, line)
			continue
		}

		orgUser := findOrgUser(line.User, orgUsers)
		if orgUser != nil {
			line.User.Email = &orgUser.Profile.Email
			line.User.Username = &orgUser.Name
			line.User.FirstName = &orgUser.Profile.FirstName
			line.User.LastName = &orgUser.Profile.LastName
			line.User.Position = &orgUser.Profile.Title
			userLines = append(userLines, line)
			continue
		}

		// If the user is not found in either the database or org users, we can skip this user
		userLines = append(userLines, line)
	}

	return userLines
}

// only comparing the username / email here because those are the primary values
// that are imported into the database.
func findDatabaseUser(user *UserImportData, usersDump []User) *User {
	for _, u := range usersDump {
		// username is the slack ID when using the import file.
		if strings.ToLower(u.Username) == toLowerString(user.Username) {
			return &u
		}

		if strings.ToLower(u.Email) == toLowerString(user.Email) {
			return &u
		}
	}
	return nil

}
func findOrgUser(user *UserImportData, orgUsers []OrgUser) *OrgUser {
	for _, ou := range orgUsers {

		emailUsername := strings.Split(toLowerString(user.Email), "@")[0]
		if strings.ToLower(ou.ID) == toLowerString(user.Username) {
			return &ou
		}

		// ou.Name is the username on slack export
		if strings.ToLower(ou.Name) == toLowerString(user.Username) {
			return &ou
		}

		if strings.ToLower(ou.Profile.Email) == toLowerString(user.Email) {
			return &ou
		}

		// sometimes the first `@` might be the user's username
		if strings.ToLower(ou.Name) == emailUsername {
			return &ou
		}

		// another way to potentially find a user if both initial sides of the username are the same.
		if strings.Split(strings.ToLower(ou.Profile.Email), "@")[0] == emailUsername {
			return &ou
		}
	}
	return nil
}

func toLowerString(s *string) string {
	return fmt.Sprintf("%v", s)
}

// This function writes the updated user information back into a new JSONL file
func writeOutputJsonlFile(filePath string, users []LineImportData) {
	file, err := os.Create(filePath)
	if err != nil {
		panic(err)
	}

	defer closeFile(file)

	for _, user := range users {
		jsonlStr, err := json.Marshal(user)
		if err != nil {
			// Handle error
			fmt.Println("Error marshalling JSONL line:", err)
			continue
		}
		_, err = file.WriteString(string(jsonlStr) + "\n")
		if err != nil {
			// Handle error
			fmt.Println("Error writing JSONL line:", err)
			return
		}
	}
}

func closeFile(file *os.File) {
	err := file.Close()
	if err != nil {
		fmt.Println("Error closing file:", err)
	}
}