Awk command to remove duplicate lines, based on a field. Explanation at http://sudarmuthu.com/blog/remove-duplicate-lines-based-on-a-field

arrays.txt

# Before starting 
x = {}

# After line 1
x = {
	CTO => 1
    }

# After line 2
x = {
	CTO => 1
	Manager => 1
    }

# After line 3
x = {
	CTO => 1
	Manager => 1
	CEO => 1
    }

# After line 4
x = {
	CTO => 1
	Manager => 2
	CEO => 1
    }

# After line 5
x = {
	CTO => 1
	Manager => 2
	CEO => 1
	CFO => 1
    }

command.sh

awk '!x[$2]++' filename

expanded.sh

{
if (x[$2] == 0 ) 
	print
x[$2]++
}

input.txt

Tom	CTO	32
Harry	Manager	45        -> Manager field is duplicate
Krish	CEO	50
Bob	Manager	49        -> Manager field is duplicate
Patrick	CFO	20

output.txt

Tom	CTO	32
Harry	Manager	45
Krish	CEO	50
Patrick	CFO	20