Scans an input text file that has been sorted alphabetically and outputs all unique lines to an output text file.

RemoveDuplicateLines.vbs

' Results: Scans READ_FILE and outputs all unique lines to WRITE_FILE.
' Assumes: Text file has been sorted alphabetically
' TODO: Universal search for line so file doesn't need to be sorted.

Dim READ_FILE, WRITE_FILE
READ_FILE = "C:\Input\textfile.txt"
WRITE_FILE = "C:\Output\textfile.txt"

Set fso = CreateObject("Scripting.FileSystemObject")
Set inputFile = fso.OpenTextFile(READ_FILE, 1) '1 = for reading
Set outputFile = fso.OpenTextFile(WRITE_FILE, 2) '2 = for writing

strLast = ""
Do Until inputFile.AtEndOfStream
  strLine = inputFile.ReadLine

  If strLine <> strLast Then
    outputFile.WriteLine strLine
    strLast = strLine
  End If
Loop

inputFile.Close
outputFile.Close

Set outputFile = Nothing
Set inputFile = Nothing
Set fso = Nothing