Skip to content

Instantly share code, notes, and snippets.

@resilar
Created March 10, 2018 08:11
Show Gist options
  • Save resilar/925ea268d9e15e79fb9cbfd5ab9d2d79 to your computer and use it in GitHub Desktop.
Save resilar/925ea268d9e15e79fb9cbfd5ab9d2d79 to your computer and use it in GitHub Desktop.
Remove duplicate lines without sorting
// rustc -O makeuniq.rs
// cat .bash_history | ./makeuniq > .bash_history_uniq
use std::io::{self, Write};
use std::io::prelude::BufRead;
use std::collections::HashSet;
fn main() {
let mut uniqs = HashSet::new();
let stdin = io::stdin();
let stdout = io::stdout();
let mut handle = stdout.lock();
for line in stdin.lock().split(b'\n') {
let line = line.unwrap();
if uniqs.insert(line.clone()) {
handle.write(&line).unwrap();
handle.write(b"\n").unwrap();
}
}
}
@resilar
Copy link
Author

resilar commented Mar 19, 2020

Iterator adapter iter.uniq() implementation

use std::cmp::Eq;
use std::hash::Hash;
use std::io::Write;
use std::io::prelude::BufRead;
use std::collections::HashSet;

struct UniqIter<I: Iterator> {
    inner: I,
    set: HashSet<I::Item>
}

impl<I> Iterator for UniqIter<I> where I: Iterator, I::Item: Eq + Hash + Clone {
    type Item = I::Item;
    fn next(&mut self) -> Option<Self::Item> {
        while let Some(item) = self.inner.next() {
            if self.set.insert(item.clone()) {
                return Some(item);
            }
        }
        None
    }
}

trait UniqIterator<I: IntoIterator> {
    fn uniq(self) -> UniqIter<I::IntoIter>;
}

impl<I: IntoIterator> UniqIterator<I> for I where I::Item: Eq + Hash + Clone {
    fn uniq(self) -> UniqIter<I::IntoIter> {
        UniqIter {
            inner: self.into_iter(),
            set: HashSet::new()
        }
    }
}

fn main() -> std::result::Result<(), std::io::Error> {
    let stdin = std::io::stdin();
    let stdout = std::io::stdout();
    let mut stdout = stdout.lock();
    for line in stdin.lock().split(b'\n').map(|ln| ln.unwrap()).uniq() {
        stdout.write(&line)?;
        stdout.write(b"\n")?;
    }
    Ok(())
}

#[test]
fn uniq_adapter() {
    assert_eq!(vec![1, 2, 2, 1].uniq().collect::<Vec<i32>>(), vec![1, 2]);
    assert_eq!(vec![4, 2, 2, 4].uniq().collect::<Vec<i32>>(), vec![4, 2]);
    assert_eq!(vec![3, 2, 1].uniq().collect::<Vec<i32>>(), vec![3, 2, 1]);
    assert_eq!(vec![6, 6, 6].iter().uniq().collect::<Vec<&i8>>(), vec![&6]);
    assert_eq!(vec![1, 2, 3, 4, 3, 2, 1].uniq().fold(0, |acc, x| acc + x), 10);
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment