#!/usr/bin/python

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2011  Laurent Peuch  <cortex@worlddomination.be>

import re
from pipetk import PipeTemplate

class UniqWithRT(PipeTemplate):
    WITH_ENDL = False
    def __init__(self):
        self.anti_duplications = set()

    def process(self, line):
        cleaned_line = "".join(line.split("-")[:-1])
        cleaned_line = re.sub("^[^:]+: ", lambda x: "", cleaned_line)
        cleaned_line = re.sub("[MRL]T *@[^ :,]+[: ,]*", lambda x: "", cleaned_line)
        if cleaned_line not in self.anti_duplications:
            self.anti_duplications.add(cleaned_line)
            yield line

if __name__ == "__main__":
    UniqWithRT().run()

# vim:set shiftwidth=4 tabstop=4 expandtab:
