sh/remove-duplicates (445 bytes)
1 # sort the lines, then remove duplicates that are next to eachoter 2 sort file.txt | uniq 3 4 # without resorting (can blow the stack if too large) 5 awk '!seen[$0]++' file.txt 6 # or 7 cat file.txt | awk '!seen[$0]++' >newfile.txt 8 9 # GNU sed can do it without blowing the stack. 10 # POSIX sed might blow 11 sed -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' 12 13 # TODO: 14 # number the lines, sort, uniq, re-unsort (best done in a real programming language)