removed duplicate entries inside common wordlists (tables & columns) and added a script which does that automatically

extra/shutils/duplicates.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+"""
+$Id: fingerprint.py 2463 2010-11-30 22:40:25Z inquisb $
+
+Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+# Removes duplicate entries in wordlist like files
+
+import sys
+
+if len(sys.argv) > 0:
+
+    items = list()
+    f = open(sys.argv[1], 'r')
+
+    for item in f.readlines():
+        item = item.strip()
+        if item in items:
+            if item:
+                print item
+        items.append(item)
+
+    f.close()
+
+    f = open(sys.argv[1], 'w+')
+    f.writelines("\n".join(items))
+    f.close()

extra/shutils/duplicates.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# $Id$
-
-# Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
-# See the file 'doc/COPYING' for copying permission
-
-# Lists all duplicate entries inside a file
-cat $1 | uniq -d

extra/shutils/id.sh

@@ -6,4 +6,4 @@
 # See the file 'doc/COPYING' for copying permission
 
 # Adds SVN property 'Id' to project files
-find ../../. -type f -name "*.py" -exec svn propset svn:keywords "Id" '{}' \;
+find ../../. -type f -name "*.py" -exec svn propset svn:keywords "Id" '{}' \;