Fix encoding issues related to Python 3 upgrade.

Python 3 str() is a unicode already, so can't be decoded.
subprocess.Popen needs to be in text mode (universal_newlines is the
oldest compatible kwarg for this) in order to do line-based buffering.
In general, all the filesystem encoding stuff we were doing is done by
Python itself now.

zenmap/install_scripts/utils/version_update.py

@@ -89,7 +89,7 @@ def update_date(base_dir):
 
 def update_version(base_dir, version):
     print(">>> Updating %s" % os.path.join(base_dir, VERSION))
-    vf = open(os.path.join(base_dir, VERSION), "wb")
+    vf = open(os.path.join(base_dir, VERSION), "w")
     print(version, file=vf)
     vf.close()
     print(">>> Updating %s" % os.path.join(base_dir, VERSION_PY))

zenmap/radialnet/core/XMLHandler.py

@@ -67,12 +67,6 @@ import xml.sax.saxutils
 from xml.sax.xmlreader import AttributesImpl as Attributes
 
 
-def convert_to_utf8(text):
-    """
-    """
-    return text.encode('utf8', 'replace')
-
-
 class XMLNode:
     """
     """
@@ -289,7 +283,7 @@ class XMLReader(xml.sax.ContentHandler):
 
         # putting attributes and values in node
         for attr in attrs.getNames():
-            node.add_attr(attr, convert_to_utf8(attrs.get(attr).strip()))
+            node.add_attr(attr, attrs.get(attr).strip())
 
         # who is my father?
         if len(self.__status) > 0:
@@ -303,7 +297,7 @@ class XMLReader(xml.sax.ContentHandler):
     def endElement(self, name):
         """
         """
-        self.__status[-1].set_text(convert_to_utf8(self.__text.strip()))
+        self.__status[-1].set_text(self.__text.strip())
 
         self.__text = ""
         self.__status.pop()

zenmap/radialnet/gui/RadialNet.py

@@ -76,8 +76,6 @@ from radialnet.core.Graph import Node
 from radialnet.gui.NodeWindow import NodeWindow
 from radialnet.gui.Image import Icons, get_pixels_for_cairo_image_surface
 
-from zenmapCore.BasePaths import fs_enc
-
 REGION_COLORS = [(1.0, 0.0, 0.0), (1.0, 1.0, 0.0), (0.0, 1.0, 0.0)]
 REGION_RED = 0
 REGION_YELLOW = 1
@@ -249,9 +247,7 @@ class RadialNet(Gtk.DrawingArea):
         self.__draw(context)
 
         if type == FILE_TYPE_PNG:
-            # write_to_png requires a str, not unicode, in py2cairo 1.8.10 and
+            self.surface.write_to_png(file)
-            # earlier.
-            self.surface.write_to_png(fs_enc(file))
 
         self.surface.flush()
         self.surface.finish()

zenmap/radialnet/gui/SaveDialog.py

@@ -83,7 +83,7 @@ for type in TYPES:
         EXTENSIONS[type[2]] = type[1]
 
 
-class SaveDialog(zenmapGUI.FileChoosers.UnicodeFileChooserDialog):
+class SaveDialog(Gtk.FileChooserDialog):
     def __init__(self):
         """
         """

zenmap/share/zenmap/config/zenmap.conf

@@ -58,8 +58,8 @@ underline = 0
 enable_highlight = True
 
 [paths]
-nmap_command_path = nmap
+nmap_command_path = ../nmap
-ndiff_command_path = ndiff
+ndiff_command_path = ../ndiff/ndiff
 
 [port_list_highlight]
 regex = ^PORT\s+STATE\s+SERVICE(\s+VERSION)?.*

zenmap/share/zenmap/locale/xgettext-profile_editor.py

@@ -17,7 +17,7 @@ directory = None
 
 
 def escape(s):
-    return '"' + s.encode("UTF-8").replace('"', '\\"') + '"'
+    return '"' + s.replace('"', '\\"') + '"'
 
 
 def output_msgid(msgid, locator):

zenmap/zenmapCore/BasePaths.py

@@ -64,29 +64,6 @@ import sys
 from zenmapCore.Name import APP_NAME
 
 
-def fs_dec(s):  # This is unused now
-    """Decode s from the filesystem decoding, handling various possible
-    errors."""
-    enc = sys.getfilesystemencoding()
-    if enc is None:
-        enc = "UTF-8"
-    return s.decode(enc)
-
-
-def fs_enc(u):
-    """Encode u to the filesystem decoding, handling various possible
-    errors."""
-    enc = sys.getfilesystemencoding()
-    if enc is None:
-        enc = "UTF-8"
-    return u.encode(enc)
-
-# We can't just use os.path.expanduser(u"~") to get a unicode version of the
-# home directory, because os.path.expanduser doesn't properly decode the raw
-# byte string from the file system encoding. You get a UnicodeDecodeError on
-# systems like Windows where the file system encoding is different from the
-# result of sys.getdefaultencoding(). So we call os.path.expanduser with a
-# plain string and decode it from the filesystem encoding.
 HOME = os.path.expanduser("~")
 
 # The base_paths dict in this file gives symbolic names to various files. For

zenmap/zenmapCore/Diff.py

@@ -122,7 +122,7 @@ class NdiffCommand(subprocess.Popen):
                 filename_b
                 ]
         self.stdout_file = tempfile.TemporaryFile(
-                mode="rb",
+                mode="r",
                 prefix=APP_NAME + "-ndiff-",
                 suffix=".xml"
                 )
@@ -132,6 +132,7 @@ class NdiffCommand(subprocess.Popen):
         subprocess.Popen.__init__(
                 self,
                 command_list,
+                universal_newlines=True,
                 stdout=self.stdout_file,
                 stderr=self.stdout_file,
                 env=env,
@@ -167,7 +168,7 @@ def ndiff(scan_a, scan_b):
                 suffix=".xml"
                 )
         temporary_filenames.append(filename_a)
-        f = os.fdopen(fd, "wb")
+        f = os.fdopen(fd, "w")
         scan_a.write_xml(f)
         f.close()
     else:
@@ -179,7 +180,7 @@ def ndiff(scan_a, scan_b):
                 suffix=".xml"
                 )
         temporary_filenames.append(filename_b)
-        f = os.fdopen(fd, "wb")
+        f = os.fdopen(fd, "w")
         scan_b.write_xml(f)
         f.close()
     else:

zenmap/zenmapCore/NmapCommand.py

@@ -87,39 +87,6 @@ paths_config = PathsConfig()
 log.debug(">>> Platform: %s" % sys.platform)
 
 
-def wrap_file_in_preferred_encoding(f):
-    """Wrap an open file to automatically decode its contents when reading from
-    the encoding given by locale.getpreferredencoding, or just return the file
-    if that doesn't work.
-
-    The nmap executable will write its output in whatever the system encoding
-    is. Nmap's output is usually all ASCII, but time zone it prints can be in a
-    different encoding. If it is not decoded correctly it will be displayed as
-    garbage characters. This function assists in reading the Nmap output. We
-    don't know for sure what the encoding used is, but we take a best guess and
-    decode the output into a proper unicode object so that the screen display
-    and XML writer interpret it correctly."""
-
-    try:
-        preferredencoding = locale.getpreferredencoding()
-    except locale.Error:
-        # This can happen if the LANG environment variable is set to something
-        # weird.
-        preferredencoding = None
-
-    if preferredencoding is not None:
-        try:
-            reader = codecs.getreader(preferredencoding)
-            return reader(f, "replace")
-        except LookupError:
-            # The lookup failed. This can happen if the preferred encoding is
-            # unknown ("X-MAC-KOREAN" has been observed). Ignore it and return
-            # the unwrapped file.
-            log.debug("Unknown encoding \"%s\"." % preferredencoding)
-
-    return f
-
-
 def escape_nmap_filename(filename):
     """Escape '%' characters so they are not interpreted as strftime format
     specifiers, which are not supported by Zenmap."""
@@ -235,8 +202,8 @@ class NmapCommand(object):
         # We don't need a file name for stdout output, just a handle. A
         # TemporaryFile is deleted as soon as it is closed, and in Unix is
         # unlinked immediately after creation so it's not even visible.
-        f = tempfile.TemporaryFile(mode="rb", prefix=APP_NAME + "-stdout-")
+        f = tempfile.TemporaryFile(mode="r", prefix=APP_NAME + "-stdout-")
-        self.stdout_file = wrap_file_in_preferred_encoding(f)
+        self.stdout_file = f
         if stderr is None:
             stderr = f
 
@@ -260,6 +227,7 @@ class NmapCommand(object):
                 startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
 
         self.command_process = subprocess.Popen(command_list, bufsize=1,
+                                     universal_newlines=True,
                                      stdin=subprocess.PIPE,
                                      stdout=f,
                                      stderr=stderr,

zenmap/zenmapCore/NmapParser.py

@@ -1034,7 +1034,7 @@ class NmapParserSAX(ParserBasics, ContentHandler):
     def write_xml_to_file(self, filename):
         """Write the XML representation of this scan to the file whose name is
         given."""
-        fd = open(filename, "wb")
+        fd = open(filename, "w")
         self.write_xml(fd)
         fd.close()
 

zenmap/zenmapGUI/DiffCompare.py

@@ -333,7 +333,7 @@ class DiffWindow(Gtk.Window):
                     message_format=_("Error running ndiff"),
                     secondary_text=_(
                         "There was an error running the ndiff program.\n\n"
-                        ) + str(e).decode(sys.getdefaultencoding(), "replace"))
+                        ) + str(e))
                 alert.run()
                 alert.destroy()
             else:
@@ -364,7 +364,7 @@ class DiffWindow(Gtk.Window):
             if status == 0 or status == 1:
                 # Successful completion.
                 try:
-                    diff = self.ndiff_process.get_scan_diff().decode("utf-8")
+                    diff = self.ndiff_process.get_scan_diff()
                 except zenmapCore.Diff.NdiffParseException as e:
                     alert = HIGAlertDialog(
                         message_format=_("Error parsing ndiff output"),

zenmap/zenmapGUI/FileChoosers.py

@@ -99,39 +99,19 @@ class ScriptFileFilter(Gtk.FileFilter):
         self.set_name(_("NSE scripts (%s)") % ", ".join(patterns))
 
 
-class UnicodeFileChooserDialog(Gtk.FileChooserDialog):
+class AllFilesFileChooserDialog(Gtk.FileChooserDialog):
-    """This is a base class for file choosers. It is designed to ease the
-    retrieval of Unicode file names. On most platforms, the file names returned
-    are encoded in the encoding given by sys.getfilesystemencoding(). On
-    Windows, they are returned in UTF-8, even though using the UTF-8 file name
-    results in a file not found error. The get_filename method of this class
-    handles the decoding automatically."""
-    def get_filename(self):
-        filename = Gtk.FileChooserDialog.get_filename(self)
-        if sys.platform == "win32":
-            encoding = "UTF-8"
-        else:
-            encoding = sys.getfilesystemencoding() or "UTF-8"
-        try:
-            filename = filename.decode(encoding)
-        except Exception:
-            pass
-        return filename
-
-
-class AllFilesFileChooserDialog(UnicodeFileChooserDialog):
     def __init__(self, title="", parent=None,
                  action=Gtk.FileChooserAction.OPEN,
                  buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                           Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None):
 
-        UnicodeFileChooserDialog.__init__(self, title=title, parent=parent,
+        Gtk.FileChooserDialog.__init__(self, title=title, parent=parent,
                                           action=action, buttons=buttons)
         self.set_default_response(Gtk.ResponseType.OK)
         self.add_filter(AllFilesFileFilter())
 
 
-class ResultsFileSingleChooserDialog(UnicodeFileChooserDialog):
+class ResultsFileSingleChooserDialog(Gtk.FileChooserDialog):
     """This results file choose only allows the selection of single files, not
     directories."""
     def __init__(self, title="", parent=None,
@@ -139,34 +119,34 @@ class ResultsFileSingleChooserDialog(UnicodeFileChooserDialog):
                  buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                           Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None):
 
-        UnicodeFileChooserDialog.__init__(self, title=title, parent=parent,
+        Gtk.FileChooserDialog.__init__(self, title=title, parent=parent,
                                           action=action, buttons=buttons)
         self.set_default_response(Gtk.ResponseType.OK)
         for f in (ResultsFileFilter(), AllFilesFileFilter()):
             self.add_filter(f)
 
 
-class ResultsFileChooserDialog(UnicodeFileChooserDialog):
+class ResultsFileChooserDialog(Gtk.FileChooserDialog):
     def __init__(self, title="", parent=None,
                  action=Gtk.FileChooserAction.OPEN,
                  buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                           "Open Directory", RESPONSE_OPEN_DIRECTORY,
                           Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None):
 
-        UnicodeFileChooserDialog.__init__(self, title=title, parent=parent,
+        Gtk.FileChooserDialog.__init__(self, title=title, parent=parent,
                                           action=action, buttons=buttons)
         self.set_default_response(Gtk.ResponseType.OK)
         for f in (ResultsFileFilter(), AllFilesFileFilter()):
             self.add_filter(f)
 
 
-class ScriptFileChooserDialog(UnicodeFileChooserDialog):
+class ScriptFileChooserDialog(Gtk.FileChooserDialog):
     def __init__(self, title="", parent=None,
                  action=Gtk.FileChooserAction.OPEN,
                  buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                           Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None):
 
-        UnicodeFileChooserDialog.__init__(self, title=title, parent=parent,
+        Gtk.FileChooserDialog.__init__(self, title=title, parent=parent,
                                           action=action, buttons=buttons)
         self.set_default_response(Gtk.ResponseType.OK)
         self.set_select_multiple(True)
@@ -174,7 +154,7 @@ class ScriptFileChooserDialog(UnicodeFileChooserDialog):
             self.add_filter(f)
 
 
-class SaveResultsFileChooserDialog(UnicodeFileChooserDialog):
+class SaveResultsFileChooserDialog(Gtk.FileChooserDialog):
     TYPES = (
         (_("By extension"), None, None),
         (_("Nmap XML format (.xml)"), "xml", ".xml"),
@@ -192,7 +172,7 @@ class SaveResultsFileChooserDialog(UnicodeFileChooserDialog):
                  buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                           Gtk.STOCK_SAVE, Gtk.ResponseType.OK), backend=None):
 
-        UnicodeFileChooserDialog.__init__(self, title=title, parent=parent,
+        Gtk.FileChooserDialog.__init__(self, title=title, parent=parent,
                                           action=action, buttons=buttons)
 
         types_store = Gtk.ListStore.new([str, str, str])
@@ -245,23 +225,23 @@ class SaveResultsFileChooserDialog(UnicodeFileChooserDialog):
         return filetype
 
 
-class DirectoryChooserDialog(UnicodeFileChooserDialog):
+class DirectoryChooserDialog(Gtk.FileChooserDialog):
     def __init__(self, title="", parent=None,
                  action=Gtk.FileChooserAction.SELECT_FOLDER,
                  buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                           Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None):
 
-        UnicodeFileChooserDialog.__init__(self, title=title, parent=parent,
+        Gtk.FileChooserDialog.__init__(self, title=title, parent=parent,
                                           action=action, buttons=buttons)
         self.set_default_response(Gtk.ResponseType.OK)
 
 
-class SaveToDirectoryChooserDialog(UnicodeFileChooserDialog):
+class SaveToDirectoryChooserDialog(Gtk.FileChooserDialog):
     def __init__(self, title="", parent=None,
                  action=Gtk.FileChooserAction.SELECT_FOLDER,
                  buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                           Gtk.STOCK_SAVE, Gtk.ResponseType.OK), backend=None):
 
-        UnicodeFileChooserDialog.__init__(self, title=title, parent=parent,
+        Gtk.FileChooserDialog.__init__(self, title=title, parent=parent,
                                           action=action, buttons=buttons)
         self.set_default_response(Gtk.ResponseType.OK)

zenmap/zenmapGUI/ScanInterface.py

@@ -65,7 +65,6 @@ from gi.repository import Gtk, GLib
 import errno
 import os
 import time
-import sys
 
 # Prevent loading PyXML
 import xml
@@ -478,7 +477,7 @@ class ScanInterface(HIGVBox):
         try:
             command_execution.run_scan()
         except OSError as e:
-            text = str(e.strerror, errors='replace')
+            text = e.strerror
             # Handle ENOENT specially.
             if e.errno == errno.ENOENT:
                 # nmap_command_path comes from zenmapCore.NmapCommand.
@@ -486,9 +485,6 @@ class ScanInterface(HIGVBox):
                 if path_env is None:
                     default_paths = []
                 else:
-                    fsencoding = sys.getfilesystemencoding()
-                    if fsencoding:
-                        path_env = path_env.decode(fsencoding, 'replace')
                     default_paths = path_env.split(os.pathsep)
                 text += "\n\n{}\n\n{}".format(
                         _("This means that the nmap executable was "

zenmap/zenmapGUI/ScriptInterface.py

@@ -65,7 +65,6 @@ gi.require_version("Gtk", "3.0")
 from gi.repository import Gtk, GLib
 
 import os
-import sys
 import tempfile
 
 # Prevent loading PyXML
@@ -134,8 +133,7 @@ class ScriptHelpXMLContentHandler (xml.sax.handler.ContentHandler):
             if "path" not in attrs:
                 raise ValueError(
                         '"directory" element did not have "path" attribute')
-            path = attrs["path"].encode("raw_unicode_escape").decode(
+            path = attrs["path"]
-                    sys.getfilesystemencoding())
             if dirname == "scripts":
                 self.scripts_dir = path
             elif dirname == "nselib":
@@ -229,7 +227,7 @@ class ScriptInterface:
         # Separate stderr to avoid breaking XML parsing with "Warning: File
         # ./nse_main.lua exists, but Nmap is using...".
         stderr = tempfile.TemporaryFile(
-                mode="rb", prefix=APP_NAME + "-script-help-stderr-")
+                mode="r", prefix=APP_NAME + "-script-help-stderr-")
         log.debug("Script interface: running %s" % repr(command_string))
         nmap_process = NmapCommand(command_string)
         try: