From 9e4d6f5f5c3a0425e0745b18cfa334030c99f0c0 Mon Sep 17 00:00:00 2001 From: dmiller Date: Wed, 7 Dec 2022 20:34:07 +0000 Subject: [PATCH] Fix encoding issues related to Python 3 upgrade. Python 3 str() is a unicode already, so can't be decoded. subprocess.Popen needs to be in text mode (universal_newlines is the oldest compatible kwarg for this) in order to do line-based buffering. In general, all the filesystem encoding stuff we were doing is done by Python itself now. --- .../install_scripts/utils/version_update.py | 2 +- zenmap/radialnet/core/XMLHandler.py | 10 +--- zenmap/radialnet/gui/RadialNet.py | 6 +-- zenmap/radialnet/gui/SaveDialog.py | 2 +- zenmap/share/zenmap/config/zenmap.conf | 4 +- .../zenmap/locale/xgettext-profile_editor.py | 2 +- zenmap/zenmapCore/BasePaths.py | 23 --------- zenmap/zenmapCore/Diff.py | 7 +-- zenmap/zenmapCore/NmapCommand.py | 38 ++------------- zenmap/zenmapCore/NmapParser.py | 2 +- zenmap/zenmapGUI/DiffCompare.py | 4 +- zenmap/zenmapGUI/FileChoosers.py | 48 ++++++------------- zenmap/zenmapGUI/ScanInterface.py | 6 +-- zenmap/zenmapGUI/ScriptInterface.py | 6 +-- 14 files changed, 35 insertions(+), 125 deletions(-) diff --git a/zenmap/install_scripts/utils/version_update.py b/zenmap/install_scripts/utils/version_update.py index 327e97f12..417d0f9e9 100644 --- a/zenmap/install_scripts/utils/version_update.py +++ b/zenmap/install_scripts/utils/version_update.py @@ -89,7 +89,7 @@ def update_date(base_dir): def update_version(base_dir, version): print(">>> Updating %s" % os.path.join(base_dir, VERSION)) - vf = open(os.path.join(base_dir, VERSION), "wb") + vf = open(os.path.join(base_dir, VERSION), "w") print(version, file=vf) vf.close() print(">>> Updating %s" % os.path.join(base_dir, VERSION_PY)) diff --git a/zenmap/radialnet/core/XMLHandler.py b/zenmap/radialnet/core/XMLHandler.py index 704061f09..26561dafa 100644 --- a/zenmap/radialnet/core/XMLHandler.py +++ b/zenmap/radialnet/core/XMLHandler.py @@ -67,12 +67,6 @@ import xml.sax.saxutils from xml.sax.xmlreader import AttributesImpl as Attributes -def convert_to_utf8(text): - """ - """ - return text.encode('utf8', 'replace') - - class XMLNode: """ """ @@ -289,7 +283,7 @@ class XMLReader(xml.sax.ContentHandler): # putting attributes and values in node for attr in attrs.getNames(): - node.add_attr(attr, convert_to_utf8(attrs.get(attr).strip())) + node.add_attr(attr, attrs.get(attr).strip()) # who is my father? if len(self.__status) > 0: @@ -303,7 +297,7 @@ class XMLReader(xml.sax.ContentHandler): def endElement(self, name): """ """ - self.__status[-1].set_text(convert_to_utf8(self.__text.strip())) + self.__status[-1].set_text(self.__text.strip()) self.__text = "" self.__status.pop() diff --git a/zenmap/radialnet/gui/RadialNet.py b/zenmap/radialnet/gui/RadialNet.py index 60bb76eac..8bbb44a8d 100644 --- a/zenmap/radialnet/gui/RadialNet.py +++ b/zenmap/radialnet/gui/RadialNet.py @@ -76,8 +76,6 @@ from radialnet.core.Graph import Node from radialnet.gui.NodeWindow import NodeWindow from radialnet.gui.Image import Icons, get_pixels_for_cairo_image_surface -from zenmapCore.BasePaths import fs_enc - REGION_COLORS = [(1.0, 0.0, 0.0), (1.0, 1.0, 0.0), (0.0, 1.0, 0.0)] REGION_RED = 0 REGION_YELLOW = 1 @@ -249,9 +247,7 @@ class RadialNet(Gtk.DrawingArea): self.__draw(context) if type == FILE_TYPE_PNG: - # write_to_png requires a str, not unicode, in py2cairo 1.8.10 and - # earlier. - self.surface.write_to_png(fs_enc(file)) + self.surface.write_to_png(file) self.surface.flush() self.surface.finish() diff --git a/zenmap/radialnet/gui/SaveDialog.py b/zenmap/radialnet/gui/SaveDialog.py index e1e278330..47db1916e 100644 --- a/zenmap/radialnet/gui/SaveDialog.py +++ b/zenmap/radialnet/gui/SaveDialog.py @@ -83,7 +83,7 @@ for type in TYPES: EXTENSIONS[type[2]] = type[1] -class SaveDialog(zenmapGUI.FileChoosers.UnicodeFileChooserDialog): +class SaveDialog(Gtk.FileChooserDialog): def __init__(self): """ """ diff --git a/zenmap/share/zenmap/config/zenmap.conf b/zenmap/share/zenmap/config/zenmap.conf index a282ef32a..7f02afc21 100644 --- a/zenmap/share/zenmap/config/zenmap.conf +++ b/zenmap/share/zenmap/config/zenmap.conf @@ -58,8 +58,8 @@ underline = 0 enable_highlight = True [paths] -nmap_command_path = nmap -ndiff_command_path = ndiff +nmap_command_path = ../nmap +ndiff_command_path = ../ndiff/ndiff [port_list_highlight] regex = ^PORT\s+STATE\s+SERVICE(\s+VERSION)?.* diff --git a/zenmap/share/zenmap/locale/xgettext-profile_editor.py b/zenmap/share/zenmap/locale/xgettext-profile_editor.py index c18c636e8..2ce09a69f 100755 --- a/zenmap/share/zenmap/locale/xgettext-profile_editor.py +++ b/zenmap/share/zenmap/locale/xgettext-profile_editor.py @@ -17,7 +17,7 @@ directory = None def escape(s): - return '"' + s.encode("UTF-8").replace('"', '\\"') + '"' + return '"' + s.replace('"', '\\"') + '"' def output_msgid(msgid, locator): diff --git a/zenmap/zenmapCore/BasePaths.py b/zenmap/zenmapCore/BasePaths.py index d72e395cf..7254aa3e7 100644 --- a/zenmap/zenmapCore/BasePaths.py +++ b/zenmap/zenmapCore/BasePaths.py @@ -64,29 +64,6 @@ import sys from zenmapCore.Name import APP_NAME -def fs_dec(s): # This is unused now - """Decode s from the filesystem decoding, handling various possible - errors.""" - enc = sys.getfilesystemencoding() - if enc is None: - enc = "UTF-8" - return s.decode(enc) - - -def fs_enc(u): - """Encode u to the filesystem decoding, handling various possible - errors.""" - enc = sys.getfilesystemencoding() - if enc is None: - enc = "UTF-8" - return u.encode(enc) - -# We can't just use os.path.expanduser(u"~") to get a unicode version of the -# home directory, because os.path.expanduser doesn't properly decode the raw -# byte string from the file system encoding. You get a UnicodeDecodeError on -# systems like Windows where the file system encoding is different from the -# result of sys.getdefaultencoding(). So we call os.path.expanduser with a -# plain string and decode it from the filesystem encoding. HOME = os.path.expanduser("~") # The base_paths dict in this file gives symbolic names to various files. For diff --git a/zenmap/zenmapCore/Diff.py b/zenmap/zenmapCore/Diff.py index f37e96564..7e4b081bd 100644 --- a/zenmap/zenmapCore/Diff.py +++ b/zenmap/zenmapCore/Diff.py @@ -122,7 +122,7 @@ class NdiffCommand(subprocess.Popen): filename_b ] self.stdout_file = tempfile.TemporaryFile( - mode="rb", + mode="r", prefix=APP_NAME + "-ndiff-", suffix=".xml" ) @@ -132,6 +132,7 @@ class NdiffCommand(subprocess.Popen): subprocess.Popen.__init__( self, command_list, + universal_newlines=True, stdout=self.stdout_file, stderr=self.stdout_file, env=env, @@ -167,7 +168,7 @@ def ndiff(scan_a, scan_b): suffix=".xml" ) temporary_filenames.append(filename_a) - f = os.fdopen(fd, "wb") + f = os.fdopen(fd, "w") scan_a.write_xml(f) f.close() else: @@ -179,7 +180,7 @@ def ndiff(scan_a, scan_b): suffix=".xml" ) temporary_filenames.append(filename_b) - f = os.fdopen(fd, "wb") + f = os.fdopen(fd, "w") scan_b.write_xml(f) f.close() else: diff --git a/zenmap/zenmapCore/NmapCommand.py b/zenmap/zenmapCore/NmapCommand.py index e6768dc83..4daa4a0f3 100644 --- a/zenmap/zenmapCore/NmapCommand.py +++ b/zenmap/zenmapCore/NmapCommand.py @@ -87,39 +87,6 @@ paths_config = PathsConfig() log.debug(">>> Platform: %s" % sys.platform) -def wrap_file_in_preferred_encoding(f): - """Wrap an open file to automatically decode its contents when reading from - the encoding given by locale.getpreferredencoding, or just return the file - if that doesn't work. - - The nmap executable will write its output in whatever the system encoding - is. Nmap's output is usually all ASCII, but time zone it prints can be in a - different encoding. If it is not decoded correctly it will be displayed as - garbage characters. This function assists in reading the Nmap output. We - don't know for sure what the encoding used is, but we take a best guess and - decode the output into a proper unicode object so that the screen display - and XML writer interpret it correctly.""" - - try: - preferredencoding = locale.getpreferredencoding() - except locale.Error: - # This can happen if the LANG environment variable is set to something - # weird. - preferredencoding = None - - if preferredencoding is not None: - try: - reader = codecs.getreader(preferredencoding) - return reader(f, "replace") - except LookupError: - # The lookup failed. This can happen if the preferred encoding is - # unknown ("X-MAC-KOREAN" has been observed). Ignore it and return - # the unwrapped file. - log.debug("Unknown encoding \"%s\"." % preferredencoding) - - return f - - def escape_nmap_filename(filename): """Escape '%' characters so they are not interpreted as strftime format specifiers, which are not supported by Zenmap.""" @@ -235,8 +202,8 @@ class NmapCommand(object): # We don't need a file name for stdout output, just a handle. A # TemporaryFile is deleted as soon as it is closed, and in Unix is # unlinked immediately after creation so it's not even visible. - f = tempfile.TemporaryFile(mode="rb", prefix=APP_NAME + "-stdout-") - self.stdout_file = wrap_file_in_preferred_encoding(f) + f = tempfile.TemporaryFile(mode="r", prefix=APP_NAME + "-stdout-") + self.stdout_file = f if stderr is None: stderr = f @@ -260,6 +227,7 @@ class NmapCommand(object): startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW self.command_process = subprocess.Popen(command_list, bufsize=1, + universal_newlines=True, stdin=subprocess.PIPE, stdout=f, stderr=stderr, diff --git a/zenmap/zenmapCore/NmapParser.py b/zenmap/zenmapCore/NmapParser.py index 3968cd112..3f6b44edb 100644 --- a/zenmap/zenmapCore/NmapParser.py +++ b/zenmap/zenmapCore/NmapParser.py @@ -1034,7 +1034,7 @@ class NmapParserSAX(ParserBasics, ContentHandler): def write_xml_to_file(self, filename): """Write the XML representation of this scan to the file whose name is given.""" - fd = open(filename, "wb") + fd = open(filename, "w") self.write_xml(fd) fd.close() diff --git a/zenmap/zenmapGUI/DiffCompare.py b/zenmap/zenmapGUI/DiffCompare.py index 825b86f1a..9f6d01871 100644 --- a/zenmap/zenmapGUI/DiffCompare.py +++ b/zenmap/zenmapGUI/DiffCompare.py @@ -333,7 +333,7 @@ class DiffWindow(Gtk.Window): message_format=_("Error running ndiff"), secondary_text=_( "There was an error running the ndiff program.\n\n" - ) + str(e).decode(sys.getdefaultencoding(), "replace")) + ) + str(e)) alert.run() alert.destroy() else: @@ -364,7 +364,7 @@ class DiffWindow(Gtk.Window): if status == 0 or status == 1: # Successful completion. try: - diff = self.ndiff_process.get_scan_diff().decode("utf-8") + diff = self.ndiff_process.get_scan_diff() except zenmapCore.Diff.NdiffParseException as e: alert = HIGAlertDialog( message_format=_("Error parsing ndiff output"), diff --git a/zenmap/zenmapGUI/FileChoosers.py b/zenmap/zenmapGUI/FileChoosers.py index d98022235..590b4461e 100644 --- a/zenmap/zenmapGUI/FileChoosers.py +++ b/zenmap/zenmapGUI/FileChoosers.py @@ -99,39 +99,19 @@ class ScriptFileFilter(Gtk.FileFilter): self.set_name(_("NSE scripts (%s)") % ", ".join(patterns)) -class UnicodeFileChooserDialog(Gtk.FileChooserDialog): - """This is a base class for file choosers. It is designed to ease the - retrieval of Unicode file names. On most platforms, the file names returned - are encoded in the encoding given by sys.getfilesystemencoding(). On - Windows, they are returned in UTF-8, even though using the UTF-8 file name - results in a file not found error. The get_filename method of this class - handles the decoding automatically.""" - def get_filename(self): - filename = Gtk.FileChooserDialog.get_filename(self) - if sys.platform == "win32": - encoding = "UTF-8" - else: - encoding = sys.getfilesystemencoding() or "UTF-8" - try: - filename = filename.decode(encoding) - except Exception: - pass - return filename - - -class AllFilesFileChooserDialog(UnicodeFileChooserDialog): +class AllFilesFileChooserDialog(Gtk.FileChooserDialog): def __init__(self, title="", parent=None, action=Gtk.FileChooserAction.OPEN, buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None): - UnicodeFileChooserDialog.__init__(self, title=title, parent=parent, + Gtk.FileChooserDialog.__init__(self, title=title, parent=parent, action=action, buttons=buttons) self.set_default_response(Gtk.ResponseType.OK) self.add_filter(AllFilesFileFilter()) -class ResultsFileSingleChooserDialog(UnicodeFileChooserDialog): +class ResultsFileSingleChooserDialog(Gtk.FileChooserDialog): """This results file choose only allows the selection of single files, not directories.""" def __init__(self, title="", parent=None, @@ -139,34 +119,34 @@ class ResultsFileSingleChooserDialog(UnicodeFileChooserDialog): buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None): - UnicodeFileChooserDialog.__init__(self, title=title, parent=parent, + Gtk.FileChooserDialog.__init__(self, title=title, parent=parent, action=action, buttons=buttons) self.set_default_response(Gtk.ResponseType.OK) for f in (ResultsFileFilter(), AllFilesFileFilter()): self.add_filter(f) -class ResultsFileChooserDialog(UnicodeFileChooserDialog): +class ResultsFileChooserDialog(Gtk.FileChooserDialog): def __init__(self, title="", parent=None, action=Gtk.FileChooserAction.OPEN, buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, "Open Directory", RESPONSE_OPEN_DIRECTORY, Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None): - UnicodeFileChooserDialog.__init__(self, title=title, parent=parent, + Gtk.FileChooserDialog.__init__(self, title=title, parent=parent, action=action, buttons=buttons) self.set_default_response(Gtk.ResponseType.OK) for f in (ResultsFileFilter(), AllFilesFileFilter()): self.add_filter(f) -class ScriptFileChooserDialog(UnicodeFileChooserDialog): +class ScriptFileChooserDialog(Gtk.FileChooserDialog): def __init__(self, title="", parent=None, action=Gtk.FileChooserAction.OPEN, buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None): - UnicodeFileChooserDialog.__init__(self, title=title, parent=parent, + Gtk.FileChooserDialog.__init__(self, title=title, parent=parent, action=action, buttons=buttons) self.set_default_response(Gtk.ResponseType.OK) self.set_select_multiple(True) @@ -174,7 +154,7 @@ class ScriptFileChooserDialog(UnicodeFileChooserDialog): self.add_filter(f) -class SaveResultsFileChooserDialog(UnicodeFileChooserDialog): +class SaveResultsFileChooserDialog(Gtk.FileChooserDialog): TYPES = ( (_("By extension"), None, None), (_("Nmap XML format (.xml)"), "xml", ".xml"), @@ -192,7 +172,7 @@ class SaveResultsFileChooserDialog(UnicodeFileChooserDialog): buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_SAVE, Gtk.ResponseType.OK), backend=None): - UnicodeFileChooserDialog.__init__(self, title=title, parent=parent, + Gtk.FileChooserDialog.__init__(self, title=title, parent=parent, action=action, buttons=buttons) types_store = Gtk.ListStore.new([str, str, str]) @@ -245,23 +225,23 @@ class SaveResultsFileChooserDialog(UnicodeFileChooserDialog): return filetype -class DirectoryChooserDialog(UnicodeFileChooserDialog): +class DirectoryChooserDialog(Gtk.FileChooserDialog): def __init__(self, title="", parent=None, action=Gtk.FileChooserAction.SELECT_FOLDER, buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OPEN, Gtk.ResponseType.OK), backend=None): - UnicodeFileChooserDialog.__init__(self, title=title, parent=parent, + Gtk.FileChooserDialog.__init__(self, title=title, parent=parent, action=action, buttons=buttons) self.set_default_response(Gtk.ResponseType.OK) -class SaveToDirectoryChooserDialog(UnicodeFileChooserDialog): +class SaveToDirectoryChooserDialog(Gtk.FileChooserDialog): def __init__(self, title="", parent=None, action=Gtk.FileChooserAction.SELECT_FOLDER, buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_SAVE, Gtk.ResponseType.OK), backend=None): - UnicodeFileChooserDialog.__init__(self, title=title, parent=parent, + Gtk.FileChooserDialog.__init__(self, title=title, parent=parent, action=action, buttons=buttons) self.set_default_response(Gtk.ResponseType.OK) diff --git a/zenmap/zenmapGUI/ScanInterface.py b/zenmap/zenmapGUI/ScanInterface.py index 1cc272518..19c2c02bd 100644 --- a/zenmap/zenmapGUI/ScanInterface.py +++ b/zenmap/zenmapGUI/ScanInterface.py @@ -65,7 +65,6 @@ from gi.repository import Gtk, GLib import errno import os import time -import sys # Prevent loading PyXML import xml @@ -478,7 +477,7 @@ class ScanInterface(HIGVBox): try: command_execution.run_scan() except OSError as e: - text = str(e.strerror, errors='replace') + text = e.strerror # Handle ENOENT specially. if e.errno == errno.ENOENT: # nmap_command_path comes from zenmapCore.NmapCommand. @@ -486,9 +485,6 @@ class ScanInterface(HIGVBox): if path_env is None: default_paths = [] else: - fsencoding = sys.getfilesystemencoding() - if fsencoding: - path_env = path_env.decode(fsencoding, 'replace') default_paths = path_env.split(os.pathsep) text += "\n\n{}\n\n{}".format( _("This means that the nmap executable was " diff --git a/zenmap/zenmapGUI/ScriptInterface.py b/zenmap/zenmapGUI/ScriptInterface.py index d2443a29b..1021b61dd 100644 --- a/zenmap/zenmapGUI/ScriptInterface.py +++ b/zenmap/zenmapGUI/ScriptInterface.py @@ -65,7 +65,6 @@ gi.require_version("Gtk", "3.0") from gi.repository import Gtk, GLib import os -import sys import tempfile # Prevent loading PyXML @@ -134,8 +133,7 @@ class ScriptHelpXMLContentHandler (xml.sax.handler.ContentHandler): if "path" not in attrs: raise ValueError( '"directory" element did not have "path" attribute') - path = attrs["path"].encode("raw_unicode_escape").decode( - sys.getfilesystemencoding()) + path = attrs["path"] if dirname == "scripts": self.scripts_dir = path elif dirname == "nselib": @@ -229,7 +227,7 @@ class ScriptInterface: # Separate stderr to avoid breaking XML parsing with "Warning: File # ./nse_main.lua exists, but Nmap is using...". stderr = tempfile.TemporaryFile( - mode="rb", prefix=APP_NAME + "-script-help-stderr-") + mode="r", prefix=APP_NAME + "-script-help-stderr-") log.debug("Script interface: running %s" % repr(command_string)) nmap_process = NmapCommand(command_string) try: