Update Windows zenmap build

mswin32/Makefile

@@ -57,9 +57,9 @@ stage-common: $(NMAPDIR)/CHANGELOG $(NMAPDIR)/LICENSE $(NMAPDIR)/nmap-mac-prefix
 # Install the ndiff batch file wrapper in the zip distribution.
 	cp $(NMAPDIR)/ndiff/ndiff.py nmap-$(NMAP_VERSION)/ndiff.py
 	cp python-wrap.bat nmap-$(NMAP_VERSION)/ndiff.bat
-	make -C $(ZENMAP_WINDIR)
+	$(MAKE) -C $(ZENMAP_WINDIR)
-	rm -rf nmap-$(NMAP_VERSION)/zenmap-w64
+	rm -rf nmap-$(NMAP_VERSION)/zenmap
-	cp -R $(ZENMAP_WINDIR)/dist/zenmap-w64 nmap-$(NMAP_VERSION)/
+	cp -R $(ZENMAP_WINDIR)/dist/zenmap-w64/mingw64 nmap-$(NMAP_VERSION)/zenmap
 	cp $(NMAPDIR)/zenmap/README nmap-$(NMAP_VERSION)/ZENMAP_README
 	cp $(NMAPDIR)/zenmap/COPYING_HIGWIDGETS nmap-$(NMAP_VERSION)
 	cp $(NMAPDIR)/ndiff/README nmap-$(NMAP_VERSION)/NDIFF_README
@@ -76,7 +76,7 @@ stage-nmap-main: build-nmap${OEM_SUFFIX}
 	mkdir -p nmap-$(NMAP_VERSION)${OEM_SUFFIX}
 	cp $(VCCONFIG)${OEM_SUFFIX}/nmap.exe nmap-$(NMAP_VERSION)${OEM_SUFFIX}/
 
-sign-files: nmap-$(NMAP_VERSION)-oem/nmap.exe nmap-$(NMAP_VERSION)/nmap.exe nmap-$(NMAP_VERSION)/ncat.exe nmap-$(NMAP_VERSION)/nping.exe nmap-$(NMAP_VERSION)/zenmap-w64/zenmap.exe nmap-$(NMAP_VERSION)/libssh2.dll nmap-$(NMAP_VERSION)/libcrypto-3.dll nmap-$(NMAP_VERSION)/libssl-3.dll nmap-$(NMAP_VERSION)/zlibwapi.dll
+sign-files: nmap-$(NMAP_VERSION)-oem/nmap.exe nmap-$(NMAP_VERSION)/nmap.exe nmap-$(NMAP_VERSION)/ncat.exe nmap-$(NMAP_VERSION)/nping.exe nmap-$(NMAP_VERSION)/libssh2.dll nmap-$(NMAP_VERSION)/libcrypto-3.dll nmap-$(NMAP_VERSION)/libssl-3.dll nmap-$(NMAP_VERSION)/zlibwapi.dll
 	# TODO: evaluate whether we should also sign the Python stuff for Zenmap, Ndiff
 	"$(SIGNTOOL)" $(SIGNTOOL_ARGS) $^
 	touch sign-files
@@ -113,7 +113,7 @@ bundle-zip-main: sign-files
 	       || true
 	cp npcap-$(NPCAP_VERSION)${OEM_SUFFIX}.exe nmap-$(NMAP_VERSION)${OEM_SUFFIX}
 	zip -x nmap-$(NMAP_VERSION)${OEM_SUFFIX}/ZENMAP_README \
-	  nmap-$(NMAP_VERSION)${OEM_SUFFIX}/zenmap-w64/\* \
+	  nmap-$(NMAP_VERSION)${OEM_SUFFIX}/zenmap/\* \
 	  nmap-$(NMAP_VERSION)${OEM_SUFFIX}/COPYING_HIGWIDGETS \
 	  nmap-$(NMAP_VERSION)${OEM_SUFFIX}/Uninstall.exe \
 	  nmap-$(NMAP_VERSION)${OEM_SUFFIX}/icon1.ico \
@@ -125,4 +125,5 @@ LICENSE.formatted: $(NMAPDIR)/LICENSE
 
 clean:
 	cmd /c Build.bat Clean "$(VCCONFIG)${OEM_SUFFIX}"
+	$(MAKE) -C $(ZENMAP_WINDIR) clean
 	rm -rf make-vars.make nmap-$(NMAP_VERSION) nmap-$(NMAP_VERSION)-oem Debug Release Release-oem msbuild.log LICENSE.formatted

zenmap/install_scripts/utils/get_deps.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import modulefinder
+import configparser
+import sys
+import os.path
+import site
+import encodings
+
+site_package_deps = ("zenmapCore", "zenmapGUI", "radialnet", "gi", "cairo")
+
+# These items are unneeded, large, and on macOS _ssl causes dependency problems.
+pyd_remove = ("_decimal", "_ssl", "_testcapi")
+
+def module_paths(mods):
+    for m in mods:
+        if m.__name__ in pyd_remove:
+            continue
+        elif getattr(m, "__file__", None) and ('..' not in m.__file__):
+            yield m.__file__
+
+def get_deps():
+    # Start with pygobject and zenmap itself
+    sitedirs = site.getsitepackages()
+    files = set(os.path.join(sitedirs[0], name) for name in site_package_deps)
+
+    # These items are missed by modulefinder
+    files.add(encodings.__path__[0]) # All encodings just in case
+    for path in module_paths((site, site._sitebuiltins)):
+        files.add(path)
+
+    # Now use modulefinder to get the rest
+    mfind = modulefinder.ModuleFinder()
+    mfind.run_script('../../zenmap')
+    for path in module_paths(mfind.modules.values()):
+        parent = os.path.dirname(path)
+        found_parent = False
+        # If a parent dir is already included, don't bother listing the file.
+        while parent not in sys.path:
+            if parent in files:
+                found_parent = True
+                break
+            parent = os.path.dirname(parent)
+        if not found_parent:
+            files.add(path)
+    return files
+
+def read_cfg(filename):
+    cfg = configparser.ConfigParser()
+    cfg.read(filename)
+    return cfg
+
+def write_cfg(cfg, filename):
+    with open(filename, "w") as f:
+        cfg.write(f)
+
+def update_cfg(cfg, files):
+    filestr = "\nmingw*".join((f.removeprefix(sys.prefix) for f in files))
+    oldvalue = cfg.get('bundle', 'nodelete')
+    cfg.set('bundle', 'nodelete', oldvalue + "\nmingw*" + filestr)
+
+
+if __name__ == "__main__":
+    files = get_deps()
+    if sys.platform == "win32":
+        cfg = read_cfg(sys.argv[2])
+        update_cfg(cfg, files)
+        write_cfg(cfg, sys.argv[1])
+    #elif sys.platform == "darwin":
+    else:
+        raise NotImplementedError

zenmap/install_scripts/windows/Makefile

@@ -1,11 +1,15 @@
-all: dist/zenmap-w64-deletelog.txt mingw-w64-x86_64-python-zenmap-%.pkg.tar.zst
+export MSYS2_ARG_CONV_EXCL=""
+all: dist/zenmap-w64-deletelog.txt mingw-w64-x86_64-python-zenmap-*.pkg.tar.zst
 	./build.sh
 
 mingw-w64-x86_64-python-zenmap-%.pkg.tar.zst: PKGBUILD
 	makepkg-mingw -RdfL
 
-dist/zenmap-w64-deletelog.txt: styrene.cfg
+dist/zenmap-w64-deletelog.txt: tmp.cfg mingw-w64-x86_64-python-zenmap-*.pkg.tar.zst
-	styrene -p . -o $@ styrene.cfg --no-exe --no-zip
+	styrene -p . -o $(dir $@) tmp.cfg --no-exe --no-zip
+
+tmp.cfg: styrene.cfg ../utils/get_deps.py
+	python3 ../utils/get_deps.py $@ $<
 
 clean:
-	rm -rf pkg/ src/ dist/ mingw-w64-*-package.log* mingw-w64-*.zst
+	rm -rf tmp.cfg pkg/ src/ dist/ mingw-w64-*-package.log* mingw-w64-*.zst

zenmap/install_scripts/windows/build.sh

@@ -36,71 +36,34 @@ pip3 install .
 # make the minimal msys2 environment
 #styrene -p . -o "$BUILDDIR" styrene.cfg --no-exe --no-zip
 
-PACKAGEDIR=$BUILDDIR/zenmap-w64
+PACKAGEDIR=$BUILDDIR/zenmap-w64/mingw64
-PYTHONLIB=$(ls -d $PACKAGEDIR/mingw64/lib/python3.*)
+PYTHONLIB=$(ls -d $PACKAGEDIR/lib/python3.*)
-# Clean up unused Python modules
-shopt -s globstar
-rm -rf "$PYTHONLIB"/distutils/
-rm -rf "$PYTHONLIB"/pydoc_data/
-rm -rf "$PYTHONLIB"/ctypes/
-rm -rf "$PYTHONLIB"/asyncio/
-rm -rf "$PYTHONLIB"/multiprocessing/
-rm -rf "$PYTHONLIB"/html/
-rm -rf "$PYTHONLIB"/curses/
-rm -rf "$PYTHONLIB"/ensurepip/
-rm -rf "$PYTHONLIB"/idlelib/
-rm -rf "$PYTHONLIB"/lib2to3/
-rm -rf "$PYTHONLIB"/msilib/
-rm -rf "$PYTHONLIB"/venv/
-rm -rf "$PYTHONLIB"/xmlrpc/
-rm -rf "$PYTHONLIB"/concurrent/
-rm -rf "$PYTHONLIB"/json/
-rm -rf "$PYTHONLIB"/test/
-rm -rf "$PYTHONLIB"/tkinter/
-rm -rf "$PYTHONLIB"/turtledemo/
-rm -rf "$PYTHONLIB"/wsgiref/
-rm -rf "$PYTHONLIB"/Tools/
-rm -rf "$PYTHONLIB"/config-3.*/
-rm -rf "$PYTHONLIB"/zoneinfo/
-# Remove some of the larger unused items
-rm "$PYTHONLIB"/_pydecimal.py
-rm "$PYTHONLIB"/turtle.py
-rm "$PYTHONLIB"/lib-dynload/_decimal.*.pyd
-rm "$PYTHONLIB"/lib-dynload/_testcapi.*.pyd
-rm -rf "$PYTHONLIB"/site-packages/**/*libxml2*
 
 # Remove compiled bytecode, recompile in legacy locations, allowing for removal of source.
 # See PEP-3147
 find "$PYTHONLIB"  -depth \( -name 'zenmap*' -o -name 'radialnet' \) -prune -o -name __pycache__ -exec rm -rf '{}' \;
 # Exit code not reliable
-"$BUILDDIR/zenmap-w64/mingw64/bin/python.exe" -m compileall -b -x 'zenmapGUI|zenmapCore|radialnet' "$PYTHONLIB" || true
+python -m compileall -b -x 'zenmapGUI|zenmapCore|radialnet' "$PYTHONLIB" #|| true
 
 # Remove source if compiled is available, except for Zenmap itself:
 find "$PYTHONLIB" \( -name 'zenmap*' -o -name 'radialnet' \) -prune -o \( -name '*.pyc' -print \) | while read pyc; do
-pysrc="${pyc%.pyc}.py"
+rm -f "${pyc%.pyc}.py"
-if [ -f "$pysrc" ]; then
-	rm "$pysrc"
-fi
 done
 
 # Now compile Zenmap using default (not legacy) location.
 # If we had used legacy location, python.exe tries to write out the PEP-3147
 # location anyway when source is available.
-"$BUILDDIR/zenmap-w64/mingw64/bin/python.exe" -m compileall "$PYTHONLIB"/site-packages || true
+python -m compileall "$PYTHONLIB"/site-packages #|| true
 
 # Remove some of the larger unused items
-rm "$PACKAGEDIR"/mingw64/bin/libGLESv*.dll
+rm -f "$PACKAGEDIR"/bin/win7appid.exe
-rm "$PACKAGEDIR"/mingw64/bin/libEGL.dll
+
 # strip binaries
-find "$PACKAGEDIR" \( -name '*.exe' -o -name '*.dll' \) -exec strip -g '{}' \;
+find "$PACKAGEDIR" \( -name '*.exe' -o -name '*.dll' -o -name '*.pyd' \) -exec strip -g '{}' \;
 
-# Remove pacman database
+# Create cache files as needed
-rm -rf "$PACKAGEDIR/var/lib/pacman/"
+cd "$PACKAGEDIR"
+export GDK_PIXBUF_MODULEDIR=$(ls -d lib/gdk-pixbuf-2.0/2.*/loaders)
+gdk-pixbuf-query-loaders > "$GDK_PIXBUF_MODULEDIR".cache
+gtk-update-icon-cache share/icons/hicolor
 
-# fake a new install script for what we removed:
-mkdir -p "$PACKAGEDIR/var/lib/pacman/local/zenmap-fake-pkg/"
-cat >"$PACKAGEDIR/var/lib/pacman/local/zenmap-fake-pkg/install" <<EOF
-post_install() {
-    mingw64/bin/gtk-update-icon-cache -q -t -f mingw64/share/icons/hicolor
-}
-EOF

zenmap/install_scripts/windows/styrene.cfg

@@ -59,35 +59,104 @@ nodelete =
     # The postinst scripts that GTK needs, followed by their binaries.
     mingw*/share/glib*/schemas
 
-    mingw*/bin/gtk-update-icon-cache*.exe
+    #mingw*/bin/gtk-update-icon-cache*.exe
-    usr/bin/mkdir.exe
+    #usr/bin/mkdir.exe
-    usr/bin/msys-2*.dll
+    #usr/bin/msys-2*.dll
-    usr/share/libalpm
+    #usr/share/libalpm
 
-    var/lib/pacman/local/mingw-w64-*-hicolor-icon-theme-*-*/install
+    #var/lib/pacman/local/mingw-w64-*-hicolor-icon-theme-*-*/install
 
     # Runtime requirements for the demos etc.
     mingw*/etc/gtk-3.0
-    mingw*/bin/*.dll
+    # Required DLLs. Checked with:
+    # * recursive `dumpbin /imports` on known python/gtk dlls
+    # * recursive `dumpbin /imports` on pyd files in lib/python3.10
+    # * `g-ir-inspect --print-shlibs` for typelibs listed below
+    mingw*/bin/libatk-1.0-0.dll
+    mingw*/bin/libbrotlicommon.dll
+    mingw*/bin/libbrotlidec.dll
+    mingw*/bin/libbz2-1.dll
+    mingw*/bin/libcairo-2.dll
+    mingw*/bin/libcairo-gobject-2.dll
+    mingw*/bin/libdatrie-1.dll
+    mingw*/bin/libepoxy-0.dll
+    mingw*/bin/libexpat-1.dll
+    mingw*/bin/libffi-8.dll
+    mingw*/bin/libfontconfig-1.dll
+    mingw*/bin/libfreetype-6.dll
+    mingw*/bin/libfribidi-0.dll
+    mingw*/bin/libgcc_s_seh-1.dll
+    mingw*/bin/libgdk-3-0.dll
+    mingw*/bin/libgdk_pixbuf-2.0-0.dll
+    mingw*/bin/libgio-2.0-0.dll
+    mingw*/bin/libgirepository-1.0-1.dll
+    mingw*/bin/libglib-2.0-0.dll
+    mingw*/bin/libgmodule-2.0-0.dll
+    mingw*/bin/libgobject-2.0-0.dll
+    mingw*/bin/libgraphite2.dll
+    mingw*/bin/libgtk-3-0.dll
+    mingw*/bin/libharfbuzz-0.dll
+    mingw*/bin/libharfbuzz-gobject-0.dll
+    mingw*/bin/libiconv-2.dll
+    mingw*/bin/libintl-8.dll
+    mingw*/bin/libpango-1.0-0.dll
+    mingw*/bin/libpangocairo-1.0-0.dll
+    mingw*/bin/libpangoft2-1.0-0.dll
+    mingw*/bin/libpangowin32-1.0-0.dll
+    mingw*/bin/libpcre2-8-0.dll
+    mingw*/bin/libpixman-1-0.dll
+    mingw*/bin/libpng16-16.dll
+    mingw*/bin/libpython3.10.dll
+    mingw*/bin/libsqlite3-0.dll
+    mingw*/bin/libstdc++-6.dll
+    mingw*/bin/libthai-0.dll
+    mingw*/bin/libwinpthread-1.dll
+    mingw*/bin/zlib1.dll
 
     mingw*/lib/gtk-3.0
-    mingw*/lib/gdk-pixbuf-2.0
+    mingw*/lib/gdk-pixbuf-2.0/*/loaders/libpixbufloader-png.dll
+    mingw*/lib/gdk-pixbuf-2.0/*/loaders/libpixbufloader-png.dll.a
+    mingw*/lib/gdk-pixbuf-2.0/*/loaders/libpixbufloader-gif.dll
+    mingw*/lib/gdk-pixbuf-2.0/*/loaders/libpixbufloader-gif.dll.a
 
 
     mingw*/share/icons/hicolor/index.theme
 
     # python-gobject stuff
-    mingw*/lib/girepository-*
+    mingw*/lib/girepository-*/Atk-1.0.typelib
-    mingw*/lib/python3.*
+    mingw*/lib/girepository-*/GLib-2.0.typelib
+    mingw*/lib/girepository-*/GModule-2.0.typelib
+    mingw*/lib/girepository-*/GObject-2.0.typelib
+    mingw*/lib/girepository-*/Gdk-3.0.typelib
+    mingw*/lib/girepository-*/GdkPixbuf-2.0.typelib
+    mingw*/lib/girepository-*/Gio-2.0.typelib
+    mingw*/lib/girepository-*/Gtk-3.0.typelib
+    mingw*/lib/girepository-*/HarfBuzz-0.0.typelib
+    mingw*/lib/girepository-*/Pango-1.0.typelib
+    mingw*/lib/girepository-*/PangoCairo-1.0.typelib
+    mingw*/lib/girepository-*/PangoOT-1.0.typelib
+    mingw*/lib/girepository-*/cairo-1.0.typelib
+    mingw*/lib/girepository-*/freetype2-2.0.typelib
+
     # To avoid terminal window, use pythonw.exe
     mingw*/bin/pythonw.exe
     # For debugging use python.exe
-    #mingw*/bin/python.exe
+    mingw*/bin/python.exe
-    mingw*/bin/libpython*.dll
     # Zenmap's files (mostly in site-packages above)
     mingw*/share/zenmap
     # mingw*/bin/zenmap
 
+    # The python modules here will be added by script:
+    # mingw*/lib/python3.*
+    # mingw*/lib/python3.*/encodings
+    # mingw*/lib/python3.*/site.py
+    # mingw*/lib/python3.*/_sitebuiltins.py
+    # mingw*/lib/python3.*/site-packages/zenmapCore
+    # mingw*/lib/python3.*/site-packages/zenmapGUI
+    # mingw*/lib/python3.*/site-packages/radialnet
+    # mingw*/lib/python3.*/site-packages/gi
+    # mingw*/lib/python3.*/site-packages/cairo
+
 # Local launcher definitions
 
 ; You can define new app launchers here, or extend the launchers in