diff --git a/scan_engine.cc b/scan_engine.cc
index 46b7a3758..bbb6a3e1e 100644
--- a/scan_engine.cc
+++ b/scan_engine.cc
@@ -2731,10 +2731,6 @@ void ultra_scan(std::vector<Target *> &Targets, const struct scan_lists *ports,
                 stype scantype, struct timeout_info *to) {
   o.current_scantype = scantype;
 
-   /* Load up _all_ payloads into a mapped table. Only needed for raw scans. */
-
-  init_payloads();
-
   if (Targets.size() == 0) {
     return;
   }
@@ -2751,6 +2747,11 @@ void ultra_scan(std::vector<Target *> &Targets, const struct scan_lists *ports,
 
   UltraScanInfo USI(Targets, ports, scantype);
 
+  /* Load up _all_ payloads into a mapped table. Only needed for raw scans. */
+  if (USI.udp_scan) {
+    init_payloads();
+  }
+
   if (USI.gstats->numprobes <= 0) {
     if (o.debugging) {
       log_write(LOG_STDOUT, "Skipping %s: no probes to send\n", scantype2str(scantype));