Added IOCP integration for Nsock, engine IOCP is the default engine on Windows

nbase/nbase_misc.c

@@ -257,7 +257,7 @@ int inheritable_socket(int af, int style, int protocol) {
   /* WSASocket is just like socket, except that the sockets it creates are
      inheritable by subprocesses (such as are created by CreateProcess), while
      those created by socket are not. */
-  return WSASocket(af, style, protocol, NULL, 0, 0);
+  return WSASocket(af, style, protocol, NULL, 0, WSA_FLAG_OVERLAPPED);
 #else
   return socket(af, style, protocol);
 #endif

nsock/include/nsock_winconfig.h

@@ -102,6 +102,7 @@
  /* WSAPoll() isn't available before Vista */
 #if defined(_WIN32_WINNT) && (_WIN32_WINNT >= 0x0600)
 #define HAVE_POLL 1
+#define HAVE_IOCP 1
 #endif
 
 #endif /* NSOCK_WINCONFIG_H */

nsock/nsock.vcxproj

@@ -186,6 +186,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="src\engine_epoll.c" />
+    <ClCompile Include="src\engine_iocp.c" />
     <ClCompile Include="src\engine_kqueue.c" />
     <ClCompile Include="src\engine_poll.c" />
     <ClCompile Include="src\engine_select.c" />

nsock/src/engine_epoll.c

@@ -85,11 +85,12 @@
 /* --- ENGINE INTERFACE PROTOTYPES --- */
 static int epoll_init(struct npool *nsp);
 static void epoll_destroy(struct npool *nsp);
-static int epoll_iod_register(struct npool *nsp, struct niod *iod, int ev);
+static int epoll_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev);
 static int epoll_iod_unregister(struct npool *nsp, struct niod *iod);
-static int epoll_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr);
+static int epoll_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr);
 static int epoll_loop(struct npool *nsp, int msec_timeout);
 
+extern struct io_operations posix_io_operations;
 
 /* ---- ENGINE DEFINITION ---- */
 struct io_engine engine_epoll = {
@@ -99,7 +100,8 @@ struct io_engine engine_epoll = {
   epoll_iod_register,
   epoll_iod_unregister,
   epoll_iod_modify,
-  epoll_loop
+  epoll_loop,
+  &posix_io_operations
 };
 
 
@@ -159,7 +161,7 @@ void epoll_destroy(struct npool *nsp) {
   free(einfo);
 }
 
-int epoll_iod_register(struct npool *nsp, struct niod *iod, int ev) {
+int epoll_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev) {
   int sd;
   struct epoll_event epev;
   struct epoll_engine_info *einfo = (struct epoll_engine_info *)nsp->engine_data;
@@ -204,7 +206,7 @@ int epoll_iod_unregister(struct npool *nsp, struct niod *iod) {
   return 1;
 }
 
-int epoll_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr) {
+int epoll_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr) {
   int sd;
   struct epoll_event epev;
   int new_events;

nsock/src/engine_iocp.c

@@ -0,0 +1,779 @@
+/***************************************************************************
+ * engine_iocp.c -- I/O Completion Ports based IO engine.                  *
+ *                                                                         *
+ ***********************IMPORTANT NSOCK LICENSE TERMS***********************
+ *                                                                         *
+ * The nsock parallel socket event library is (C) 1999-2016 Insecure.Com   *
+ * LLC This library is free software; you may redistribute and/or          *
+ * modify it under the terms of the GNU General Public License as          *
+ * published by the Free Software Foundation; Version 2.  This guarantees  *
+ * your right to use, modify, and redistribute this software under certain *
+ * conditions.  If this license is unacceptable to you, Insecure.Com LLC   *
+ * may be willing to sell alternative licenses (contact                    *
+ * sales@insecure.com ).                                                   *
+ *                                                                         *
+ * As a special exception to the GPL terms, Insecure.Com LLC grants        *
+ * permission to link the code of this program with any version of the     *
+ * OpenSSL library which is distributed under a license identical to that  *
+ * listed in the included docs/licenses/OpenSSL.txt file, and distribute   *
+ * linked combinations including the two. You must obey the GNU GPL in all *
+ * respects for all of the code used other than OpenSSL.  If you modify    *
+ * this file, you may extend this exception to your version of the file,   *
+ * but you are not obligated to do so.                                     *
+ *                                                                         *
+ * If you received these files with a written license agreement stating    *
+ * terms other than the (GPL) terms above, then that alternative license   *
+ * agreement takes precedence over this comment.                           *
+ *                                                                         *
+ * Source is provided to this software because we believe users have a     *
+ * right to know exactly what a program is going to do before they run it. *
+ * This also allows you to audit the software for security holes.          *
+ *                                                                         *
+ * Source code also allows you to port Nmap to new platforms, fix bugs,    *
+ * and add new features.  You are highly encouraged to send your changes   *
+ * to the dev@nmap.org mailing list for possible incorporation into the    *
+ * main distribution.  By sending these changes to Fyodor or one of the    *
+ * Insecure.Org development mailing lists, or checking them into the Nmap  *
+ * source code repository, it is understood (unless you specify otherwise) *
+ * that you are offering the Nmap Project (Insecure.Com LLC) the           *
+ * unlimited, non-exclusive right to reuse, modify, and relicense the      *
+ * code.  Nmap will always be available Open Source, but this is important *
+ * because the inability to relicense code has caused devastating problems *
+ * for other Free Software projects (such as KDE and NASM).  We also       *
+ * occasionally relicense the code to third parties as discussed above.    *
+ * If you wish to specify special license conditions of your               *
+ * contributions, just say so when you send them.                          *
+ *                                                                         *
+ * This program is distributed in the hope that it will be useful, but     *
+ * WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU       *
+ * General Public License v2.0 for more details                            *
+ * (http://www.gnu.org/licenses/gpl-2.0.html).                             *
+ *                                                                         *
+ ***************************************************************************/
+
+/* $Id$ */
+
+#if WIN32
+#include "nsock_winconfig.h"
+#endif
+
+#if HAVE_IOCP
+
+#include <Winsock2.h>
+#include <Mswsock.h>
+
+#include "nsock_internal.h"
+#include "nsock_log.h"
+
+#if HAVE_PCAP
+#include "nsock_pcap.h"
+#endif
+
+
+/* --- ENGINE INTERFACE PROTOTYPES --- */
+static int iocp_init(struct npool *nsp);
+static void iocp_destroy(struct npool *nsp);
+static int iocp_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev);
+static int iocp_iod_unregister(struct npool *nsp, struct niod *iod);
+static int iocp_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr);
+static int iocp_loop(struct npool *nsp, int msec_timeout);
+
+int iocp_iod_connect(struct npool *nsp, int sockfd, const struct sockaddr *addr, socklen_t addrlen);
+int iocp_iod_read(struct npool *nsp, int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen);
+int iocp_iod_write(struct npool *nsp, int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+
+struct io_operations iocp_io_operations = {
+  iocp_iod_connect,
+  iocp_iod_read,
+  iocp_iod_write
+};
+
+/* ---- ENGINE DEFINITION ---- */
+struct io_engine engine_iocp = {
+  "iocp",
+  iocp_init,
+  iocp_destroy,
+  iocp_iod_register,
+  iocp_iod_unregister,
+  iocp_iod_modify,
+  iocp_loop,
+  &iocp_io_operations
+};
+
+/*
+* Engine specific data structure
+*/
+struct iocp_engine_info {
+  /* The handle to the Completion Port*/
+  HANDLE iocp;
+
+  /* We put the current eov to be processed here in order to be retrieved by nsock_core */
+  struct extended_overlapped *eov;
+
+  /* The overlapped_entry list used to retrieve completed packets from the port */
+  OVERLAPPED_ENTRY *eov_list;
+  unsigned long capacity;
+
+  /* How many Completion Packets we actually retreieved */
+  unsigned long entries_removed;
+
+  gh_list_t active_eovs;
+  gh_list_t free_eovs;
+};
+
+struct extended_overlapped {
+  /* Overlapped structure used for overlapped operations */
+  OVERLAPPED ov;
+
+  /* Did we get an error when we initiated the operation?
+  Put the error code here and post it to the main loop */
+  int err;
+
+  /* The event may have expired and was recycled, we can't trust
+  a pointer to the nevent structure to tell us the real nevent */
+  nsock_event_id nse_id;
+
+  /* A pointer to the event */
+  struct nevent *nse;
+
+  /* Needed for WSARecv/WSASend */
+  WSABUF wsabuf;
+
+  /* This is the buffer we will read data in */
+  char *readbuf;
+
+  /* The struct npool keeps track of EOVs that have been allocated so that it
+  * can destroy them if the msp is deleted.  This pointer makes it easy to
+  * remove this struct extended_overlapped from the allocated list when necessary */
+  gh_lnode_t nodeq;
+
+  int eov_received;
+};
+
+/* --- INTERNAL PROTOTYPES --- */
+static void iterate_through_event_lists(struct npool *nsp);
+static void terminate_overlapped_event(struct npool *nsp, struct nevent *nse);
+static void initiate_overlapped_event(struct npool *nsp, struct nevent *nse);
+static int get_overlapped_result(struct npool *nsp, int fd, const void *buffer, size_t count);
+static void force_operation(struct npool *nsp, struct nevent *nse);
+static void free_eov(struct npool *nsp, struct extended_overlapped *eov);
+static int map_faulty_errors(int err);
+
+/* defined in nsock_core.c */
+void process_iod_events(struct npool *nsp, struct niod *nsi, int ev);
+void process_event(struct npool *nsp, gh_list_t *evlist, struct nevent *nse, int ev);
+void process_expired_events(struct npool *nsp);
+#if HAVE_PCAP
+#ifndef PCAP_CAN_DO_SELECT
+int pcap_read_on_nonselect(struct npool *nsp);
+#endif
+#endif
+
+/* defined in nsock_event.c */
+void update_first_events(struct nevent *nse);
+
+
+extern struct timeval nsock_tod;
+
+int iocp_init(struct npool *nsp) {
+  struct iocp_engine_info *iinfo;
+
+  iinfo = (struct iocp_engine_info *)safe_malloc(sizeof(struct iocp_engine_info));
+
+  gh_list_init(&iinfo->active_eovs);
+  gh_list_init(&iinfo->free_eovs);
+
+  iinfo->iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, NULL, 0);
+  iinfo->capacity = 10;
+  iinfo->eov = NULL;
+  iinfo->entries_removed = 0;
+  iinfo->eov_list = (OVERLAPPED_ENTRY *)safe_malloc(iinfo->capacity * sizeof(OVERLAPPED_ENTRY));
+  nsp->engine_data = (void *)iinfo;
+
+  return 1;
+}
+
+void iocp_destroy(struct npool *nsp) {
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+
+  assert(iinfo != NULL);
+
+  struct extended_overlapped *eov;
+  gh_lnode_t *current, *next;
+
+  while ((current = gh_list_pop(&iinfo->active_eovs))) {
+    eov = container_of(current, struct extended_overlapped, nodeq);
+    if (eov->readbuf) {
+      free(eov->readbuf);
+      eov->readbuf = NULL;
+    }
+    free(eov);
+  }
+
+  while ((current = gh_list_pop(&iinfo->free_eovs))) {
+    eov = container_of(current, struct extended_overlapped, nodeq);
+    free(eov);
+  }
+
+  gh_list_free(&iinfo->active_eovs);
+  gh_list_free(&iinfo->free_eovs);
+
+  CloseHandle(iinfo->iocp);
+  free(iinfo->eov_list);
+
+  free(iinfo);
+}
+
+int iocp_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev) {
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+  HANDLE result;
+
+  assert(!IOD_PROPGET(iod, IOD_REGISTERED));
+  iod->watched_events = ev;
+  result = CreateIoCompletionPort((HANDLE)iod->sd, iinfo->iocp, NULL, 0);
+  assert(result);
+
+  IOD_PROPSET(iod, IOD_REGISTERED);
+
+  initiate_overlapped_event(nsp, nse);
+
+  return 1;
+}
+
+/* Sadly a socket can't be unassociated with a completion port */
+int iocp_iod_unregister(struct npool *nsp, struct niod *iod) {
+
+  if (IOD_PROPGET(iod, IOD_REGISTERED)) {
+    /* Nuke all uncompleted operations on that iod */
+    CancelIo((HANDLE)iod->sd);
+    IOD_PROPCLR(iod, IOD_REGISTERED);
+  }
+
+  return 1;
+}
+
+int iocp_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr) {
+  int new_events;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+
+  assert((ev_set & ev_clr) == 0);
+  assert(IOD_PROPGET(iod, IOD_REGISTERED));
+
+  new_events = iod->watched_events;
+  new_events |= ev_set;
+  new_events &= ~ev_clr;
+
+  if (ev_set != EV_NONE)
+    initiate_overlapped_event(nsp, nse);
+  else if (ev_clr != EV_NONE)
+    terminate_overlapped_event(nsp, nse);
+
+  if (new_events == iod->watched_events)
+    return 1; /* nothing to do */
+
+  iod->watched_events = new_events;
+
+  return 1;
+}
+
+int iocp_loop(struct npool *nsp, int msec_timeout) {
+  int event_msecs; /* msecs before an event goes off */
+  int combined_msecs;
+  int sock_err = 0;
+  BOOL bRet;
+  int total_events;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+
+  assert(msec_timeout >= -1);
+
+  if (nsp->events_pending == 0)
+    return 0; /* No need to wait on 0 events ... */
+
+
+  struct nevent *nse;
+
+  /* Make sure the preallocated space for the retrieved events is big enough */
+  total_events = gh_list_count(&nsp->connect_events) + gh_list_count(&nsp->read_events) + gh_list_count(&nsp->write_events);
+  if (iinfo->capacity < total_events) {
+    iinfo->capacity *= 2;
+    iinfo->eov_list = (OVERLAPPED_ENTRY *)safe_realloc(iinfo->eov_list, iinfo->capacity * sizeof(OVERLAPPED_ENTRY));
+  }
+
+  nsock_log_debug_all("wait for events");
+
+  nse = next_expirable_event(nsp);
+  if (!nse)
+    event_msecs = -1; /* None of the events specified a timeout */
+  else
+    event_msecs = MAX(0, TIMEVAL_MSEC_SUBTRACT(nse->timeout, nsock_tod));
+
+#if HAVE_PCAP
+#ifndef PCAP_CAN_DO_SELECT
+  /* Force a low timeout when capturing packets on systems where
+  * the pcap descriptor is not select()able. */
+  if (gh_list_count(&nsp->pcap_read_events) > 0)
+  if (event_msecs > PCAP_POLL_INTERVAL)
+    event_msecs = PCAP_POLL_INTERVAL;
+#endif
+#endif
+
+  /* We cast to unsigned because we want -1 to be very high (since it means no
+  * timeout) */
+  combined_msecs = MIN((unsigned)event_msecs, (unsigned)msec_timeout);
+
+#if HAVE_PCAP
+#ifndef PCAP_CAN_DO_SELECT
+  /* do non-blocking read on pcap devices that doesn't support select()
+  * If there is anything read, just leave this loop. */
+  if (pcap_read_on_nonselect(nsp)) {
+    /* okay, something was read. */
+  }
+  else
+#endif
+#endif
+  /* It is mandatory these values are reset before calling GetQueuedCompletionStatusEx */
+  iinfo->entries_removed = 0;
+  memset(iinfo->eov_list, 0, iinfo->capacity * sizeof(OVERLAPPED_ENTRY));
+  bRet = GetQueuedCompletionStatusEx(iinfo->iocp, iinfo->eov_list, iinfo->capacity, &iinfo->entries_removed, combined_msecs, FALSE);
+
+  gettimeofday(&nsock_tod, NULL); /* Due to iocp delay */
+  if (!bRet) {
+    sock_err = socket_errno();
+    if (!iinfo->eov && sock_err != WAIT_TIMEOUT) {
+      nsock_log_error("nsock_loop error %d: %s", sock_err, socket_strerror(sock_err));
+      nsp->errnum = sock_err;
+      return -1;
+    } 
+  }
+
+  iterate_through_event_lists(nsp);
+
+  return 1;
+}
+
+
+/* ---- INTERNAL FUNCTIONS ---- */
+
+/* Iterate through all the event lists (such as connect_events, read_events,
+* timer_events, etc) and take action for those that have completed (due to
+* timeout, i/o, etc) */
+void iterate_through_event_lists(struct npool *nsp) {
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+
+  for (int i = 0; i < iinfo->entries_removed; i++) {
+
+    iinfo->eov = (struct extended_overlapped *)iinfo->eov_list[i].lpOverlapped;
+    /* We can't rely on iinfo->entries_removed to tell us the real number of
+     * events to process */
+    if (!iinfo->eov || !iinfo->eov->nse)
+      continue;
+
+    /* We check if this is from a cancelled operation */
+    if (iinfo->eov->nse->id != iinfo->eov->nse_id ||
+        iinfo->eov->nse->event_done) {
+      free_eov(nsp, iinfo->eov);
+      iinfo->eov = NULL;
+      continue;
+    }
+
+    if (!HasOverlappedIoCompleted((OVERLAPPED *)iinfo->eov))
+      continue;
+ 
+    struct niod *nsi = iinfo->eov->nse->iod;
+    struct nevent *nse = iinfo->eov->nse;
+    gh_list_t *evlist = NULL;
+    int ev = 0;
+
+    switch (nse->type) {
+      case NSE_TYPE_CONNECT:
+      case NSE_TYPE_CONNECT_SSL:
+        ev = EV_READ;
+        evlist = &nsp->connect_events;
+        break;
+      case NSE_TYPE_READ:
+        ev = EV_READ;
+        evlist = &nsp->read_events;
+        break;
+      case NSE_TYPE_WRITE:
+        ev = EV_WRITE;
+        evlist = &nsp->write_events;
+        break;
+    }
+
+    /* Setting the connect error for nsock_core to get in handle_connect_result */
+    if (nse->type == NSE_TYPE_CONNECT || nse->type == NSE_TYPE_CONNECT_SSL) {
+      setsockopt(nse->iod->sd, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, NULL, 0);
+      DWORD dwRes;
+      if (!GetOverlappedResult((HANDLE)nse->iod->sd, (LPOVERLAPPED)iinfo->eov, &dwRes, FALSE)) {
+        int err = map_faulty_errors(socket_errno());
+        if (err)
+          setsockopt(nse->iod->sd, SOL_SOCKET, SO_ERROR, (char *)&err, sizeof(err));
+      }
+    }
+
+    process_event(nsp, evlist, nse, ev);
+
+    if (nse->event_done) {
+      /* event is done, remove it from the event list and update IOD pointers
+      * to the first events of each kind */
+      update_first_events(nse);
+      gh_list_remove(evlist, &nse->nodeq_io);
+      gh_list_append(&nsp->free_events, &nse->nodeq_io);
+
+      if (nse->timeout.tv_sec)
+        gh_heap_remove(&nsp->expirables, &nse->expire);
+    } else
+      initiate_overlapped_event(nsp, nse);
+
+    if (nsi->state == NSIOD_STATE_DELETED) {
+      gh_list_remove(&nsp->active_iods, &nsi->nodeq);
+      gh_list_prepend(&nsp->free_iods, &nsi->nodeq);
+    }
+
+    iinfo->eov = NULL;
+  }
+
+  /* iterate through timers and expired events */
+  process_expired_events(nsp);
+}
+
+static int errcode_is_failure(int err) {
+#ifndef WIN32
+  return err != EINTR && err != EAGAIN && err != EBUSY;
+#else
+  return err != EINTR && err != EAGAIN && err != WSA_IO_PENDING && err != ERROR_NETNAME_DELETED;
+#endif
+}
+
+static int map_faulty_errors(int err) {
+  /* This actually happens https://svn.boost.org/trac/boost/ticket/10744 */
+  switch (err) {
+    case ERROR_NETWORK_UNREACHABLE: return WSAENETUNREACH;
+    case ERROR_HOST_UNREACHABLE: return WSAEHOSTUNREACH;
+    case ERROR_CONNECTION_REFUSED: return WSAECONNREFUSED;
+    case ERROR_SEM_TIMEOUT: return WSAETIMEDOUT;
+  }
+  return err;
+}
+
+static struct extended_overlapped *new_eov(struct npool *nsp, struct nevent *nse) {
+  struct extended_overlapped *eov;
+  gh_lnode_t *lnode;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+
+  lnode = gh_list_pop(&iinfo->free_eovs);
+  if (!lnode)
+    eov = (struct extended_overlapped *)safe_malloc(sizeof(struct extended_overlapped));
+  else
+    eov = container_of(lnode, struct extended_overlapped, nodeq);
+
+  memset(eov, 0, sizeof(struct extended_overlapped));
+  nse->eov = eov;
+  eov->nse = nse;
+  eov->nse_id = nse->id;
+  eov->err = 0;
+  eov->eov_received = false;
+  gh_list_prepend(&iinfo->active_eovs, &eov->nodeq);
+
+  /* Make the read buffer equal to the size of the buffer in do_actual_read() */
+  if (nse->type == NSE_TYPE_READ && !eov->readbuf && !nse->iod->ssl)
+    eov->readbuf = (char*)safe_malloc(READ_BUFFER_SZ * sizeof(char));
+
+  return eov;
+}
+
+/* This needs to be called after getting the overlapped event in */
+static void free_eov(struct npool *nsp, struct extended_overlapped *eov) {
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+  struct nevent *nse = eov->nse;
+
+  gh_list_remove(&iinfo->active_eovs, &eov->nodeq);
+
+  if (eov->readbuf) {
+    free(eov->readbuf);
+    eov->readbuf = NULL;
+  }
+
+  gh_list_prepend(&iinfo->free_eovs, &eov->nodeq);
+
+  eov->nse = NULL;
+  if (nse)
+    nse->eov = NULL;
+}
+
+
+static void call_connect_overlapped(struct npool *nsp, struct nevent *nse) {
+  BOOL ok;
+  DWORD numBytes = 0;
+  int one = 1;
+  SOCKET sock = nse->iod->sd;
+  GUID guid = WSAID_CONNECTEX;
+  struct sockaddr_in addr;
+  LPFN_CONNECTEX ConnectExPtr = NULL;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nse->iod->nsp->engine_data;
+  struct extended_overlapped *eov = new_eov(nsp, nse);
+  int ret;
+  struct sockaddr_storage *ss = &nse->iod->peer;
+  size_t sslen = nse->iod->peerlen;
+
+  if (nse->iod->lastproto != IPPROTO_TCP) {
+    if (connect(sock, (struct sockaddr *)ss, sslen) == -1) {
+      int err = socket_errno();
+      nse->event_done = 1;
+      nse->status = NSE_STATUS_ERROR;
+      nse->errnum = err;
+    } else {
+      force_operation(nsp, nse);
+    }
+    return;
+  }
+
+  ret = WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER,
+    (void*)&guid, sizeof(guid), (void*)&ConnectExPtr, sizeof(ConnectExPtr),
+    &numBytes, NULL, NULL);
+  if (ret)
+    fatal("Error initiating event type(%d)", nse->type);
+
+  ret = setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char *)&one, sizeof(one));
+  if (ret == -1) {
+    int err = socket_errno();
+    nse->event_done = 1;
+    nse->status = NSE_STATUS_ERROR;
+    nse->errnum = err;
+    return;
+  }
+
+  /* ConnectEx doesn't automatically bind the socket */
+  memset(&addr, 0, sizeof(addr));
+  addr.sin_family = AF_INET;
+  addr.sin_addr.s_addr = INADDR_ANY;
+  addr.sin_port = 0;
+  if (!nse->iod->locallen) {
+    ret = bind(sock, (SOCKADDR*)&addr, sizeof(addr));
+    if (ret) {
+      int err = socket_errno();
+      nse->event_done = 1;
+      nse->status = NSE_STATUS_ERROR;
+      nse->errnum = err;
+      return;
+    }
+  }
+
+  ok = ConnectExPtr(sock, (SOCKADDR*)ss, sslen, NULL, 0, NULL, (LPOVERLAPPED)eov);
+  if (!ok) {
+    int err = socket_errno();
+    if (err != ERROR_IO_PENDING) {
+      nse->event_done = 1;
+      nse->status = NSE_STATUS_ERROR;
+      nse->errnum = err;
+    }
+  }
+}
+
+static void call_read_overlapped(struct nevent *nse) {
+  DWORD flags = 0;
+  int err = 0;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nse->iod->nsp->engine_data;
+
+  struct extended_overlapped *eov = new_eov(nse->iod->nsp, nse);
+
+  eov->wsabuf.buf = eov->readbuf;
+  eov->wsabuf.len = READ_BUFFER_SZ;
+
+  err = WSARecvFrom(nse->iod->sd, &eov->wsabuf, 1, NULL, &flags,
+    (struct sockaddr *)&nse->iod->peer, (LPINT)&nse->iod->peerlen, (LPOVERLAPPED)eov, NULL);
+  if (err) {
+    err = socket_errno();
+    if (errcode_is_failure(err)) {
+      eov->err = err;
+      /* Send the error to the main loop to be picked up by the appropriate handler */
+      BOOL bRet = PostQueuedCompletionStatus(iinfo->iocp, -1, (ULONG_PTR)nse->iod, (LPOVERLAPPED)eov);
+      if (!bRet)
+        fatal("Error initiating event type(%d)", nse->type);
+    }
+  }
+}
+
+static void call_write_overlapped(struct nevent *nse) {
+  int err;
+  char *str;
+  int bytesleft;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nse->iod->nsp->engine_data;
+
+  struct extended_overlapped *eov = new_eov(nse->iod->nsp, nse);
+
+  str = fs_str(&nse->iobuf) + nse->writeinfo.written_so_far;
+  bytesleft = fs_length(&nse->iobuf) - nse->writeinfo.written_so_far;
+
+  eov->wsabuf.buf = str;
+  eov->wsabuf.len = bytesleft;
+
+  if (nse->writeinfo.dest.ss_family == AF_UNSPEC)
+    err = WSASend(nse->iod->sd, &eov->wsabuf, 1, NULL, 0, (LPWSAOVERLAPPED)eov, NULL);
+  else
+    err = WSASendTo(nse->iod->sd, &eov->wsabuf, 1, NULL, 0,
+    (struct sockaddr *)&nse->writeinfo.dest, (int)nse->writeinfo.destlen,
+    (LPWSAOVERLAPPED)eov, NULL);
+  if (err) {
+    err = socket_errno();
+    if (errcode_is_failure(err)) {
+      eov->err = err;
+      /* Send the error to the main loop to be picked up by the appropriate handler */
+      BOOL bRet = PostQueuedCompletionStatus(iinfo->iocp, -1, (ULONG_PTR)nse->iod, (LPOVERLAPPED)eov);
+      if (!bRet)
+        fatal("Error initiating event type(%d)", nse->type);
+    }
+  }
+}
+
+/* Anything that isn't an overlapped operation uses this to get processed by the main loop */
+static void force_operation(struct npool *nsp, struct nevent *nse) {
+  BOOL bRet;
+  struct extended_overlapped *eov;
+
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+  eov = new_eov(nse->iod->nsp, nse);
+
+  bRet = PostQueuedCompletionStatus(iinfo->iocp, 0, (ULONG_PTR)nse->iod, (LPOVERLAPPED)eov);
+  if (!bRet)
+    fatal("Error initiating event type(%d)", nse->type);
+}
+
+/* Either initiate a I/O read or force a SSL_read */
+static void initiate_read(struct npool *nsp, struct nevent *nse) {
+  if (!nse->iod->ssl)
+    call_read_overlapped(nse);
+  else
+    force_operation(nsp, nse);
+}
+
+/* Either initiate a I/O write or force a SSL_write */
+static void initiate_write(struct npool *nsp, struct nevent *nse) {
+  if (!nse->iod->ssl)
+    call_write_overlapped(nse);
+  else
+    force_operation(nsp, nse);
+}
+
+/* Force a PCAP read */
+static void initiate_pcap_read(struct npool *nsp, struct nevent *nse) {
+  force_operation(nsp, nse);
+}
+
+static void initiate_connect(struct npool *nsp, struct nevent *nse) {
+  int sslconnect_inprogress = 0;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+
+#if HAVE_OPENSSL
+  sslconnect_inprogress = nse->type == NSE_TYPE_CONNECT_SSL && nse->iod &&
+    (nse->sslinfo.ssl_desire == SSL_ERROR_WANT_READ ||
+    nse->sslinfo.ssl_desire == SSL_ERROR_WANT_WRITE);
+#endif
+
+  if (sslconnect_inprogress)
+    force_operation(nsp, nse);
+  else
+    call_connect_overlapped(nsp, nse);
+}
+
+/* Start the overlapped I/O operation */
+static void initiate_overlapped_event(struct npool *nsp, struct nevent *nse) {
+  if (nse->eov)
+    terminate_overlapped_event(nsp, nse);
+
+  switch (nse->type) {
+  case NSE_TYPE_CONNECT:
+  case NSE_TYPE_CONNECT_SSL:
+    initiate_connect(nsp, nse);
+    break;
+  case NSE_TYPE_READ:
+    initiate_read(nsp, nse);
+    break;
+  case NSE_TYPE_WRITE:
+    initiate_write(nsp, nse);
+    break;
+#if HAVE_PCAP
+  case NSE_TYPE_PCAP_READ:
+    initiate_pcap_read(nsp, nse);
+    break;
+#endif
+  default: fatal("Event type(%d) not supported by engine IOCP\n", nse->type);
+  }
+}
+
+/* Terminate an overlapped I/O operation that expired */
+static void terminate_overlapped_event(struct npool *nsp, struct nevent *nse) {
+  bool eov_done = true;
+
+  if (nse->eov) {
+    if (!HasOverlappedIoCompleted((LPOVERLAPPED)nse->eov)) {
+      CancelIoEx((HANDLE)nse->iod->sd, (LPOVERLAPPED)nse->eov);
+      eov_done = false;
+    }
+
+    if (eov_done)
+      free_eov(nsp, nse->eov);
+  }
+}
+
+/* Retrieve the ammount of bytes transferred or set the appropriate error */
+static int get_overlapped_result(struct npool *nsp, int fd, const void *buffer, size_t count) {
+  char *buf = (char *)buffer;
+  DWORD dwRes = 0;
+  int err;
+  static struct extended_overlapped *old_eov = NULL;
+  struct iocp_engine_info *iinfo = (struct iocp_engine_info *)nsp->engine_data;
+
+  struct extended_overlapped *eov = iinfo->eov;
+  struct nevent *nse = eov->nse;
+
+  /* If the operation failed at initialization, set the error for nsock_core.c to see */
+  if (eov->err) {
+    SetLastError(map_faulty_errors(eov->err));
+    return -1;
+  }
+
+  if (!GetOverlappedResult((HANDLE)fd, (LPOVERLAPPED)eov, &dwRes, FALSE)) {
+    err = socket_errno();
+    if (errcode_is_failure(err)) {
+      eov->eov_received = true;
+      SetLastError(map_faulty_errors(err));
+      return -1;
+    }
+  }
+  eov->eov_received = true;
+
+  if (nse->type == NSE_TYPE_READ && buf)
+    memcpy(buf, eov->wsabuf.buf, dwRes);
+
+  /* If the read buffer wasn't big enough, subsequent calls from do_actual_read will make us
+  read with recvfrom the rest of the returned data */
+  if (nse->type == NSE_TYPE_READ && dwRes == eov->wsabuf.len && old_eov == eov) {
+    struct sockaddr_storage peer;
+    socklen_t peerlen = sizeof(peer);
+    dwRes = recvfrom(fd, buf, READ_BUFFER_SZ, 0, (struct sockaddr *)&peer, &peerlen);
+  }
+
+  if (!nse->type == NSE_TYPE_READ || (nse->type == NSE_TYPE_READ && dwRes < eov->wsabuf.len)) {
+    old_eov = NULL;
+  } else if (nse->type == NSE_TYPE_READ && dwRes == eov->wsabuf.len) {
+    old_eov = eov;
+  }
+
+  return dwRes;
+}
+
+int iocp_iod_connect(struct npool *nsp, int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
+  return 0;
+}
+
+int iocp_iod_read(struct npool *nsp, int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen) {
+  return get_overlapped_result(nsp, sockfd, buf, len);
+}
+
+int iocp_iod_write(struct npool *nsp, int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen) {
+  return get_overlapped_result(nsp, sockfd, buf, len);
+}
+
+#endif /* HAVE_IOCP */

nsock/src/engine_kqueue.c

@@ -78,11 +78,12 @@
 /* --- ENGINE INTERFACE PROTOTYPES --- */
 static int kqueue_init(struct npool *nsp);
 static void kqueue_destroy(struct npool *nsp);
-static int kqueue_iod_register(struct npool *nsp, struct niod *iod, int ev);
+static int kqueue_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev);
 static int kqueue_iod_unregister(struct npool *nsp, struct niod *iod);
-static int kqueue_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr);
+static int kqueue_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr);
 static int kqueue_loop(struct npool *nsp, int msec_timeout);
 
+extern struct io_operations posix_io_operations;
 
 /* ---- ENGINE DEFINITION ---- */
 struct io_engine engine_kqueue = {
@@ -92,7 +93,8 @@ struct io_engine engine_kqueue = {
   kqueue_iod_register,
   kqueue_iod_unregister,
   kqueue_iod_modify,
-  kqueue_loop
+  kqueue_loop,
+  &posix_io_operations
 };
 
 
@@ -151,7 +153,7 @@ void kqueue_destroy(struct npool *nsp) {
   free(kinfo);
 }
 
-int kqueue_iod_register(struct npool *nsp, struct niod *iod, int ev) {
+int kqueue_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev) {
   struct kqueue_engine_info *kinfo = (struct kqueue_engine_info *)nsp->engine_data;
 
   assert(!IOD_PROPGET(iod, IOD_REGISTERED));
@@ -185,7 +187,7 @@ int kqueue_iod_unregister(struct npool *nsp, struct niod *iod) {
 
 #define EV_SETFLAG(_set, _ev) (((_set) & (_ev)) ? (EV_ADD|EV_ENABLE) : (EV_ADD|EV_DISABLE))
 
-int kqueue_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr) {
+int kqueue_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr) {
   struct kevent kev[2];
   int new_events, i;
   struct kqueue_engine_info *kinfo = (struct kqueue_engine_info *)nsp->engine_data;

nsock/src/engine_poll.c

@@ -106,13 +106,14 @@
   #define POLL_X_FLAGS (POLLERR | POLLHUP)
 #endif /* POLLRDHUP */
 
+extern struct io_operations posix_io_operations;
 
 /* --- ENGINE INTERFACE PROTOTYPES --- */
 static int poll_init(struct npool *nsp);
 static void poll_destroy(struct npool *nsp);
-static int poll_iod_register(struct npool *nsp, struct niod *iod, int ev);
+static int poll_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev);
 static int poll_iod_unregister(struct npool *nsp, struct niod *iod);
-static int poll_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr);
+static int poll_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr);
 static int poll_loop(struct npool *nsp, int msec_timeout);
 
 
@@ -124,7 +125,8 @@ struct io_engine engine_poll = {
   poll_iod_register,
   poll_iod_unregister,
   poll_iod_modify,
-  poll_loop
+  poll_loop,
+  &posix_io_operations
 };
 
 
@@ -212,7 +214,7 @@ void poll_destroy(struct npool *nsp) {
   free(pinfo);
 }
 
-int poll_iod_register(struct npool *nsp, struct niod *iod, int ev) {
+int poll_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev) {
   struct poll_engine_info *pinfo = (struct poll_engine_info *)nsp->engine_data;
   int sd;
 
@@ -265,7 +267,7 @@ int poll_iod_unregister(struct npool *nsp, struct niod *iod) {
   return 1;
 }
 
-int poll_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr) {
+int poll_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr) {
   int sd;
   int new_events;
   struct poll_engine_info *pinfo = (struct poll_engine_info *)nsp->engine_data;

nsock/src/engine_select.c

@@ -67,13 +67,15 @@
 #include "nsock_pcap.h"
 #endif
 
+extern struct io_operations posix_io_operations;
+
 
 /* --- ENGINE INTERFACE PROTOTYPES --- */
 static int select_init(struct npool *nsp);
 static void select_destroy(struct npool *nsp);
-static int select_iod_register(struct npool *nsp, struct niod *iod, int ev);
+static int select_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev);
 static int select_iod_unregister(struct npool *nsp, struct niod *iod);
-static int select_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr);
+static int select_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr);
 static int select_loop(struct npool *nsp, int msec_timeout);
 
 
@@ -85,7 +87,8 @@ struct io_engine engine_select = {
   select_iod_register,
   select_iod_unregister,
   select_iod_modify,
-  select_loop
+  select_loop,
+  &posix_io_operations
 };
 
 
@@ -153,11 +156,11 @@ void select_destroy(struct npool *nsp) {
   free(nsp->engine_data);
 }
 
-int select_iod_register(struct npool *nsp, struct niod *iod, int ev) {
+int select_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev) {
   assert(!IOD_PROPGET(iod, IOD_REGISTERED));
 
   iod->watched_events = ev;
-  select_iod_modify(nsp, iod, ev, EV_NONE);
+  select_iod_modify(nsp, iod, nse, ev, EV_NONE);
   IOD_PROPSET(iod, IOD_REGISTERED);
   return 1;
 }
@@ -196,7 +199,7 @@ int select_iod_unregister(struct npool *nsp, struct niod *iod) {
   return 1;
 }
 
-int select_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr) {
+int select_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr) {
   int sd;
   struct select_engine_info *sinfo = (struct select_engine_info *)nsp->engine_data;
 

nsock/src/nsock_connect.c

@@ -253,7 +253,7 @@ void nsock_connect_internal(struct npool *ms, struct nevent *nse, int type, int
       memcpy(&iod->peer, ss, sslen);
     iod->peerlen = sslen;
 
-    if (connect(iod->sd, (struct sockaddr *)ss, sslen) == -1) {
+    if (ms->engine->io_operations->iod_connect(ms, iod->sd, (struct sockaddr *)ss, sslen) == -1) {
       int err = socket_errno();
 
       if (proto == IPPROTO_UDP || (err != EINPROGRESS && err != EAGAIN)) {

nsock/src/nsock_core.c

@@ -86,7 +86,6 @@
 #include "nsock_pcap.h"
 #endif
 
-
 /* Nsock time of day -- we update this at least once per nsock_loop round (and
  * after most calls that are likely to block).  Other nsock files should grab
  * this */
@@ -178,7 +177,7 @@ static int socket_count_dec_ssl_desire(struct nevent *nse) {
  * If this counter reaches zero, the event won't be watched anymore by the
  * IO engine for this IOD.
  */
-static void update_events(struct niod * iod, struct npool *ms, int ev_inc, int ev_dec) {
+static void update_events(struct niod * iod, struct npool *ms, struct nevent *nse, int ev_inc, int ev_dec) {
   int setmask, clrmask, ev_temp;
 
   /* Filter out events that belong to both sets. */
@@ -204,9 +203,9 @@ static void update_events(struct niod * iod, struct npool *ms, int ev_inc, int e
 
   if (!IOD_PROPGET(iod, IOD_REGISTERED)) {
     assert(clrmask == EV_NONE);
-    nsock_engine_iod_register(ms, iod, setmask);
+    nsock_engine_iod_register(ms, iod, nse, setmask);
   } else {
-    nsock_engine_iod_modify(ms, iod, setmask, clrmask);
+    nsock_engine_iod_modify(ms, iod, nse, setmask, clrmask);
   }
 }
 
@@ -287,6 +286,7 @@ static int iod_add_event(struct niod *iod, struct nevent *nse) {
     default:
       fatal("Unknown event type (%d) for IOD #%lu\n", nse->type, iod->id);
   }
+
   return 0;
 }
 
@@ -427,7 +427,7 @@ void handle_connect_result(struct npool *ms, struct nevent *nse, enum nse_status
     ev |= socket_count_read_dec(iod);
     ev |= socket_count_write_dec(iod);
     ev |= EV_EXCEPT;
-    update_events(iod, ms, EV_NONE, ev);
+    update_events(iod, ms, nse, EV_NONE, ev);
   }
 
 #if HAVE_OPENSSL
@@ -446,7 +446,7 @@ void handle_connect_result(struct npool *ms, struct nevent *nse, enum nse_status
       int ev;
 
       ev = socket_count_dec_ssl_desire(nse);
-      update_events(iod, ms, EV_NONE, ev);
+      update_events(iod, ms, nse, EV_NONE, ev);
     }
 
     rc = SSL_connect(iod->ssl);
@@ -468,11 +468,11 @@ void handle_connect_result(struct npool *ms, struct nevent *nse, enum nse_status
       if (rc == -1 && sslerr == SSL_ERROR_WANT_READ) {
         nse->sslinfo.ssl_desire = sslerr;
         socket_count_read_inc(iod);
-        update_events(iod, ms, EV_READ, EV_NONE);
+        update_events(iod, ms, nse, EV_READ, EV_NONE);
       } else if (rc == -1 && sslerr == SSL_ERROR_WANT_WRITE) {
         nse->sslinfo.ssl_desire = sslerr;
         socket_count_write_inc(iod);
-        update_events(iod, ms, EV_WRITE, EV_NONE);
+        update_events(iod, ms, nse, EV_WRITE, EV_NONE);
       } else if (!(options & SSL_OP_NO_SSLv2)) {
         int saved_ev;
 
@@ -488,7 +488,7 @@ void handle_connect_result(struct npool *ms, struct nevent *nse, enum nse_status
         close(iod->sd);
         nsock_connect_internal(ms, nse, SOCK_STREAM, iod->lastproto, &iod->peer,
                                iod->peerlen, nsock_iod_get_peerport(iod));
-        nsock_engine_iod_register(ms, iod, saved_ev);
+        nsock_engine_iod_register(ms, iod, nse, saved_ev);
 
         /* Use SSL_free here because SSL_clear keeps session info, which
          * doesn't work when changing SSL versions (as we're clearly trying to
@@ -501,7 +501,7 @@ void handle_connect_result(struct npool *ms, struct nevent *nse, enum nse_status
         SSL_set_options(iod->ssl, options | SSL_OP_NO_SSLv2);
         socket_count_read_inc(nse->iod);
         socket_count_write_inc(nse->iod);
-        update_events(iod, ms, EV_READ|EV_WRITE, EV_NONE);
+        update_events(iod, ms, nse, EV_READ|EV_WRITE, EV_NONE);
         nse->sslinfo.ssl_desire = SSL_ERROR_WANT_CONNECT;
       } else {
         nsock_log_info("EID %li %s",
@@ -543,10 +543,7 @@ void handle_write_result(struct npool *ms, struct nevent *nse, enum nse_status s
       res = SSL_write(iod->ssl, str, bytesleft);
     else
 #endif
-      if (nse->writeinfo.dest.ss_family == AF_UNSPEC)
+      res = ms->engine->io_operations->iod_write(ms, nse->iod->sd, str, bytesleft, 0, (struct sockaddr *)&nse->writeinfo.dest, (int)nse->writeinfo.destlen);
-        res = send(nse->iod->sd, str, bytesleft, 0);
-      else
-        res = sendto(nse->iod->sd, str, bytesleft, 0, (struct sockaddr *)&nse->writeinfo.dest, (int)nse->writeinfo.destlen);
     if (res == bytesleft) {
       nse->event_done = 1;
       nse->status = NSE_STATUS_SUCCESS;
@@ -562,14 +559,14 @@ void handle_write_result(struct npool *ms, struct nevent *nse, enum nse_status s
 
           evclr = socket_count_dec_ssl_desire(nse);
           socket_count_read_inc(iod);
-          update_events(iod, ms, EV_READ, evclr);
+          update_events(iod, ms, nse, EV_READ, evclr);
           nse->sslinfo.ssl_desire = err;
         } else if (err == SSL_ERROR_WANT_WRITE) {
           int evclr;
 
           evclr = socket_count_dec_ssl_desire(nse);
           socket_count_write_inc(iod);
-          update_events(iod, ms, EV_WRITE, evclr);
+          update_events(iod, ms, nse, EV_WRITE, evclr);
           nse->sslinfo.ssl_desire = err;
         } else {
           /* Unexpected error */
@@ -601,7 +598,7 @@ void handle_write_result(struct npool *ms, struct nevent *nse, enum nse_status s
     else
 #endif
       ev |= socket_count_write_dec(nse->iod);
-    update_events(nse->iod, ms, EV_NONE, ev);
+    update_events(nse->iod, ms, nse, EV_NONE, ev);
   }
 }
 
@@ -613,7 +610,7 @@ void handle_timer_result(struct npool *ms, struct nevent *nse, enum nse_status s
 
 /* Returns -1 if an error, otherwise the number of newly written bytes */
 static int do_actual_read(struct npool *ms, struct nevent *nse) {
-  char buf[8192];
+  char buf[READ_BUFFER_SZ];
   int buflen = 0;
   struct niod *iod = nse->iod;
   int err = 0;
@@ -627,9 +624,9 @@ static int do_actual_read(struct npool *ms, struct nevent *nse) {
     do {
       struct sockaddr_storage peer;
       socklen_t peerlen;
-
       peerlen = sizeof(peer);
-      buflen = recvfrom(iod->sd, buf, sizeof(buf), 0, (struct sockaddr *)&peer, &peerlen);
+
+      buflen = ms->engine->io_operations->iod_read(ms, iod->sd, buf, sizeof(buf), 0, (struct sockaddr *)&peer, &peerlen);
 
       /* Using recv() was failing, at least on UNIX, for non-network sockets
        * (i.e. stdin) in this case, a read() is done - as on ENOTSOCK we may
@@ -712,14 +709,14 @@ static int do_actual_read(struct npool *ms, struct nevent *nse) {
 
         evclr = socket_count_dec_ssl_desire(nse);
         socket_count_read_inc(iod);
-        update_events(iod, ms, EV_READ, evclr);
+        update_events(iod, ms, nse, EV_READ, evclr);
         nse->sslinfo.ssl_desire = err;
       } else if (err == SSL_ERROR_WANT_WRITE) {
         int evclr;
 
         evclr = socket_count_dec_ssl_desire(nse);
         socket_count_write_inc(iod);
-        update_events(iod, ms, EV_WRITE, evclr);
+        update_events(iod, ms, nse, EV_WRITE, evclr);
         nse->sslinfo.ssl_desire = err;
       } else {
         /* Unexpected error */
@@ -820,7 +817,7 @@ void handle_read_result(struct npool *ms, struct nevent *nse, enum nse_status st
     else
 #endif
       ev |= socket_count_read_dec(nse->iod);
-    update_events(nse->iod, ms, EV_NONE, ev);
+    update_events(nse->iod, ms, nse, EV_NONE, ev);
   }
 }
 
@@ -861,7 +858,7 @@ void handle_pcap_read_result(struct npool *ms, struct nevent *nse, enum nse_stat
     int ev;
 
     ev = socket_count_readpcap_dec(iod);
-    update_events(iod, ms, EV_NONE, ev);
+    update_events(iod, ms, nse, EV_NONE, ev);
   }
 }
 
@@ -1266,7 +1263,7 @@ void nsock_pool_add_event(struct npool *nsp, struct nevent *nse) {
         assert(nse->iod->sd >= 0);
         socket_count_read_inc(nse->iod);
         socket_count_write_inc(nse->iod);
-        update_events(nse->iod, nsp, EV_READ|EV_WRITE|EV_EXCEPT, EV_NONE);
+        update_events(nse->iod, nsp, nse, EV_READ|EV_WRITE|EV_EXCEPT, EV_NONE);
       }
       iod_add_event(nse->iod, nse);
       break;
@@ -1275,7 +1272,7 @@ void nsock_pool_add_event(struct npool *nsp, struct nevent *nse) {
       if (!nse->event_done) {
         assert(nse->iod->sd >= 0);
         socket_count_read_inc(nse->iod);
-        update_events(nse->iod, nsp, EV_READ, EV_NONE);
+        update_events(nse->iod, nsp, nse, EV_READ, EV_NONE);
 #if HAVE_OPENSSL
         if (nse->iod->ssl)
           nse->sslinfo.ssl_desire = SSL_ERROR_WANT_READ;
@@ -1288,7 +1285,7 @@ void nsock_pool_add_event(struct npool *nsp, struct nevent *nse) {
       if (!nse->event_done) {
         assert(nse->iod->sd >= 0);
         socket_count_write_inc(nse->iod);
-        update_events(nse->iod, nsp, EV_WRITE, EV_NONE);
+        update_events(nse->iod, nsp, nse, EV_WRITE, EV_NONE);
 #if HAVE_OPENSSL
         if (nse->iod->ssl)
           nse->sslinfo.ssl_desire = SSL_ERROR_WANT_WRITE;
@@ -1309,7 +1306,7 @@ void nsock_pool_add_event(struct npool *nsp, struct nevent *nse) {
       if (mp->pcap_desc >= 0) { /* pcap descriptor present */
         if (!nse->event_done) {
           socket_count_readpcap_inc(nse->iod);
-          update_events(nse->iod, nsp, EV_READ, EV_NONE);
+          update_events(nse->iod, nsp, nse, EV_READ, EV_NONE);
         }
         nsock_log_debug_all("PCAP NSE #%lu: Adding event to READ_EVENTS", nse->id);
 

nsock/src/nsock_engines.c

@@ -63,6 +63,13 @@
 
 #include "nsock_internal.h"
 
+#if HAVE_IOCP
+  extern struct io_engine engine_iocp;
+  #define ENGINE_IOCP &engine_iocp,
+#else
+  #define ENGINE_IOCP
+#endif /* HAVE_IOCP */
+
 #if HAVE_EPOLL
   extern struct io_engine engine_epoll;
   #define ENGINE_EPOLL &engine_epoll,
@@ -94,12 +101,34 @@ static struct io_engine *available_engines[] = {
   ENGINE_EPOLL
   ENGINE_KQUEUE
   ENGINE_POLL
+  ENGINE_IOCP
   ENGINE_SELECT
   NULL
 };
 
 static char *engine_hint;
 
+int posix_iod_connect(struct npool *nsp, int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
+  return connect(sockfd, addr, addrlen);
+}
+
+int posix_iod_read(struct npool *nsp, int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen) {
+  return recvfrom(sockfd, (char *)buf, len, flags, src_addr, addrlen);
+}
+
+int posix_iod_write(struct npool *nsp, int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen) {
+  struct sockaddr_storage *dest = (struct sockaddr_storage *)dest_addr;
+  if (dest->ss_family == AF_UNSPEC)
+    return send(sockfd, (char *)buf, len, flags);
+  else
+    return sendto(sockfd, (char *)buf, len, flags, dest_addr, addrlen);
+}
+
+struct io_operations posix_io_operations = {
+  posix_iod_connect,
+  posix_iod_read,
+  posix_iod_write
+};
 
 struct io_engine *get_io_engine(void) {
   struct io_engine *engine = NULL;
@@ -145,6 +174,9 @@ int nsock_set_default_engine(char *engine) {
 
 const char *nsock_list_engines(void) {
   return
+#if HAVE_IOCP
+  "iocp "
+#endif
 #if HAVE_EPOLL
   "epoll "
 #endif

nsock/src/nsock_internal.h

@@ -109,6 +109,7 @@
 
 
 /* ------------------- CONSTANTS ------------------- */
+#define READ_BUFFER_SZ 8192
 
 enum nsock_read_types {
   NSOCK_READLINES,
@@ -360,8 +361,21 @@ struct nevent {
    * that other crap */
   unsigned int event_done: 1;
   unsigned int eof: 1;
+  
+#if HAVE_IOCP
+  struct extended_overlapped *eov;
+#endif
 };
 
+struct io_operations {
+  int(*iod_connect)(struct npool *nsp, int sockfd, const struct sockaddr *addr, socklen_t addrlen);
+
+  int(*iod_read)(struct npool *nsp, int sockfd, void *buf, size_t len, int flags,
+    struct sockaddr *src_addr, socklen_t *addrlen);
+
+  int(*iod_write)(struct npool *nsp, int sockfd, const void *buf, size_t len, int flags,
+    const struct sockaddr *dest_addr, socklen_t addrlen);
+};
 
 struct io_engine {
   /* Human readable identifier for this engine. */
@@ -374,7 +388,7 @@ struct io_engine {
   void (*destroy)(struct npool *nsp);
 
   /* Register a new IOD to the engine */
-  int (*iod_register)(struct npool *nsp, struct niod *iod, int ev);
+  int(*iod_register)(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev);
 
   /* Remove a registered IOD */
   int(*iod_unregister)(struct npool *nsp, struct niod *iod);
@@ -382,10 +396,13 @@ struct io_engine {
   /* Modify events for a registered IOD.
    *  - ev_set represent the events to add
    *  - ev_clr represent the events to delete (if set) */
-  int (*iod_modify)(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr);
+  int (*iod_modify)(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr);
 
   /* Main engine loop */
   int (*loop)(struct npool *nsp, int msec_timeout);
+
+  /* I/O operations */
+  struct io_operations *io_operations;
 };
 
 /* ----------- NSOCK I/O ENGINE CONVENIENCE WRAPPERS ------------ */
@@ -398,16 +415,16 @@ static inline void nsock_engine_destroy(struct npool *nsp) {
   return;
 }
 
-static inline int nsock_engine_iod_register(struct npool *nsp, struct niod *iod, int ev) {
+static inline int nsock_engine_iod_register(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev) {
-  return nsp->engine->iod_register(nsp, iod, ev);
+  return nsp->engine->iod_register(nsp, iod, nse, ev);
 }
 
 static inline int nsock_engine_iod_unregister(struct npool *nsp, struct niod *iod) {
   return nsp->engine->iod_unregister(nsp, iod);
 }
 
-static inline int nsock_engine_iod_modify(struct npool *nsp, struct niod *iod, int ev_set, int ev_clr) {
+static inline int nsock_engine_iod_modify(struct npool *nsp, struct niod *iod, struct nevent *nse, int ev_set, int ev_clr) {
-  return nsp->engine->iod_modify(nsp, iod, ev_set, ev_clr);
+  return nsp->engine->iod_modify(nsp, iod, nse, ev_set, ev_clr);
 }
 
 static inline int nsock_engine_loop(struct npool *nsp, int msec_timeout) {