mTCP support for SPDK

README.spdk

@@ -0,0 +1,37 @@
+========================================================================
+ README for mTCP development with spdk
+========================================================================
+
+1. Why mTCP changes
+
+mTCP is a highly scalable user-level TCP stack for multicore systems.
+It requires per-thread architecture and starts "RunMainLoop" for each
+thread with dead-loop handling.
+
+However, spdk will register poller per thread to handle receiving and
+events and so it requires mtcp to supply a separated recv/send handler.
+It's necessary to split RunMainLoop to two APIs: RunMainLoop and
+mtcp_run_instance. Original mTCP apps still use RunMainLoop and it's not
+affected and spdk will use mtcp_run_instance for its poller.
+
+2. How to enable spdk support
+
+Add option "--enable-spdk" to choose spdk support enable or not.
+Also introduce mtcp_crossbuild.sh for reference.
+
+3. SPDK changes
+
+For SPDK, it needs more changes. 
+
+- Add mtcp sock interface. Current now it only supports posix/vpp sock
+  interface. Just follow the framework to add mtcp sock interface.
+- Change connection schedule mode to support mtcp per-thread arch.
+
+Now upstream is ongoing.
+
+========================================================================
+
+                   Contact: mtcp-user at list.ndsl.kaist.edu
+                             April 2, 2015. 
+                 EunYoung Jeong <notav at ndsl.kaist.edu>
+		 M. Asim Jamshed <ajamshed at ndsl.kaist.edu>

apps/example/Makefile.in

@@ -7,9 +7,13 @@ PS=@PSIO@
 NETMAP=@NETMAP@
 ONVM=@ONVM@
 CFLAGS=@CFLAGS@
+LDFLAGS=@LDFLAGS@
+
+# If ARCH is not defined, retrive from system
+ARCH ?= $(shell uname -m)
 
 # Add arch-specific optimization
-ifeq ($(shell uname -m),x86_64)
+ifeq ($(ARCH),x86_64)
 LIBS += -m64
 endif
 
@@ -44,7 +48,7 @@ endif
 ifeq ($(DPDK),1)
 DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
 DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
-LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS}
+LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS} ${LDFLAGS}
 endif
 
 # onvm-specific variables

configure.ac

@@ -63,6 +63,17 @@ AC_FUNC_MMAP
 AC_CHECK_FUNC([clock_gettime],,AC_MSG_ERROR([librt library is missing]))
 AC_CHECK_FUNCS([bzero getpagesize gettimeofday memmove memset munmap select socket strchr strerror strstr strtol],,AC_MSG_ERROR([glibc library is missing]))
 
+# Reset SPDK to 0
+AC_SUBST(SPDK, 0)
+
+dnl Example of default-disabled feature
+AC_ARG_ENABLE([spdk],
+	AS_HELP_STRING([--enable-spdk], [Enable SPDK Support]))
+
+AS_IF([test "x$enable_spdk" = "xyes"], [
+	    AC_SUBST(SPDK, 1)
+])
+
 # Reset DPDK to 0
 AC_SUBST(DPDK, 0)
 # Reset enforcement value

dpdk-iface-kmod/Makefile

@@ -6,9 +6,12 @@ endif
 ifeq ($(RTE_TARGET),)
 $(error "Please define RTE_TARGET environment variable")
 endif
+
+RTE_KERNELDIR ?= /lib/modules/$(shell uname -r)/build/
+
 #-------------------------------------------------------------------------#
 include $(RTE_SDK)/mk/rte.vars.mk
-CC=gcc
+CC=$(CROSS)gcc
 obj-m=dpdk_iface.o
 DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
 DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
@@ -25,7 +28,7 @@ else
 endif
 #-------------------------------------------------------------------------#
 all: dpdk_iface.c $(appname) $(appname).c
-	make -C /lib/modules/$(shell uname -r)/build/ M=$(PWD) modules
+	make ARCH=${ARCH} CROSS_COMPILE=${CROSS} -C ${RTE_KERNELDIR} M=$(PWD) modules
 
 $(appname): $(appname).c
 	$(MSG) "  CC $<"
@@ -34,7 +37,7 @@ $(appname): $(appname).c
 	-L$(DPDK_LIB) ${DPDK_MACHINE_LDFLAGS} -lpthread
 
 clean:
-	make -C /lib/modules/$(shell uname -r)/build/ M=$(PWD) clean
+	make -C ${RTE_KERNELDIR} M=$(PWD) clean
 	$(MSG) "  CLEAN  $(appname)"
 	$(HIDE) rm -rf *~ *.o *.ko dpdk_iface_main
 

dpdk-iface-kmod/dpdk_iface_main.c

@@ -22,7 +22,11 @@
 typedef struct {
 	PciDevice pd;
 	struct rte_eth_dev_info dev_details;
+#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0)
 	struct ether_addr ports_eth_addr;
+#else
+	struct rte_ether_addr ports_eth_addr;
+#endif
 } DevInfo;
 
 static DevInfo di[RTE_MAX_ETHPORTS];	
@@ -260,7 +264,11 @@ main(int argc, char **argv)
 	ret = rte_eal_init(rte_argc, rte_argv);
 
 	/* get total count of detected ethernet ports */
+#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
 	num_devices = rte_eth_dev_count();
+#else
+	num_devices = rte_eth_dev_count_avail();
+#endif
 	if (num_devices == 0) {
 		fprintf(stderr, "No Ethernet port detected!\n");
 		exit(EXIT_FAILURE);

mtcp/src/Makefile.in

@@ -4,6 +4,7 @@
 
 ### TARGET ###
 PS=@PSIO@
+SPDK=@SPDK@
 DPDK=@DPDK@
 ENFORCE_RX_IDLE=@ENFORCE_RX_IDLE@
 NETMAP=@NETMAP@
@@ -19,7 +20,10 @@ MTCP_HDR = mtcp_api.h mtcp_epoll.h
 GCC=@CC@
 
 ### FLAGS ###
-ifeq ($(shell uname -m),x86_64)
+
+# If ARCH is not defined, retrive from system
+ARCH ?= $(shell uname -m)
+ifeq ($(ARCH),x86_64)
 GCC_OPT = -m64
 else
 GCC_OPT =
@@ -50,6 +54,10 @@ GCC_OPT += -DNDEBUG -g -O3 -DNETSTAT -DINFO -DDBGERR -DDBGCERR
 #GCC_OPT += -DNDEBUG -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR
 GCC_OPT += $(DBG_OPT)
 
+ifeq ($(SPDK),1)
+GCC_OPT += -DENABLE_SPDK
+endif
+
 ifeq ($(LRO),1)
 GCC_OPT += -DENABLELRO
 endif

mtcp/src/api.c

@@ -1284,7 +1284,7 @@ mtcp_recv(mctx_t mctx, int sockid, char *buf, size_t len, int flags)
         return ret;
 }
 /*----------------------------------------------------------------------------*/
-inline ssize_t
+ssize_t
 mtcp_read(mctx_t mctx, int sockid, char *buf, size_t len)
 {
 	return mtcp_recv(mctx, sockid, buf, len, 0);