nhttpd: let sendfile() support files >= 2GB

src/nhttpd/Makefile.am

@@ -1,6 +1,6 @@
 SUBDIRS = yhttpd_core yhttpd_mods tuxboxapi web
 
-AM_CPPFLAGS = -fno-rtti -fno-exceptions
+AM_CPPFLAGS = -fno-rtti -fno-exceptions -D_FILE_OFFSET_BITS=64
 
 AM_CPPFLAGS += \
 	-I$(srcdir) \

src/nhttpd/tuxboxapi/coolstream/Makefile.am

@@ -1,4 +1,4 @@
-AM_CPPFLAGS = -fno-rtti -fno-exceptions -D__STDC_FORMAT_MACROS
+AM_CPPFLAGS = -fno-rtti -fno-exceptions -D__STDC_FORMAT_MACROS -D_FILE_OFFSET_BITS=64
 
 AM_CPPFLAGS += \
 	-I$(top_builddir) \

src/nhttpd/tuxboxapi/dbox/Makefile.am

@@ -1,4 +1,4 @@
-AM_CPPFLAGS = -fno-rtti -fno-exceptions
+AM_CPPFLAGS = -fno-rtti -fno-exceptions -D_FILE_OFFSET_BITS=64
 
 AM_CPPFLAGS += \
 	-I$(top_srcdir)/lib \

src/nhttpd/yhttpd_core/Makefile.am

@@ -19,7 +19,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/lib/libcoolstream
 endif
 endif
 
-AM_CPPFLAGS += -fno-rtti -fno-exceptions
+AM_CPPFLAGS += -fno-rtti -fno-exceptions -D_FILE_OFFSET_BITS=64
 
 noinst_LIBRARIES = libyhttpd.a
 

src/nhttpd/yhttpd_core/yhook.cpp

@@ -304,8 +304,8 @@ std::string CyhookHandler::BuildHeader(bool cache) {
 
 			strftime(timeStr, sizeof(timeStr), RFC1123FMT, gmtime(&mod_time));
 			result += string_printf(
-					"Last-Modified: %s\r\nContent-Length: %ld\r\n", timeStr,
+					"Last-Modified: %s\r\nContent-Length: %lld\r\n", timeStr,
-					GetContentLength());
+					(long long) GetContentLength());
 		}
 		result += "\r\n"; // End of Header
 		break;

src/nhttpd/yhttpd_core/yhook.h

@@ -47,6 +47,7 @@
 //=============================================================================
 #ifndef __yhttpd_yhook_h__
 #define __yhttpd_yhook_h__
+#include <unistd.h>
 // C++
 #include <string>
 #include <list>
@@ -128,7 +129,7 @@ public:
 	HttpResponseType httpStatus;		// http-status code for response
 	std::string 	ResponseMimeType;	// mime-type for response
 	std::string 	NewURL;				// new URL for Redirection
-	long		ContentLength;			// Length of Response Body
+	off_t		ContentLength;			// Length of Response Body
 	time_t 		LastModified;			// Last Modified Time of Item to send / -1 dynamic content
 	std::string	Sendfile;				// Path & Name (local os style) of file to send
 	bool		keep_alive;
@@ -173,8 +174,8 @@ public:
 	void SetError(HttpResponseType responseType, THandleStatus _status)
 		{SetError(responseType); status = _status;}
 	// others
-	long GetContentLength()
+	off_t GetContentLength()
-		{return (status==HANDLED_SENDFILE)?ContentLength : (long)yresult.length();}
+		{return (status==HANDLED_SENDFILE)?ContentLength : (off_t)yresult.length();}
 	// output methods
 	std::string BuildHeader(bool cache = false);
 	void addResult(const std::string& result) 	{yresult += result;}

src/nhttpd/yhttpd_core/ysocket.cpp

@@ -12,6 +12,7 @@
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <sys/ioctl.h>
+#include <sys/stat.h>
 #include <errno.h>
 #include <signal.h>
 #include <unistd.h>
@@ -316,25 +317,40 @@ int CySocket::SendFile(int filed) {
 #ifdef Y_CONFIG_HAVE_SENDFILE
 	// does not work with SSL !!!
 	off_t start = 0;
-	off_t end = lseek(filed,0,SEEK_END);
+	struct stat st;
-	int written = 0;
+	fstat(filed, &st);
-	if((written = ::sendfile(sock,filed,&start,end)) == -1)
+	size_t end = st.st_size;
-	{
+	off_t written = 0;
-		perror("sendfile failed\n");
+	off_t left = end;
-		return false;
+	while (left > 0) {
-	}
+		// Split sendfile() transfer to smaller chunks to reduce memory-mapping requirements --martii
-	else
+		if((written = ::sendfile(sock,filed,&start,0x8000000)) == -1) {
-	BytesSend += written;
+			perror("sendfile failed");
-#else
+			if (errno != EINVAL)
-	char sbuf[1024];
+				return false;
-	unsigned int r = 0;
+			break;
-	while ((r = read(filed, sbuf, 1024)) > 0) {
+		} else {
-		if (Send(sbuf, r) < 0) {
+			BytesSend += written;
-			perror("sendfile failed\n");
+			left -= written;
-			return false;
 		}
 	}
+	if (left) {
+		::lseek(filed, start, SEEK_SET);
 #endif // Y_CONFIG_HAVE_SENDFILE
+
+		char sbuf[65536];
+		int r;
+		while ((r = read(filed, sbuf, 65536)) > 0) {
+			if (Send(sbuf, r) < 0) {
+				perror("send failed");
+				return false;
+			}
+			BytesSend += written;
+		}
+#ifdef Y_CONFIG_HAVE_SENDFILE
+	}
+#endif // Y_CONFIG_HAVE_SENDFILE
+
 	log_level_printf(9, "<Sock:SendFile>: Bytes:%ld\n", BytesSend);
 	return true;
 }

src/nhttpd/yhttpd_mods/Makefile.am

@@ -10,7 +10,7 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/lib/libconfigfile \
 	@FREETYPE_CFLAGS@ 
 
-AM_CPPFLAGS += -fno-rtti -fno-exceptions
+AM_CPPFLAGS += -fno-rtti -fno-exceptions -D_FILE_OFFSET_BITS=64
 
 noinst_LIBRARIES = libyhttpdmods.a
 

src/nhttpd/yhttpd_mods/mod_weblog.cpp

@@ -74,8 +74,7 @@ bool CmWebLog::OpenLogFile() {
 			if (LogFormat == "ELF") {
 				printf("#Version: 1.0\n");
 				printf("#Remarks: yhttpd" WEBSERVERNAME "\n");
-				printf(
+				printf("#Fields: c-ip username date time x-request cs-uri sc-status cs-method bytes time-taken x-time-request x-time-response cached\n");
-						"#Fields: c-ip username date time x-request cs-uri sc-status cs-method bytes time-taken x-time-request x-time-response cached\n");
 			}
 		}
 		pthread_mutex_unlock(&WebLog_mutex);
@@ -147,7 +146,7 @@ void CmWebLog::AddLogEntry_CLF(CyhookHandler *hh)
 	std::string c_ip = 			hh->UrlData["clientaddr"].c_str();
 	std::string request_startline = 	hh->UrlData["startline"].c_str();
 	int s_status = hh->httpStatus;
-	int bytes	= hh->GetContentLength();
+	off_t bytes	= hh->GetContentLength();
 
 	struct tm *time_now;
 	time_t now = time(NULL);
@@ -156,12 +155,12 @@ void CmWebLog::AddLogEntry_CLF(CyhookHandler *hh)
 	time_now = localtime(&now);
 	strftime(request_time, 80, "[%d/%b/%Y:%H:%M:%S]", time_now);
 
-	printf("%s - - %s \"%s\" %d %d\n",
+	printf("%s - - %s \"%s\" %d %lld\n",
 		c_ip.c_str(),
 		request_time,
 		request_startline.c_str(),
 		s_status,
-		bytes);
+		(long long) bytes);
 }
 
 //-----------------------------------------------------------------------------
@@ -319,7 +318,7 @@ void CmWebLog::AddLogEntry_ELF(CyhookHandler *hh)
 	std::string request_startline = 	hh->UrlData["startline"].c_str();
 	std::string cs_uri			= hh->UrlData["fullurl"];
 	int sc_status = hh->httpStatus;
-	int bytes	= hh->GetContentLength();
+	off_t bytes	= hh->GetContentLength();
 	int cached = (hh->HookVarList["CacheCategory"].empty()) ? 0 : 1;
 
 	struct tm *time_now;
@@ -339,7 +338,7 @@ void CmWebLog::AddLogEntry_ELF(CyhookHandler *hh)
 	std::string time_taken_response = hh->HookVarList["enlapsed_response"];
 	long time_taken = atoi(time_taken_request.c_str()) + atoi(time_taken_response.c_str());
 
-	printf("%s %s %s \"%s\" %s %d %s %d %ld %s %s %d\n",
+	printf("%s %s %s \"%s\" %s %d %s %d %ld %s %s %lld\n",
 		c_ip.c_str(),
 		_date,
 		_time,
@@ -347,7 +346,7 @@ void CmWebLog::AddLogEntry_ELF(CyhookHandler *hh)
 		cs_uri.c_str(),
 		sc_status,
 		cs_method.c_str(),
-		bytes,
+		(long long) bytes,
 		time_taken,
 		time_taken_request.c_str(),
 		time_taken_response.c_str(),