nttpd: allow sendfile() to cope with files >= 2GB; introduce range support

(This re-applies most of the patches I've reverted about two hours ago.)

src/nhttpd/yhttpd_core/Makefile.am

@@ -19,7 +19,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/lib/libcoolstream
 endif
 endif
 
-AM_CPPFLAGS += -fno-rtti -fno-exceptions
+AM_CPPFLAGS += -fno-rtti -fno-exceptions -D_FILE_OFFSET_BITS=64
 
 noinst_LIBRARIES = libyhttpd.a
 

src/nhttpd/yhttpd_core/yhook.cpp

@@ -249,7 +249,7 @@ std::string CyhookHandler::BuildHeader(bool cache) {
 		}
 	// print Status-line
 	result = string_printf(HTTP_PROTOCOL " %d %s\r\nContent-Type: %s\r\n",httpStatus, responseString, ResponseMimeType.c_str());
-	log_level_printf(2, "Respose: HTTP/1.1 %d %s\r\nContent-Type: %s\r\n",
+	log_level_printf(2, "Response: HTTP/1.1 %d %s\r\nContent-Type: %s\r\n",
 			httpStatus, responseString, ResponseMimeType.c_str());
 
 	switch (httpStatus) {
@@ -287,15 +287,13 @@ std::string CyhookHandler::BuildHeader(bool cache) {
 		if(keep_alive)
 		result += "Connection: keep-alive\r\n";
 		else
-		result += "Connection: close\r\n";
-#else
-		result += "Connection: close\r\n";
 #endif
+			result += "Connection: close\r\n";
 		// gzipped ?
 		if (UrlData["fileext"] == "gz")
 			result += "Content-Encoding: gzip\r\n";
 		// content-len, last-modified
-		if (httpStatus == HTTP_NOT_MODIFIED || httpStatus == HTTP_NOT_FOUND)
+		if (httpStatus == HTTP_NOT_MODIFIED || httpStatus == HTTP_NOT_FOUND || httpStatus == HTTP_REQUEST_RANGE_NOT_SATISFIABLE)
 			result += "Content-Length: 0\r\n";
 		else if (GetContentLength() > 0) {
 			time_t mod_time = time(NULL);
@@ -303,9 +301,14 @@ std::string CyhookHandler::BuildHeader(bool cache) {
 				mod_time = LastModified;
 
 			strftime(timeStr, sizeof(timeStr), RFC1123FMT, gmtime(&mod_time));
-			result += string_printf(
-					"Last-Modified: %s\r\nContent-Length: %ld\r\n", timeStr,
-					GetContentLength());
+			result += string_printf("Last-Modified: %s\r\n", timeStr);
+			if (status == HANDLED_SENDFILE) {
+				result += string_printf("Accept-Ranges: bytes\r\n");
+				result += string_printf("Content-Length: %lld\r\n", RangeEnd - RangeStart + 1);
+				if (httpStatus == HTTP_PARTIAL_CONTENT)
+					result += string_printf("Content-Range: bytes %lld-%lld/%lld\r\n", RangeStart, RangeEnd, ContentLength);
+			} else
+				result += string_printf("Content-Length: %lld\r\n", GetContentLength());
 		}
 		result += "\r\n"; // End of Header
 		break;
@@ -320,6 +323,8 @@ std::string CyhookHandler::BuildHeader(bool cache) {
 		case HTTP_ACCEPTED:
 		case HTTP_NO_CONTENT:
 		case HTTP_NOT_FOUND:
+		case HTTP_PARTIAL_CONTENT:
+		case HTTP_REQUEST_RANGE_NOT_SATISFIABLE:
 		case HTTP_INTERNAL_SERVER_ERROR:
 			break;
 

src/nhttpd/yhttpd_core/yhook.h

@@ -47,6 +47,7 @@
 //=============================================================================
 #ifndef __yhttpd_yhook_h__
 #define __yhttpd_yhook_h__
+#include <unistd.h>
 // C++
 #include <string>
 #include <list>
@@ -128,7 +129,9 @@ public:
 	HttpResponseType httpStatus;		// http-status code for response
 	std::string 	ResponseMimeType;	// mime-type for response
 	std::string 	NewURL;				// new URL for Redirection
-	long		ContentLength;			// Length of Response Body
+	off_t		ContentLength;			// Length of Response Body
+	off_t		RangeStart;			// Start of range, used for sendfile only
+	off_t		RangeEnd;			// End of range, used for sendfile only
 	time_t 		LastModified;			// Last Modified Time of Item to send / -1 dynamic content
 	std::string	Sendfile;				// Path & Name (local os style) of file to send
 	bool		keep_alive;
@@ -141,7 +144,7 @@ public:
 	CStringList 	HookVarList;		// Variables in Hook-Handling passing to other Hooks
 	THttp_Method 	Method;				// HTTP Method (requested)
 	// constructor & deconstructor
-	CyhookHandler(){ContentLength = 0; keep_alive = 0; _outIndent = 0;status = HANDLED_NONE;Method = M_UNKNOWN;httpStatus =  HTTP_NIL;outType = plain;};
+	CyhookHandler(){ContentLength = 0; RangeStart = 0; RangeEnd = -1; keep_alive = 0; _outIndent = 0;status = HANDLED_NONE;Method = M_UNKNOWN;httpStatus =  HTTP_NIL;outType = plain;};
 	virtual ~CyhookHandler(){};
 
 	// hook slot handler
@@ -173,8 +176,8 @@ public:
 	void SetError(HttpResponseType responseType, THandleStatus _status)
 		{SetError(responseType); status = _status;}
 	// others
-	long GetContentLength()
-		{return (status==HANDLED_SENDFILE)?ContentLength : (long)yresult.length();}
+	off_t GetContentLength()
+		{return (status==HANDLED_SENDFILE)?ContentLength : (off_t)yresult.length();}
 	// output methods
 	std::string BuildHeader(bool cache = false);
 	void addResult(const std::string& result) 	{yresult += result;}

src/nhttpd/yhttpd_core/yresponse.cpp

@@ -140,9 +140,13 @@ bool CWebserverResponse::SendResponse() {
 		//			cache = false;
 		Write(Connection->HookHandler.BuildHeader(cache));
 		if (Connection->Method != M_HEAD)
-			Sendfile(Connection->Request.UrlData["url"]);
+			Sendfile(Connection->Request.UrlData["url"], Connection->HookHandler.RangeStart, Connection->HookHandler.RangeEnd - Connection->HookHandler.RangeStart + 1);
 		return true;
 	}
+	if (Connection->HookHandler.status == HANDLED_SENDFILE && Connection->HookHandler.httpStatus == HTTP_REQUEST_RANGE_NOT_SATISFIABLE) {
+		SendError(Connection->HookHandler.httpStatus);
+		return false;
+	}
 
 	// arrived here? = error!
 	SendError( HTTP_NOT_FOUND);
@@ -193,13 +197,13 @@ bool CWebserverResponse::WriteLn(char const *text) {
 }
 
 //-----------------------------------------------------------------------------
-bool CWebserverResponse::Sendfile(std::string filename) {
+bool CWebserverResponse::Sendfile(std::string filename, off_t start, off_t end) {
 	if (Connection->RequestCanceled)
 		return false;
 	int filed = open(filename.c_str(), O_RDONLY);
 	if (filed != -1) //can access file?
 	{
-		if (!Connection->sock->SendFile(filed))
+		if (!Connection->sock->SendFile(filed, start, end))
 			Connection->RequestCanceled = true;
 		close(filed);
 	}

src/nhttpd/yhttpd_core/yresponse.h

@@ -24,7 +24,7 @@ private:
 
 protected:
 	bool WriteData(char const *data, long length);
-	bool Sendfile(std::string filename);
+	bool Sendfile(std::string filename, off_t start = 0, off_t end = -1);
 	std::string	redirectURI;		// URI for redirection else: empty
 
 public:

src/nhttpd/yhttpd_core/ysocket.cpp

@@ -5,6 +5,7 @@
 
 #include <cstring>
 #include <cstdio>
+#include <algorithm>
 
 // system
 #include <arpa/inet.h>
@@ -12,6 +13,7 @@
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <sys/ioctl.h>
+#include <sys/stat.h>
 #include <errno.h>
 #include <signal.h>
 #include <unistd.h>
@@ -310,31 +312,46 @@ bool CySocket::CheckSocketOpen() {
 // BASIC Send File over Socket for FILE*
 // fd is an opened FILE-Descriptor
 //-----------------------------------------------------------------------------
-int CySocket::SendFile(int filed) {
+bool CySocket::SendFile(int filed, off_t start, off_t size) {
 	if (!isValid)
 		return false;
-#ifdef Y_CONFIG_HAVE_SENDFILE
 	// does not work with SSL !!!
-	off_t start = 0;
-	off_t end = lseek(filed,0,SEEK_END);
-	int written = 0;
-	if((written = ::sendfile(sock,filed,&start,end)) == -1)
-	{
-		perror("sendfile failed\n");
+	struct stat st;
+	fstat(filed, &st);
+	off_t left = st.st_size - start;
+	off_t written = 0;
+	if (size > -1 && size < left)
+		left = size;
+#ifdef Y_CONFIG_HAVE_SENDFILE
+	while (left > 0) {
+		// split sendfile() transfer to smaller chunks to reduce memory-mapping requirements
+		if((written = ::sendfile(sock, filed, &start, std::min((off_t) 0x8000000LL, left))) == -1) {
+			if (errno != EPIPE)
+				perror("sendfile failed");
+			if (errno != EINVAL)
 		return false;
+			break;
+		} else {
+			BytesSend += written;
+			left -= written;
 	}
-	else
-	BytesSend += written;
-#else
-	char sbuf[1024];
-	unsigned int r = 0;
-	while ((r = read(filed, sbuf, 1024)) > 0) {
-		if (Send(sbuf, r) < 0) {
-			perror("sendfile failed\n");
-			return false;
-		}
 	}
 #endif // Y_CONFIG_HAVE_SENDFILE
+	if (left > 0) {
+		::lseek(filed, start, SEEK_SET);
+
+		char sbuf[65536];
+		while (left && (written = read(filed, sbuf, std::min((off_t) sizeof(sbuf), left))) > 0) {
+			if (Send(sbuf, written) < 0) {
+				if (errno != EPIPE)
+					perror("send failed");
+				return false;
+			}
+			BytesSend += written;
+			left -= written;
+		}
+	}
+
 	log_level_printf(9, "<Sock:SendFile>: Bytes:%ld\n", BytesSend);
 	return true;
 }

src/nhttpd/yhttpd_core/ysocket.h

@@ -71,7 +71,7 @@ public:
 	bool 		CheckSocketOpen();				// check if socket was closed by client
 	
 	// send & receive
-	int 		SendFile(int filed);				// Send a File
+	bool 		SendFile(int filed, off_t start = 0, off_t size = -1); // Send a File
 	std::string 	ReceiveBlock();					// receive a Block. Look at length
 	unsigned int	ReceiveFileGivenLength(int filed, unsigned int _length); // Receive File of given length
 	std::string 	ReceiveLine();					// receive until "\n"