HPCC-8761 Add HTTP 304 support to HTTP requests

This fix adds HTTP 304 support to ESP HTTP requests to improve ESP
performance. When an HTTP GetFile request is received, ESP reads
the timestamp of the file. If the request does not contain an etag
which contains the timestamp, ESP will create the etag and send
the etag with the content of the file. Otherwise, ESP will not
read the file content and send an HTTP 304 'Not Modified' response
to the client. If the HTTP request contains a 'if-modified-since'
header without an etag header, ESP will only change whether the
value of 'if-modified-since' matches the file's modified time or
not. If matches, an HTTP 304 response will be sent.

Signed-off-by: Kevin Wang <kevin.wang@lexisnexis.com>

esp/bindings/http/platform/httpbinding.cpp

@@ -2117,9 +2117,10 @@ int EspHttpBinding::formatResultsPage(IEspContext &context, const char *serv, co
 
 bool EspHttpBinding::setContentFromFile(IEspContext &context, CHttpResponse &resp, const char *filepath)
 {
-    StringBuffer mimetype;
+    StringBuffer mimetype, etag, lastModified;
     MemoryBuffer content;
-    if (httpContentFromFile(filepath, mimetype, content))
+    bool modified = false;
+    if (httpContentFromFile(filepath, mimetype, content, modified, lastModified, etag))
     {
         resp.setContent(content.length(), content.toByteArray());
         resp.setContentType(mimetype.str());

esp/bindings/http/platform/httpservice.cpp

@@ -853,22 +853,18 @@ int CEspHttpServer::onGetFile(CHttpRequest* request, CHttpResponse* response, co
         if (pathlen>5 && !stricmp(path+pathlen-4, ".php"))
             return onRunCGI(request, response, path);
         
-        StringBuffer mimetype;
+        StringBuffer mimetype, etag, lastModified;
         MemoryBuffer content;
+        bool modified = true;
+        request->getHeader("If-None-Match", etag);
+        request->getHeader("If-Modified-Since", lastModified);
 
         StringBuffer filepath(getCFD());
         filepath.append("files/");
         filepath.append(path);
-        if (httpContentFromFile(filepath.str(), mimetype, content))
+        if (httpContentFromFile(filepath.str(), mimetype, content, modified, lastModified, etag))
         {
-            response->setContent(content.length(), content.toByteArray());
-            response->setContentType(mimetype.str());
-
-            //if file being requested is an image file then set its expiration to a year
-            //so browser does not keep reloading it
-            const char* pchExt = strrchr(path, '.');
-            if (pchExt && (!stricmp(++pchExt, "gif") || !stricmp(pchExt, "jpg") || !stricmp(pchExt, "png")))
-                response->addHeader("Cache-control",  "max-age=31536000");
+            response->setHTTPContent(modified, lastModified.str(), etag.str(), mimetype.str(), content);
         }
         else
         {
@@ -884,14 +880,16 @@ int CEspHttpServer::onGetXslt(CHttpRequest* request, CHttpResponse* response, co
         if (!request || !response || !path)
             return -1;
         
-        StringBuffer mimetype;
+        StringBuffer mimetype, etag, lastModified;
         MemoryBuffer content;
+        bool modified = true;
+        request->getHeader("If-None-Match", etag);
+        request->getHeader("If-Modified-Since", lastModified);
 
         StringBuffer filepath;
-        if (httpContentFromFile(filepath.append(getCFD()).append("smc_xslt/").append(path).str(), mimetype, content) || httpContentFromFile(filepath.clear().append(getCFD()).append("xslt/").append(path).str(), mimetype, content))
+        if (httpContentFromFile(filepath.append(getCFD()).append("smc_xslt/").append(path).str(), mimetype, content, modified, lastModified.clear(), etag) || httpContentFromFile(filepath.clear().append(getCFD()).append("xslt/").append(path).str(), mimetype, content, modified, lastModified.clear(), etag))
         {
-            response->setContent(content.length(), content.toByteArray());
-            response->setContentType(mimetype.str());
+            response->setHTTPContent(modified, lastModified.str(), etag.str(), mimetype.str(), content);
         }
         else
         {
@@ -962,14 +960,14 @@ int CEspHttpServer::onGet()
 {   
     if (m_request && m_request->queryParameters()->hasProp("config_") && m_viewConfig)
     {
-        StringBuffer mimetype;
+        StringBuffer mimetype, etag, lastModified;
         MemoryBuffer content;
-        httpContentFromFile("esp.xml", mimetype, content);
-
+        bool modified = true;
+        m_request->getHeader("If-None-Match", etag);
+        m_request->getHeader("If-Modified-Since", lastModified);
+        httpContentFromFile("esp.xml", mimetype, content, modified, lastModified, etag);
         m_response->setVersion(HTTP_VERSION);
-        m_response->setContent(content.length(), content.toByteArray());
-        m_response->setContentType(HTTP_TYPE_APPLICATION_XML_UTF8);
-        m_response->setStatus(HTTP_STATUS_OK);
+        m_response->setHTTPContent(modified, lastModified.str(), etag.str(), HTTP_TYPE_APPLICATION_XML_UTF8, content);
         m_response->send();
     }
     else

esp/bindings/http/platform/httptransport.cpp

@@ -37,7 +37,7 @@ IEspHttpException* createEspHttpException(int code, const char *_msg, const char
     return new CEspHttpException(code, _msg, _httpstatus);
 }
 
-bool httpContentFromFile(const char *filepath, StringBuffer &mimetype, MemoryBuffer &fileContents)
+bool httpContentFromFile(const char *filepath, StringBuffer &mimetype, MemoryBuffer &fileContents, bool &checkModifiedTime, StringBuffer &lastModified, StringBuffer &etag)
 {
     StringBuffer strfile(filepath);
 
@@ -49,6 +49,38 @@ bool httpContentFromFile(const char *filepath, StringBuffer &mimetype, MemoryBuf
     Owned<IFile> file = createIFile(strfile.str());
     if (file && file->isFile())
     {
+        if (checkModifiedTime)
+        {
+            CDateTime createTime, accessedTime, modifiedTime;
+            file->getTime( &createTime,  &modifiedTime, &accessedTime);
+            if ((lastModified.length() < 1) || (etag.length() > 0))
+            {
+                StringBuffer etagSource, etagForThisFile;
+                modifiedTime.getString(lastModified.clear());
+                etagSource.appendf("%s+%s", filepath, lastModified.str());
+                etagForThisFile.append(hashc((unsigned char *)etagSource.str(), etagSource.length(), 0));
+                if ( !streq(etagForThisFile.str(), etag.str()))
+                    etag.set(etagForThisFile);
+                else
+                {
+                    checkModifiedTime = false;
+                    return true;
+                }
+            }
+            else
+            {
+                StringBuffer lastModifiedForThisFile;
+                modifiedTime.getString(lastModifiedForThisFile);
+                if ( !streq(lastModifiedForThisFile.str(), lastModified.str()))
+                    lastModified.set(lastModifiedForThisFile);
+                else
+                {
+                    checkModifiedTime = false;
+                    return true;
+                }
+            }
+        }
+
         Owned<IFileIO> io = file->open(IFOread);
         if (io)
         {
@@ -2217,9 +2249,12 @@ int CHttpResponse::processHeaders(IMultiException *me)
 
 bool CHttpResponse::httpContentFromFile(const char *filepath)
 {
-    StringBuffer mimetype;
+    StringBuffer mimetype, etag, lastModified;
     MemoryBuffer content;
-    bool ok = ::httpContentFromFile(filepath, mimetype, content);
+    //Set 'modified = false' here since this is called by CMySoapBinding::onGetFile() which does
+    //not need to check and set both 'etag' and 'lastModified' inside httpContentFromFile().
+    bool modified = false;
+    bool ok = ::httpContentFromFile(filepath, mimetype, content, modified, lastModified, etag);
     if (ok)
     {
         setContent(content.length(), content.toByteArray());
@@ -2234,6 +2269,39 @@ bool CHttpResponse::httpContentFromFile(const char *filepath)
     return ok;
 }
 
+void CHttpResponse::setETageCacheControl(const char *etag, const char *contenttype)
+{
+    if (etag && *etag)
+        addHeader("Etag",  etag);
+
+    if (!strncmp(contenttype, "image/", 6))
+    {
+       //if file being requested is an image file then set its expiration to a year
+       //so browser does not keep reloading it
+        addHeader("Cache-control",  "max-age=31536000");
+    }
+    else
+        addHeader("Cache-control",  "max-age=300");
+}
+
+void CHttpResponse::setHTTPContent(bool modified, const char *lastmodified, const char *etag, const char *contenttype, MemoryBuffer &content)
+{
+    if (!modified)
+    {
+        if (etag && *etag)
+            setETageCacheControl(etag, contenttype);
+        setStatus(HTTP_STATUS_NOT_MODIFIED);
+    }
+    else
+    {
+        addHeader("Last-Modified", lastmodified);
+        setETageCacheControl(etag, contenttype);
+        setContent(content.length(), content.toByteArray());
+        setContentType(contenttype);
+        setStatus(HTTP_STATUS_OK);
+    }
+}
+
 int CHttpResponse::receive(IMultiException *me)
 {
     // If it's receiving a response, it's behaving as the client side of this conversation.

esp/bindings/http/platform/httptransport.hpp

@@ -97,7 +97,7 @@
 //#define DEBUG_HTTP_
 #endif
 
-bool httpContentFromFile(const char *filepath, StringBuffer &mimetype, MemoryBuffer &fileContents);
+bool httpContentFromFile(const char *filepath, StringBuffer &mimetype, MemoryBuffer &fileContents, bool &checkmodifiedTime, StringBuffer &lastModified, StringBuffer &etag);
 bool xmlContentFromFile(const char *filepath, const char *stylesheet, StringBuffer &fileContents);
 
 

esp/bindings/http/platform/httptransport.ipp

@@ -404,6 +404,8 @@ public:
     virtual int sendException(IEspHttpException* e);
 
     void setTimeOut(unsigned int timeout);
+    void setETageCacheControl(const char *etag, const char *contenttype);
+    void setHTTPContent(bool modified, const char *lastModified, const char *etag, const char *contenttype, MemoryBuffer &content);
 };
 
 inline bool canRedirect(CHttpRequest &req)