HPCC-18655 Clean ESP cache after certain ESP methods

1. ESP may cache responses into multiple cache servers (groups).
2. in the ecm files, ESP methods may be defined to clean one or
more cache servers before sending out a response.

Multiple changes are added based on Gavin's comments. ESP only
caches ESP responses if there is cache configuration. An ESP may
be configured to eithor have one default cache server or multiple
named cache servers. ESP may not be started if the cache server
cannot be accessed. To avoid runtime delay, ESP cache clients
are created when ESP starts. To avoid overhead on systems that
do not have memcached installed, a quick memcached available
check is added. The cache clients are moved from EspHttpBinding
to CEspServer because multiple EspHttpBindings may share one
cache client. The cache client may be accessed from outside of
EspHttpBinding.

The ensureESPCache configuration is removed since ESP may not
be started if the cache server is configured but cannot be
accessed. Also cleaned some code.

Revise based on review:

1. Allow a client to force refresh using URL parameter: not_read_esp_cache;
2. Support method level cache group;
3. Not cache WUFullResult;
4. Clean cache when WUQuerysetImport is called;
5. The cache_second for WUDetailsMeta should be 600.

Revise based on review:

1. Remove extra cache_group from .ecm file;
2. Replace not_read_esp_cache with dirty_cache;
3. Replace 'MapStringTo<IEspCache*> cacheClientMap' with
   'MapStringToMyClass<IEspCache> cacheClientMap', as well
   as related code;
4. Fix a bug in hidlcomp.cpp.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/bindings/http/platform/httpbinding.cpp

@@ -807,10 +807,10 @@ const char* EspHttpBinding::createESPCacheID(CHttpRequest* request, StringBuffer
     return cacheID.str();
 }
 
-bool EspHttpBinding::sendFromESPCache(CHttpRequest* request, CHttpResponse* response, const char* cacheID)
+bool EspHttpBinding::sendFromESPCache(IEspCache* cacheClient, CHttpRequest* request, CHttpResponse* response, const char* cacheID)
 {
     StringBuffer content, contentType;
-    if (!espCacheClient->readResponseCache(cacheID, content.clear(), contentType.clear()))
+    if (!cacheClient->readResponseCache(cacheID, content.clear(), contentType.clear()))
         ESPLOG(LogMax, "Failed to read from ESP Cache for %s.", request->queryServiceMethod());
     if (content.isEmpty() || contentType.isEmpty())
         return false;
@@ -822,38 +822,53 @@ bool EspHttpBinding::sendFromESPCache(CHttpRequest* request, CHttpResponse* resp
     return true;
 }
 
-void EspHttpBinding::addToESPCache(CHttpRequest* request, CHttpResponse* response, const char* cacheID)
+void EspHttpBinding::addToESPCache(IEspCache* cacheClient, CHttpRequest* request, CHttpResponse* response, const char* cacheID, unsigned cacheSeconds)
 {
-    unsigned cacheSeconds = 0;
-    const char* method = request->queryServiceMethod();
-    if (!queryCacheSeconds(method, cacheSeconds)) //no cache required for this method
-        return;
-
     StringBuffer content, contentType;
     response->getContent(content);
     response->getContentType(contentType);
-    if (espCacheClient->cacheResponse(cacheID, cacheSeconds, content.str(), contentType.str()))
-        ESPLOG(LogMax, "AddTo ESP Cache for %s.", method);
+    if (cacheClient->cacheResponse(cacheID, cacheSeconds, content.str(), contentType.str()))
+        ESPLOG(LogMax, "AddTo ESP Cache for %s.", request->queryServiceMethod());
     else
-        ESPLOG(LogMax, "Failed to add ESP Cache for %s.", method);
+        ESPLOG(LogMax, "Failed to add ESP Cache for %s.", request->queryServiceMethod());
+}
+
+void EspHttpBinding::clearCacheByGroupID(const char *ids)
+{
+    if (isEmptyString(ids))
+        return;
+
+    IEspContainer *espContainer = getESPContainer();
+    if (!espContainer->hasCacheClient())
+        return;
+
+    ESPLOG(LogMax, "clearCacheByGroupID %s.", ids);
+    StringArray errorMsgs;
+    espContainer->clearCacheByGroupID(ids, errorMsgs);
+    if (errorMsgs.length() > 0)
+    {
+        ForEachItemIn(i, errorMsgs)
+            DBGLOG("%s", errorMsgs.item(i));
+    }
 }
 
 void EspHttpBinding::handleHttpPost(CHttpRequest *request, CHttpResponse *response)
 {
     StringBuffer cacheID;
-    if (cacheMethods > 0)
-    {
-        unsigned cacheSeconds = 0;
-        const char* method = request->queryServiceMethod();
-        if (queryCacheSeconds(method, cacheSeconds)) //ESP cache is needed for this method
-        {
-            if (!espCacheClient)
-                espCacheClient.setown(createESPCache(espCacheInitString.get()));
-            if (espCacheClient)
-                createESPCacheID(request, cacheID);
-            if (!cacheID.isEmpty() && sendFromESPCache(request, response, cacheID.str()))
-                return;
-        }
+    unsigned cacheSeconds = 0;
+    IEspCache *cacheClient = nullptr;
+    IEspContext &context = *request->queryContext();
+
+    IEspContainer *espContainer = getESPContainer();
+    if (espContainer->hasCacheClient() && (cacheMethods > 0)
+        && queryCacheSeconds(request->queryServiceMethod(), cacheSeconds)) //ESP cache is needed for this method
+    {
+        cacheClient = (IEspCache*) espContainer->queryCacheClient(getCacheGroupID(request->queryServiceMethod()));
+        if (cacheClient)
+            createESPCacheID(request, cacheID);
+        if (!cacheID.isEmpty() && !context.queryRequestParameters()->queryProp("dirty_cache")
+            && sendFromESPCache(cacheClient, request, response, cacheID.str()))
+            return;
     }
 
     if(request->isSoapMessage()) 
@@ -867,7 +882,7 @@ void EspHttpBinding::handleHttpPost(CHttpRequest *request, CHttpResponse *respon
         onPost(request, response);
 
     if (!cacheID.isEmpty())
-        addToESPCache(request, response, cacheID.str());
+        addToESPCache(cacheClient, request, response, cacheID.str(), cacheSeconds);
 }
 
 int EspHttpBinding::onGet(CHttpRequest* request, CHttpResponse* response)

esp/bindings/http/platform/httpbinding.hpp

@@ -143,8 +143,8 @@ private:
     StringAttrMapping desc_map;
     StringAttrMapping help_map;
 
-    Owned<IEspCache> espCacheClient;
-    StringAttr espCacheInitString;
+    StringAttr cacheGroupID;
+    StringAttrMapping cacheMethodGroupIDs;
     unsigned cacheMethods = 0;
     MapStringTo<unsigned> cacheSecondsMap;
     MapStringTo<bool> cacheGlobalMap;
@@ -152,8 +152,8 @@ private:
     bool queryCacheSeconds(const char *method, unsigned& cacheSecond);
     bool queryCacheGlobal(const char *method);
     const char* createESPCacheID(CHttpRequest* request, StringBuffer& cacheID);
-    void addToESPCache(CHttpRequest* request, CHttpResponse* response, const char* cacheID);
-    bool sendFromESPCache(CHttpRequest* request, CHttpResponse* response, const char* cacheID);
+    void addToESPCache(IEspCache* cacheClient, CHttpRequest* request, CHttpResponse* response, const char* cacheID, unsigned cacheSecond);
+    bool sendFromESPCache(IEspCache* cacheClient, CHttpRequest* request, CHttpResponse* response, const char* cacheID);
 
     StringAttr              processName;
     StringAttr              domainName;
@@ -231,15 +231,29 @@ public:
     //starting and the WsWorkunits lib is loading.
     void setCacheTimeout(const char *method, unsigned timeoutSeconds, bool global)
     {
-        //Disable http caching until it has been rethought - it makes the system unusable.
-#if 0
         StringBuffer key(method);
         cacheSecondsMap.setValue(key.toUpperCase().str(), timeoutSeconds);
         cacheMethods++;
         if (global)
             cacheGlobalMap.setValue(key.str(), global);
-#endif
     }
+    void setCacheGroupID(const char *method, const char *id)
+    {
+        if (isEmptyString(method))
+            cacheGroupID.set(id);
+        else
+        {
+            StringBuffer key(method);
+            cacheMethodGroupIDs.setValue(key.toUpperCase().str(), id);
+        }
+    }
+    const char *getCacheGroupID(const char *method)
+    {
+        StringBuffer key(method);
+        StringAttr *idStr = cacheMethodGroupIDs.getValue(key.toUpperCase().str());
+        return idStr ? idStr->get() : cacheGroupID.get();
+    }
+    void clearCacheByGroupID(const char *id);
 
     int onGetConfig(IEspContext &context, CHttpRequest* request, CHttpResponse* response);
 

esp/platform/espcache.hpp

@@ -37,6 +37,7 @@ interface IEspCache : extends IInterface
 {
     virtual bool cacheResponse(const char* cacheID, const unsigned cacheSeconds, const char* content, const char* contentType) = 0;
     virtual bool readResponseCache(const char* cacheID, StringBuffer& content, StringBuffer& contentType) = 0;
+    virtual void flush(unsigned when) = 0;
 };
 
 extern esp_http_decl IEspCache* createESPCache(const char* setting);

esp/platform/espcfg.cpp

@@ -1002,22 +1002,32 @@ IEspPlugin* CEspConfig::getPlugin(const char* name)
     return NULL;
 }
 
-bool CEspConfig::checkESPCache()
+void CEspConfig::checkESPCache(IEspServer& server)
 {
-    bool espCacheAvailable = false ;
-    list<binding_cfg*>::iterator iter = m_bindings.begin();
-    while (iter!=m_bindings.end())
+    const char* cacheInitString = m_cfg->queryProp("@espCacheInitString");
+    IPropertyTree* espCacheCfg = m_cfg->queryBranch("ESPCache");
+    if (!espCacheCfg && isEmptyString(cacheInitString))
+        return;
+
+    if (!espCacheCfg)
     {
-        binding_cfg& xcfg = **iter;
-        if (xcfg.bind->getCacheMethodCount() > 0)
-        {
-            Owned<IEspCache> espCache = createESPCache(m_cfg->queryProp("@espCacheInitString"));
-            espCacheAvailable = (espCache != nullptr);
-            break;
-        }
-        iter++;
+        if (!server.addCacheClient("default", cacheInitString))
+            throw MakeStringException(-1, "Failed in checking ESP cache service using %s", cacheInitString);
+        return;
+    }
+    Owned<IPropertyTreeIterator> iter = espCacheCfg->getElements("Group");
+    ForEach(*iter)
+    {
+        IPropertyTree& espCacheGroup = iter->query();
+        const char* id = espCacheGroup.queryProp("@id");
+        const char* initString = espCacheGroup.queryProp("@initString");
+        if (isEmptyString(id))
+            throw MakeStringException(-1, "ESP cache ID not defined");
+        if (isEmptyString(initString))
+            throw MakeStringException(-1, "ESP cache initStrings not defined");
+        if (!server.addCacheClient(id, initString))
+            throw MakeStringException(-1, "Failed in checking ESP cache service using %s", initString);
     }
-    return espCacheAvailable;
 }
 
 bool CEspConfig::reSubscribeESPToDali()

esp/platform/espcfg.ipp

@@ -258,9 +258,6 @@ public:
         loadBindings();
         if(useDali)
             startEsdlMonitor();
-
-        if (m_cfg->getPropBool("@ensureESPCache", false) && !checkESPCache())
-            throw MakeStringException(-1, "Failed in checking ESP cache service using %s", m_cfg->queryProp("@espCacheInitString"));
     }
 
     bool reSubscribeESPToDali();
@@ -268,7 +265,7 @@ public:
     bool detachESPFromDali(bool force);
     bool attachESPToDali();
     bool canAllBindingsDetachFromDali();
-    bool checkESPCache();
+    void checkESPCache(IEspServer& server);
     IEspPlugin* getPlugin(const char* name);
 
     void loadBuiltIns();

esp/platform/espp.cpp

@@ -390,7 +390,7 @@ int init_main(int argc, char* argv[])
 
             config->loadAll();
             config->bindServer(*server.get(), *server.get()); 
-            
+            config->checkESPCache(*server.get());
         }
         catch(IException* e)
         {

esp/platform/espp.hpp

@@ -64,6 +64,8 @@ private:
     bool m_SEHMappingEnabled;
     CEspConfig* m_config;
     CriticalSection m_BindingCritSect;
+    unsigned countCacheClients = 0;
+    MapStringToMyClass<IEspCache> cacheClientMap;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -304,6 +306,41 @@ public:
     }
     virtual void sendSnmpMessage(const char* msg) { throwUnexpected(); }
 
+    virtual bool addCacheClient(const char *id, const char *cacheInitString)
+    {
+        Owned<IEspCache> cacheClient = createESPCache(cacheInitString);
+        if (!cacheClient)
+            return false;
+        cacheClientMap.setValue(id, cacheClient);
+        countCacheClients++;
+        return true;
+    }
+    virtual bool hasCacheClient()
+    {
+        return countCacheClients > 0;
+    }
+    virtual const void *queryCacheClient(const char* id)
+    {
+        return countCacheClients > 1 ? cacheClientMap.getValue(id) : nullptr;
+    }
+    virtual void clearCacheByGroupID(const char *ids, StringArray& errorMsgs)
+    {
+        StringArray idList;
+        idList.appendListUniq(ids, ",");
+        ForEachItemIn(i, idList)
+        {
+            const char *id = idList.item(i);
+            IEspCache* cacheClient = (IEspCache*) queryCacheClient(id);
+            if (cacheClient)
+                cacheClient->flush(0);
+            else
+            {
+                VStringBuffer msg("Failed to get ESPCache client %s.", id);
+                errorMsgs.append(msg);
+            }
+        }
+    }
+
     virtual bool reSubscribeESPToDali()
     {
         return m_config->reSubscribeESPToDali();

esp/scm/esp.ecm

@@ -234,6 +234,9 @@ interface IEspContainer : extends IInterface
     virtual bool attachESPToDali() = 0;
     virtual bool isAttachedToDali() = 0;
     virtual bool isSubscribedToDali() = 0;
+    virtual bool hasCacheClient() = 0;
+    virtual const void* queryCacheClient(const char* id) = 0;
+    virtual void clearCacheByGroupID(const char* ids, StringArray& errorMsgs) = 0;
 };
 
 interface IEspRpcBinding;
@@ -371,6 +374,7 @@ SCMinterface IEspServer(IInterface)
    IEspRpcBinding* queryBinding(const char* name);
    const char* getProcName();
    virtual IPropertyTree* queryProcConfig();
+   bool addCacheClient(const char* id, const char* initString);
 };
 
 SCMinterface IEspServiceCfg(IInterface)

esp/scm/ws_topology.ecm

@@ -612,7 +612,7 @@ ESPresponse [nil_remove, exceptions_inline] TpDropZoneQueryResponse
     ESParray<ESPstruct TpDropZone>    TpDropZones;
 };
 
-ESPservice [auth_feature("DEFERRED"), noforms, version("1.27"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
+ESPservice [auth_feature("DEFERRED"), noforms, version("1.27"), cache_group("ESPWsTP"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
 {
     ESPmethod [cache_seconds(180), cache_global(1), resp_xsl_default("/esp/xslt/targetclusters.xslt")] TpTargetClusterQuery(TpTargetClusterQueryRequest, TpTargetClusterQueryResponse);
     ESPmethod [cache_seconds(180), cache_global(1), resp_xsl_default("/esp/xslt/topology.xslt")] TpClusterQuery(TpClusterQueryRequest, TpClusterQueryResponse);
@@ -624,8 +624,8 @@ ESPservice [auth_feature("DEFERRED"), noforms, version("1.27"), exceptions_inlin
 
     ESPmethod [cache_seconds(180), cache_global(1), min_ver("1.26")] TpDropZoneQuery(TpDropZoneQueryRequest, TpDropZoneQueryResponse);
     ESPmethod [cache_seconds(180), cache_global(1), resp_xsl_default("/esp/xslt/services.xslt")] TpServiceQuery(TpServiceQueryRequest, TpServiceQueryResponse);
-    ESPmethod TpSetMachineStatus(TpSetMachineStatusRequest, TpSetMachineStatusResponse);
-    ESPmethod TpSwapNode(TpSwapNodeRequest, TpSwapNodeResponse);
+    ESPmethod [clear_cache_group] TpSetMachineStatus(TpSetMachineStatusRequest, TpSetMachineStatusResponse);
+    ESPmethod [clear_cache_group] TpSwapNode(TpSwapNodeRequest, TpSwapNodeResponse);
     ESPmethod [cache_seconds(180)] TpXMLFile(TpXMLFileRequest, TpXMLFileResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/tplog.xslt")] TpLogFile(TpLogFileRequest, TpLogFileResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/tplogdisplay.xslt")] TpLogFileDisplay(TpLogFileRequest, TpLogFileResponse);

esp/scm/ws_workunits.ecm

@@ -2225,7 +2225,7 @@ ESPresponse [exceptions_inline] WUEclDefinitionActionResponse
 // ----------------------------------------------------------------------------------
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.73"), default_client_version("1.73"),
+    version("1.73"), default_client_version("1.73"), cache_group("ESPWsWUs"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);
@@ -2237,10 +2237,10 @@ ESPservice [
     ESPmethod [cache_seconds(30), resp_xsl_default("/esp/xslt/graph_gvc.xslt")]     WUGVCGraphInfo(WUGVCGraphInfoRequest, WUGVCGraphInfoResponse);
     ESPmethod [description("Stub for Ajax GVC Graph."), help(""), resp_xsl_default("/esp/xslt/GvcGraph.xslt")] GVCAjaxGraph(GVCAjaxGraphRequest, GVCAjaxGraphResponse);
     ESPmethod [cache_seconds(30), resp_xsl_default("/esp/xslt/result.xslt")]        WUResult(WUResultRequest, WUResultResponse);
-    ESPmethod [cache_seconds(30)] WUFullResult(WUFullResultRequest, WUFullResultResponse);
+    ESPmethod WUFullResult(WUFullResultRequest, WUFullResultResponse);
     ESPmethod [cache_seconds(30)] WUResultView(WUResultViewRequest, WUResultViewResponse);
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/wuid_jobs.xslt")]     WUJobList(WUJobListRequest, WUJobListResponse);
-    ESPmethod [resp_xsl_default("/esp/xslt/wuaction_results.xslt")] WUAction(WUActionRequest, WUActionResponse);
+    ESPmethod [clear_cache_group, resp_xsl_default("/esp/xslt/wuaction_results.xslt")] WUAction(WUActionRequest, WUActionResponse);
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/scheduledwus.xslt")] WUShowScheduled(WUShowScheduledRequest, WUShowScheduledResponse);
 
     ESPmethod [cache_seconds(30)] WUResultSummary(WUResultSummaryRequest, WUResultSummaryResponse);
@@ -2250,26 +2250,26 @@ ESPservice [
     ESPmethod WUClusterJobQueueLOG(WUClusterJobQueueLOGRequest, WUClusterJobQueueLOGResponse);
     ESPmethod [cache_seconds(60)] WUClusterJobXLS(WUClusterJobXLSRequest, WUClusterJobXLSResponse);
     ESPmethod [cache_seconds(60)] WUClusterJobSummaryXLS(WUClusterJobSummaryXLSRequest, WUClusterJobSummaryXLSResponse);
-    ESPmethod [auth_feature("OwnWorkunitsAccess:WRITE")] WUCreate(WUCreateRequest, WUCreateResponse);
-    ESPmethod [auth_feature("OwnWorkunitsAccess:WRITE")] WUCreateAndUpdate(WUUpdateRequest, WUUpdateResponse);
-    ESPmethod WUUpdate(WUUpdateRequest, WUUpdateResponse);
-    ESPmethod WUDelete(WUDeleteRequest, WUDeleteResponse);
-    ESPmethod WUSubmit(WUSubmitRequest, WUSubmitResponse);
-    ESPmethod WUSchedule(WUScheduleRequest, WUScheduleResponse);
-    ESPmethod WUPushEvent(WUPushEventRequest, WUPushEventResponse);
-    ESPmethod WUDeployWorkunit(WUDeployWorkunitRequest, WUDeployWorkunitResponse);
-
-    ESPmethod WUAbort(WUAbortRequest, WUAbortResponse);
-    ESPmethod WUProtect(WUProtectRequest, WUProtectResponse);
+    ESPmethod [clear_cache_group, auth_feature("OwnWorkunitsAccess:WRITE")] WUCreate(WUCreateRequest, WUCreateResponse);
+    ESPmethod [clear_cache_group, auth_feature("OwnWorkunitsAccess:WRITE")] WUCreateAndUpdate(WUUpdateRequest, WUUpdateResponse);
+    ESPmethod [clear_cache_group] WUUpdate(WUUpdateRequest, WUUpdateResponse);
+    ESPmethod [clear_cache_group] WUDelete(WUDeleteRequest, WUDeleteResponse);
+    ESPmethod [clear_cache_group] WUSubmit(WUSubmitRequest, WUSubmitResponse);
+    ESPmethod [clear_cache_group] WUSchedule(WUScheduleRequest, WUScheduleResponse);
+    ESPmethod [clear_cache_group] WUPushEvent(WUPushEventRequest, WUPushEventResponse);
+    ESPmethod [clear_cache_group] WUDeployWorkunit(WUDeployWorkunitRequest, WUDeployWorkunitResponse);
+
+    ESPmethod [clear_cache_group] WUAbort(WUAbortRequest, WUAbortResponse);
+    ESPmethod [clear_cache_group] WUProtect(WUProtectRequest, WUProtectResponse);
     ESPmethod [min_ver("1.70")] WURecreateQuery(WURecreateQueryRequest, WURecreateQueryResponse);
-    ESPmethod WUResubmit(WUResubmitRequest, WUResubmitResponse); //????
-    ESPmethod WURun(WURunRequest, WURunResponse);
+    ESPmethod [clear_cache_group] WUResubmit(WUResubmitRequest, WUResubmitResponse); //????
+    ESPmethod [clear_cache_group] WURun(WURunRequest, WURunResponse);
 
     ESPmethod WUExport(WUExportRequest, WUExportResponse);
     ESPmethod WUWaitCompiled(WUWaitRequest, WUWaitResponse);
     ESPmethod WUWaitComplete(WUWaitRequest, WUWaitResponse);
     ESPmethod WUSyntaxCheckECL(WUSyntaxCheckRequest, WUSyntaxCheckResponse);
-    ESPmethod WUCompileECL(WUCompileECLRequest, WUCompileECLResponse);
+    ESPmethod [clear_cache_group] WUCompileECL(WUCompileECLRequest, WUCompileECLResponse);
 
     //ESPmethod WUAction(WUActionRequest, WUActionResponse);
     ESPmethod [cache_seconds(60)]WUFile(WULogFileRequest, WULogFileResponse);
@@ -2281,26 +2281,26 @@ ESPservice [
     ESPmethod [cache_seconds(30)] WUGetDependancyTrees(WUGetDependancyTreesRequest, WUGetDependancyTreesResponse);
 
     ESPmethod [cache_seconds(60)] WUListLocalFileRequired(WUListLocalFileRequiredRequest, WUListLocalFileRequiredResponse);
-    ESPmethod WUAddLocalFileToWorkunit(WUAddLocalFileToWorkunitRequest, WUAddLocalFileToWorkunitResponse);
+    ESPmethod [clear_cache_group] WUAddLocalFileToWorkunit(WUAddLocalFileToWorkunitRequest, WUAddLocalFileToWorkunitResponse);
     ESPmethod WUCDebug(WUDebugRequest, WUDebugResponse);
 
     ESPmethod [resp_xsl_default("/esp/xslt/WUPublishWorkunit.xslt")] WUPublishWorkunit(WUPublishWorkunitRequest, WUPublishWorkunitResponse);
-    ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/WUQuerysets.xslt")] WUQuerysets(WUQuerysetsRequest, WUQuerysetsResponse);
-    ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/WUQuerysetQueries.xslt")] WUQuerysetDetails(WUQuerySetDetailsRequest, WUQuerySetDetailsResponse);
-    ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/WUQueryDetails.xslt")] WUQueryDetails(WUQueryDetailsRequest, WUQueryDetailsResponse);
+    ESPmethod [cache_group("ESPWsWUQS"), cache_seconds(60), resp_xsl_default("/esp/xslt/WUQuerysets.xslt")] WUQuerysets(WUQuerysetsRequest, WUQuerysetsResponse);
+    ESPmethod [cache_group("ESPWsWUQS"), cache_seconds(60), resp_xsl_default("/esp/xslt/WUQuerysetQueries.xslt")] WUQuerysetDetails(WUQuerySetDetailsRequest, WUQuerySetDetailsResponse);
+    ESPmethod [cache_group("ESPWsWUQS"), cache_seconds(60), resp_xsl_default("/esp/xslt/WUQueryDetails.xslt")] WUQueryDetails(WUQueryDetailsRequest, WUQueryDetailsResponse);
     ESPmethod [cache_seconds(60)] WUMultiQuerysetDetails(WUMultiQuerySetDetailsRequest, WUMultiQuerySetDetailsResponse);
-    ESPmethod [min_ver("1.71")] WUQuerysetImport(WUQuerysetImportRequest, WUQuerysetImportResponse);
-    ESPmethod [min_ver("1.71")] WUQuerysetExport(WUQuerysetExportRequest, WUQuerysetExportResponse);
-    ESPmethod WUQuerysetQueryAction(WUQuerySetQueryActionRequest, WUQuerySetQueryActionResponse);
-    ESPmethod WUQuerysetAliasAction(WUQuerySetAliasActionRequest, WUQuerySetAliasActionResponse);
-    ESPmethod WUQuerysetCopyQuery(WUQuerySetCopyQueryRequest, WUQuerySetCopyQueryResponse);
-    ESPmethod WUCopyQuerySet(WUCopyQuerySetRequest, WUCopyQuerySetResponse);
-    ESPmethod [resp_xsl_default("/esp/xslt/WUCopyLogicalFiles.xslt")] WUCopyLogicalFiles(WUCopyLogicalFilesRequest, WUCopyLogicalFilesResponse);
+    ESPmethod [clear_cache_group("ESPWsWUQS"), min_ver("1.71")] WUQuerysetImport(WUQuerysetImportRequest, WUQuerysetImportResponse);
+    ESPmethod [cache_group("ESPWsWUQS"), min_ver("1.71")] WUQuerysetExport(WUQuerysetExportRequest, WUQuerysetExportResponse);
+    ESPmethod [clear_cache_group("ESPWsWUQS")] WUQuerysetQueryAction(WUQuerySetQueryActionRequest, WUQuerySetQueryActionResponse);
+    ESPmethod [clear_cache_group("ESPWsWUQS")] WUQuerysetAliasAction(WUQuerySetAliasActionRequest, WUQuerySetAliasActionResponse);
+    ESPmethod [clear_cache_group("ESPWsWUQS")] WUQuerysetCopyQuery(WUQuerySetCopyQueryRequest, WUQuerySetCopyQueryResponse);
+    ESPmethod [clear_cache_group("ESPWsWUQS")] WUCopyQuerySet(WUCopyQuerySetRequest, WUCopyQuerySetResponse);
+    ESPmethod [clear_cache_group("ESPWsWUQS"), resp_xsl_default("/esp/xslt/WUCopyLogicalFiles.xslt")] WUCopyLogicalFiles(WUCopyLogicalFilesRequest, WUCopyLogicalFilesResponse);
     ESPmethod WUQueryConfig(WUQueryConfigRequest, WUQueryConfigResponse);
-    ESPmethod [cache_seconds(60)] WUListQueries(WUListQueriesRequest, WUListQueriesResponse);
-    ESPmethod [min_ver("1.59")] WUUpdateQueryEntry(WUUpdateQueryEntryRequest, WUUpdateQueryEntryResponse);
-    ESPmethod [cache_seconds(60)]WUQueryFiles(WUQueryFilesRequest, WUQueryFilesResponse);
-    ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/QueriesUsingFile.xslt")] WUListQueriesUsingFile(WUListQueriesUsingFileRequest, WUListQueriesUsingFileResponse);
+    ESPmethod [cache_group("ESPWsWUQS"), cache_seconds(60)] WUListQueries(WUListQueriesRequest, WUListQueriesResponse);
+    ESPmethod [clear_cache_group("ESPWsWUQS"), min_ver("1.59")] WUUpdateQueryEntry(WUUpdateQueryEntryRequest, WUUpdateQueryEntryResponse);
+    ESPmethod [cache_group("ESPWsWUQS"), cache_seconds(60)] WUQueryFiles(WUQueryFilesRequest, WUQueryFilesResponse);
+    ESPmethod [cache_group("ESPWsWUQS"), cache_seconds(60), resp_xsl_default("/esp/xslt/QueriesUsingFile.xslt")] WUListQueriesUsingFile(WUListQueriesUsingFileRequest, WUListQueriesUsingFileResponse);
     ESPmethod WUCreateZAPInfo(WUCreateZAPInfoRequest, WUCreateZAPInfoResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUZAPInfoForm.xslt")] WUGetZAPInfo(WUGetZAPInfoRequest, WUGetZAPInfoResponse);
     ESPmethod WUCheckFeatures(WUCheckFeaturesRequest, WUCheckFeaturesResponse);
@@ -2309,8 +2309,8 @@ ESPservice [
     ESPmethod [cache_seconds(60), min_ver("1.57")] WUGetArchiveFile(WUGetArchiveFileRequest, WUGetArchiveFileResponse);
     ESPmethod [cache_seconds(60), min_ver("1.61")] WUGetNumFileToCopy(WUGetNumFileToCopyRequest, WUGetNumFileToCopyResponse);
     ESPmethod [min_ver("1.71")] WUDetails(WUDetailsRequest, WUDetailsResponse);
-    ESPmethod [cache_seconds(600),min_ver("1.71")] WUDetailsMeta(WUDetailsMetaRequest, WUDetailsMetaResponse);
-    ESPmethod [min_ver("1.72")] WUEclDefinitionAction(WUEclDefinitionActionRequest, WUEclDefinitionActionResponse);
+    ESPmethod [cache_seconds(600), min_ver("1.71")] WUDetailsMeta(WUDetailsMetaRequest, WUDetailsMetaResponse);
+    ESPmethod [clear_cache_group, min_ver("1.72")] WUEclDefinitionAction(WUEclDefinitionActionRequest, WUEclDefinitionActionResponse);
 };
 
 

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -3036,7 +3036,19 @@ int WUSchedule::run()
                             now.setNow();
                             cw.getTimeScheduled(dt);
                             if (now.compare(dt)>=0)
+                            {
                                 runWorkUnit(cw.queryWuid(), cw.queryClusterName());
+                                if (m_container->hasCacheClient())
+                                {
+                                    StringArray errorMsgs;
+                                    m_container->clearCacheByGroupID("ESPWsWUs", errorMsgs);
+                                    if (errorMsgs.length() > 0)
+                                    {
+                                        ForEachItemIn(i, errorMsgs)
+                                            DBGLOG("%s", errorMsgs.item(i));
+                                    }
+                                }
+                            }
                         }
                         catch(IException *e)
                         {

initfiles/componentfiles/configxml/esp.xsd.in

@@ -894,13 +894,6 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
-            <xs:attribute name="ensureESPCache" type="xs:boolean" use="optional" default="false">
-                <xs:annotation>
-                    <xs:appinfo>
-                        <tooltip>If true, ESP will not be started if no ESP cache service is available.</tooltip>
-                    </xs:appinfo>
-                </xs:annotation>
-            </xs:attribute>
             <xs:attribute name="espCacheInitString" type="xs:string" use="optional">
                 <xs:annotation>
                     <xs:appinfo>

tools/hidl/hidlcomp.cpp

@@ -5653,7 +5653,8 @@ void EspServInfo::write_esp_binding()
 
     outf("\nvoid C%sSoapBinding::init_strings()\n", name_);
     outs("{\n");
-    
+
+    bool cacheDefined = false;
     for (mthi=methods;mthi!=NULL;mthi=mthi->next)
     {
         StrBuffer val;
@@ -5670,13 +5671,28 @@ void EspServInfo::write_esp_binding()
         int cacheGlobal = mthi->getMetaInt("cache_global", 0);
         int cacheSeconds = mthi->getMetaInt("cache_seconds", -1);
         if (cacheSeconds > -1) {
+            cacheDefined = true;
             if (cacheGlobal > 0)
                 outf("\tsetCacheTimeout(\"%s\", %d, 1);\n", mthi->getName(), cacheSeconds);
             else
                 outf("\tsetCacheTimeout(\"%s\", %d, 0);\n", mthi->getName(), cacheSeconds);
             outs("\tm_cacheMethodCount++;\n");
+
+            StrBuffer methodCacheGroupID;
+            mthi->getMetaStringValue(methodCacheGroupID,"cache_group");
+            if (methodCacheGroupID.length() > 0)
+                outf("\tsetCacheGroupID(\"%s\", \"%s\");\n", mthi->getName(), methodCacheGroupID.str());
         }
     }
+    StrBuffer serviceCacheGroupID;
+    if (cacheDefined)
+    {
+        getMetaStringValue(serviceCacheGroupID,"cache_group");
+        if (serviceCacheGroupID.length() == 0)
+            serviceCacheGroupID.set(name_);
+        outf("\tsetCacheGroupID(nullptr, \"%s\");\n", serviceCacheGroupID.str());
+    }
+
     outs("}\n");
     
     outf("\nint C%sSoapBinding::processRequest(IRpcMessage* rpc_call, IRpcMessage* rpc_response)\n", name_);
@@ -5736,6 +5752,14 @@ void EspServInfo::write_esp_binding()
 
         writeAccessMap(servicefeatureurl.str(),name_, 2);
 
+        StrBuffer clearCacheGroupIDs;
+        if (mthi->hasMetaTag("clear_cache_group"))
+        {
+            StrBuffer cCGIDs;
+            mthi->getMetaStringValue(cCGIDs,"clear_cache_group");
+            if (cacheDefined || (cCGIDs.length() != 0))
+                clearCacheGroupIDs.set((cCGIDs.length() != 0) ? cCGIDs.str() : serviceCacheGroupID.str());
+        }
         //begin try block
         if (bHandleExceptions)
         {
@@ -5756,6 +5780,8 @@ void EspServInfo::write_esp_binding()
                 outf("\t\t\tif( accessmap.ordinality() > 0 )\n\t\t\t\tonFeaturesAuthorize(context, accessmap, \"%s\", \"%s\");\n", name_, mthi->getName());
 
             outf("\t\t\tiserv->on%s(context, *esp_request, *esp_response);\n", mthi->getName());
+            if (clearCacheGroupIDs.length() > 0)
+                outf("\t\t\tclearCacheByGroupID(\"%s\");\n", clearCacheGroupIDs.str());
 
             outs("\t\t}\n");
             
@@ -5774,6 +5800,8 @@ void EspServInfo::write_esp_binding()
             if (servicefeatureurl.length() != 0)
                 outf("\t\tif( accessmap.ordinality() > 0 )\n\t\t\tonFeaturesAuthorize(context, accessmap, \"%s\", \"%s\");\n", name_, mthi->getName());
             outf("\t\tiserv->on%s(*rpc_call->queryContext(), *esp_request, *esp_response);\n", mthi->getName());
+            if (clearCacheGroupIDs.length() > 0)
+                outf("\t\tclearCacheByGroupID(\"%s\");\n", clearCacheGroupIDs.str());
             outs("\t\tresponse->set_status(SOAP_OK);\n");
         }
 
@@ -6147,6 +6175,15 @@ void EspServInfo::write_esp_binding()
             bClientXslt=(respXsl!=NULL);
         }
 
+        StrBuffer clearCacheGroupIDs;
+        if (mthi->hasMetaTag("clear_cache_group"))
+        {
+            StrBuffer cCGIDs;
+            mthi->getMetaStringValue(cCGIDs,"clear_cache_group");
+            if (cacheDefined || (cCGIDs.length() != 0))
+                clearCacheGroupIDs.set((cCGIDs.length() != 0) ? cCGIDs.str() : serviceCacheGroupID.str());
+        }
+
         bool bHandleExceptions =  0 != mthi->getMetaInt("exceptions_inline", 0) || mthi->getMetaInt("http_exceptions_inline", 0);
         if (!bHandleExceptions)
             bHandleExceptions = 0 != getMetaInt("exceptions_inline", 0) || getMetaInt("http_exceptions_inline", 0);
@@ -6188,6 +6225,8 @@ void EspServInfo::write_esp_binding()
                 if (mthi->getMetaInt("do_not_log",0))
                     outf("\t\t\t\tcontext.queryRequestParameters()->setProp(\"do_not_log\",1);\n");
                 outf("\t\t\t\tiserv->on%s(context, *esp_request.get(), *resp);\n", mthi->getName());
+                if (clearCacheGroupIDs.length() > 0)
+                    outf("\t\t\t\tclearCacheByGroupID(\"%s\");\n", clearCacheGroupIDs.str());
                 outs("\t\t\t}\n");
                 
                 write_catch_blocks(mthi, ct_httpresp, 3);
@@ -6197,6 +6236,8 @@ void EspServInfo::write_esp_binding()
                 if (servicefeatureurl.length() != 0)
                     outf("\t\t\tif(accessmap.ordinality()>0)\n\t\t\t\tonFeaturesAuthorize(context, accessmap, \"%s\", \"%s\");\n", name_, mthi->getName());
                 outf("\t\t\tiserv->on%s(*request->queryContext(), *esp_request.get(), *resp);\n", mthi->getName());
+                if (clearCacheGroupIDs.length() > 0)
+                    outf("\t\t\tclearCacheByGroupID(\"%s\");\n", clearCacheGroupIDs.str());
             }
 
             outs("\t\t}\n");
@@ -6219,6 +6260,8 @@ void EspServInfo::write_esp_binding()
                 outs("\t\t\ttry\n");
                 outs("\t\t\t{\n");
                 outf("\t\t\t\tiserv->on%s(*request->queryContext(), *esp_request.get(), *esp_response.get());\n", mthi->getName());
+                if (clearCacheGroupIDs.length() > 0)
+                    outf("\t\t\t\tclearCacheByGroupID(\"%s\");\n", clearCacheGroupIDs.str());
                 outs("\t\t\t}\n");
                 
                 write_catch_blocks(mthi, ct_httpresp,3);
@@ -6226,6 +6269,8 @@ void EspServInfo::write_esp_binding()
             else
             {
                 outf("\t\t\t\tiserv->on%s(*request->queryContext(), *esp_request.get(), *esp_response.get());\n", mthi->getName());
+                if (clearCacheGroupIDs.length() > 0)
+                    outf("\t\t\tclearCacheByGroupID(\"%s\");\n", clearCacheGroupIDs.str());
             }
 
             outs("\t\t\tif (canRedirect(*request) && esp_response->getRedirectUrl() && *esp_response->getRedirectUrl())\n");

tools/hidl/hidlcomp.h

@@ -982,6 +982,11 @@ public:
         response_ =strdup(name);
     }
 
+    bool hasMetaTag(const char *tag)
+    {
+        return findMetaTag(tags,tag)!=NULL;
+    }
+
     const char *getMetaString(const char *tag, const char *def_val)
     {
         return ::getMetaString(tags, tag, def_val);