Merge branch 'candidate-8.2.x' into candidate-8.4.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcmd/eclcmd_common.hpp

@@ -54,6 +54,9 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_SSL "--ssl"
 #define ECLOPT_SSL_S "-ssl"
 
+#define ECLOPT_SOURCE_SSL "--source-ssl"
+#define ECLOPT_SOURCE_NO_SSL "--source-no-ssl"
+
 #define ECLOPT_PORT "--port"
 #define ECLOPT_PORT_INI "eclWatchPort"
 #define ECLOPT_PORT_ENV "ECL_WATCH_PORT"
@@ -266,7 +269,7 @@ public:
         if (usesESP)
             fprintf(stdout,
                 "   -s, --server=<ip>      IP of server running ecl services (eclwatch)\n"
-                "   -ssl, --ssl            Use SSL to secure the connection to the server\n"
+                "   -ssl, --ssl            Use SSL to secure the connection to the server(s)\n"
                 "   --port=<port>          ECL services port\n"
                 "   -u, --username=<name>  Username for accessing ecl services\n"
                 "   -pw, --password=<pw>   Password for accessing ecl services\n"

ecl/eclcmd/queries/ecl-queries.cpp

@@ -454,8 +454,13 @@ public:
                 continue;
             if (iter.matchOption(optSourceProcess, ECLOPT_SOURCE_PROCESS))
                 continue;
+
             if (iter.matchFlag(optActivate, ECLOPT_ACTIVATE)||iter.matchFlag(optActivate, ECLOPT_ACTIVATE_S))
                 continue;
+            if (iter.matchFlag(optSourceSSL, ECLOPT_SOURCE_SSL))
+                continue;
+            if (iter.matchFlag(optSourceNoSSL, ECLOPT_SOURCE_NO_SSL))
+                continue;
             if (iter.matchFlag(optSuspendPrevious, ECLOPT_SUSPEND_PREVIOUS)||iter.matchFlag(optSuspendPrevious, ECLOPT_SUSPEND_PREVIOUS_S))
                 continue;
             if (iter.matchFlag(optDeletePrevious, ECLOPT_DELETE_PREVIOUS)||iter.matchFlag(optDeletePrevious, ECLOPT_DELETE_PREVIOUS_S))
@@ -537,6 +542,15 @@ public:
         return true;
     }
 
+    inline bool useSSLForSource()
+    {
+        if (optSourceSSL)
+            return true;
+        if (optSourceNoSSL)
+            return false;
+        return optSSL; //default to whether we use SSL to call ESP
+    }
+
     virtual int processCMD()
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
@@ -565,6 +579,9 @@ public:
         req->setAllowForeignFiles(optAllowForeign);
         req->setIncludeFileErrors(true);
 
+        //default to same tcp/tls as our ESP connection, but can be changed using --source-ssl or --source-no-ssl
+        req->setSourceSSL(useSSLForSource());
+
         if (optTimeLimit != (unsigned) -1)
             req->setTimeLimit(optTimeLimit);
         if (optWarnTimeLimit != (unsigned) -1)
@@ -607,6 +624,8 @@ public:
             "                          in the form: //ip:port/queryset/query\n"
             "                          or: queryset/query\n"
             "   <target>               Name of target cluster to copy the query to\n"
+            "   --source-ssl           Use SSL when connecting to source (default if --ssl is used)\n"
+            "   --source-no-ssl        Do not use SSL when connecting to source (default if --ssl is NOT used)\n"
             "   --no-files             Do not copy DFS file information for referenced files\n"
             "   --daliip=<ip>          Remote Dali DFS to use for copying file information\n"
             "                          (only required if remote environment version < 3.8)\n"
@@ -656,6 +675,8 @@ private:
     bool optDontAppendCluster; //Undesirable but here temporarily because DALI may have locking issues
     bool optDontCopyFiles;
     bool optAllowForeign;
+    bool optSourceSSL = false; //user explicitly turning on SSL for accessing the remote source location (ssl defaults to use SSL if we are hitting ESP via SSL)
+    bool optSourceNoSSL = false; //user explicitly turning OFF SSL for accessing the remote source location (ssl defaults to not use SSL if we are not hitting ESP via SSL)
 };
 
 class EclCmdQueriesCopyQueryset : public EclCmdCommon
@@ -686,6 +707,10 @@ public:
                 }
                 continue;
             }
+            if (iter.matchFlag(optSourceSSL, ECLOPT_SOURCE_SSL))
+                continue;
+            if (iter.matchFlag(optSourceNoSSL, ECLOPT_SOURCE_NO_SSL))
+                continue;
             if (iter.matchOption(optDaliIP, ECLOPT_DALIIP))
                 continue;
             if (iter.matchOption(optSourceProcess, ECLOPT_SOURCE_PROCESS))
@@ -724,6 +749,15 @@ public:
         return true;
     }
 
+    inline bool useSSLForSource()
+    {
+        if (optSourceSSL)
+            return true;
+        if (optSourceNoSSL)
+            return false;
+        return optSSL; //default to whether we use SSL to call ESP
+    }
+
     virtual int processCMD()
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
@@ -744,6 +778,9 @@ public:
         req->setAllowForeignFiles(optAllowForeign);
         req->setIncludeFileErrors(true);
 
+        //default to same tcp/tls as our ESP connection, but can be changed using --source-ssl or --source-no-ssl
+        req->setSourceSSL(useSSLForSource());
+
         Owned<IClientWUCopyQuerySetResponse> resp = client->WUCopyQuerySet(req);
         int ret = outputMultiExceptionsEx(resp->getExceptions());
         if (outputQueryFileCopyErrors(resp->getFileErrors()))
@@ -788,6 +825,8 @@ public:
             "   <source_target>        Name of local (or path to remote) target cluster to"
             "                          copy queries from\n"
             "   <destination_target>   Target cluster to copy queries to\n"
+            "   --source-ssl           Use SSL when connecting to source (default if --ssl is used)\n"
+            "   --source-no-ssl        Do not use SSL when connecting to source (default if --ssl is NOT used)\n"
             "   --all                  Copy both active and inactive queries\n"
             "   --no-files             Do not copy DFS file information for referenced files\n"
             "   --daliip=<ip>          Remote Dali DFS to use for copying file information\n"
@@ -815,6 +854,8 @@ private:
     bool optDontCopyFiles;
     bool optAllowForeign;
     bool optAllQueries;
+    bool optSourceSSL = false; //user explicitly turning on SSL for accessing the remote source location (ssl defaults to use SSL if we are hitting ESP via SSL)
+    bool optSourceNoSSL = false; //user explicitly turning OFF SSL for accessing the remote source location (ssl defaults to not use SSL if we are not hitting ESP via SSL)
 };
 
 class EclCmdQueriesConfig : public EclCmdCommon

esp/scm/ws_workunits_queryset_req_resp.ecm

@@ -432,6 +432,7 @@ ESPrequest [nil_remove] WUQuerySetCopyQueryRequest
     bool UpdateCloneFrom(false); //explicity wan't to change where roxie will grab from
     bool AppendCluster(true); //file exists on other local cluster, add new one, make optional in case of locking issues, but should be made to work
     bool IncludeFileErrors(false);
+    bool SourceSSL(false);
 };
 
 ESPresponse [exceptions_inline] WUQuerySetCopyQueryResponse
@@ -456,6 +457,7 @@ ESPrequest [nil_remove] WUCopyQuerySetRequest
     bool UpdateCloneFrom(false); //explicity wan't to change where roxie will grab from
     bool AppendCluster(true); //file exists on other local cluster, add new one, make optional in case of locking issues, but should be made to work
     bool IncludeFileErrors(false);
+    bool SourceSSL(false);
 };
 
 ESPresponse [exceptions_inline] WUCopyQuerySetResponse

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -65,13 +65,13 @@ void ensureInputString(const char* input, bool lowerCase, StringBuffer& inputStr
         inputStr.toLowerCase();
 }
 
-static IClientWsWorkunits *ensureWsWorkunitsClient(IClientWsWorkunits *ws, IEspContext *ctx, const char *netAddress)
+static IClientWsWorkunits *ensureWsWorkunitsClient(IClientWsWorkunits *ws, IEspContext *ctx, const char *netAddress, bool useSSL)
 {
     if (ws)
         return LINK(ws);
     StringBuffer url;
     if (netAddress && *netAddress)
-        url.appendf("http://%s%s/WsWorkunits", netAddress, (!strchr(netAddress, ':')) ? ":8010" : "");
+        url.appendf("%s://%s%s/WsWorkunits", useSSL ? "https" : "http", netAddress, (!strchr(netAddress, ':')) ? ":8010" : "");
     else
     {
         if (!ctx)
@@ -79,7 +79,7 @@ static IClientWsWorkunits *ensureWsWorkunitsClient(IClientWsWorkunits *ws, IEspC
         StringBuffer ip;
         short port = 0;
         ctx->getServAddress(ip, port);
-        url.appendf("http://%s:%d/WsWorkunits", ip.str(), port);
+        url.appendf("%s://%s:%d/WsWorkunits", useSSL ? "https" : "http", ip.str(), port);
     }
     Owned<IClientWsWorkunits> cws = createWsWorkunitsClient();
     cws->addServiceUrl(url);
@@ -88,9 +88,9 @@ static IClientWsWorkunits *ensureWsWorkunitsClient(IClientWsWorkunits *ws, IEspC
     return cws.getClear();
 }
 
-IClientWUQuerySetDetailsResponse *fetchQueryDetails(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *target, const char *queryid)
+IClientWUQuerySetDetailsResponse *fetchQueryDetails(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *target, const char *queryid, bool useSSL)
 {
-    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress);
+    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress, useSSL);
 
     //using existing WUQuerysetDetails rather than extending WUQueryDetails, to support copying query meta data from prior releases
     Owned<IClientWUQuerySetDetailsRequest> reqQueryInfo = ws->createWUQuerysetDetailsRequest();
@@ -101,9 +101,9 @@ IClientWUQuerySetDetailsResponse *fetchQueryDetails(IClientWsWorkunits *_ws, IEs
     return ws->WUQuerysetDetails(reqQueryInfo);
 }
 
-void fetchRemoteWorkunit(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer)
+void fetchRemoteWorkunit(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer, bool useSSL)
 {
-    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress);
+    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress, useSSL);
     Owned<IClientWULogFileRequest> req = ws->createWUFileRequest();
     if (queryset && *queryset)
         req->setQuerySet(queryset);
@@ -131,11 +131,11 @@ void fetchRemoteWorkunit(IClientWsWorkunits *_ws, IEspContext *ctx, const char *
         ep.getUrlStr(daliServer);
 }
 
-void fetchRemoteWorkunitAndQueryDetails(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer, Owned<IClientWUQuerySetDetailsResponse> &respQueryInfo)
+void fetchRemoteWorkunitAndQueryDetails(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer, Owned<IClientWUQuerySetDetailsResponse> &respQueryInfo, bool useSSL)
 {
-    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress);
-    fetchRemoteWorkunit(ws, ctx, netAddress, queryset, query, wuid, name, xml, dllname, dll, daliServer);
-    respQueryInfo.setown(fetchQueryDetails(ws, ctx, netAddress, queryset, query));
+    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress, useSSL);
+    fetchRemoteWorkunit(ws, ctx, netAddress, queryset, query, wuid, name, xml, dllname, dll, daliServer, useSSL);
+    respQueryInfo.setown(fetchQueryDetails(ws, ctx, netAddress, queryset, query, useSSL));
 }
 
 void doWuFileCopy(IClientFileSpray &fs, IEspWULogicalFileCopyInfo &info, const char *logicalname, const char *cluster, bool isRoxie, bool supercopy)
@@ -2748,12 +2748,12 @@ bool splitQueryPath(const char *path, StringBuffer &netAddress, StringBuffer &qu
     return true;
 }
 
-IPropertyTree *fetchRemoteQuerySetInfo(IEspContext *context, const char *srcAddress, const char *srcTarget)
+IPropertyTree *fetchRemoteQuerySetInfo(IEspContext *context, const char *srcAddress, const char *srcTarget, bool useSSL)
 {
     if (!srcAddress || !*srcAddress || !srcTarget || !*srcTarget)
         return NULL;
 
-    VStringBuffer url("http://%s%s/WsWorkunits/WUQuerysetDetails.xml?ver_=1.51&QuerySetName=%s&FilterType=All", srcAddress, (!strchr(srcAddress, ':')) ? ":8010" : "", srcTarget);
+    VStringBuffer url("%s://%s%s/WsWorkunits/WUQuerysetDetails.xml?ver_=1.51&QuerySetName=%s&FilterType=All", useSSL ? "https" : "http", srcAddress, (!strchr(srcAddress, ':')) ? ":8010" : "", srcTarget);
 
     Owned<IHttpClientContext> httpCtx = getHttpClientContext();
     Owned<IHttpClient> httpclient = httpCtx->createHttpClient(NULL, url);
@@ -2778,11 +2778,11 @@ IPropertyTree *fetchRemoteQuerySetInfo(IEspContext *context, const char *srcAddr
 class QueryCloner
 {
 public:
-    QueryCloner(IEspContext *_context, const char *address, const char *source, const char *_target) :
-        context(_context), target(_target), srcAddress(address)
+    QueryCloner(IEspContext *_context, const char *address, const char *source, const char *_target, bool _useSSL) :
+        context(_context), target(_target), srcAddress(address), useSSL(_useSSL)
     {
         if (srcAddress.length())
-            srcQuerySet.setown(fetchRemoteQuerySetInfo(context, srcAddress, source));
+            srcQuerySet.setown(fetchRemoteQuerySetInfo(context, srcAddress, source, useSSL));
         else
             srcQuerySet.setown(getQueryRegistry(source, true));
         if (!srcQuerySet)
@@ -2824,7 +2824,7 @@ public:
         StringBuffer dllname;
         StringBuffer fetchedName;
         StringBuffer remoteDfs;
-        fetchRemoteWorkunit(NULL, context, srcAddress.str(), NULL, NULL, wuid, fetchedName, xml, dllname, dll, remoteDfs);
+        fetchRemoteWorkunit(NULL, context, srcAddress.str(), NULL, NULL, wuid, fetchedName, xml, dllname, dll, remoteDfs, useSSL);
         deploySharedObject(*context, wuid, dllname, target, queryName, dll, queryDirectory, xml.str(), false);
 
         SCMStringBuffer existingQueryId;
@@ -3042,6 +3042,7 @@ private:
     StringAttr process;
     StringAttr queryDirectory;
     bool cloneFilesEnabled = false;
+    bool useSSL = false;
     unsigned updateFlags = 0;
     StringArray locations;
 
@@ -3070,7 +3071,7 @@ bool CWsWorkunitsEx::onWUCopyQuerySet(IEspContext &context, IEspWUCopyQuerySetRe
 
     DBGLOG("%s copying queryset %s from %s target %s", context.queryUserId(), target, srcAddress.str(), srcTarget.str());
 
-    QueryCloner cloner(&context, srcAddress, srcTarget, target);
+    QueryCloner cloner(&context, srcAddress, srcTarget, target, req.getSourceSSL());
     cloner.setQueryDirectory(queryDirectory);
 
     SCMStringBuffer process;
@@ -3144,7 +3145,7 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
         MemoryBuffer dll;
         StringBuffer dllname;
         StringBuffer queryName;
-        fetchRemoteWorkunitAndQueryDetails(NULL, &context, srcAddress.str(), srcQuerySet.str(), srcQuery.str(), NULL, queryName, xml, dllname, dll, remoteIP, sourceQueryInfoResp);
+        fetchRemoteWorkunitAndQueryDetails(NULL, &context, srcAddress.str(), srcQuerySet.str(), srcQuery.str(), NULL, queryName, xml, dllname, dll, remoteIP, sourceQueryInfoResp, req.getSourceSSL());
         if (sourceQueryInfoResp && sourceQueryInfoResp->getQuerysetQueries().ordinality())
             srcInfo = &sourceQueryInfoResp->getQuerysetQueries().item(0);
         if (srcInfo)
@@ -3157,7 +3158,7 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
     {
         //Could get the atributes without soap call, but this creates a common data structure shared with fetching remote query info
         //Get query attributes before resolveQueryAlias, to avoid deadlock
-        sourceQueryInfoResp.setown(fetchQueryDetails(NULL, &context, NULL, srcQuerySet, srcQuery));
+        sourceQueryInfoResp.setown(fetchQueryDetails(NULL, &context, NULL, srcQuerySet, srcQuery, req.getSourceSSL()));
         if (sourceQueryInfoResp && sourceQueryInfoResp->getQuerysetQueries().ordinality())
             srcInfo = &sourceQueryInfoResp->getQuerysetQueries().item(0);