HPCC-27152 Helm support for generating remote client certificates for services

New remote issuer requires an additional private key pair
"hpcc-remote-issuer-key-pair", which can be created as follows:

> openssl req -x509 -newkey rsa:2048 -nodes -keyout remoteca.key -sha256 -days 1825 -out remoteca.crt -config examples/certmanager/ca-req.cfg

> kubectl create secret tls hpcc-remote-issuer-key-pair --cert=remoteca.crt --key=remoteca.key

Example values config:
---------------------------
esp:
- name: eclwatch
  application: eclwatch
  auth: none
  replicas: 1
  remoteClients:
  - name: myclient
    #organization is ptional
    organization: mycompany

For client, example ecl.ini entries:

ecl.ini
----------------------
eclSSL=true
eclClientCert=/home/jdoe/myclient/tls.crt
eclClientPrivateKey=/home/jdoe/myclient/tls.key
eclCACert=/home/jdoe/myclient/ca.crt

example command line parameters:
> ecl run --ssl --cert=/home/jdoe/myclient/tls.crt --key=/home/jdoe/myclient/tls.key --cacert=/home/jdoe/myclient/ca.crt hthor myquery.ecl

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexisrisk.com>

ecl/ecl-package/ecl-package.cpp

@@ -102,7 +102,7 @@ public:
     {
         Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
         Owned<IClientActivatePackageRequest> request = packageProcessClient->createActivatePackageRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setPackageMap(optPackageMap);
@@ -193,7 +193,7 @@ public:
     {
         Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
         Owned<IClientDeActivatePackageRequest> request = packageProcessClient->createDeActivatePackageRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setPackageMap(optPackageMap);
@@ -263,7 +263,7 @@ public:
     {
         Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
         Owned<IClientListPackageRequest> request = packageProcessClient->createListPackageRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setProcess("*");
@@ -354,7 +354,7 @@ public:
     {
         Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
         Owned<IClientGetPackageRequest> request = packageProcessClient->createGetPackageRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setProcess("*");
@@ -446,7 +446,7 @@ public:
 
         Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
         Owned<IClientDeletePackageRequest> request = packageProcessClient->createDeletePackageRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setPackageMap(optPackageMap);
@@ -579,7 +579,7 @@ public:
         fprintf(stdout, "\n ... adding package map %s now\n\n", optFileName.str());
 
         Owned<IClientAddPackageRequest> request = packageProcessClient->createAddPackageRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setActivate(optActivate);
         request->setInfo(pkgInfo);
@@ -736,7 +736,7 @@ public:
         fprintf(stdout, "\n ... copy package map %s to %s\n\n", optSrcPath.str(), optTarget.str());
 
         Owned<IClientCopyPackageMapRequest> request = packageProcessClient->createCopyPackageMapRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setSourcePath(optSrcPath);
         request->setTarget(optTarget);
@@ -920,7 +920,7 @@ public:
     {
         Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
         Owned<IClientValidatePackageRequest> request = packageProcessClient->createValidatePackageRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         if (optFileName.length())
         {
@@ -1143,7 +1143,7 @@ public:
     {
         Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
         Owned<IClientGetQueryFileMappingRequest> request = packageProcessClient->createGetQueryFileMappingRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setQueryName(optQueryId);
@@ -1304,7 +1304,7 @@ public:
         fprintf(stdout, "\n ... adding packagemap %s part %s from file %s\n\n", optPMID.get(), optPartName.get(), optFileName.get());
 
         Owned<IClientAddPartToPackageMapRequest> request = packageProcessClient->createAddPartToPackageMapRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setPackageMap(optPMID);
@@ -1442,7 +1442,7 @@ public:
 
         Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
         Owned<IClientRemovePartFromPackageMapRequest> request = packageProcessClient->createRemovePartFromPackageMapRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setPackageMap(optPMID);
@@ -1542,7 +1542,7 @@ public:
     {
         Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
         Owned<IClientGetPartFromPackageMapRequest> request = packageProcessClient->createGetPartFromPackageMapRequest();
-        setCmdRequestTimeouts(request->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(request->rpc());
 
         request->setTarget(optTarget);
         request->setPackageMap(optPMID);

ecl/eclcmd/eclcmd_common.cpp

@@ -354,8 +354,24 @@ eclCmdOptionMatchIndicator EclCmdCommon::matchCommandLineOption(ArgvIterator &it
     if (iter.matchFlag(optVerbose, ECLOPT_VERBOSE) || iter.matchFlag(optVerbose, ECLOPT_VERBOSE_S))
         return EclCmdOptionMatch;
     if (iter.matchFlag(optSSL, ECLOPT_SSL) || iter.matchFlag(optSSL, ECLOPT_SSL_S))
+    {
+        sslOptProvided = true;
+        return EclCmdOptionMatch;
+    }
+
+    if (iter.matchOption(optClientCert, ECLOPT_CLIENT_CERT))
+        return EclCmdOptionMatch;
+    if (iter.matchOption(optClientPrivateKey, ECLOPT_CLIENT_PRIVATE_KEY))
+        return EclCmdOptionMatch;
+    if (iter.matchOption(optCACert, ECLOPT_CA_CERT))
         return EclCmdOptionMatch;
 
+    if (iter.matchFlag(optAcceptSelfSigned, ECLOPT_ACCEPT_SELFSIGNED))
+    {
+        selfsignedOptProvided = true;
+        return EclCmdOptionMatch;
+    }
+
     StringAttr tempArg;
     if (iter.matchOption(tempArg, "-brk"))
     {
@@ -378,6 +394,14 @@ bool EclCmdCommon::finalizeOptions(IProperties *globals)
     extractEclCmdOption(optServer, globals, ECLOPT_SERVER_ENV, ECLOPT_SERVER_INI, ECLOPT_SERVER_DEFAULT, NULL);
     extractEclCmdOption(optPort, globals, ECLOPT_PORT_ENV, ECLOPT_PORT_INI, ECLOPT_PORT_DEFAULT, NULL);
     extractEclCmdOption(optUsername, globals, ECLOPT_USERNAME_ENV, ECLOPT_USERNAME_INI, NULL, NULL);
+    extractEclCmdOption(optClientCert, globals, ECLOPT_CLIENT_CERT_ENV, ECLOPT_CLIENT_CERT_INI, NULL, NULL);
+    extractEclCmdOption(optClientPrivateKey, globals, ECLOPT_CLIENT_PRIVATE_KEY_ENV, ECLOPT_CLIENT_PRIVATE_KEY_INI, NULL, NULL);
+    extractEclCmdOption(optCACert, globals, ECLOPT_CA_CERT_ENV, ECLOPT_CA_CERT_INI, NULL, NULL);
+    if (!sslOptProvided)
+        extractEclCmdOption(optSSL, globals, ECLOPT_SSL_ENV, ECLOPT_SSL_INI, false);
+    if (!selfsignedOptProvided)
+        extractEclCmdOption(optAcceptSelfSigned, globals, ECLOPT_ACCEPT_SELFSIGNED_ENV, ECLOPT_ACCEPT_SELFSIGNED_INI, false);
+
 
     //if an empty password was explicitly provided, optPasswordProvided would still be set
     if (!optPasswordProvided)
@@ -390,7 +414,11 @@ bool EclCmdCommon::finalizeOptions(IProperties *globals)
         if (pw.length())
             optPassword.set(pw);
     }
-
+    if (!optClientCert.isEmpty() && optClientPrivateKey.isEmpty())
+    {
+        fprintf(stdout, "\n ... Adding a client certificate requires adding a client private key.\n");
+        return false;
+    }
     if (!optVerbose)
     {
         Owned<ILogMsgFilter> filter = getCategoryLogMsgFilter(MSGAUD_user, MSGCLS_error);

ecl/eclcmd/eclcmd_common.hpp

@@ -51,10 +51,31 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_SERVER_INI "eclWatchIP"
 #define ECLOPT_SERVER_ENV "ECL_WATCH_IP"
 #define ECLOPT_SERVER_DEFAULT "."
+
 #define ECLOPT_SSL "--ssl"
 #define ECLOPT_SSL_S "-ssl"
+#define ECLOPT_SSL_INI "eclSSL"
+#define ECLOPT_SSL_ENV "ECL_SSL"
+
+//The following TLS options could be made more verbose, but these match CURL for convenience
+#define ECLOPT_CLIENT_CERT "--cert"
+#define ECLOPT_CA_CERT "--cacert"
+#define ECLOPT_CLIENT_PRIVATE_KEY "--key"
+
+#define ECLOPT_CLIENT_CERT_INI "eclClientCert"
+#define ECLOPT_CLIENT_CERT_ENV "ECL_CLIENT_CERT"
+#define ECLOPT_CLIENT_PRIVATE_KEY_INI "eclClientPrivateKey"
+#define ECLOPT_CLIENT_PRIVATE_KEY_ENV "ECL_CLIENT_PRIVATE_KEY"
+
+#define ECLOPT_CA_CERT_INI "eclCACert"
+#define ECLOPT_CA_CERT_ENV "ECL_CA_CERT"
+
+#define ECLOPT_ACCEPT_SELFSIGNED "--accept-self-signed"
+#define ECLOPT_ACCEPT_SELFSIGNED_INI "eclAcceptSelfSigned"
+#define ECLOPT_ACCEPT_SELFSIGNED_ENV "ECL_ACCEPT_SELF_SIGNED"
 
 #define ECLOPT_SOURCE_SSL "--source-ssl"
+
 #define ECLOPT_SOURCE_NO_SSL "--source-no-ssl"
 
 #define ECLOPT_PORT "--port"
@@ -264,6 +285,11 @@ public:
     }
     virtual eclCmdOptionMatchIndicator matchCommandLineOption(ArgvIterator &iter, bool finalAttempt=false);
     virtual bool finalizeOptions(IProperties *globals);
+    inline void setRpcOptions(IEspClientRpcSettings &rpc, unsigned int waitMS = 0)
+    {
+        setRpcRequestTimeouts(rpc, waitMS, optWaitConnectMs, optWaitReadSec);
+        setRpcSSLOptions(rpc, optSSL, optClientCert, optClientPrivateKey, optCACert, optAcceptSelfSigned);
+    }
 
     virtual void usage()
     {
@@ -275,6 +301,10 @@ public:
             fprintf(stdout,
                 "   -s, --server=<ip>      IP of server running ecl services (eclwatch)\n"
                 "   -ssl, --ssl            Use SSL to secure the connection to the server(s)\n"
+                "   --accept-self-signed   Allow SSL servers to use self signed certificates\n"
+                "   --cert                 Path to file containing SSL client certificate\n"
+                "   --key                  Path to file containing SSL client certificate private key\n"
+                "   --cacert               Path to file containing SSL CA certificate\n"
                 "   --port=<port>          ECL services port\n"
                 "   -u, --username=<name>  Username for accessing ecl services\n"
                 "   -pw, --password=<pw>   Password for accessing ecl services\n"
@@ -287,11 +317,18 @@ public:
     StringAttr optPort;
     StringAttr optUsername;
     StringAttr optPassword;
+    StringAttr optClientCert;
+    StringAttr optClientPrivateKey;
+    StringAttr optCACert;
+
     bool optPasswordProvided = false;
     unsigned optWaitConnectMs = 0;
     unsigned optWaitReadSec = 0;
     bool optVerbose;
     bool optSSL;
+    bool sslOptProvided = false;
+    bool optAcceptSelfSigned = false;
+    bool selfsignedOptProvided = false;
     bool usesESP;
 };
 
@@ -421,14 +458,4 @@ template <class Iface> Iface *intClient(Iface *client, EclCmdCommon &cmd, const
 #define createCmdClient(SN, cmd) intClient<IClient##SN>(create##SN##Client(), cmd, #SN, NULL);
 #define createCmdClientExt(SN, cmd, urlTail) intClient<IClient##SN>(create##SN##Client(), cmd, #SN, urlTail);
 
-inline void setCmdRequestTimeouts(IEspClientRpcSettings &rpc, unsigned waitMs, unsigned waitConnectMs, unsigned waitReadSec)
-{
-    if (waitMs==(unsigned)-1)
-        waitMs=0;
-    if (waitConnectMs || waitMs)
-        rpc.setConnectTimeOutMs(waitConnectMs ? waitConnectMs : waitMs);
-    if (waitReadSec || waitMs)
-        rpc.setReadTimeOutSecs(waitReadSec ? waitReadSec : (waitMs / 1000));
-}
-
 #endif

ecl/eclcmd/eclcmd_core.cpp

@@ -66,12 +66,12 @@ void expandDefintionsAsDebugValues(const IArrayOf<IEspNamedValue> & definitions,
 
 }
 
-void checkFeatures(IClientWsWorkunits *client, bool &useCompression, int &major, int &minor, int &point, unsigned waitMs, unsigned waitConnectMs, unsigned waitReadSec)
+void checkFeatures(EclCmdCommon &cmd, IClientWsWorkunits *client, bool &useCompression, int &major, int &minor, int &point, unsigned waitMs, unsigned waitConnectMs, unsigned waitReadSec)
 {
     try
     {
         Owned<IClientWUCheckFeaturesRequest> req = client->createWUCheckFeaturesRequest();
-        setCmdRequestTimeouts(req->rpc(), waitMs, waitConnectMs, waitReadSec);
+        cmd.setRpcOptions(req->rpc(), waitMs);
         Owned<IClientWUCheckFeaturesResponse> resp = client->WUCheckFeatures(req);
         useCompression = resp->getDeployment().getUseCompression();
         major = resp->getBuildVersionMajor();
@@ -93,7 +93,7 @@ bool doDeploy(EclCmdWithEclTarget &cmd, IClientWsWorkunits *client, unsigned wai
     int minor = 0;
     int point = 0;
     bool useCompression = false;
-    checkFeatures(client, useCompression, major, minor, point, 0, cmd.optWaitConnectMs, cmd.optWaitReadSec);
+    checkFeatures(cmd, client, useCompression, major, minor, point, 0, cmd.optWaitConnectMs, cmd.optWaitReadSec);
 
     bool compressed = false;
     if (useCompression)
@@ -110,7 +110,7 @@ bool doDeploy(EclCmdWithEclTarget &cmd, IClientWsWorkunits *client, unsigned wai
 
     StringBuffer objType(compressed ? "compressed_" : ""); //change compressed type string so old ESPs will fail gracefully
     Owned<IClientWUDeployWorkunitRequest> req = client->createWUDeployWorkunitRequest();
-    setCmdRequestTimeouts(req->rpc(), waitMs, cmd.optWaitConnectMs, cmd.optWaitReadSec);
+    cmd.setRpcOptions(req->rpc(), waitMs);
 
     switch (cmd.optObj.type)
     {
@@ -444,7 +444,7 @@ public:
 
         unsigned clientRemaining = remaining + 100; //give the ESP method time to return from timeout before hard client stop
         Owned<IClientWUPublishWorkunitRequest> req = client->createWUPublishWorkunitRequest();
-        setCmdRequestTimeouts(req->rpc(), clientRemaining, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), clientRemaining);
         req->setWuid(wuid.str());
         if (optDeletePrevious)
             req->setActivate(CWUQueryActivationMode_ActivateDeletePrevious);
@@ -783,7 +783,7 @@ public:
             fputs("Getting Workunit Information\n", stdout);
 
         Owned<IClientWUInfoRequest> req = client->createWUInfoRequest();
-        setCmdRequestTimeouts(req->rpc(), optWaitTime, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optWaitTime);
 
         req->setWuid(wuid);
         req->setIncludeExceptions(true);
@@ -839,7 +839,7 @@ public:
         if (optVerbose)
             fputs("Retrieving Results\n", stdout);
         Owned<IClientWUFullResultRequest> req = client->createWUFullResultRequest();
-        setCmdRequestTimeouts(req->rpc(), optWaitTime, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optWaitTime);
 
         req->setWuid(wuid);
         req->setNoRootTag(optNoRoot);
@@ -864,7 +864,7 @@ public:
         int minor = 0;
         int point = 0;
         bool useCompression = false;
-        checkFeatures(client, useCompression, major, minor, point, optWaitTime, optWaitConnectMs, optWaitReadSec);
+        checkFeatures(*this, client, useCompression, major, minor, point, optWaitTime, optWaitConnectMs, optWaitReadSec);
         bool optimized = !optPre64 && ((major>=7) || (major==6 && minor>=3));
 
         try
@@ -925,7 +925,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClientExt(WsWorkunits, *this, "?upload_"); //upload_ disables maxRequestEntityLength
         Owned<IClientWURunRequest> req = client->createWURunRequest();
-        setCmdRequestTimeouts(req->rpc(), optWaitTime, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optWaitTime);
         req->setCloneWorkunit(true);
         req->setNoRootTag(optNoRoot);
 
@@ -1155,7 +1155,7 @@ public:
     int gatherLegacyServerResults(IClientWsWorkunits* client, const char *wuid)
     {
         Owned<IClientWUInfoRequest> req = client->createWUInfoRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setWuid(wuid);
         req->setIncludeExceptions(true);
@@ -1205,7 +1205,7 @@ public:
     int getAndOutputResults(IClientWsWorkunits* client, const char *wuid)
     {
         Owned<IClientWUFullResultRequest> req = client->createWUFullResultRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setWuid(wuid);
         req->setNoRootTag(optNoRoot);
@@ -1226,7 +1226,7 @@ public:
         int minor = 0;
         int point = 0;
         bool useCompression = false;
-        checkFeatures(client, useCompression, major, minor, point, 0, optWaitConnectMs, optWaitReadSec);
+        checkFeatures(*this, client, useCompression, major, minor, point, 0, optWaitConnectMs, optWaitReadSec);
 
         if (!optPre64 && (major>=6 && minor>=3))
             return getAndOutputResults(client, optWuid);
@@ -1284,7 +1284,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQuerySetQueryActionRequest> req = client->createWUQuerysetQueryActionRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         IArrayOf<IEspQuerySetQueryActionItem> queries;
         Owned<IEspQuerySetQueryActionItem> item = createQuerySetQueryActionItem();
@@ -1334,7 +1334,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQuerySetQueryActionRequest> req = client->createWUQuerysetQueryActionRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setQuerySetName(optQuerySet.get());
         req->setAction("Delete");
@@ -1384,7 +1384,7 @@ public:
         StringBuffer s;
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQuerySetAliasActionRequest> req = client->createWUQuerysetAliasActionRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         IArrayOf<IEspQuerySetAliasActionItem> aliases;
         Owned<IEspQuerySetAliasActionItem> item = createQuerySetAliasActionItem();
@@ -1502,7 +1502,7 @@ public:
 
         // Abort
         Owned<IClientWUAbortRequest> req = client->createWUAbortRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
 
         req->setWuids(wuids);
@@ -1603,7 +1603,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQueryRequest> req = client->createWUQueryRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
 
         if (optName.isEmpty())
@@ -1688,7 +1688,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQueryRequest> req = client->createWUQueryRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         if (optName.isEmpty())
             return 0;
@@ -1780,7 +1780,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQueryRequest> req = client->createWUQueryRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         if (optName.isEmpty())
         {

ecl/eclcmd/queries/ecl-queries.cpp

@@ -256,7 +256,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUMultiQuerySetDetailsRequest> req = client->createWUMultiQuerysetDetailsRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setQuerySetName(optTargetCluster.get());
         req->setClusterName(optTargetCluster.get());
@@ -388,7 +388,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQueryFilesRequest> req = client->createWUQueryFilesRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setTarget(optTarget.get());
         req->setQueryId(optQuery.get());
@@ -569,7 +569,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQuerySetCopyQueryRequest> req = client->createWUQuerysetCopyQueryRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setSource(optSourceQueryPath.get());
         req->setTarget(optTargetCluster.get());
@@ -776,7 +776,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUCopyQuerySetRequest> req = client->createWUCopyQuerySetRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setActiveOnly(!optAllQueries);
         req->setSource(optSourceQuerySet.get());
@@ -947,7 +947,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQueryConfigRequest> req = client->createWUQueryConfigRequest();
-        setCmdRequestTimeouts(req->rpc(), optMsToWait, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optMsToWait);
 
         req->setTarget(optTargetCluster.get());
         req->setQueryId(optQueryId.get());
@@ -1154,7 +1154,7 @@ public:
 
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this); //upload_ disables maxRequestEntityLength
         Owned<IClientWURecreateQueryRequest> req = client->createWURecreateQueryRequest();
-        setCmdRequestTimeouts(req->rpc(), optMsToWait, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optMsToWait);
 
         if (optDeletePrevious)
             req->setActivate(CWUQueryActivationMode_ActivateDeletePrevious);
@@ -1361,7 +1361,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQuerysetExportRequest> req = client->createWUQuerysetExportRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setTarget(optTarget);
         req->setActiveOnly(optActiveOnly);
@@ -1493,7 +1493,7 @@ public:
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
         Owned<IClientWUQuerysetImportRequest> req = client->createWUQuerysetImportRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         MemoryBuffer compressed;
         fastLZCompressToBuffer(compressed, content.length()+1, content.str());

ecl/eclcmd/roxie/ecl-roxie.cpp

@@ -184,7 +184,7 @@ public:
     {
         Owned<IClientWsSMC> client = createCmdClient(WsSMC, *this);
         Owned<IClientRoxieControlCmdRequest> req = client->createRoxieControlCmdRequest();
-        setCmdRequestTimeouts(req->rpc(), optMsToWait, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optMsToWait);
 
         req->setWait(optMsToWait);
         req->setProcessCluster(optProcess);
@@ -294,7 +294,7 @@ public:
     {
         Owned<IClientWsSMC> client = createCmdClient(WsSMC, *this);
         Owned<IClientRoxieControlCmdRequest> req = client->createRoxieControlCmdRequest();
-        setCmdRequestTimeouts(req->rpc(), optMsToWait, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optMsToWait);
 
         req->setWait(optMsToWait);
         req->setProcessCluster(optProcess);
@@ -418,7 +418,7 @@ public:
     {
         Owned<IClientWsSMC> client = createCmdClient(WsSMC, *this);
         Owned<IClientRoxieXrefCmdRequest> req = client->createRoxieXrefCmdRequest();
-        setCmdRequestTimeouts(req->rpc(), optMsToWait, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc(), optMsToWait);
 
         req->setWait(optMsToWait);
         req->setRoxieCluster(optCluster);
@@ -520,7 +520,7 @@ public:
     {
         Owned<IClientWsDFUXRef> client = createCmdClientExt(WsDFUXRef, *this, "?ver_=1.29");
         Owned<IClientDFUXRefUnusedFilesRequest> req = client->createDFUXRefUnusedFilesRequest();
-        setCmdRequestTimeouts(req->rpc(), 0, optWaitConnectMs, optWaitReadSec);
+        setRpcOptions(req->rpc());
 
         req->setProcessCluster(optProcess);
         req->setCheckPackageMaps(optCheckPackageMaps);

esp/bindings/SOAP/Platform/soapbind.cpp

@@ -269,6 +269,30 @@ int CHttpSoapBinding::HandleSoapRequest(CHttpRequest* request, CHttpResponse* re
 }
 
 
+static IPropertyTree *createSecClientConfig(const char *clientCertPath, const char *clientPrivateKey, const char *caCertsPath, bool acceptSelfSigned)
+{
+    Owned<IPropertyTree> info = createPTree();
+
+    if (!isEmptyString(clientCertPath))
+    {
+        info->setProp("certificate", clientCertPath);
+        if (!isEmptyString(clientPrivateKey))
+            info->setProp("privatekey", clientPrivateKey);
+    }
+
+    IPropertyTree *verify = ensurePTree(info, "verify");
+    if (!isEmptyString(caCertsPath))
+    {
+        IPropertyTree *ca = ensurePTree(verify, "ca_certificates");
+        ca->setProp("@path", caCertsPath);
+    }
+    verify->setPropBool("@enable", true);
+    verify->setPropBool("@accept_selfsigned", acceptSelfSigned);
+    verify->setProp("trusted_peers", "anyone");
+
+    return info.getClear();
+}
+
 void CSoapRequestBinding::post(const char *proxy, const char* url, IRpcResponseBinding& response, const char *soapaction)
 {
     CRpcCall rpccall;
@@ -286,6 +310,8 @@ void CSoapRequestBinding::post(const char *proxy, const char* url, IRpcResponseB
         soapclient.setReadTimeoutSecs(readTimeoutSecs_);
     if (mtls_secret_.length())
         soapclient.setMtlsSecretName(mtls_secret_);
+    if (client_cert_.length() || ca_certs_.length() || accept_self_signed_)
+        soapclient.setSecureSocketConfig(createSecClientConfig(client_cert_, client_priv_key_, ca_certs_, accept_self_signed_));
 
     soapclient.setUsernameToken(soap_getUserId(), soap_getPassword(), soap_getRealm());
 

esp/bindings/SOAP/Platform/soapbind.hpp

@@ -132,6 +132,11 @@ private:
     StringBuffer password_;
     StringBuffer realm_;
     StringBuffer mtls_secret_;
+    StringBuffer client_cert_;
+    StringBuffer client_priv_key_;
+    StringBuffer ca_certs_;
+    bool accept_self_signed_ = false;
+
     unsigned connectTimeoutMs_ = 0;
     unsigned readTimeoutSecs_ = 0;
 
@@ -176,6 +181,14 @@ public:
     void setMtlsSecretName(const char *name){mtls_secret_.set(name);}
     const char *getMtlsSecretName(){return mtls_secret_.str();}
 
+    void setCACertificates(const char *path) override {ca_certs_.set(path);}
+    virtual void setClientCertificate(const char *certPath, const char *privateKeyPath) override
+    {
+        client_cert_.set(certPath);
+        client_priv_key_.set(privateKeyPath);
+    }
+    virtual void setAcceptSelfSigned(bool acceptSelfSigned) override {accept_self_signed_=acceptSelfSigned;}
+
     void post(const char *proxy, const char* url, IRpcResponseBinding& response, const char *action=NULL);
 
     void post(IRpcResponseBinding& response)
@@ -189,6 +202,39 @@ public:
     }
 };
 
+inline void setRpcSSLOptions(IEspClientRpcSettings &rpc, bool useSSL, const char *clientCert, const char *clientPrivateKey, const char *caCert, bool acceptSelfSigned)
+{
+    if (useSSL)
+    {
+        if (!isEmptyString(clientCert))
+        {
+            if (!checkFileExists(clientCert))
+                throw makeStringExceptionV(-1,"Client certificate not found %s.", clientCert);
+            if (isEmptyString(clientPrivateKey))
+                throw makeStringException(-1,"Client private key not provided.");
+            if (!checkFileExists(clientPrivateKey))
+                throw makeStringExceptionV(-1,"Client private key not found %s.", clientPrivateKey);
+            rpc.setClientCertificate(clientCert, clientPrivateKey);
+        }
+        if (!isEmptyString(caCert))
+        {
+            if (!checkFileExists(caCert))
+                throw makeStringExceptionV(-1,"CA certificate not found %s.", caCert);
+            rpc.setCACertificates(caCert);
+        }
+        rpc.setAcceptSelfSigned(acceptSelfSigned);
+    }
+}
+
+inline void setRpcRequestTimeouts(IEspClientRpcSettings &rpc, unsigned waitMs, unsigned waitConnectMs, unsigned waitReadSec)
+{
+    if (waitMs==(unsigned)-1)
+        waitMs=0;
+    if (waitConnectMs || waitMs)
+        rpc.setConnectTimeOutMs(waitConnectMs ? waitConnectMs : waitMs);
+    if (waitReadSec || waitMs)
+        rpc.setReadTimeOutSecs(waitReadSec ? waitReadSec : (waitMs / 1000));
+}
 
 class esp_http_decl CSoapBinding : public CEspBinding
 {

esp/bindings/SOAP/client/soapclient.cpp

@@ -145,10 +145,14 @@ int CSoapClient::postRequest(const char* contenttype, const char* soapaction, IR
     Owned<CSoapResponse> soap_response;
     soap_response.setown(new CSoapResponse);
 
+    Owned<IHttpClientContext> httpctx;
     //Send request and get response
     if(m_transportclient.get() == NULL)
     {
-        Owned<IHttpClientContext> httpctx = getHttpClientSecretContext(m_mtls_secret);
+        if (m_sec_config)
+            httpctx.setown(createHttpClientContext(LINK(m_sec_config)));
+        else
+            httpctx.setown(getHttpClientSecretContext(m_mtls_secret));
         Owned<IHttpClient> httpclient = httpctx->createHttpClient(call->getProxy(), call->get_url());
         if (m_disableKeepAlive)
             httpclient->disableKeepAlive();
@@ -357,7 +361,11 @@ int CSoapClient::postRequest(IRpcMessage & rpccall, StringBuffer & responsebuf,
     //Send request and get response
     if(m_transportclient.get() == NULL)
     {
-        Owned<IHttpClientContext> httpctx = getHttpClientContext();
+        Owned<IHttpClientContext> httpctx;
+        if (m_sec_config)
+            httpctx.setown(createHttpClientContext(LINK(m_sec_config)));
+        else
+            httpctx.setown(getHttpClientContext());
         IHttpClient* httpclient = httpctx->createHttpClient(call->getProxy(), call->get_url());
         m_transportclient.setown(httpclient);
         if (m_disableKeepAlive)

esp/bindings/SOAP/client/soapclient.hpp

@@ -37,6 +37,7 @@ private:
     StringAttr                  m_realm;
     StringAttr                  m_mtls_secret;
     bool                        m_disableKeepAlive;
+    Owned<IPropertyTree> m_sec_config;
     Owned<ITransportClient> m_transportclient;
     unsigned m_connectTimeoutMs = 0;
     unsigned m_readTimeoutSecs = 0;
@@ -75,6 +76,11 @@ public:
         return m_mtls_secret.str();
     }
 
+    void setSecureSocketConfig(IPropertyTree *config)
+    {
+        m_sec_config.setown(config);
+    }
+
     virtual int setUsernameToken(const char* userid, const char* password, const char* realm);
     virtual void disableKeepAlive() { m_disableKeepAlive = true;}
 

esp/scm/esp.ecm

@@ -332,6 +332,9 @@ SCMinterface IEspClientRpcSettings(IEspStruct)
     unsigned getReadTimeOutSecs();
     void setMtlsSecretName(const char *name);
     const char *getMtlsSecretName();
+    void setClientCertificate(const char *certPath, const char *privateKeyPath);
+    void setCACertificates(const char *path);
+    void setAcceptSelfSigned(bool accept);
 };
 
 SCMinterface IEspResponse(IEspStruct)

helm/hpcc/templates/_helpers.tpl

@@ -826,6 +826,10 @@ Generate instance queue names
 {{ end -}}
 {{- end -}}
 
+{{- define "hpcc.usesRemoteClientCertificates" -}}
+  {{- if (hasKey . "remoteClients") -}}{{- if (.remoteClients) -}} true {{- end -}}{{- end -}}
+{{- end -}}
+
 {{/*
 Generate service entries for TLS
 */}}
@@ -838,7 +842,8 @@ Generate service entries for TLS
     {{- if and ($externalService) (hasKey .component "certificate") }}
   tls: true
     {{- else }}
-      {{- $issuerName := ternary "public" "local" $externalService }}
+      {{- $externalIssuerName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" . )) -}}
+      {{- $issuerName := ternary $externalIssuerName "local" $externalService }}
       {{- $certificates := (.root.Values.certificates | default dict) -}}
       {{- if not $certificates.enabled }}
   tls: false
@@ -883,7 +888,7 @@ Generate list of available services
   class: esp
   type: {{ $esp.application }}
   port: {{ $esp.service.servicePort }}
-  {{- include "hpcc.addTLSServiceEntries" (dict "root" $ "service" $esp "component" $esp "visibility" $esp.service.visibility) }}
+  {{- include "hpcc.addTLSServiceEntries" (dict "root" $ "service" $esp "component" $esp "visibility" $esp.service.visibility "remoteClients" $esp.remoteClients) }}
 {{ end -}}
 {{- range $dali := $.Values.dali -}}
 {{- $sashaServices := $dali.services | default dict -}}
@@ -1341,7 +1346,8 @@ use "public" or "local"
 {{- define "hpcc.addCertificate" }}
 {{- if (.root.Values.certificates | default dict).enabled -}}
 {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
-{{- $issuerName := .issuer | default (ternary "public" "local" $externalCert) -}}
+{{- $externalIssuerName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" . )) -}}
+{{- $issuerName := .issuer | default (ternary $externalIssuerName "local" $externalCert) -}}
 {{- if eq (include "hpcc.isIssuerEnabled" (dict "root" .root "issuer" $issuerName)) "true" -}}
 {{- $issuer := get .root.Values.certificates.issuers $issuerName -}}
 {{- if $issuer -}}
@@ -1408,6 +1414,121 @@ spec:
 {{- end }}
 
 {{/*
+Builds the commonName for a client certificate.  Used in creation of both certificate and access control list.
+  Pass in root, client (name), instance (myeclwatch), component (eclwatch), visibility, external (bool, optional)
+*/}}
+{{- define "hpcc.getClientCommonName" -}}
+ {{- if (.root.Values.certificates | default dict).enabled -}}
+  {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
+  {{- $issuerName := .issuer | default (ternary "remote" "local" $externalCert) -}}
+  {{- if ne (include "hpcc.isIssuerEnabled" (dict "root" .root "issuer" $issuerName)) "true" -}}
+   {{- $_ := fail (printf "Issuer '%s' for client certificates not enabled." $issuerName) -}}
+  {{- else -}}
+   {{- $issuer := get .root.Values.certificates.issuers $issuerName -}}
+   {{- if not $issuer -}}
+    {{- $_ := fail (printf "Issuer '%s' for client certificates not found." $issuerName) -}}
+   {{- else -}}
+    {{- $namespace := .root.Release.Namespace -}}
+    {{- $service := (.service | default dict) -}}
+    {{- $domain := ( $service.domain | default $issuer.domain | default $namespace | default "default" ) -}}
+    {{- .client }}@{{ .instance }}.{{ .component }}.{{ $domain }}
+   {{- end -}}
+  {{- end -}}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+Turns an array of remoteClients into a | delimited string to be used for the trusted_peers element of SecureSocket settings.
+  Pass in root, remoteClients, instance (myeclwatch), component (eclwatch), visibility
+*/}}
+{{- define "hpcc.getTrustedPeerString" -}}
+ {{- if not .remoteClients -}}
+  anyone
+ {{- else -}}
+  {{/* Turn remoteClients array into one single array element which is a | delimited string */}}
+  {{- $instance := .instance -}}
+  {{- $component := .component -}}
+  {{- $visibility := .visibility -}}
+  {{- $root := .root }}
+  {{- range $remoteClient := .remoteClients -}}
+   {{- include "hpcc.getClientCommonName" (dict "root" $root "client" $remoteClient.name "instance" $instance "component" $component "visibility" $visibility) -}}|
+  {{- end -}}
+ {{- end -}}
+{{- end }}
+
+{{/*
+Use cert-manager to create a public certificate and private key for use as
+remote client certificates.
+Adding the following to ESP (Roxie support to be added later)
+  remoteClients:
+  - name: myRemoteClient
+Will generate certificates that can be deployed to the remote client.
+Will cause ESP to require client certificates when a socket connects.
+Will create a TLS based access control list which ESP will check to make sure a connections client certificate is enabled.
+
+Pass in root, client (name), organization (optional), instance (myeclwatch), component (eclwatch), visibility
+*/}}
+{{- define "hpcc.addClientCertificate" }}
+ {{- if (.root.Values.certificates | default dict).enabled -}}
+  {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
+  {{- $issuerName := .issuer | default (ternary "remote" "local" $externalCert) -}}
+  {{- if eq (include "hpcc.isIssuerEnabled" (dict "root" .root "issuer" $issuerName)) "true" -}}
+   {{- $issuer := get .root.Values.certificates.issuers $issuerName -}}
+   {{- if not $issuer -}}
+    {{- $_ := fail (printf "Issuer %s for client certificates not found." $issuerName) -}}
+   {{- else -}}
+    {{- if not $issuer.enabled -}}
+     {{- $_ := fail (printf "Issuer %s for client certificates not enabled." $issuerName) -}}
+    {{- end }}
+    {{- $namespace := .root.Release.Namespace -}}
+    {{- $service := (.service | default dict) -}}
+    {{- $domain := ( $service.domain | default $issuer.domain | default $namespace | default "default" ) -}}
+    {{- $instance := .instance -}}
+    {{- $component := .component -}}
+    {{- $client := .client -}}
+    {{- $organization := .organization -}}
+    {{- if not $externalCert -}}
+     {{- $_ := fail (printf "Remote certificate defined for non external facing service %s - %s." $component $instance) -}}
+    {{- end }}
+
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: client-{{ $issuerName }}-{{ $component }}-{{ $instance }}-{{ $client }}-cert
+  namespace: {{ $namespace }}
+spec:
+  # Secret names are always required.
+  secretName: client-{{ $issuerName }}-{{ $component }}-{{ $instance }}-{{ $client }}-tls
+  duration: 2160h # 90d
+  renewBefore: 360h # 15d
+  subject:
+    organizations:
+    {{- if $organization }}
+    - {{ $organization }}
+    {{- else }}
+    - HPCC Client
+    {{- end }}
+  commonName: {{ include "hpcc.getClientCommonName" . }}
+  isCA: false
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS1
+    size: 2048
+  usages:
+    - client auth
+  uris:
+  - spiffe://hpcc-client.{{ $client }}/{{ $domain }}/{{ $component }}/{{ $instance }}
+  issuerRef:
+    name: {{ $issuer.name }}
+    kind: {{ $issuer.kind }}
+    group: cert-manager.io
+---
+   {{- end }}
+  {{- end }}
+ {{- end }}
+{{- end }}
+
+{{/*
 Experimental: Use certmanager to generate a key for roxie udp encryption.
 A public certificate and private key are generated under /opt/HPCCSystems/secrets/certificates/udp.
 Current udp encryption design would only use the private key.
@@ -1465,7 +1586,8 @@ use "public" or "local"
 */}}
 {{- define "hpcc.addCertificateVolumeMount" -}}
 {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
-{{- $issuerName := .issuer | default (ternary "public" "local" $externalCert) -}}
+{{- $externalIssuerName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" . )) -}}
+{{- $issuerName := .issuer | default (ternary $externalIssuerName "local" $externalCert) -}}
 {{- /*
     A .certificate parameter means the user explicitly configured a certificate to use
     otherwise check if certificate generation is enabled
@@ -1488,7 +1610,8 @@ use "public" or "local"
 */}}
 {{- define "hpcc.addCertificateVolume" -}}
 {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
-{{- $issuerName := .issuer | default (ternary "public" "local" $externalCert) -}}
+{{- $externalIssuerName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" . )) -}}
+{{- $issuerName := .issuer | default (ternary $externalIssuerName "local" $externalCert) -}}
 {{- /*
     A .certificate parameter means the user explicitly configured a certificate to use
     otherwise check if certificate generation is enabled

helm/hpcc/templates/esp.yaml

@@ -38,14 +38,24 @@ data:
 {{- include "hpcc.generateMetricsConfig" . | indent 6 }}
 {{- if and .root.Values.certificates .root.Values.certificates.enabled }}
  {{- $externalCert := (ne (include "hpcc.isVisibilityPublic" (dict "root" .root "visibility" .me.service.visibility)) "") -}}
- {{- $issuerName := ternary "public" "local" $externalCert -}}
+ {{- $externalIssuerName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" .me )) -}}
+ {{- $issuerName := ternary $externalIssuerName "local" $externalCert -}}
  {{- if not (hasKey .me "tls" )}}
       tls: {{ include "hpcc.isIssuerEnabled" (dict "root" .root "issuer" $issuerName) }}
  {{- end }}
       tls_config:
  {{- if $externalCert }}
-        certificate: /opt/HPCCSystems/secrets/certificates/public/tls.crt
-        privatekey: /opt/HPCCSystems/secrets/certificates/public/tls.key
+        certificate: /opt/HPCCSystems/secrets/certificates/{{ $issuerName }}/tls.crt
+        privatekey: /opt/HPCCSystems/secrets/certificates/{{ $issuerName }}/tls.key
+   {{- if (hasKey .me "remoteClients" )}}
+        verify:
+          enable: true
+          address_match: false
+          accept_selfsigned: false
+          trusted_peers: [ {{ include "hpcc.getTrustedPeerString" (dict "root" .root "remoteClients" .me.remoteClients "instance" .me.name "component" .me.application "visibility" .me.service.visibility) }} ]
+          ca_certificates:
+            path: /opt/HPCCSystems/secrets/certificates/{{ $issuerName }}/ca.crt
+   {{- end }}
  {{- else }}
         certificate: /opt/HPCCSystems/secrets/certificates/local/tls.crt
         privatekey: /opt/HPCCSystems/secrets/certificates/local/tls.key
@@ -55,7 +65,7 @@ data:
           accept_selfsigned: false
           trusted_peers: [ anyone ]
           ca_certificates:
-            path: "/opt/HPCCSystems/secrets/certificates/local/ca.crt"
+            path: /opt/HPCCSystems/secrets/certificates/local/ca.crt
  {{- end }}
 {{- else if not (hasKey .me "tls" )}}
       tls: false
@@ -144,7 +154,7 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
-{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility) | indent 8 }}
+{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients) | indent 8 }}
 {{- if $commonCtx.externalCert }}
  {{- include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "external" false) | nindent 8 }}
 {{- end }}
@@ -155,7 +165,7 @@ spec:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
-{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility) | indent 6 }}
+{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients) | indent 6 }}
 {{- if $commonCtx.externalCert }}
  {{- include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "external" false) | nindent 6 }}
 {{- end }}
@@ -178,12 +188,19 @@ kind: ConfigMap
 {{- end }}
 {{- end }}
 
-{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" $application "visibility" .service.visibility) }}
+{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" $application "visibility" .service.visibility "remoteClients" .remoteClients) }}
+
 {{- if $commonCtx.externalCert }}
  {{- include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" $application "external" false) }}
 {{- end }}
 {{- if $generateSigningCert }}
  {{- include "hpcc.addCertificate" (dict "root" $ "name" "global" "component" $application "issuer" "signing") }}
 {{- end }}
+
+{{- $instance := .name -}}
+{{- $visibility := .service.visibility -}}
+{{- range $remoteClient := .remoteClients }}
+ {{ include "hpcc.addClientCertificate" (dict "root" $ "client" $remoteClient.name "organization" $remoteClient.organization "instance" $instance "component" $application "visibility" $visibility) }}
+{{- end }}
 {{- end }}
 {{- end }}

helm/hpcc/values.schema.json

@@ -1133,6 +1133,9 @@
         "bindingInfo": {
           "description": "Customizable binding options",
           "type": "object"
+        },
+        "remoteClients": {
+          "$ref": "#/definitions/remoteClients"
         }
       }
     },
@@ -1263,7 +1266,6 @@
           "type": "string",
           "description": "Name of the secret which contains the TLS certificate.  Custom configuration instead of using, or overriding cert-manager certificate."
         },
-        
         "allFilesDynamic": { 
           "type": "boolean",
           "default": false,
@@ -2582,6 +2584,18 @@
         },
         "additionalProperties": false
       }
+    },
+    "remoteClients": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Remote client name"
+          }
+        }
+      }
     }
   }
 }

helm/hpcc/values.yaml

@@ -244,6 +244,18 @@ certificates:
       ## for information on what spec should contain.
       spec:
         selfSigned: {}
+    remote:
+      name: hpcc-remote-issuer
+      ## set enabled to true if adding remoteClients for any components
+      enabled: false
+      ## kind can be changed to ClusterIssue to refer to a ClusterIssuer. https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer
+      kind: Issuer
+      ## do not define spec (set spec: null), to reference an Issuer resource that already exists in the cluster
+      ## change spec if you'd like to change how certificates get issued... see ## https://cert-manager.io/docs/configuration/#supported-issuer-types
+      ## for information on what spec should contain.
+      spec:
+        ca:
+          secretName: hpcc-remote-issuer-key-pair
     signing: # intended to be used for signing/verification purposes only, e.g. by dafilesrv
       name: hpcc-signing-issuer
       ## kind can be changed to ClusterIssue to refer to a ClusterIssuer. https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer
@@ -483,6 +495,11 @@ esp:
   application: eclwatch
   auth: none
   replicas: 1
+# Add remote clients to generated client certificates and make the ESP require that one of the generated certificates is provided by a client in order to connect
+#   When setting up remote clients make sure that certificates.issuers.remote.enabled is set to true.
+# remoteClients:
+# - name: myclient
+#   organization: mycompany
   service:
     ## port can be used to change the local port used by the pod. If omitted, the default port (8880) is used
     port: 8888
@@ -550,6 +567,8 @@ esp:
   application: sql2ecl
   auth: none
   replicas: 1
+# remoteClients:
+# - name: sqlclient111
   service:
     visibility: local
     servicePort: 8510