HPCC-24546 Support for Hashicorp vaults and kubernetes secrets

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexisrisk.com>

common/remote/hooks/azure/azurefile.cpp

@@ -22,6 +22,7 @@
 #include "jfile.hpp"
 #include "jregexp.hpp"
 #include "jstring.hpp"
+#include "jsecrets.hpp"
 #include "jlog.hpp"
 #include "azurefile.hpp"
 
@@ -518,7 +519,7 @@ static std::shared_ptr<azure::storage_lite::blob_client> getClient(const char *
         {
             StringBuffer secretName;
             secretName.append("azure-").append(accountName);
-            getSecret(keyTemp, secretName, "key");
+            getSecretValue(keyTemp, "storage", secretName, "key", true);
             //Trim trailing whitespace/newlines in case the secret has been entered by hand e.g. on bare metal
             size32_t len = keyTemp.length();
             for (;;)

common/thorhelper/thorsoapcall.cpp

@@ -18,6 +18,7 @@
 #include "jliball.hpp"
 #include "jqueue.tpp"
 #include "jisem.hpp"
+#include "jsecrets.hpp"
 
 #include "rtlformat.hpp"
 
@@ -279,7 +280,7 @@ public:
         free(fullText);
     }
 
-    unsigned getUrls(UrlArray &array)
+    unsigned getUrls(UrlArray &array, const char *basic_credentials=nullptr)
     {
         if (fullText)
         {
@@ -289,7 +290,10 @@ public:
             char *url = strtok_r(copyFullText, "|", &saveptr);
             while (url != NULL)
             {
-                array.append(*new Url(url));
+                Owned<Url> item = new Url(url);
+                if (basic_credentials)
+                    item->userPasswordPair.set(basic_credentials);
+                array.append(*item.getClear());
                 url = strtok_r(NULL, "|", &saveptr);
             }
 
@@ -805,8 +809,11 @@ private:
     static CriticalSection secureContextCrit;
     static Owned<ISecureSocketContext> secureContext;
 
+    Owned<ISecureSocketContext> customSecureContext;
+
     CTimeMon timeLimitMon;
     bool complete, timeLimitExceeded;
+    bool customClientCert = false;
     IRoxieAbortMonitor * roxieAbortMonitor;
 
 protected:
@@ -865,15 +872,6 @@ public:
             xpathHints.setown(createPTreeFromXMLString(s.get()));
         }
 
-        StringAttr proxyAddress;
-        proxyAddress.set(s.setown(helper->getProxyAddress()));
-        if (!proxyAddress.isEmpty())
-        {
-            UrlListParser proxyUrlListParser(proxyAddress);
-            if (0 == proxyUrlListParser.getUrls(proxyUrlArray))
-                throw MakeStringException(0, "SOAPCALL PROXYADDRESS specified no URLs");
-        }
-
         if (wscType == STsoap)
         {
             soapaction.set(s.setown(helper->getSoapAction()));
@@ -941,46 +939,100 @@ public:
         else
             rowTransformer = NULL;
 
+        StringBuffer proxyAddress;
+        proxyAddress.set(s.setown(helper->getProxyAddress()));
+
         OwnedRoxieString hosts(helper->getHosts());
-        UrlListParser urlListParser(hosts);
-        if ((numUrls = urlListParser.getUrls(urlArray)) > 0)
+        if (isEmptyString(hosts))
+            throw MakeStringException(0, "%sCALL specified no URLs",wscType == STsoap ? "SOAP" : "HTTP");
+        if (0==strncmp(hosts, "secret:", 7))
         {
-            if (wscMode == SCrow)
+            const char *finger = hosts.get()+7;
+            if (isEmptyString(finger))
+                throw MakeStringException(0, "%sCALL HTTP-CONNECT SECRET specified with no name", wscType == STsoap ? "SOAP" : "HTTP");
+            if (!proxyAddress.isEmpty())
+                throw MakeStringException(0, "%sCALL PROXYADDRESS can't be used with HTTP-CONNECT secrets", wscType == STsoap ? "SOAP" : "HTTP");
+            StringAttr vaultId;
+            const char *thumb = strchr(finger, ':');
+            if (thumb)
             {
-                numRowThreads = 1;
-
-                numUrlThreads = helper->numParallelThreads();
-                if (numUrlThreads == 0)
-                    numUrlThreads = 1;
-                else if (numUrlThreads > MAXWSCTHREADS)
-                    numUrlThreads = MAXWSCTHREADS;
-
-                numRecordsPerBatch = 1;
+                vaultId.set(finger, thumb-finger);
+                finger = thumb + 1;
             }
-            else
+            StringBuffer secretName("http-connect-");
+            secretName.append(finger);
+            Owned<IPropertyTree> secret = (vaultId.isEmpty()) ? getSecret("ecl", secretName) : getVaultSecret("ecl", vaultId, secretName, nullptr);
+            if (!secret)
+                throw MakeStringException(0, "%sCALL %s SECRET not found", wscType == STsoap ? "SOAP" : "HTTP", secretName.str());
+
+            StringBuffer url;
+            getSecretKeyValue(url, secret, "url");
+            if (url.isEmpty())
+                throw MakeStringException(0, "%sCALL %s HTTP SECRET must contain url", wscType == STsoap ? "SOAP" : "HTTP", secretName.str());
+            UrlListParser urlListParser(url);
+            StringBuffer auth;
+            getSecretKeyValue(auth, secret, "username");
+            if (auth.length())
             {
-                unsigned totThreads = helper->numParallelThreads();
-                if (totThreads < 1)
-                    totThreads = 2; // default to 2 threads
-                else if (totThreads > MAXWSCTHREADS)
-                    totThreads = MAXWSCTHREADS;
-
-                numUrlThreads = (numUrls < totThreads)? numUrls: totThreads;
-
-                numRowThreads = totThreads / numUrlThreads;
-                if (numRowThreads < 1)
-                    numRowThreads = 1;
-                else if (numRowThreads > MAXWSCTHREADS)
-                    numRowThreads = MAXWSCTHREADS;
-
-                numRecordsPerBatch = helper->numRecordsPerBatch();
-                if (numRecordsPerBatch < 1)
-                    numRecordsPerBatch = 1;
+                if (strchr(auth, ':'))
+                    throw MakeStringException(0, "%sCALL HTTP-CONNECT SECRET username contains illegal colon", wscType == STsoap ? "SOAP" : "HTTP");
+                auth.append(':');
+                getSecretKeyValue(auth, secret, "password");
             }
+            urlListParser.getUrls(urlArray, auth);
+            proxyAddress.set(secret->queryProp("proxy"));
+            getSecretKeyValue(proxyAddress.clear(), secret, "proxy");
         }
         else
+        {
+            UrlListParser urlListParser(hosts);
+            urlListParser.getUrls(urlArray);
+        }
+
+        numUrls = urlArray.ordinality();
+        if (numUrls == 0)
             throw MakeStringException(0, "%sCALL specified no URLs",wscType == STsoap ? "SOAP" : "HTTP");
 
+        if (!proxyAddress.isEmpty())
+        {
+            UrlListParser proxyUrlListParser(proxyAddress);
+            if (0 == proxyUrlListParser.getUrls(proxyUrlArray))
+                throw MakeStringException(0, "%sCALL proxy address specified no URLs",wscType == STsoap ? "SOAP" : "HTTP");
+        }
+
+        if (wscMode == SCrow)
+        {
+            numRowThreads = 1;
+
+            numUrlThreads = helper->numParallelThreads();
+            if (numUrlThreads == 0)
+                numUrlThreads = 1;
+            else if (numUrlThreads > MAXWSCTHREADS)
+                numUrlThreads = MAXWSCTHREADS;
+
+            numRecordsPerBatch = 1;
+        }
+        else
+        {
+            unsigned totThreads = helper->numParallelThreads();
+            if (totThreads < 1)
+                totThreads = 2; // default to 2 threads
+            else if (totThreads > MAXWSCTHREADS)
+                totThreads = MAXWSCTHREADS;
+
+            numUrlThreads = (numUrls < totThreads)? numUrls: totThreads;
+
+            numRowThreads = totThreads / numUrlThreads;
+            if (numRowThreads < 1)
+                numRowThreads = 1;
+            else if (numRowThreads > MAXWSCTHREADS)
+                numRowThreads = MAXWSCTHREADS;
+
+            numRecordsPerBatch = helper->numRecordsPerBatch();
+            if (numRecordsPerBatch < 1)
+                numRecordsPerBatch = 1;
+        }
+
         for (unsigned i=0; i<numRowThreads; i++)
             threads.append(*new CWSCHelperThread(this));
     }
@@ -1047,19 +1099,26 @@ public:
     }
     inline IEngineRowAllocator * queryOutputAllocator() const { return outputAllocator; }
 #ifdef _USE_OPENSSL
-    ISecureSocket *createSecureSocket(ISocket *sock)
+    ISecureSocketContext *ensureSecureContext(Owned<ISecureSocketContext> &ownedSC)
     {
+        if (!ownedSC)
         {
-            CriticalBlock b(secureContextCrit);
-            if (!secureContext)
-            {
-                if (clientCert != NULL)
-                    secureContext.setown(createSecureSocketContextEx(clientCert->certificate, clientCert->privateKey, clientCert->passphrase, ClientSocket));
-                else
-                    secureContext.setown(createSecureSocketContext(ClientSocket));
-            }
+            if (clientCert != NULL)
+                ownedSC.setown(createSecureSocketContextEx(clientCert->certificate, clientCert->privateKey, clientCert->passphrase, ClientSocket));
+            else
+                ownedSC.setown(createSecureSocketContext(ClientSocket));
         }
-        return secureContext->createSecureSocket(sock);
+        return ownedSC.get();
+    }
+    ISecureSocketContext *ensureStaticSecureContext()
+    {
+        CriticalBlock b(secureContextCrit);
+        return ensureSecureContext(secureContext);
+    }
+    ISecureSocket *createSecureSocket(ISocket *sock)
+    {
+        ISecureSocketContext *sc = (customClientCert) ? ensureSecureContext(customSecureContext) : ensureStaticSecureContext();
+        return sc->createSecureSocket(sock);
     }
 #endif
     bool isTimeLimitExceeded(unsigned *_remainingMS)

dockerfiles/platform-build-base/Dockerfile

@@ -79,7 +79,8 @@ RUN apt-get install -y \
   vim \
   gdb \
   software-properties-common \  
-  lsb-release
+  lsb-release \
+  jq
 
 RUN curl https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
 RUN apt-add-repository "deb https://deb.nodesource.com/node_12.x $(lsb_release -sc) main"

helm/examples/secrets/README.md

@@ -0,0 +1,176 @@
+# Containerized HPCC Systems Secrets
+
+This example demonstrates HPCC use use of Kubernetes and Hashicorp Vault secrets.
+
+This example assumes you are starting from a linux command shell in the HPCC-Platform/helm directory.  From there you will find the example files and this README file in the examples/secrets directory.
+
+## Hashicorp Vault support:
+
+This example uses Hashicorp vault.  The following steps can be used to set up a development mode only instance of vault just for the purposes of this example.  This makes it easy to test out vault functionality without going through the much more extensive configuration process for a production ready vault installation.
+
+## Install hashicorp vault command line client:
+
+https://learn.hashicorp.com/tutorials/vault/getting-started-install
+
+--------------------------------------------------------------------------------------------------------
+
+## Install hashicorp vault service in dev mode:
+
+This is for development only, never deploy this way in production.
+Deploying in dev mode sets up an in memory kv store that won't persist secret values across restart, and the vault will automatically be unsealed.
+
+In dev mode the default root token is simply the string "root".
+
+Add Hashicorp helm repo:
+
+```bash
+helm repo add hashicorp https://helm.releases.hashicorp.com
+```
+
+Install vault server.
+
+```bash
+helm install vault hashicorp/vault --set "server.dev.enabled=true"
+```
+
+## Setting up vault
+
+Tell the vault command line application the server location (dev mode is http, default location is https)
+
+```bash
+export VAULT_ADDR=http://127.0.0.1:8200
+```
+
+In a separate terminal window start vault port forwarding.
+
+```bash
+kubectl port-forward vault-0 8200:8200
+```
+
+Login to the vault command line using the vault root token (development mode defaults to "root"):
+
+```bash
+vault login root
+```
+
+If you don't provide the token on the command line you will be prompted to input the value and it will be hidden from view.
+
+
+## Configure vault kubernetes auth
+
+Enabling kubernetes auth will allow k8s nodes to access the vault via their kubernetes.io access tokens.
+
+```bash
+vault auth enable kubernetes
+```
+
+Exec into the vault-0 pod:
+
+```bash
+kubectl exec -it vault-0 /bin/sh
+```
+
+Configure vault kubernetes auth:
+
+```bash
+vault write auth/kubernetes/config \
+   token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
+   kubernetes_host=https://${KUBERNETES_PORT_443_TCP_ADDR}:443 \
+   kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+```
+
+Exit from the vault-0 pod:
+
+```bash
+exit
+```
+
+Setup vault auth policy granting access to the ecl secrets locations we plan to use:
+
+```bash
+vault policy write hpcc-kv-ro examples/secrets/hpcc_vault_policies.hcl
+```
+
+Setup hpcc-vault-access auth role within the default service account (if necessary change bound_service_account "names" and "namespace" to match the service account the HPCC deployment is using):
+
+```bash
+vault write auth/kubernetes/role/hpcc-vault-access \
+        bound_service_account_names="*" \
+        bound_service_account_namespaces=default \
+        policies=hpcc-kv-ro \
+        ttl=24h
+```
+
+## HTTP-CONNECT Secrets:
+
+This example focuses on ECL secrets to provide HTTP connection strings and credentials for ECL SOAPCALL and HTTPCALL commands.
+
+These secrets are prefixed with the string "http-connect-" requiring this prefix ensures that HTTPCALL/SOAPCALL only accesses secrets which are intended for this use.
+
+HTTP-CONNECT secrets consist of a url string and optional additional secrets associated with that URL.  Requiring the url to be part of the secret prevents credentials from being easily hijacked via an HTTPCALL to an arbitrary location.  Instead the credentials are explicitly associated with the provided url.
+
+Besides the URL values can currently be set for proxy (trusted for keeping these secrets), username, and password.
+
+## Creating the HTTP-CONNECT Secrets
+
+## Create example vault secret:
+
+Create example vault secrets:
+
+```bash
+vault kv put secret/ecl/http-connect-vaultsecret url=@examples/secrets/url-basic username=@examples/secrets/username password=@examples/secrets/password
+```
+
+The following vault secret will be hidden by our "local" kubernetes secret below by default.  But we can ask for it directly in our HTTPCALL (see "httpcall_vault.ecl" example).
+
+```bash
+vault kv put secret/ecl/http-connect-basicsecret url=@examples/secrets/url-basic username=@examples/secrets/username password=@examples/secrets/password
+```
+
+Create example kubernetes secret:
+
+```bash
+kubectl create secret generic http-connect-basicsecret --from-file=url=examples/secrets/url-basic --from-file=examples/secrets/username --from-file=examples/secrets/password
+```
+
+## Installing the HPCC with the HTTP-CONNECT Secrets added to ECL components
+
+Install the HPCC helm chart with the secrets just defined added to all components that run ECL.
+
+```bash
+helm install myhpcc hpcc/ --set global.image.version=latest -f examples/secrets/values-secrets.yaml
+```
+
+Use kubectl to check the status of the deployed pods.  Wait until all pods are running before continuing.
+
+```bash
+kubectl get pods
+```
+
+## Using the created secrets via HTTPCALL from within ECL code
+
+If you don't already have the HPCC client tools installed please install them now:
+
+https://hpccsystems.com/download#HPCC-Platform
+
+--------------------------------------------------------------------------------------------------------
+
+The following ecl commands will run the three example ECL files on hthor.
+
+```bash
+ecl run hthor examples/secrets/httpcall_secret.ecl
+
+ecl run hthor examples/secrets/httpcall_vault.ecl
+
+ecl run hthor examples/secrets/httpcall_vault_direct.ecl
+```
+
+For each job the expected result would be:
+
+```xml
+<Result>
+<Dataset name='Result 1'>
+ <Row><authenticated>true</authenticated></Row>
+</Dataset>
+</Result>
+```

helm/examples/secrets/hpcc_vault_policies.hcl

@@ -0,0 +1,11 @@
+path "secret/data/ecl/*" {
+    capabilities = ["read"]
+}
+
+path "secret/data/ecl-user/*" {
+    capabilities = ["read"]
+}
+
+path "secret/data/storage/*" {
+    capabilities = ["read"]
+}

helm/examples/secrets/httpcall_secret.ecl

@@ -0,0 +1,5 @@
+ServiceOutRecord := RECORD
+    string authenticated {XPATH('authenticated')};
+END;
+
+output(HTTPCALL('secret:basicsecret','GET', 'application/json', ServiceOutRecord, XPATH('/'), LOG));

helm/examples/secrets/httpcall_vault.ecl

@@ -0,0 +1,7 @@
+ServiceOutRecord := RECORD
+    string authenticated {XPATH('authenticated')};
+END;
+
+//call out using our http-connect-vaultsecret stored in our ecl vault
+
+output(HTTPCALL('secret:vaultsecret','GET', 'application/json', ServiceOutRecord, XPATH('/'), LOG));

helm/examples/secrets/httpcall_vault_direct.ecl

@@ -0,0 +1,8 @@
+ServiceOutRecord := RECORD
+    string authenticated {XPATH('authenticated')};
+END;
+
+
+//call out using our http-connect-basicsecret.. but get it directly from the ecl vault
+
+output(HTTPCALL('secret:my-ecl-vault:basicsecret','GET', 'application/json', ServiceOutRecord, XPATH('/'), LOG));

helm/examples/secrets/password

@@ -0,0 +1 @@
+password

helm/examples/secrets/url-basic

@@ -0,0 +1 @@
+https://postman-echo.com/basic-auth

helm/examples/secrets/username

@@ -0,0 +1 @@
+postman

helm/examples/secrets/values-secrets.yaml

@@ -0,0 +1,22 @@
+# Overrides for http-connect secrets in hpcc.
+# NB: The "hpcc-connect-testsecret" should be created before installing the helm chart.
+
+secrets:
+  ecl:
+    http-connect-basicsecret: "http-connect-basicsecret"
+
+vaults:
+  storage:
+    - name: my-storage-vault
+      url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/storage/${secret}
+      kind: kv-v2
+  ecl:
+    #vault using kubernetes auth
+    - name: my-ecl-vault
+      url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/ecl/${secret}
+      kind: kv-v2
+    #vault using provided token auth
+#    - name: ecl-vault-direct-auth
+#      url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/ecl/${secret}
+#      kind: kv-v2
+#      client-secret: "hpcc-vault-client"

helm/hpcc/templates/_helpers.tpl

@@ -290,7 +290,7 @@ Add the secret volume mounts for a component
  {{- if (has $category $categories) -}}
 {{- range $secretid, $secretname := $key -}}
 - name: secret-{{ $secretid }}
-  mountPath: /opt/HPCCSystems/secrets/{{ $secretid }}
+  mountPath: /opt/HPCCSystems/secrets/{{ $category }}/{{ $secretid }}
 {{ end -}}
  {{- end -}}
 {{- end -}}
@@ -335,6 +335,28 @@ readinessProbe:
   periodSeconds: 10
 {{ end -}}
 
+
+{{/*
+Generate vault info
+*/}}
+{{- define "hpcc.generateVaultConfig" -}}
+{{- $categories := .categories -}}
+vaults:
+{{- range  $categoryname, $category := .root.Values.vaults -}}
+ {{- if (has $categoryname $categories) }}
+  {{ $categoryname }}:
+  {{- range $vault := . }}
+    - name: {{ $vault.name }}
+      kind: {{ $vault.kind }}
+      url: {{ $vault.url }}
+    {{- if index $vault "client-secret" }}
+      client-secret: {{ index $vault "client-secret" }}
+    {{- end -}}
+  {{- end -}}
+ {{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Return a value indicating whether a storage plane is defined or not.
 */}}

helm/hpcc/templates/dali.yaml

@@ -33,13 +33,13 @@ spec:
         - name: {{ .name }}-configmap-volume
           mountPath: /etc/config
 {{ include "hpcc.addDaliVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" ) ) | indent 8 }}
       volumes:
       - name: {{ .name }}-configmap-volume
         configMap:
           name: {{ .name }}-configmap
 {{ include "hpcc.addDaliVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" ) ) | indent 6 }}
 ---
 kind: ConfigMap 
 apiVersion: v1 

helm/hpcc/templates/eclagent.yaml

@@ -35,12 +35,12 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 6 }}
 ---
 kind: ConfigMap 
 apiVersion: v1 
@@ -55,6 +55,7 @@ data:
     {{ $apptype }}:
 {{ toYaml (omit . "logging") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" .) | indent 6 }}
+{{ include "hpcc.generateVaultConfig" (dict "root" $ "categories" (list "storage" "ecl" "ecl-user" ) ) | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 {{- if not .useChildProcesses }} 
@@ -94,12 +95,12 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 12 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 12 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 12 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 12 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 12 }}
           volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 10 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 10 }}
 {{ include "hpcc.addDllVolume" $ | indent 10 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 10 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 10 }}
           restartPolicy: Never
       backoffLimit: 0
 {{- end }}      

helm/hpcc/templates/eclccserver.yaml

@@ -35,14 +35,14 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" ) ) | indent 8 }}
         - name: "hpccbundles"
           mountPath: "/home/hpcc/.HPCCSystems"
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" ) ) | indent 6 }}
       - name: hpccbundles
         emptyDir: {}
 ---
@@ -97,12 +97,12 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 12 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 12 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 12 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" ) ) | indent 12 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" ) ) | indent 12 }}
           volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 10 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 10 }}
 {{ include "hpcc.addDllVolume" $ | indent 10 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" ) ) | indent 10 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" ) ) | indent 10 }}
           restartPolicy: Never
       backoffLimit: 0
 {{- end }}      

helm/hpcc/templates/esp.yaml

@@ -34,12 +34,12 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "storage" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "storage" ) ) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "storage" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "storage" ) ) | indent 6 }}
 ---
 kind: ConfigMap 
 apiVersion: v1 
@@ -55,8 +55,10 @@ data:
 {{ include "hpcc.generateConfigMapQueues" $ | indent 6 }}
       services:
 {{ include "hpcc.generateConfigMapServices" $ | indent 6 }}
+{{ include "hpcc.generateVaultConfig" (dict "root" $ "categories" (list "storage" "esp" ) ) | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
+
 ---
 apiVersion: v1
 kind: Service

helm/hpcc/templates/localroxie.yaml

@@ -53,12 +53,12 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 6 }}
 ---
 {{- range $service := $roxie.services }}
 {{- if ne (int $service.port)  0 }}
@@ -90,6 +90,7 @@ data:
     roxie:
 {{ toYaml (omit $roxie "logging") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" $roxie) | indent 6 }}
+{{ include "hpcc.generateVaultConfig" (dict "root" $ "categories" (list "storage" "ecl" "ecl-user" ) ) | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 ---

helm/hpcc/templates/roxie.yaml

@@ -111,6 +111,7 @@ data:
       topologyServers: "{{ $toponame }}:{{ $topoport }}"
       resolveLocally: false
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" $roxie) | indent 6 }}
+{{ include "hpcc.generateVaultConfig" (dict "root" $ "categories" (list "storage" "ecl" "ecl-user" ) ) | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 ---
@@ -164,12 +165,12 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" $roxie | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 6 }}
 ---
 
 {{ end -}}
@@ -230,12 +231,12 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" $roxie ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" $roxie | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" $roxie ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 6 }}
 
 ---
 

helm/hpcc/templates/thor.yaml

@@ -47,12 +47,12 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 6 }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -86,12 +86,12 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 8 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 8 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 6 }}
 {{ include "hpcc.addDllVolume" $ | indent 6 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 6 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 6 }}
 ---
 kind: ConfigMap 
 apiVersion: v1 
@@ -103,6 +103,7 @@ data:
     thor:
 {{ toYaml (omit $thorScope "logging") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" $thorScope) | indent 6 }}
+{{ include "hpcc.generateVaultConfig" (dict "root" $ "categories" (list "storage" "ecl" "ecl-user" ) ) | indent 6 }}
     {{ $agentAppType }}:
 {{- if $thorScope.multiJobLinger }}
       multiJobLinger: true
@@ -112,9 +113,11 @@ data:
         type: "thor"
         width: {{ $thor.numSlaves }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" $hthorScope) | indent 6 }}
+{{ include "hpcc.generateVaultConfig" (dict "root" $ "categories" (list "storage" "ecl" "ecl-user" ) ) | indent 6 }}
     eclagent: # main agent Q handler
 {{ toYaml (omit $eclAgentScope "logging") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" $eclAgentScope) | indent 6 }}
+{{ include "hpcc.generateVaultConfig" (dict "root" $ "categories" (list "storage" "ecl" "ecl-user" ) ) | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 
@@ -171,12 +174,12 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 12 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 12 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 12 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 12 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 12 }}
           volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 10 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 10 }}
 {{ include "hpcc.addDllVolume" $ | indent 10 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 10 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 10 }}
           restartPolicy: Never
       backoffLimit: 0
 {{- end }}
@@ -222,12 +225,12 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 12 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 12 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 12 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 12 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 12 }}
           volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 10 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 10 }}
 {{ include "hpcc.addDllVolume" $ | indent 10 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 10 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 10 }}
           restartPolicy: Never
       backoffLimit: 0
 
@@ -269,12 +272,12 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 12 }}
 {{ include "hpcc.addDataVolumeMount" (dict "root" $ "me" . ) | indent 12 }}
 {{ include "hpcc.addDllVolumeMount" $ | indent 12 }}
-{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 12 }}
+{{ include "hpcc.addSecretVolumeMounts" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 12 }}
           volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 10 }}
 {{ include "hpcc.addDataVolume" (dict "root" $ "me" . ) | indent 10 }}
 {{ include "hpcc.addDllVolume" $ | indent 10 }}
-{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "all" "ecl" "storage" ) ) | indent 10 }}
+{{ include "hpcc.addSecretVolumes" (dict "root" $ "categories" (list "system" "ecl" "storage" ) ) | indent 10 }}
           restartPolicy: Never
       backoffLimit: 0
 

helm/hpcc/values.schema.json

@@ -49,12 +49,34 @@
         "ecl": {
           "$ref": "#/definitions/secrets"
         },
-        "all": {
+        "system": {
           "$ref": "#/definitions/secrets"
         }
       },
       "additionalProperties": false
     },
+    "vaults": {
+      "description": "configuration for vaults accessed by the components",
+      "type": "object",
+      "properties": {
+        "timeout": {
+          "type": "integer"
+        },
+        "storage": {
+          "$ref": "#/definitions/vaultCategory"
+        },
+        "esp": {
+          "$ref": "#/definitions/vaultCategory"
+        },
+        "ecl": {
+          "$ref": "#/definitions/vaultCategory"
+        },
+        "ecl-user": {
+          "$ref": "#/definitions/vaultCategory"
+        }
+      },
+      "additionalProperties": false
+    },
     "bundles": {
       "description": "bundles",
       "type": "array",
@@ -257,6 +279,42 @@
         }
       ]
     },
+    "vaultCategory": {
+      "description": "set of vaults under a given category",
+      "oneOf": [
+        {
+            "type": "array",
+            "items": { "$ref": "#/definitions/vault" }
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "vault": {
+      "description": "information about an individual vault",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "the name of the vault",
+          "type": "string"
+        },
+        "url": {
+          "description": "the url used to access the vault",
+          "type": "string"
+        },
+        "kind": {
+          "type": "string",
+          "enum": ["kv-v2", "kv-v1"]
+        },
+        "client-secret": {
+          "description": "optional name of kubernetes secret that will provide the vault client token",
+          "type": "string"
+        }
+      },
+      "required": [ "name", "url" ],
+      "additionalProperties": false
+    },
     "hostgroups": {
       "oneOf": [
         {

helm/hpcc/values.yaml

@@ -98,8 +98,30 @@ secrets:
   ecl:
     ## Category for secrets published to all components that run ecl
 
-  all:
-    ## Category for secrets published to all components
+  system:
+    ## Category for secrets published to all components for system level useage
+
+## The vaults section mirrors the secret section but leverages vault for the storage of secrets.
+## There is an additional category for vaults named "ecl-user".  In the future "ecl-user" vault
+## secrets will be readable directly from ECL code.  Other secret categories are read internally
+## by system components and not exposed directly to ECL code.
+##
+## For each vault:
+##   name is the name that is is accessed by within the platform
+##   url is the url used to read a secret from the vault.
+##   kind is the type of vault being accessed, or the protocol to use to access the secrets
+##   client_secret a kubernetes level secret that contains the client_token used to retrive secrets.
+##       if a client_secret is not provided "vault kubernetes auth" will be attempted.
+
+vaults:
+  storage:
+
+  ecl:
+
+  ecl-user:
+    #ECL code will have direct access to these secrets
+
+  esp:
 
 bundles: []
 ## Specifying bundles here will cause the indicated bundles to be downloaded and installed automatically

system/httplib/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 yhirose
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

system/httplib/README.md

@@ -0,0 +1,651 @@
+cpp-httplib
+===========
+
+[![](https://github.com/yhirose/cpp-httplib/workflows/test/badge.svg)](https://github.com/yhirose/cpp-httplib/actions)
+
+A C++11 single-file header-only cross platform HTTP/HTTPS library.
+
+It's extremely easy to setup. Just include **httplib.h** file in your code!
+
+Server Example
+--------------
+
+```c++
+#include <httplib.h>
+
+int main(void)
+{
+  using namespace httplib;
+
+  Server svr;
+
+  svr.Get("/hi", [](const Request& req, Response& res) {
+    res.set_content("Hello World!", "text/plain");
+  });
+
+  svr.Get(R"(/numbers/(\d+))", [&](const Request& req, Response& res) {
+    auto numbers = req.matches[1];
+    res.set_content(numbers, "text/plain");
+  });
+
+  svr.Get("/body-header-param", [](const Request& req, Response& res) {
+    if (req.has_header("Content-Length")) {
+      auto val = req.get_header_value("Content-Length");
+    }
+    if (req.has_param("key")) {
+      auto val = req.get_param_value("key");
+    }
+    res.set_content(req.body, "text/plain");
+  });
+
+  svr.Get("/stop", [&](const Request& req, Response& res) {
+    svr.stop();
+  });
+
+  svr.listen("localhost", 1234);
+}
+```
+
+`Post`, `Put`, `Delete` and `Options` methods are also supported.
+
+### Bind a socket to multiple interfaces and any available port
+
+```cpp
+int port = svr.bind_to_any_port("0.0.0.0");
+svr.listen_after_bind();
+```
+
+### Static File Server
+
+```cpp
+// Mount / to ./www directory
+auto ret = svr.set_mount_point("/", "./www");
+if (!ret) {
+  // The specified base directory doesn't exist...
+}
+
+// Mount /public to ./www directory
+ret = svr.set_mount_point("/public", "./www");
+
+// Mount /public to ./www1 and ./www2 directories
+ret = svr.set_mount_point("/public", "./www1"); // 1st order to search
+ret = svr.set_mount_point("/public", "./www2"); // 2nd order to search
+
+// Remove mount /
+ret = svr.remove_mount_point("/");
+
+// Remove mount /public
+ret = svr.remove_mount_point("/public");
+```
+
+```cpp
+// User defined file extension and MIME type mappings
+svr.set_file_extension_and_mimetype_mapping("cc", "text/x-c");
+svr.set_file_extension_and_mimetype_mapping("cpp", "text/x-c");
+svr.set_file_extension_and_mimetype_mapping("hh", "text/x-h");
+```
+
+The followings are built-in mappings:
+
+| Extension |     MIME Type          |
+| :-------- | :--------------------- |
+| txt       | text/plain             |
+| html, htm | text/html              |
+| css       | text/css               |
+| jpeg, jpg | image/jpg              |
+| png       | image/png              |
+| gif       | image/gif              |
+| svg       | image/svg+xml          |
+| ico       | image/x-icon           |
+| json      | application/json       |
+| pdf       | application/pdf        |
+| js        | application/javascript |
+| wasm      | application/wasm       |
+| xml       | application/xml        |
+| xhtml     | application/xhtml+xml  |
+
+NOTE: These the static file server methods are not thread safe.
+
+### Logging
+
+```cpp
+svr.set_logger([](const auto& req, const auto& res) {
+  your_logger(req, res);
+});
+```
+
+### Error handler
+
+```cpp
+svr.set_error_handler([](const auto& req, auto& res) {
+  auto fmt = "<p>Error Status: <span style='color:red;'>%d</span></p>";
+  char buf[BUFSIZ];
+  snprintf(buf, sizeof(buf), fmt, res.status);
+  res.set_content(buf, "text/html");
+});
+```
+
+### 'multipart/form-data' POST data
+
+```cpp
+svr.Post("/multipart", [&](const auto& req, auto& res) {
+  auto size = req.files.size();
+  auto ret = req.has_file("name1");
+  const auto& file = req.get_file_value("name1");
+  // file.filename;
+  // file.content_type;
+  // file.content;
+});
+```
+
+### Receive content with Content receiver
+
+```cpp
+svr.Post("/content_receiver",
+  [&](const Request &req, Response &res, const ContentReader &content_reader) {
+    if (req.is_multipart_form_data()) {
+      MultipartFormDataItems files;
+      content_reader(
+        [&](const MultipartFormData &file) {
+          files.push_back(file);
+          return true;
+        },
+        [&](const char *data, size_t data_length) {
+          files.back().content.append(data, data_length);
+          return true;
+        });
+    } else {
+      std::string body;
+      content_reader([&](const char *data, size_t data_length) {
+        body.append(data, data_length);
+        return true;
+      });
+      res.set_content(body, "text/plain");
+    }
+  });
+```
+
+### Send content with Content provider
+
+```cpp
+const size_t DATA_CHUNK_SIZE = 4;
+
+svr.Get("/stream", [&](const Request &req, Response &res) {
+  auto data = new std::string("abcdefg");
+
+  res.set_content_provider(
+    data->size(), // Content length
+    "text/plain", // Content type
+    [data](size_t offset, size_t length, DataSink &sink) {
+      const auto &d = *data;
+      sink.write(&d[offset], std::min(length, DATA_CHUNK_SIZE));
+      return true; // return 'false' if you want to cancel the process.
+    },
+    [data] { delete data; });
+});
+```
+
+Without content length:
+
+```cpp
+svr.Get("/stream", [&](const Request &req, Response &res) {
+  res.set_content_provider(
+    "text/plain", // Content type
+    [&](size_t offset, size_t length, DataSink &sink) {
+      if (/* there is still data */) {
+        std::vector<char> data;
+        // prepare data...
+        sink.write(data.data(), data.size());
+      } else {
+        done(); // No more data
+      }
+      return true; // return 'false' if you want to cancel the process.
+    });
+});
+```
+
+### Chunked transfer encoding
+
+```cpp
+svr.Get("/chunked", [&](const Request& req, Response& res) {
+  res.set_chunked_content_provider(
+    [](size_t offset, DataSink &sink) {
+      sink.write("123", 3);
+      sink.write("345", 3);
+      sink.write("789", 3);
+      sink.done(); // No more data
+      return true; // return 'false' if you want to cancel the process.
+    }
+  );
+});
+```
+
+### 'Expect: 100-continue' handler
+
+As default, the server sends `100 Continue` response for `Expect: 100-continue` header.
+
+```cpp
+// Send a '417 Expectation Failed' response.
+svr.set_expect_100_continue_handler([](const Request &req, Response &res) {
+  return 417;
+});
+```
+
+```cpp
+// Send a final status without reading the message body.
+svr.set_expect_100_continue_handler([](const Request &req, Response &res) {
+  return res.status = 401;
+});
+```
+
+### Keep-Alive connection
+
+```cpp
+svr.set_keep_alive_max_count(2); // Default is 5
+```
+
+### Timeout
+
+```c++
+svr.set_read_timeout(5, 0); // 5 seconds
+svr.set_write_timeout(5, 0); // 5 seconds
+svr.set_idle_interval(0, 100000); // 100 milliseconds
+```
+
+### Set maximum payload length for reading request body
+
+```c++
+svr.set_payload_max_length(1024 * 1024 * 512); // 512MB
+```
+
+### Server-Sent Events
+
+Please see [Server example](https://github.com/yhirose/cpp-httplib/blob/master/example/ssesvr.cc) and [Client example](https://github.com/yhirose/cpp-httplib/blob/master/example/ssecli.cc).
+
+### Default thread pool support
+
+
+`ThreadPool` is used as a default task queue, and the default thread count is set to value from `std::thread::hardware_concurrency()`.
+
+You can change the thread count by setting `CPPHTTPLIB_THREAD_POOL_COUNT`.
+
+### Override the default thread pool with yours
+
+```cpp
+class YourThreadPoolTaskQueue : public TaskQueue {
+public:
+  YourThreadPoolTaskQueue(size_t n) {
+    pool_.start_with_thread_count(n);
+  }
+
+  virtual void enqueue(std::function<void()> fn) override {
+    pool_.enqueue(fn);
+  }
+
+  virtual void shutdown() override {
+    pool_.shutdown_gracefully();
+  }
+
+private:
+  YourThreadPool pool_;
+};
+
+svr.new_task_queue = [] {
+  return new YourThreadPoolTaskQueue(12);
+};
+```
+
+Client Example
+--------------
+
+```c++
+#include <httplib.h>
+#include <iostream>
+
+int main(void)
+{
+  httplib::Client cli("localhost", 1234);
+
+  if (auto res = cli.Get("/hi")) {
+    if (res->status == 200) {
+      std::cout << res->body << std::endl;
+    }
+  } else {
+    auto err = res.error();
+    ...
+  }
+}
+```
+
+NOTE: Constructor with scheme-host-port string is now supported!
+
+```c++
+httplib::Client cli("localhost");
+httplib::Client cli("localhost:8080");
+httplib::Client cli("http://localhost");
+httplib::Client cli("http://localhost:8080");
+httplib::Client cli("https://localhost");
+```
+
+### GET with HTTP headers
+
+```c++
+httplib::Headers headers = {
+  { "Accept-Encoding", "gzip, deflate" }
+};
+auto res = cli.Get("/hi", headers);
+```
+or
+```c++
+cli.set_default_headers({
+  { "Accept-Encoding", "gzip, deflate" }
+});
+auto res = cli.Get("/hi");
+```
+
+### POST
+
+```c++
+res = cli.Post("/post", "text", "text/plain");
+res = cli.Post("/person", "name=john1&note=coder", "application/x-www-form-urlencoded");
+```
+
+### POST with parameters
+
+```c++
+httplib::Params params;
+params.emplace("name", "john");
+params.emplace("note", "coder");
+
+auto res = cli.Post("/post", params);
+```
+ or
+
+```c++
+httplib::Params params{
+  { "name", "john" },
+  { "note", "coder" }
+};
+
+auto res = cli.Post("/post", params);
+```
+
+### POST with Multipart Form Data
+
+```c++
+httplib::MultipartFormDataItems items = {
+  { "text1", "text default", "", "" },
+  { "text2", "aωb", "", "" },
+  { "file1", "h\ne\n\nl\nl\no\n", "hello.txt", "text/plain" },
+  { "file2", "{\n  \"world\", true\n}\n", "world.json", "application/json" },
+  { "file3", "", "", "application/octet-stream" },
+};
+
+auto res = cli.Post("/multipart", items);
+```
+
+### PUT
+
+```c++
+res = cli.Put("/resource/foo", "text", "text/plain");
+```
+
+### DELETE
+
+```c++
+res = cli.Delete("/resource/foo");
+```
+
+### OPTIONS
+
+```c++
+res = cli.Options("*");
+res = cli.Options("/resource/foo");
+```
+
+### Timeout
+
+```c++
+cli.set_connection_timeout(0, 300000); // 300 milliseconds
+cli.set_read_timeout(5, 0); // 5 seconds
+cli.set_write_timeout(5, 0); // 5 seconds
+```
+
+### Receive content with Content receiver
+
+```c++
+std::string body;
+
+auto res = cli.Get("/large-data",
+  [&](const char *data, size_t data_length) {
+    body.append(data, data_length);
+    return true;
+  });
+```
+
+```cpp
+std::string body;
+
+auto res = cli.Get(
+  "/stream", Headers(),
+  [&](const Response &response) {
+    EXPECT_EQ(200, response.status);
+    return true; // return 'false' if you want to cancel the request.
+  },
+  [&](const char *data, size_t data_length) {
+    body.append(data, data_length);
+    return true; // return 'false' if you want to cancel the request.
+  });
+```
+
+### Send content with Content provider
+
+```cpp
+std::string body = ...;
+
+auto res = cli_.Post(
+  "/stream", body.size(),
+  [](size_t offset, size_t length, DataSink &sink) {
+    sink.write(body.data() + offset, length);
+    return true; // return 'false' if you want to cancel the request.
+  },
+  "text/plain");
+```
+
+### With Progress Callback
+
+```cpp
+httplib::Client client(url, port);
+
+// prints: 0 / 000 bytes => 50% complete
+auto res = cli.Get("/", [](uint64_t len, uint64_t total) {
+  printf("%lld / %lld bytes => %d%% complete\n",
+    len, total,
+    (int)(len*100/total));
+  return true; // return 'false' if you want to cancel the request.
+}
+);
+```
+
+![progress](https://user-images.githubusercontent.com/236374/33138910-495c4ecc-cf86-11e7-8693-2fc6d09615c4.gif)
+
+### Authentication
+
+```cpp
+// Basic Authentication
+cli.set_basic_auth("user", "pass");
+
+// Digest Authentication
+cli.set_digest_auth("user", "pass");
+
+// Bearer Token Authentication
+cli.set_bearer_token_auth("token");
+```
+
+NOTE: OpenSSL is required for Digest Authentication.
+
+### Proxy server support
+
+```cpp
+cli.set_proxy("host", port);
+
+// Basic Authentication
+cli.set_proxy_basic_auth("user", "pass");
+
+// Digest Authentication
+cli.set_proxy_digest_auth("user", "pass");
+
+// Bearer Token Authentication
+cli.set_proxy_bearer_token_auth("pass");
+```
+
+NOTE: OpenSSL is required for Digest Authentication.
+
+### Range
+
+```cpp
+httplib::Client cli("httpbin.org");
+
+auto res = cli.Get("/range/32", {
+  httplib::make_range_header({{1, 10}}) // 'Range: bytes=1-10'
+});
+// res->status should be 206.
+// res->body should be "bcdefghijk".
+```
+
+```cpp
+httplib::make_range_header({{1, 10}, {20, -1}})      // 'Range: bytes=1-10, 20-'
+httplib::make_range_header({{100, 199}, {500, 599}}) // 'Range: bytes=100-199, 500-599'
+httplib::make_range_header({{0, 0}, {-1, 1}})        // 'Range: bytes=0-0, -1'
+```
+
+### Keep-Alive connection
+
+```cpp
+httplib::Client cli("localhost", 1234);
+
+cli.Get("/hello");         // with "Connection: close"
+
+cli.set_keep_alive(true);
+cli.Get("/world");
+
+cli.set_keep_alive(false);
+cli.Get("/last-request");  // with "Connection: close"
+```
+
+### Redirect
+
+```cpp
+httplib::Client cli("yahoo.com");
+
+auto res = cli.Get("/");
+res->status; // 301
+
+cli.set_follow_location(true);
+res = cli.Get("/");
+res->status; // 200
+```
+
+### Use a specitic network interface
+
+NOTE: This feature is not available on Windows, yet.
+
+```cpp
+cli.set_interface("eth0"); // Interface name, IP address or host name
+```
+
+OpenSSL Support
+---------------
+
+SSL support is available with `CPPHTTPLIB_OPENSSL_SUPPORT`. `libssl` and `libcrypto` should be linked.
+
+NOTE: cpp-httplib currently supports only version 1.1.1.
+
+```c++
+#define CPPHTTPLIB_OPENSSL_SUPPORT
+
+SSLServer svr("./cert.pem", "./key.pem");
+
+SSLClient cli("localhost", 8080);
+cli.set_ca_cert_path("./ca-bundle.crt");
+cli.enable_server_certificate_verification(true);
+```
+
+Compression
+-----------
+
+The server can applie compression to the following MIME type contents:
+
+  * all text types except text/event-stream
+  * image/svg+xml
+  * application/javascript
+  * application/json
+  * application/xml
+  * application/xhtml+xml
+
+### Zlib Support
+
+'gzip' compression is available with `CPPHTTPLIB_ZLIB_SUPPORT`. `libz` should be linked.
+
+### Brotli Support
+
+Brotli compression is available with `CPPHTTPLIB_BROTLI_SUPPORT`. Necessary libraries should be linked.
+Please see https://github.com/google/brotli for more detail.
+
+### Compress request body on client
+
+```c++
+cli.set_compress(true);
+res = cli.Post("/resource/foo", "...", "text/plain");
+```
+
+### Compress response body on client
+
+```c++
+cli.set_decompress(false);
+res = cli.Get("/resource/foo", {{"Accept-Encoding", "gzip, deflate, br"}});
+res->body; // Compressed data
+```
+
+Split httplib.h into .h and .cc
+-------------------------------
+
+```bash
+> python3 split.py
+> ls out
+httplib.h  httplib.cc
+```
+
+NOTE
+----
+
+### g++
+
+g++ 4.8 and below cannot build this library since `<regex>` in the versions are [broken](https://stackoverflow.com/questions/12530406/is-gcc-4-8-or-earlier-buggy-about-regular-expressions).
+
+### Windows
+
+Include `httplib.h` before `Windows.h` or include `Windows.h` by defining `WIN32_LEAN_AND_MEAN` beforehand.
+
+```cpp
+#include <httplib.h>
+#include <Windows.h>
+```
+
+```cpp
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <httplib.h>
+```
+
+Note: Cygwin on Windows is not supported.
+
+License
+-------
+
+MIT license (© 2020 Yuji Hirose)
+
+Special Thanks To
+-----------------
+
+[These folks](https://github.com/yhirose/cpp-httplib/graphs/contributors) made great contributions to polish this library to totally another level from a simple toy!

system/jlib/CMakeLists.txt

@@ -80,6 +80,7 @@ set (    SRCS
          jqueue.cpp
          jregexp.cpp 
          jrowstream.cpp
+         jsecrets.cpp
          jsem.cpp 
          jset.cpp 
          jsmartsock.cpp 
@@ -151,6 +152,7 @@ set (    INCLUDES
         jrespool.tpp
         jrowstream.hpp
         jscm.hpp
+        jsecrets.hpp
         jsem.hpp
         jset.hpp
         jsmartsock.hpp
@@ -169,6 +171,7 @@ set (    INCLUDES
         jtime.ipp
         junicode.hpp
         jutil.hpp
+        ${HPCC_SOURCE_DIR}/system/httplib/httplib.h
         ${HPCC_SOURCE_DIR}/system/security/cryptohelper/cryptocommon.cpp
         ${HPCC_SOURCE_DIR}/system/security/cryptohelper/digisign.cpp
         ${HPCC_SOURCE_DIR}/system/security/cryptohelper/pke.cpp
@@ -194,6 +197,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/globalid
          ${HPCC_SOURCE_DIR}/system/security/cryptohelper
          ${HPCC_SOURCE_DIR}/system/yaml/libyaml/include
+         ${HPCC_SOURCE_DIR}/system/httplib
          ${CMAKE_CURRENT_BINARY_DIR}  # for generated jelog.h file 
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss

system/jlib/jsecrets.cpp

@@ -0,0 +1,706 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "jlog.hpp"
+#include "jutil.hpp"
+#include "jexcept.hpp"
+#include "jmutex.hpp"
+#include "jfile.hpp"
+#include "jptree.hpp"
+#include "jerror.hpp"
+#include "jsecrets.hpp"
+
+#include "build-config.h"
+
+//including cpp-httplib single header file REST client
+//  doesn't work with format-nonliteral as an error
+//
+#if defined(__clang__) || defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
+#endif
+
+#ifdef _USE_OPENSSL
+#define CPPHTTPLIB_OPENSSL_SUPPORT
+#endif
+
+#undef INVALID_SOCKET
+#include "httplib.h"
+
+#if defined(__clang__) || defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+#include <vector>
+
+enum class CVaultKind { kv_v1, kv_v2 };
+
+CVaultKind getSecretType(const char *s)
+{
+    if (isEmptyString(s))
+        return CVaultKind::kv_v2;
+    if (streq(s, "kv_v1"))
+        return CVaultKind::kv_v1;
+    return CVaultKind::kv_v2;
+}
+interface IVaultManager : extends IInterface
+{
+    virtual bool getCachedSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0;
+    virtual bool requestSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0;
+    virtual bool getCachedSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0;
+    virtual bool requestSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0;
+};
+
+static CriticalSection secretCacheCS;
+static Owned<IPropertyTree> secretCache;
+static Owned<IVaultManager> vaultManager;
+
+MODULE_INIT(INIT_PRIORITY_SYSTEM)
+{
+    secretCache.setown(createPTree());
+    return true;
+}
+
+MODULE_EXIT()
+{
+    vaultManager.clear();
+    secretCache.clear();
+}
+
+static void splitUrlAddress(const char *address, size_t len, StringBuffer &host, StringBuffer *port)
+{
+    if (!address || len==0)
+        return;
+    const char *sep = (const char *)memchr(address, ':', len);
+    if (!sep)
+        host.append(len, address);
+    else
+    {
+        host.append(sep - address, address);
+        len = len - (sep - address) - 1;
+        if (port)
+            port->append(len, sep+1);
+        else
+            host.append(':').append(len, sep+1);
+    }
+}
+
+static void splitUrlAuthority(const char *authority, size_t authorityLen, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer *port)
+{
+    if (!authority || authorityLen==0)
+        return;
+    const char *at = (const char *) memchr(authority, '@', authorityLen);
+    if (!at)
+        splitUrlAddress(authority, authorityLen, host, port);
+    else
+    {
+        size_t userinfoLen = (at - authority);
+        splitUrlAddress(at+1, authorityLen - userinfoLen - 1, host, port);
+        const char *sep = (const char *) memchr(authority, ':', at - authority);
+        if (!sep)
+            user.append(at-authority, authority);
+        else
+        {
+            user.append(sep-authority, authority);
+            size_t passwordLen = (at - sep - 1);
+            password.append(passwordLen, sep+1);
+        }
+    }
+}
+
+static inline void extractUrlProtocol(const char *&url, StringBuffer *scheme)
+{
+    if(!url)
+        throw MakeStringException(-1, "Invalid URL %s", url);
+    if (0 == strnicmp(url, "HTTPS://", 8))
+    {
+        url+=8;
+        if (scheme)
+            scheme->append("https://");
+    }
+    else if (0 == strnicmp(url, "HTTP://", 7))
+    {
+        url+=7;
+        if (scheme)
+            scheme->append("http://");
+    }
+    else
+        throw MakeStringException(-1, "Invalid URL, protocol not recognized %s", url);
+}
+
+static void splitUrlSections(const char *url, const char * &authority, size_t &authorityLen, StringBuffer &fullpath, StringBuffer *scheme)
+{
+    extractUrlProtocol(url, scheme);
+    const char* path = strchr(url, '/');
+    authority = url;
+    if (!path)
+        authorityLen = strlen(authority);
+    else
+    {
+        authorityLen = path-url;
+        fullpath.append(path);
+    }
+}
+
+extern jlib_decl void splitFullUrl(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer &port, StringBuffer &path)
+{
+    const char *authority = nullptr;
+    size_t authorityLen = 0;
+    splitUrlSections(url, authority, authorityLen, path, nullptr);
+    splitUrlAuthority(authority, authorityLen, user, password, host, &port);
+}
+
+extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &schemeHostPort, StringBuffer &path)
+{
+    const char *authority = nullptr;
+    size_t authorityLen = 0;
+    splitUrlSections(url, authority, authorityLen, path, &schemeHostPort);
+    splitUrlAuthority(authority, authorityLen, user, password, schemeHostPort, nullptr);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+
+static StringBuffer secretDirectory;
+static CriticalSection secretCS;
+
+//there are various schemes for renewing kubernetes secrets and they are likely to vary greatly in how often
+//  a secret gets updated this timeout determines the maximum amount of time before we'll pick up a change
+//  10 minutes for now we can change this as we gather more experience and user feedback
+static unsigned secretTimeoutMs = 10 * 60 * 1000;
+
+extern jlib_decl unsigned getSecretTimeout()
+{
+    return secretTimeoutMs;
+}
+
+extern jlib_decl void setSecretTimeout(unsigned timeoutMs)
+{
+    secretTimeoutMs = timeoutMs;
+}
+
+extern jlib_decl void setSecretMount(const char * path)
+{
+    if (!path)
+    {
+        getPackageFolder(secretDirectory);
+        addPathSepChar(secretDirectory).append("secrets");
+    }
+    else
+        secretDirectory.set(path);
+}
+
+static inline bool checkSecretExpired(unsigned created)
+{
+    if (!created)
+        return false;
+    unsigned age = msTick() - created;
+    return age > getSecretTimeout();
+}
+
+class CVault
+{
+private:
+    bool useKubernetesAuth = true;
+    CVaultKind kind;
+    CriticalSection vaultCS;
+    Owned<IPropertyTree> cache;
+
+    StringBuffer schemeHostPort;
+    StringBuffer path;
+    StringBuffer username;
+    StringBuffer password;
+    StringAttr name;
+    StringAttr role;
+    StringAttr token;
+
+public:
+    CVault(IPropertyTree *vault)
+    {
+        cache.setown(createPTree());
+        StringBuffer url;
+        replaceEnvVariables(url, vault->queryProp("@url"), false);
+        if (url.length())
+            splitUrlSchemeHostPort(url.str(), username, password, schemeHostPort, path);
+        name.set(vault->queryProp("@name"));
+        kind = getSecretType(vault->queryProp("@kind"));
+        if (vault->hasProp("@role"))
+            role.set(vault->queryProp("@role"));
+        else
+            role.set("hpcc-vault-access");
+        if (vault->hasProp("@client-secret"))
+        {
+            useKubernetesAuth = false;
+            //for now only support direct access token.  we can support other combinations for example login token, ldap login, etc later.
+            Owned<IPropertyTree> clientSecret = getLocalSecret("system", vault->queryProp("@client-secret"));
+            if (clientSecret)
+                token.set(clientSecret->queryProp("token"));
+        }
+    }
+    CVaultKind getVaultKind() const { return kind; }
+    void kubernetesLogin()
+    {
+        CriticalBlock block(vaultCS);
+        if (token.length())
+            return;
+        StringBuffer login_token;
+        login_token.loadFile("/var/run/secrets/kubernetes.io/serviceaccount/token");
+        if (login_token.length())
+        {
+            std::string json;
+            json.append("{\"jwt\": \"").append(login_token.str()).append("\", \"role\": \"").append(role.str()).append("\"}");
+            httplib::Client cli(schemeHostPort.str());
+            if (username.length() && password.length())
+                cli.set_basic_auth(username, password);
+            httplib::Result res = cli.Post("/v1/auth/kubernetes/login", json, "application/json");
+            if (res)
+            {
+                if (res->status == 200)
+                {
+                    const char *response = res->body.c_str();
+                    if (!isEmptyString(response))
+                    {
+                        Owned<IPropertyTree> respTree = createPTreeFromJSONString(response);
+                        if (respTree)
+                            token.set(respTree->queryProp("auth/client_token"));
+                    }
+                }
+                else
+                {
+                    Owned<IException> e = MakeStringException(0, "Vault kube auth error [%d](%d) - vault: %s - response: %s", res->status, res.error(), name.str(), res->body.c_str());
+                    OWARNLOG(e);
+                    throw e.getClear();
+                }
+            }
+        }
+    }
+    bool getCachedSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version)
+    {
+        CriticalBlock block(vaultCS);
+        IPropertyTree *tree = cache->queryPropTree(secret);
+        if (tree)
+        {
+            VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version);
+            IPropertyTree *envelope = tree->queryPropTree(vername);
+            if (!envelope)
+                return false;
+            if (checkSecretExpired((unsigned) envelope->getPropInt("@created")))
+            {
+                tree->removeTree(envelope);
+                return false;
+            }
+            const char *s = envelope->queryProp("");
+            if (!isEmptyString(s))
+            {
+                rkind = kind;
+                content.append(s);
+                return true;
+            }
+        }
+        return false;
+    }
+    void addCachedSecret(const char *content, const char *secret, const char *version)
+    {
+        VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version);
+        Owned<IPropertyTree> envelope = createPTree(vername);
+        envelope->setPropInt("@created", (int) msTick());
+        envelope->setProp("", content);
+        {
+            CriticalBlock block(vaultCS);
+            IPropertyTree *parent = ensurePTree(cache, secret);
+            parent->setPropTree(vername, envelope.getClear());
+        }
+    }
+    bool requestSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version)
+    {
+        if (isEmptyString(secret))
+            return false;
+        if (useKubernetesAuth && token.isEmpty())
+            kubernetesLogin();
+        if (token.isEmpty())
+        {
+            Owned<IException> e = MakeStringException(0, "Vault auth error - vault: %s - vault access token not provided", name.str());
+            OERRLOG(e);
+            throw e.getClear();
+        }
+        StringBuffer location(path);
+        location.replaceString("${secret}", secret);
+        location.replaceString("${version}", version ? version : "1");
+
+        httplib::Client cli(schemeHostPort.str());
+        if (username.length() && password.length())
+            cli.set_basic_auth(username.str(), password.str());
+
+        httplib::Headers headers = {
+            { "X-Vault-Token", token.str() }
+        };
+
+        if (httplib::Result res = cli.Get(location, headers))
+        {
+            if (res->status == 200)
+            {
+                rkind = kind;
+                content.append(res->body.c_str());
+                addCachedSecret(content.str(), secret, version);
+                return true;
+            }
+            else
+            {
+                DBGLOG("Vault %s error accessing secret %s.%s [%d](%d) - response: %s", name.str(), secret, version ? version : "", res->status, res.error(), res->body.c_str());
+            }
+        }
+        return false;
+    }
+};
+
+class CVaultSet
+{
+private:
+    std::map<std::string, std::unique_ptr<CVault>> vaults;
+public:
+    CVaultSet()
+    {
+    }
+    void addVault(IPropertyTree *vault)
+    {
+        const char *name = vault->queryProp("@name");
+        if (!isEmptyString(name))
+            vaults.emplace(name, std::unique_ptr<CVault>(new CVault(vault)));
+    }
+    bool getCachedSecret(CVaultKind &kind, StringBuffer &content, const char *secret, const char *version)
+    {
+        auto it = vaults.begin();
+        for (; it != vaults.end(); it++)
+        {
+            if (it->second->getCachedSecret(kind, content, secret, version))
+                return true;
+        }
+        return false;
+    }
+    bool requestSecret(CVaultKind &kind, StringBuffer &content, const char *secret, const char *version)
+    {
+        auto it = vaults.begin();
+        for (; it != vaults.end(); it++)
+        {
+            if (it->second->requestSecret(kind, content, secret, version))
+                return true;
+        }
+        return false;
+    }
+    bool getCachedSecretFromVault(const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version)
+    {
+        if (isEmptyString(vaultId))
+            return false;
+        auto it = vaults.find(vaultId);
+        if (it == vaults.end())
+            return false;
+        return it->second->getCachedSecret(kind, content, secret, version);
+    }
+    bool requestSecretFromVault(const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version)
+    {
+        if (isEmptyString(vaultId))
+            return false;
+        auto it = vaults.find(vaultId);
+        if (it == vaults.end())
+            return false;
+        return it->second->requestSecret(kind, content, secret, version);
+    }
+};
+
+class CVaultManager : public CInterfaceOf<IVaultManager>
+{
+private:
+    std::map<std::string, std::unique_ptr<CVaultSet>> categories;
+public:
+    CVaultManager()
+    {
+        IPropertyTree *config = queryComponentConfig().queryPropTree("vaults");
+        if (!config)
+            return;
+        Owned<IPropertyTreeIterator> iter = config->getElements("*");
+        ForEach (*iter)
+        {
+            IPropertyTree &vault = iter->query();
+            const char *category = vault.queryName();
+            auto it = categories.find(category);
+            if (it == categories.end())
+            {
+                auto placed = categories.emplace(category, std::unique_ptr<CVaultSet>(new CVaultSet()));
+                if (placed.second)
+                    it = placed.first;
+            }
+            if (it != categories.end())
+                it->second->addVault(&vault);
+        }
+    }
+    bool getCachedSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override
+    {
+        if (isEmptyString(category))
+            return false;
+        auto it = categories.find(category);
+        if (it == categories.end())
+            return false;
+        return it->second->getCachedSecretFromVault(vaultId, kind, content, secret, version);
+    }
+    bool requestSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override
+    {
+        if (isEmptyString(category))
+            return false;
+        auto it = categories.find(category);
+        if (it == categories.end())
+            return false;
+        return it->second->requestSecretFromVault(vaultId, kind, content, secret, version);
+    }
+
+    bool getCachedSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override
+    {
+        if (isEmptyString(category))
+            return false;
+        auto it = categories.find(category);
+        if (it == categories.end())
+            return false;
+        return it->second->getCachedSecret(kind, content, secret, version);
+    }
+    bool requestSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override
+    {
+        if (isEmptyString(category))
+            return false;
+        auto it = categories.find(category);
+        if (it == categories.end())
+            return false;
+        return it->second->requestSecret(kind, content, secret, version);
+    }
+};
+
+IVaultManager *ensureVaultManager()
+{
+    CriticalBlock block(secretCS);
+    if (!vaultManager)
+        vaultManager.setown(new CVaultManager());
+    return vaultManager;
+}
+
+static IPropertyTree *getCachedLocalSecret(const char *category, const char *name)
+{
+    if (isEmptyString(name))
+        return nullptr;
+    Owned<IPropertyTree> secret;
+    {
+        CriticalBlock block(secretCacheCS);
+        IPropertyTree *tree = secretCache->queryPropTree(category);
+        if (!tree)
+            return nullptr;
+        secret.setown(tree->getPropTree(name));
+        if (secret)
+        {
+            if (checkSecretExpired((unsigned) secret->getPropInt("@created")))
+            {
+                secretCache->removeProp(name);
+                return nullptr;
+            }
+            return secret.getClear();
+        }
+    }
+    return nullptr;
+}
+
+static void addCachedLocalSecret(const char *category, const char *name, IPropertyTree *secret)
+{
+    if (!secret || isEmptyString(name) || isEmptyString(category))
+        return;
+    secret->setPropInt("@created", (int)msTick());
+    {
+        CriticalBlock block(secretCacheCS);
+        IPropertyTree *tree = ensurePTree(secretCache, category);
+        tree->setPropTree(name, LINK(secret));
+    }
+}
+
+static const char *ensureSecretDirectory()
+{
+    CriticalBlock block(secretCS);
+    if (secretDirectory.isEmpty())
+        setSecretMount(nullptr);
+    return secretDirectory;
+}
+
+static IPropertyTree *loadLocalSecret(const char *category, const char * name)
+{
+    StringBuffer path;
+    addPathSepChar(path.append(ensureSecretDirectory())).append(category).append(PATHSEPCHAR).append(name).append(PATHSEPCHAR);
+    Owned<IDirectoryIterator> entries = createDirectoryIterator(path);
+    if (!entries || !entries->first())
+        return nullptr;
+    Owned<IPropertyTree> tree = createPTree(name);
+    tree->setPropInt("@created", (int) msTick());
+    ForEach(*entries)
+    {
+        if (entries->isDir())
+            continue;
+        StringBuffer name;
+        entries->getName(name);
+        if (!validateXMLTag(name))
+            continue;
+        MemoryBuffer content;
+        Owned<IFileIO> io = entries->get().open(IFOread);
+        read(io, 0, (size32_t)-1, content);
+        if (!content.length())
+            continue;
+        tree->setPropBin(name, content.length(), content.bufferBase());
+    }
+    addCachedLocalSecret(category, name, tree);
+    return tree.getClear();
+}
+
+extern jlib_decl IPropertyTree *getLocalSecret(const char *category, const char * name)
+{
+    Owned<IPropertyTree> tree = getCachedLocalSecret(category, name);
+    if (tree)
+        return tree.getClear();
+    return loadLocalSecret(category, name);
+}
+
+static IPropertyTree *createPTreeFromVaultSecret(const char *content, CVaultKind kind)
+{
+    if (isEmptyString(content))
+        return nullptr;
+
+    Owned<IPropertyTree> tree = createPTreeFromJSONString(content);
+    if (!tree)
+        return nullptr;
+    switch (kind)
+    {
+        case CVaultKind::kv_v1:
+            tree.setown(tree->getPropTree("data"));
+            break;
+        default:
+        case CVaultKind::kv_v2:
+            tree.setown(tree->getPropTree("data/data"));
+            break;
+    }
+    return tree.getClear();
+}
+static IPropertyTree *getCachedVaultSecret(const char *category, const char *vaultId, const char * name, const char *version)
+{
+    CVaultKind kind;
+    StringBuffer json;
+    IVaultManager *vaultmgr = ensureVaultManager();
+    if (isEmptyString(vaultId))
+    {
+        if (!vaultmgr->getCachedSecretByCategory(category, kind, json, name, version))
+            return nullptr;
+    }
+    else
+    {
+        if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version))
+            return nullptr;
+    }
+    return createPTreeFromVaultSecret(json.str(), kind);
+}
+
+static IPropertyTree *requestVaultSecret(const char *category, const char *vaultId, const char * name, const char *version)
+{
+    CVaultKind kind;
+    StringBuffer json;
+    IVaultManager *vaultmgr = ensureVaultManager();
+    if (isEmptyString(vaultId))
+    {
+        if (!vaultmgr->requestSecretByCategory(category, kind, json, name, version))
+            return nullptr;
+    }
+    else
+    {
+        if (!vaultmgr->requestSecretFromVault(category, vaultId, kind, json, name, version))
+            return nullptr;
+    }
+    return createPTreeFromVaultSecret(json.str(), kind);
+}
+
+extern jlib_decl IPropertyTree *getVaultSecret(const char *category, const char *vaultId, const char * name, const char *version)
+{
+    CVaultKind kind;
+    StringBuffer json;
+    IVaultManager *vaultmgr = ensureVaultManager();
+    if (isEmptyString(vaultId))
+    {
+        if (!vaultmgr->getCachedSecretByCategory(category, kind, json, name, version))
+            vaultmgr->requestSecretByCategory(category, kind, json, name, version);
+    }
+    else
+    {
+        if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version))
+            vaultmgr->requestSecretFromVault(category, vaultId, kind, json, name, version);
+    }
+    return createPTreeFromVaultSecret(json.str(), kind);
+}
+
+extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name)
+{
+    //check for any chached first
+    Owned<IPropertyTree> secret = getCachedLocalSecret(category, name);
+    if (!secret)
+        secret.setown(getCachedVaultSecret(category, nullptr, name, nullptr));
+    //now check local, then vaults
+    if (!secret)
+        secret.setown(loadLocalSecret(category, name));
+    if (!secret)
+        secret.setown(requestVaultSecret(category, nullptr, name, nullptr));
+    return secret.getClear();
+}
+
+extern jlib_decl bool getSecretKeyValue(MemoryBuffer & result, IPropertyTree *secret, const char * key)
+{
+    IPropertyTree *tree = secret->queryPropTree(key);
+    if (tree)
+        return tree->getPropBin(nullptr, result);
+    return false;
+}
+
+extern jlib_decl bool getSecretKeyValue(StringBuffer & result, IPropertyTree *secret, const char * key)
+{
+    IPropertyTree *tree = secret->queryPropTree(key);
+    if (!tree)
+        return false;
+    if (tree->isBinary(nullptr))
+    {
+        MemoryBuffer mb;
+        tree->getPropBin(nullptr, mb);
+        //caller implies it's a string
+        result.append(mb.length(), mb.toByteArray());
+        return true;
+    }
+    const char *value = tree->queryProp(nullptr);
+    if (value)
+    {
+        result.append(value);
+        return true;
+    }
+    return false;
+}
+
+extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category, const char * name, const char * key, bool required)
+{
+    Owned<IPropertyTree> secret = getSecret(category, name);
+    if (required && !secret)
+        throw MakeStringException(-1, "secret %s.%s not found", category, name);
+    bool found = getSecretKeyValue(result, secret, key);
+    if (required && !found)
+        throw MakeStringException(-1, "secret %s.%s missing key %s", category, name, key);
+    return true;
+}
+

system/jlib/jsecrets.hpp

@@ -0,0 +1,39 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef JSECRETS_HPP
+#define JSECRETS_HPP
+
+#include "jlib.hpp"
+#include "jstring.hpp"
+
+extern jlib_decl void setSecretMount(const char * path);
+extern jlib_decl void setSecretTimeout(unsigned timeoutMs);
+
+extern jlib_decl IPropertyTree *getLocalSecret(const char *category, const char * name);
+extern jlib_decl IPropertyTree *getVaultSecret(const char *category, const char *vaultId, const char * name, const char *version=nullptr);
+extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name);
+
+extern jlib_decl bool getSecretKeyValue(MemoryBuffer & result, IPropertyTree *secret, const char * key);
+extern jlib_decl bool getSecretKeyValue(StringBuffer & result, IPropertyTree *secret, const char * key);
+extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category, const char * name, const char * key, bool required);
+
+extern jlib_decl  void splitFullUrl(const char *url, bool &https, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer &port, StringBuffer &fullpath);
+extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &schemeHostPort, StringBuffer &path);
+
+#endif

system/jlib/jstring.cpp

@@ -957,6 +957,46 @@ StringBuffer &replaceString(StringBuffer & result, size_t lenSource, const char
     return result;
 }
 
+StringBuffer &replaceEnvVariables(StringBuffer & result, const char *source, bool exceptions, const char* delim, const char* term)
+{
+    if (isEmptyString(source) || isEmptyString(delim) || isEmptyString(term))
+        return result;
+    size_t lenDelim = strlen(delim);
+    size_t lenTerm = strlen(term);
+    size_t left = strlen(source);
+    size_t minLen = lenDelim + lenTerm + 1;
+    while (left >= minLen)
+    {
+        if (memcmp(source, delim, lenDelim)==0)
+        {
+            const char *finger = source + lenDelim;
+            const char *thumb = strstr(finger, term);
+            if (thumb)
+            {
+                StringAttr name(finger, (size_t)(thumb - finger));
+                const char *value = getenv(name);
+                if (value)
+                {
+                    result.append(value);
+                    size_t replaced = (thumb - source) + lenTerm;
+                    source = thumb + lenTerm;
+                    left -= replaced;
+                    continue;
+                }
+                if (exceptions)
+                    throw MakeStringException(-1, "Environment variable %s not set", name.str());
+            }
+        }
+        result.append(*source);
+        source++;
+        left--;
+    }
+
+    // there are no more possible replacements, make sure we keep the end of the original buffer
+    result.append(left, source);
+    return result;
+}
+
 StringBuffer &replaceStringNoCase(StringBuffer & result, size_t lenSource, const char *source, size_t lenOldStr, const char* oldStr, size_t lenNewStr, const char* newStr)
 {
     if (lenSource)

system/jlib/jstring.hpp

@@ -402,6 +402,7 @@ extern jlib_decl int utf8CharLen(unsigned char ch);
 extern jlib_decl int utf8CharLen(const unsigned char *ch, unsigned maxsize = (unsigned)-1);
 
 extern jlib_decl StringBuffer &replaceString(StringBuffer & result, size_t lenSource, const char *source, size_t lenOldStr, const char* oldStr, size_t lenNewStr, const char* newStr);
+extern jlib_decl StringBuffer &replaceEnvVariables(StringBuffer & result, const char *source, bool exceptions, const char* delim = "${env.", const char* term = "}");
 
 inline const char *encodeUtf8XML(const char *x, StringBuffer &ret, unsigned flags=false, unsigned len=(unsigned)-1)
 {

system/jlib/jutil.cpp

@@ -3030,49 +3030,6 @@ int getEnum(const char *v, const EnumMapping *map, int defval)
 
 //---------------------------------------------------------------------------------------------------------------------
 
-static StringBuffer secretDirectory;
-static CriticalSection secretCS;
-static unsigned secretTimeoutMs = UINT_MAX;
-
-//How long can secrets be cached for?
-extern jlib_decl unsigned getSecretTimeout()
-{
-    return secretTimeoutMs;
-}
-
-extern jlib_decl void setSecretTimeout(unsigned timeoutMs)
-{
-    secretTimeoutMs= timeoutMs;
-}
-
-extern jlib_decl void setSecretMount(const char * path)
-{
-    if (!path)
-    {
-        getPackageFolder(secretDirectory);
-        addPathSepChar(secretDirectory).append("secrets");
-    }
-    else
-        secretDirectory.set(path);
-}
-
-extern jlib_decl StringBuffer & getSecret(StringBuffer & result, const char * name, const char * key)
-{
-    {
-        CriticalBlock block(secretCS);
-        if (secretDirectory.isEmpty())
-            setSecretMount(nullptr);
-    }
-    //MORE: cache the secret for up to secretTimeoutMs
-    StringBuffer path;
-    addPathSepChar(path.append(secretDirectory)).append(name).append(PATHSEPCHAR).append(key);
-    Owned<IFile> file = createIFile(path);
-    result.loadFile(file);
-    return result;
-}
-
-//---------------------------------------------------------------------------------------------------------------------
-
 //#define TESTURL
 #ifdef TESTURL
 

system/jlib/jutil.hpp

@@ -441,9 +441,6 @@ extern jlib_decl bool replaceConfigurationDirectoryEntry(const char *path,const
 extern jlib_decl const char *queryCurrentProcessPath();
 
 extern jlib_decl StringBuffer &getFileAccessUrl(StringBuffer &out);
-extern jlib_decl void setSecretMount(const char * path);
-extern jlib_decl StringBuffer & getSecret(StringBuffer & result, const char * name, const char * key);
-
 
 /**
  * Locate the 'package home' directory - normally /opt/HPCCSystems - by detecting the current executable's location