Merge branch 'candidate-7.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -2623,7 +2623,7 @@ class CEnvironmentClusterInfo: implements IConstWUClusterInfo, public CInterface
     StringAttr serverQueue;
     StringAttr agentQueue;
     StringAttr agentName;
-    StringAttr eclServerName;
+    StringArray eclServerNames;
     bool legacyEclServer = false;
     StringAttr eclSchedulerName;
     StringAttr roxieProcess;
@@ -2644,7 +2644,7 @@ class CEnvironmentClusterInfo: implements IConstWUClusterInfo, public CInterface
 public:
     IMPLEMENT_IINTERFACE;
     CEnvironmentClusterInfo(const char *_name, const char *_prefix, const char *_alias, IPropertyTree *agent,
-        IPropertyTree *eclServer, bool _legacyEclServer, IPropertyTree *eclScheduler, IArrayOf<IPropertyTree> &thors, IPropertyTree *roxie)
+        IArrayOf<IPropertyTree> &eclServers, bool _legacyEclServer, IPropertyTree *eclScheduler, IArrayOf<IPropertyTree> &thors, IPropertyTree *roxie)
         : name(_name), prefix(_prefix), alias(_alias), roxieRedundancy(0), channelsPerNode(0), numberOfSlaveLogs(0), roxieReplicateOffset(1), legacyEclServer(_legacyEclServer)
     {
         StringBuffer queue;
@@ -2731,8 +2731,11 @@ public:
 #endif
         if (eclScheduler)
             eclSchedulerName.set(eclScheduler->queryProp("@name"));
-        if (eclServer)
-            eclServerName.set(eclServer->queryProp("@name"));
+        ForEachItemIn(j, eclServers)
+        {
+            const IPropertyTree &eclServer = eclServers.item(j);
+            eclServerNames.append(eclServer.queryProp("@name"));
+        }
 
         // MORE - does this need to be conditional?
         serverQueue.set(getClusterEclCCServerQueueName(queue.clear(), name));
@@ -2762,10 +2765,9 @@ public:
         str.set(agentName);
         return str;
     }
-    IStringVal & getECLServerName(IStringVal & str) const
+    const StringArray & getECLServerNames() const
     {
-        str.set(eclServerName);
-        return str;
+        return eclServerNames;
     }
     bool isLegacyEclServer() const
     {
@@ -3037,6 +3039,23 @@ extern bool isProcessCluster(const char *remoteDali, const char *process)
     return true;
 }
 
+static void getTargetClusterProcesses(const IPropertyTree *environment, const IPropertyTree *cluster, const char *clustName, const char *processType, IArrayOf<IPropertyTree> &processes)
+{
+    StringBuffer xpath;
+    Owned<IPropertyTreeIterator> processItr = cluster->getElements(processType);
+    ForEach(*processItr)
+    {
+        const char *processName = processItr->query().queryProp("@process");
+        if (isEmptyString(processName))
+            throw MakeStringException(-1, "Empty %s/@process for %s", processType, clustName);
+
+        xpath.setf("Software/%s[@name=\"%s\"]", processType, processName);
+        if (!environment->hasProp(xpath))
+            throw MakeStringException(-1, "%s %s not found", processType, processName);
+        processes.append(*environment->getPropTree(xpath.str()));
+    }
+}
+
 IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyTree *cluster)
 {
     const char *clustname = cluster->queryProp("@name");
@@ -3066,23 +3085,13 @@ IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyT
         eclScheduler = environment->queryPropTree(xpath);
     }
     bool isLegacyEclServer = false;
-    IPropertyTree *eclServer = nullptr;
-    const char *eclServerName = cluster->queryProp("EclServerProcess/@process");
-    if (!isEmptyString(eclServerName))
-    {
-        xpath.setf("Software/EclServerProcess[@name=\"%s\"]", eclServerName);
-        eclServer = environment->queryPropTree(xpath);
+    IArrayOf<IPropertyTree> eclServers;
+    getTargetClusterProcesses(environment, cluster, clustname, "EclServerProcess", eclServers);
+    if (eclServers.ordinality())
         isLegacyEclServer = true;
-    }
     else
-    {
-        const char *eclccServerName = cluster->queryProp("EclCCServerProcess/@process");
-        if (!isEmptyString(eclccServerName))
-        {
-            xpath.setf("Software/EclCCServerProcess[@name=\"%s\"]", eclccServerName);
-            eclServer = environment->queryPropTree(xpath);
-        }
-    }
+        getTargetClusterProcesses(environment, cluster, clustname, "EclCCServerProcess", eclServers);
+
     Owned<IPropertyTreeIterator> ti = cluster->getElements("ThorCluster");
     IArrayOf<IPropertyTree> thors;
     ForEach(*ti)
@@ -3096,7 +3105,7 @@ IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyT
         }
     }
     const char *roxieName = cluster->queryProp("RoxieCluster/@process");
-    return new CEnvironmentClusterInfo(clustname, prefix, cluster->queryProp("@alias"), agent, eclServer, isLegacyEclServer, eclScheduler, thors, queryRoxieProcessTree(environment, roxieName));
+    return new CEnvironmentClusterInfo(clustname, prefix, cluster->queryProp("@alias"), agent, eclServers, isLegacyEclServer, eclScheduler, thors, queryRoxieProcessTree(environment, roxieName));
 }
 
 IPropertyTree* getTopologyCluster(Owned<IPropertyTree> &envRoot, const char *clustname)

common/environment/environment.hpp

@@ -251,7 +251,7 @@ interface IConstWUClusterInfo : extends IInterface
     virtual IStringVal & getAgentQueue(IStringVal & str) const = 0;
     virtual IStringVal & getAgentName(IStringVal & str) const = 0;
     virtual IStringVal & getECLSchedulerName(IStringVal & str) const = 0;
-    virtual IStringVal & getECLServerName(IStringVal & str) const = 0;
+    virtual const StringArray & getECLServerNames() const = 0;
     virtual bool isLegacyEclServer() const = 0;
     virtual IStringVal & getServerQueue(IStringVal & str) const = 0;
     virtual IStringVal & getRoxieProcess(IStringVal & str) const = 0;

common/thorhelper/thorsoapcall.cpp

@@ -1502,6 +1502,7 @@ private:
     Owned<CSocketDataProvider> dataProvider;
     PTreeReaderOptions options;
     unsigned remainingMS;
+    cycle_t startSoapCallCycles;
 
     inline void checkRoxieAbortMonitor(IRoxieAbortMonitor * roxieAbortMonitor)
     {
@@ -2025,10 +2026,15 @@ public:
             responsePath.append(master->service).append("ResponseArray/");
         }
         responsePath.append(master->service).append("Response");
+        remainingMS = 0;
+        startSoapCallCycles = get_cycles_now();
     }
 
     ~CWSCAsyncFor()
     {
+        cycle_t endCycles = get_cycles_now();
+        __int64 elapsedNs = cycle_to_nanosec(endCycles-startSoapCallCycles);
+        master->logctx.noteStatistic(StTimeSoapcall, elapsedNs);
     }
 
     IMPLEMENT_IINTERFACE;
@@ -2051,7 +2057,7 @@ public:
         while (!master->aborted)
         {
             Owned<ISocket> socket;
-            cycle_t startCycles, endCycles;
+            cycle_t startCycles;
             startCycles = get_cycles_now();
             for (;;)
             {
@@ -2148,9 +2154,8 @@ public:
                 {
                     throw MakeStringException(-1, "Zero length response in processQuery");
                 }
-                endCycles = get_cycles_now();
+                cycle_t endCycles = get_cycles_now();
                 __int64 elapsedNs = cycle_to_nanosec(endCycles-startCycles);
-                master->logctx.noteStatistic(StTimeSoapcall, elapsedNs);
                 checkTimeLimitExceeded(&remainingMS);
                 ColumnProvider * meta = (ColumnProvider*)CreateColumnProvider((unsigned)nanoToMilli(elapsedNs), master->flags&SOAPFencoding?true:false);
                 processResponse(url, response, meta);

common/workunit/workunit.cpp

@@ -3903,7 +3903,7 @@ public:
     virtual bool aborting() const
             { return c->aborting(); }
     virtual void forceReload()
-            { UNIMPLEMENTED; }
+            { }
     virtual WUAction getAction() const
             { return c->getAction(); }
     virtual const char *queryActionDesc() const

dali/base/dadfs.cpp

@@ -5118,7 +5118,6 @@ protected:
                 subfile.setown(transaction?transaction->lookupFile(subname.str(),timeout):parent->lookup(subname.str(), udesc, false, false, false, transaction, defaultPrivilegedUser, timeout));
                 if (!subfile.get())
                     subfile.setown(transaction?transaction->lookupSuperFile(subname.str(),timeout):parent->lookupSuperFile(subname.str(),udesc,transaction,timeout));
-                containsRestrictedSubfile = containsRestrictedSubfile || subfile->isRestrictedAccess();
                 // Some files are ok not to exist
                 if (!subfile.get())
                 {
@@ -5137,14 +5136,23 @@ protected:
                             _transaction->ensureFile(subfile);
                         }
                     }
+                    else if (logicalName.isMulti())
+                    {
+                        /*
+                         * implicit superfiles, can't validate subfile presence at this point,
+                         * but will be caught if empty and not OPT later.
+                         */
+                        continue;
+                    }
                     else
                         ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: corrupt subfile file '%s' cannot be found", logicalName.get(), subname.str());
                 }
+                containsRestrictedSubfile = containsRestrictedSubfile || subfile->isRestrictedAccess();
                 subfiles.append(*subfile.getClear());
                 if (link)
                     linkSubFile(f);
             }
-            // This is *only* due to foreign files
+            // This is can happen due to missing referenced foreign files, or missing files referenced via an implicit inline superfile definition
             if (subfiles.ordinality() != n)
             {
                 IWARNLOG("CDistributedSuperFile: SuperFile %s's number of sub-files updated to %d", logicalName.get(), subfiles.ordinality());

dali/server/daserver.cpp

@@ -270,7 +270,7 @@ static bool populateWhiteListFromEnvironment(IWhiteListWriter &writer)
                     Owned<IPropertyTreeIterator> slaveIter = component.getElements("ThorSlaveProcess");
                     ForEach(*slaveIter)
                     {
-                        const char *slaveCompName = component.queryProp("@computer");
+                        const char *slaveCompName = slaveIter->query().queryProp("@computer");
                         const char *ip = resolveComputer(slaveCompName, nullptr, ipSB.clear());
                         if (ip)
                             writer.add(ip, DCR_ThorSlave);

dockerfiles/hpcc/templates/localroxie.yaml

@@ -2,6 +2,7 @@
 {{- if not $roxie.disabled  -}}
 {{- if $roxie.localSlave -}}
 {{- $name := $roxie.name -}}
+{{- $servername := printf "%s-server" $roxie.name -}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -16,6 +17,7 @@ spec:
     metadata:
       labels:
         run: {{ $roxie.name | quote }}
+        roxie-server: {{ $servername | quote }}
     spec:
       {{- include "hpcc.checkDataStorageHostMount" (dict "root" $) | indent 6 }}
       containers:
@@ -23,10 +25,21 @@ spec:
         args: [
                 {{ include "hpcc.configArg" $roxie }},
                 {{ include "hpcc.daliArg" $ }},
-                "--serverPorts={{ template "hpcc.portListToCommas" $roxie.ports }}", 
+                "--server=true", 
                 "--localSlave=true",
                 "--resolveLocally=false"
               ]
+{{- $local := dict "first" true }}
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- if $local.first }}
+{{- $_ := set $local "first" false }}
+        ports:
+{{- end }}
+        - name: {{ $service.name }}
+          containerPort: {{ $service.port }}
+{{- end }}
+{{- end }}
 {{- include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{ include "hpcc.addImageAttrs" (dict "root" $ "me" . "imagename" "roxie") | indent 8 }}
         volumeMounts:
@@ -38,6 +51,24 @@ spec:
 {{ include "hpcc.addDataVolume" $ | indent 6 }}
 {{ include "hpcc.addDllserverVolume" $ | indent 6 }}
 ---
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- $name := printf "%s-%s" $roxie.name $service.name }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name | quote }}
+spec:
+  ports:
+  - port: {{ $service.port }}
+    protocol: TCP
+    targetPort: {{ $service.port }}
+  selector:
+    roxie-server: {{ $servername | quote }}
+  type: {{ if $service.external -}} LoadBalancer {{- else -}} ClusterIP {{- end }}
+{{- end }}
+{{- end }}
+---
 kind: ConfigMap 
 apiVersion: v1 
 metadata:

dockerfiles/hpcc/templates/roxie.yaml

@@ -4,6 +4,7 @@
 {{- $toponame := printf "%s-toposerver" $roxie.name -}}
 {{- $numChannels := $roxie.numChannels | int | default 1 -}}
 {{- $topoport := $roxie.topoport | int | default 9004 -}}
+{{- $servername := printf "%s-server" $roxie.name -}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -32,6 +33,26 @@ spec:
           name: {{ .name }}-configmap
 ---
 
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- $name := printf "%s-%s" $roxie.name $service.name }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name | quote }}
+spec:
+  ports:
+  - port: {{ $service.port }}
+    protocol: TCP
+    targetPort: {{ $service.port }}
+  selector:
+    roxie-server: {{ $servername | quote }}
+  type: {{ if $service.external -}} LoadBalancer {{- else -}} ClusterIP {{- end }}
+{{- end }}
+{{- end }}
+
+---
+
 apiVersion: v1
 kind: Service
 metadata:
@@ -60,7 +81,6 @@ data:
 ---
 
 {{ if $roxie.serverReplicas -}}
-{{- $servername := printf "%s-server" $roxie.name -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -74,6 +94,7 @@ spec:
     metadata:
       labels:
         run: {{ $servername | quote }}
+        roxie-server: {{ $servername | quote }}
     spec:
       {{- include "hpcc.checkDataStorageHostMount" (dict "root" $) | indent 6 }}
       containers:
@@ -82,10 +103,21 @@ spec:
                 {{ include "hpcc.configArg" . }},
                 {{ include "hpcc.daliArg" $ }},
                 "--numChannels={{ $numChannels }}",
-                "--serverPorts={{ template "hpcc.portListToCommas" $roxie.ports }}", 
+                "--server=true", 
                 "--topologyServers={{ $toponame }}:{{ $roxie.topoport }}",
                 "--resolveLocally=false"
               ]
+{{- $local := dict "first" true }}
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- if $local.first }}
+{{- $_ := set $local "first" false }}
+        ports:
+{{- end }}
+        - name: {{ $service.name }}
+          containerPort: {{ $service.port }}
+{{- end }}
+{{- end }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{ include "hpcc.addImageAttrs" (dict "root" $ "me" $roxie "imagename" "roxie") | indent 8 }}
         volumeMounts:
@@ -108,7 +140,7 @@ kind: Deployment
 metadata:
   name: {{ $name | quote}}
 spec:
-  replicas: {{ $roxie.channelReplicas | default 2 }}
+  replicas: {{ $roxie.channelReplicas | default 1 }}
   selector:
     matchLabels:
       run: {{ $name | quote}}
@@ -116,6 +148,9 @@ spec:
     metadata:
       labels:
         run: {{ $name | quote}}
+{{- if not $roxie.serverReplicas }}        
+        roxie-server: {{ $servername | quote }}
+{{- end }}
     spec:
       {{- include "hpcc.checkDataStorageHostMount" (dict "root" $) | indent 6 }}
       containers:
@@ -124,10 +159,24 @@ spec:
                 {{ include "hpcc.configArg" $roxie }},
                 {{ include "hpcc.daliArg" $ }},
                 "--channels={{ $channel }}", 
-                "--serverPorts={{ if not $roxie.serverReplicas }}{{ template "hpcc.portListToCommas" $roxie.ports }}{{ end }}",
+                "--server={{ not $roxie.serverReplicas }}",
                 "--numChannels={{ $numChannels }}",
                 "--topologyServers={{ $toponame }}:{{ $roxie.topoport }}",
+                "--resolveLocally=false"
               ]
+{{- if not $roxie.serverReplicas }}
+{{- $local := dict "first" true }}
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- if $local.first }}
+{{- $_ := set $local "first" false }}
+        ports:
+{{- end }}
+        - name: {{ $service.name }}
+          containerPort: {{ $service.port }}
+{{- end }}
+{{- end }}
+{{- end }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{ include "hpcc.addImageAttrs" (dict "root" $ "me" $roxie "imagename" "roxie") | indent 8 }}
         volumeMounts:

dockerfiles/hpcc/values.schema.json

@@ -210,16 +210,38 @@
           "type": "string",
           "description": "The (optional) file prefix to add to relative filenames"
         },
-        "ports": {
-          "type": "array",
-          "description": "The ports to listen on",
-          "items": { "type": "integer" }
-        },
         "image": {
           "$ref": "#/definitions/image"
-        }
+        },
+        "services": {
+          "description": "Roxie query services",
+          "type": "array",
+          "items": { "$ref": "#/definitions/roxieservice" }
+        }    
       }
     },
+    "roxieservice": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "numThreads": {
+          "type": "integer"
+        },
+        "listenQueue": {
+          "type": "integer"
+        },
+        "external": {
+          "type": "boolean"
+        }
+      },
+      "required": [ "name", "port" ],
+      "additionalProperties": false
+    },
     "thor": {
       "type": "object",
       "required": [ "name" ],

dockerfiles/hpcc/values.yaml

@@ -83,14 +83,24 @@ esp:
 
 roxie:
 - name: roxie-cluster
-  disabled: true
+  disabled: false
   prefix: roxiecluster
-  ports: [9876]
+  services:
+  - name: query
+    port: 9876
+    listenQueue: 200
+    numThreads: 0
+    external: true
+  - name: on-demand
+    port: 0
   numChannels: 2
-  serverReplicas: 1
+  ## Set serverReplicas to indicate a separate replicaSet of roxie servers, with slave nodes not acting as servers
+  serverReplicas: 0
   topoReplicas: 1
   topoport: 9004
-  useAeron: true
+  ## Set localSlave to true for a scalable cluster of "single-node" roxie servers
+  localSlave: false
+  useAeron: false
 
 thor:
 - name: thor

dockerfiles/roxie/Dockerfile

@@ -23,6 +23,5 @@ FROM hpccsystems/platform-core:${BUILD_LABEL}
 USER hpcc
 RUN mkdir -p /var/lib/HPCCSystems/roxie
 WORKDIR /var/lib/HPCCSystems/roxie
-RUN roxie --init > roxie.yaml
-ENTRYPOINT ["roxie", "--config=roxie.yaml"]
+ENTRYPOINT ["roxie"]
 

docs/EN_US/ECLLanguageReference/ECLR_mods/Value-Decimal.xml

@@ -23,10 +23,10 @@
 
   <para>A packed decimal<indexterm>
       <primary>packed decimal</primary>
-    </indexterm> value of <emphasis>n</emphasis> total digits (to a maximum of
-  32). If the _<emphasis>y</emphasis> value is present, the
-  <emphasis>y</emphasis> defines the number of decimal places in the
-  value.</para>
+    </indexterm> value of <emphasis>n</emphasis> total digits. If the
+  _<emphasis>y</emphasis> value is present, the <emphasis>y</emphasis> defines
+  the number of decimal places in the value. There can be at most 32 leading
+  digits and 32 fractional digits. </para>
 
   <para>If the UNSIGNED keyword is omitted, the rightmost nibble holds the
   sign. Unsigned decimal declarations may be contracted to use the optional

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/SprayFixed.xml

@@ -67,8 +67,8 @@
         <row>
           <entry><emphasis>sourceIP</emphasis></entry>
 
-          <entry>A null-terminated string containing the IP address of the
-          file.</entry>
+          <entry>A null-terminated string containing the IP address or
+          hostname of the Dropzone where the file is located.</entry>
         </row>
 
         <row>

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/SprayJson.xml

@@ -0,0 +1,256 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="SprayJSON">
+  <title>SprayJson</title>
+
+  <para><emphasis role="bold">STD.File.SprayJson<indexterm>
+      <primary>STD.File.SprayJson</primary>
+    </indexterm><indexterm>
+      <primary>File.SprayJson</primary>
+    </indexterm><indexterm>
+      <primary>SprayJson</primary>
+    </indexterm>(</emphasis> <emphasis> sourceIP </emphasis> <emphasis
+  role="bold">, </emphasis> <emphasis>sourcepath , </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis> maxrecordsize </emphasis> <emphasis
+  role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis role="bold">
+  </emphasis> <emphasis> srcRowPath </emphasis> <emphasis role="bold">
+  </emphasis> <emphasis>, </emphasis> <emphasis role="bold">[</emphasis>
+  <emphasis> srcEncoding </emphasis> <emphasis role="bold">] </emphasis>
+  <emphasis>, </emphasis> <emphasis role="bold"> </emphasis>
+  <emphasis>destinationgroup, destinationlogicalname </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis>timeout</emphasis> <emphasis
+  role="bold">]</emphasis> <emphasis role="bold"> [</emphasis>
+  <emphasis>espserverIPport</emphasis> <emphasis role="bold">]</emphasis>
+  <emphasis> </emphasis> <emphasis role="bold">[</emphasis>
+  <emphasis>maxConnections</emphasis> <emphasis role="bold">]</emphasis>
+  <emphasis role="bold"> [</emphasis> <emphasis>allowoverwrite</emphasis>
+  <emphasis role="bold">] [</emphasis> <emphasis>replicate</emphasis>
+  <emphasis role="bold">] [</emphasis> <emphasis> compress
+  </emphasis><emphasis role="bold">] </emphasis>, <emphasis
+  role="bold">[</emphasis><emphasis>failIfNoSourceFile</emphasis><emphasis
+  role="bold">]</emphasis>, <emphasis
+  role="bold">[</emphasis><emphasis>expireDays</emphasis><emphasis
+  role="bold">] , </emphasis><emphasis role="bold">[</emphasis>
+  <emphasis>dfuServerQueue</emphasis><emphasis role="bold">] ,
+  </emphasis><emphasis role="bold">[</emphasis>
+  <emphasis>noSplit</emphasis><emphasis role="bold">])</emphasis></para>
+
+  <para><emphasis>dfuwuid</emphasis> <emphasis role="bold"> :=
+  STD.File.fSprayJson<indexterm>
+      <primary>STD.File.fSprayJson</primary>
+    </indexterm><indexterm>
+      <primary>File.fSprayJson</primary>
+    </indexterm><indexterm>
+      <primary>fSprayJson</primary>
+    </indexterm>(</emphasis> <emphasis> sourceIP</emphasis> <emphasis
+  role="bold">, </emphasis> <emphasis>sourcepath, </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis> maxrecordsize </emphasis> <emphasis
+  role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis role="bold">
+  </emphasis> <emphasis>srcRowPath</emphasis> <emphasis role="bold">
+  </emphasis> <emphasis>, </emphasis> <emphasis role="bold">[</emphasis>
+  <emphasis> srcEncoding </emphasis> <emphasis role="bold">] </emphasis>
+  <emphasis>,destinationgroup,</emphasis> <emphasis> destinationlogicalname
+  </emphasis> , <emphasis role="bold">[</emphasis>
+  <emphasis>timeout</emphasis> <emphasis role="bold">] , [</emphasis>
+  <emphasis>espserverIPport</emphasis> <emphasis role="bold">]</emphasis>
+  <emphasis> </emphasis>, <emphasis role="bold"> [</emphasis>
+  <emphasis>maxConnections</emphasis> <emphasis role="bold">] , [</emphasis>
+  <emphasis>allowoverwrite</emphasis> <emphasis role="bold">] , [</emphasis>
+  <emphasis>replicate</emphasis> <emphasis role="bold">] , [</emphasis>
+  <emphasis> compress </emphasis> <emphasis role="bold">] </emphasis>,
+  <emphasis
+  role="bold">[</emphasis><emphasis>failIfNoSourceFile</emphasis><emphasis
+  role="bold">]</emphasis>, <emphasis
+  role="bold">[</emphasis><emphasis>expireDays</emphasis><emphasis
+  role="bold">] , </emphasis><emphasis role="bold">[</emphasis>
+  <emphasis>dfuServerQueue</emphasis><emphasis role="bold">] ,
+  </emphasis><emphasis role="bold">[</emphasis>
+  <emphasis>noSplit</emphasis><emphasis role="bold">])</emphasis></para>
+
+  <informaltable colsep="1" frame="all" rowsep="1">
+    <tgroup cols="2">
+      <colspec colwidth="80.50pt" />
+
+      <colspec />
+
+      <tbody>
+        <row>
+          <entry><emphasis>sourceIP</emphasis></entry>
+
+          <entry>A null-terminated string containing the IP address or
+          hostname of the Dropzone where the file is located.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>sourcepath</emphasis></entry>
+
+          <entry>A null-terminated string containing the path and name of the
+          file.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>maxrecordsize</emphasis></entry>
+
+          <entry>Optional. An integer containing the maximum size of the
+          records in the file. If omitted, the default is 8192.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>sourceRowPath</emphasis></entry>
+
+          <entry>The JSON path that is used to delimit records in the source
+          file. Required.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>srcEncoding</emphasis></entry>
+
+          <entry>Optional. A null-terminated string containing the encoding
+          (utf8,utf8n,utf16be,utf16le,utf32be,utf32le). If omitted, the
+          default is 'utf8'</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>destinationgroup</emphasis></entry>
+
+          <entry>A null-terminated string containing the name of the group to
+          distribute the file across.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>destinationlogicalname</emphasis></entry>
+
+          <entry>A null-terminated string containing the logical name of the
+          file to create.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>timeout</emphasis></entry>
+
+          <entry>Optional. An integer value indicating the timeout setting. If
+          omitted, the default is -1. If set to zero (0), execution control
+          returns immediately to the ECL workunit without waiting for the DFU
+          workunit to complete.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>espserverIPport</emphasis></entry>
+
+          <entry>Optional. A null-terminated string containing the protocol,
+          IP, port, and directory, or the DNS equivalent, of the ESP server
+          program. This is usually the same IP and port as ECL Watch, with
+          "/FileSpray" appended. If omitted, the default is the value
+          contained in the lib_system.ws_fs_server attribute.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>maxConnections</emphasis></entry>
+
+          <entry>Optional. An integer specifying the maximum number of
+          connections. If omitted, the default is one (1).</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>allowoverwrite</emphasis></entry>
+
+          <entry>Optional. A boolean TRUE or FALSE flag indicating whether to
+          allow the new file to overwrite an existing file of the same name.
+          If omitted, the default is FALSE.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>replicate</emphasis></entry>
+
+          <entry>Optional. A boolean TRUE or FALSE flag indicating whether to
+          replicate the new file. If omitted, the default is FALSE.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>compress</emphasis></entry>
+
+          <entry>Optional. A boolean TRUE or FALSE flag indicating whether to
+          compress the new file. If omitted, the default is FALSE.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>failIfNoSourceFile</emphasis></entry>
+
+          <entry>Optional. A boolean TRUE or FALSE flag indicating whether a
+          missing file triggers a failure. If omitted, the default is
+          FALSE.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>expireDays</emphasis></entry>
+
+          <entry>Optional. Specifies the file is a temporary file to be
+          automatically deleted after the specified number of days since the
+          file was read. If omitted, the default is -1 (never expires). If set
+          to 0, the file is automatically deleted when it reaches the
+          threshold set in Sasha Server's <emphasis
+          role="bold">expiryDefault</emphasis> setting.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>dfuServerQueue</emphasis></entry>
+
+          <entry>Name of target DFU Server queue. Default is '' (empty) for
+          the first DFU queue in the environment.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>noSplit</emphasis></entry>
+
+          <entry>Optional. A boolean TRUE or FALSE flag indicating to not
+          split a file part to multiple target parts. Default is
+          FALSE.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>dfuwuid</emphasis></entry>
+
+          <entry>The attribute name to receive the null-terminated string
+          containing the DFU workunit ID (DFUWUID) generated for the
+          job.</entry>
+        </row>
+
+        <row>
+          <entry>username<emphasis> </emphasis></entry>
+
+          <entry>Optional. String containing a username to use for
+          authenticated access to the ESP process; an empty string value
+          indicates that no user authentication is required. If omitted, the
+          default is an empty string.</entry>
+        </row>
+
+        <row>
+          <entry>userPw:<emphasis> </emphasis></entry>
+
+          <entry>Optional. String containing the password to be used with the
+          user cited in the <emphasis>username</emphasis> argument; if
+          <emphasis>username</emphasis> is empty then this is ignored. If
+          omitted, the default is an empty string.</entry>
+        </row>
+
+        <row>
+          <entry>Return:<emphasis> </emphasis></entry>
+
+          <entry>fSprayJson returns a null-terminated string containing the
+          DFU workunit ID (DFUWUID).</entry>
+        </row>
+      </tbody>
+    </tgroup>
+  </informaltable>
+
+  <para>The <emphasis role="bold">SprayJson </emphasis>function takes a
+  well-formed JSON file from a landing zone and distributes it across the
+  nodes of the destination cluster, producing a well-formed JSON file on each
+  node.</para>
+
+  <para>Example:</para>
+
+  <programlisting format="linespecific">STD.File.SprayJson('10.150.50.14','/var/lib/HPCCSystems/mydropzone/colors.json',,
+      '/',,'mythor','examples::colors.json',-1,
+      'http://10.150.50.12:8010/FileSpray');</programlisting>
+</sect1>

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/SprayVariable.xml

@@ -97,8 +97,8 @@
         <row>
           <entry><emphasis>sourceIP</emphasis></entry>
 
-          <entry>A null-terminated string containing the IP address of the
-          file.</entry>
+          <entry>A null-terminated string containing the IP address or
+          hostname of the Dropzone where the file is located.</entry>
         </row>
 
         <row>

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/SprayXML.xml

@@ -78,8 +78,8 @@
         <row>
           <entry><emphasis>sourceIP</emphasis></entry>
 
-          <entry>A null-terminated string containing the IP address of the
-          file.</entry>
+          <entry>A null-terminated string containing the IP address or
+          hostname of the Dropzone where the file is located.</entry>
         </row>
 
         <row>

docs/EN_US/ECLStandardLibraryReference/SLR-includer.xml

@@ -227,6 +227,9 @@
     <xi:include href="ECLStandardLibraryReference/SLR-Mods/SprayXML.xml"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
+    <xi:include href="ECLStandardLibraryReference/SLR-Mods/SprayJson.xml"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLStandardLibraryReference/SLR-Mods/WaitDfuWorkunit.xml"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
   </chapter>

docs/EN_US/ECLWatch/TheECLWatchMan.xml

@@ -69,13 +69,13 @@
     <para>ECL Watch is a service that runs on the Enterprise Services Platform
     (ESP), a middleware component on the HPCC Systems platform.</para>
 
-    <para>ECL Watch provides an interface to the HPCC Systems platform and allows you to
-    view information and interrogate nodes to confirm all expected processes
-    are running. It is a plugin that is useful for Systems Administrators to
-    check processes, examine topology, and view logs. It is useful to ECL
-    Programmers to monitor the status of jobs and files, and other pertinent
-    information. This provides a simple view into the system and a means to
-    perform Workunit and data files maintenance.</para>
+    <para>ECL Watch provides an interface to the HPCC Systems platform and
+    allows you to view information and interrogate nodes to confirm all
+    expected processes are running. It is a plugin that is useful for Systems
+    Administrators to check processes, examine topology, and view logs. It is
+    useful to ECL Programmers to monitor the status of jobs and files, and
+    other pertinent information. This provides a simple view into the system
+    and a means to perform Workunit and data files maintenance.</para>
 
     <para>The ECL Watch interface is a browser based set of pages where you
     can access and interface with the HPCC Systems platform. To Run <emphasis
@@ -418,8 +418,20 @@
 
           <para>The <emphasis role="bold">Set Banner</emphasis> link allows
           you to set a custom banner message at the top of your browser window
-          when you open ECL Watch. You can use this feature to send messages
-          to users.</para>
+          when you open ECL Watch. Check the <emphasis
+          role="bold">Enable</emphasis> box to enable a banner. You can use
+          the banner to deliver a short message about the environment to
+          users. Customize the appearance of the message banner with the other
+          controls. Banner settings persist until the ESP Server
+          restarts.</para>
+
+          <para>The <emphasis role="bold">Set Toolbar</emphasis> link allows
+          you to customize the toolbar at the top of the ECL Watch page. Check
+          the <emphasis role="bold">Enable Environment Text </emphasis>box to
+          display the<emphasis role="bold"> Name of Environment</emphasis>
+          text at the top of the page and in the browser tab. Labeling browser
+          tabs is helpful when working with multiple environments. Toolbar
+          settings persist through restarts of the ESP Server.</para>
 
           <para>The <emphasis role="bold">Error/Warning(s)</emphasis> link
           displays a tab showing you Errors, Warnings, and Information
@@ -427,17 +439,19 @@
           bottom of the tab. A copy facility is also provided.</para>
 
           <para>The <emphasis role="bold">Release Notes</emphasis> link opens
-          a new browser tab to the HPCC Systems release notes page where you can find
-          more release specific information about the contents of each version
-          of HPCC.</para>
+          a new browser tab to the HPCC Systems release notes page where you
+          can find more release specific information about the contents of
+          each version of HPCC.</para>
 
           <para>The <emphasis role="bold">Documentation</emphasis> link opens
-          a new browser tab to the HPCC Systems documentation page, where you can view
-          and download the HPCC Systems platform documentation.</para>
+          a new browser tab to the HPCC Systems documentation page, where you
+          can view and download the HPCC Systems platform
+          documentation.</para>
 
           <para>The <emphasis role="bold">Downloads</emphasis> link opens a
-          new browser tab to the HPCC Systems downloads page, where you can find and
-          download the HPCC Systems platform, client tools, and plugins.</para>
+          new browser tab to the HPCC Systems downloads page, where you can
+          find and download the HPCC Systems platform, client tools, and
+          plugins.</para>
 
           <para>The <emphasis role="bold">Additional Resources</emphasis> link
           opens a submenu that provides links to areas on the HPCC Systems web
@@ -451,8 +465,8 @@
           system.</para>
 
           <para>The <emphasis role="bold">About</emphasis> link opens a dialog
-          to display information about the version of the HPCC Systems platform
-          installed on your server.</para>
+          to display information about the version of the HPCC Systems
+          platform installed on your server.</para>
         </sect3>
 
         <sect3 id="ECLWatch_AdvancedMenuLoggedInAs" role="nobrk">
@@ -509,8 +523,8 @@
           <sect4 id="ECLWatch_AdvancedMenuChangePassword" role="nobrk">
             <title>Change Password</title>
 
-            <para>If authentication is enabled on your HPCC Systems platform, you can
-            change your password, right from the User Details window.
+            <para>If authentication is enabled on your HPCC Systems platform,
+            you can change your password, right from the User Details window.
             <orderedlist>
                 <listitem>
                   <para>Click on your username link under the <emphasis
@@ -1077,14 +1091,14 @@
   <chapter id="ECLWatch_FilesChapter">
     <title>Files</title>
 
-    <para>This chapter contains sections dealing with HPCC Systems platform Files,
-    found on the <emphasis role="bold">Files</emphasis> link in ECL
+    <para>This chapter contains sections dealing with HPCC Systems platform
+    Files, found on the <emphasis role="bold">Files</emphasis> link in ECL
     Watch.</para>
 
-    <para>In an HPCC Systems platform, data files are partitioned across nodes. The
-    file parts, referenced using Logical Filenames, are stored in the
-    Distributed File Utility. This allows the collection of file parts to be
-    referenced as a single entity.</para>
+    <para>In an HPCC Systems platform, data files are partitioned across
+    nodes. The file parts, referenced using Logical Filenames, are stored in
+    the Distributed File Utility. This allows the collection of file parts to
+    be referenced as a single entity.</para>
 
     <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
                 xpointer="xpointer(//*[@id='ECLWatchLogicalFiles'])"
@@ -1853,9 +1867,9 @@
         <title>Installing Ganglia in ECL Watch</title>
 
         <para>In order to use Ganglia in ECL Watch, you need to have Ganglia
-        installed on your HPCC Systems platform. For details on installing Ganglia for
-        ECL Watch, refer to the <emphasis>HPCC Systems Monitoring and
-        Reporting</emphasis> manual.</para>
+        installed on your HPCC Systems platform. For details on installing
+        Ganglia for ECL Watch, refer to the <emphasis>HPCC Systems Monitoring
+        and Reporting</emphasis> manual.</para>
       </sect2>
     </sect1>
 

ecl/eclagent/eclagent.cpp

@@ -3368,11 +3368,6 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
     int retcode = 0;
     addAbortHandler(ControlHandler);
     cmdLineArgs.setown(createProperties(true));
-    if (argc==1)
-    {
-        usage();
-        return 2;
-    }
     for (int i = 1; i < argc; i++)
     {
         const char *arg = argv[i];
@@ -3383,6 +3378,11 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
     //get logfile location from agentexec.xml config file
     if (!standAloneExe)
     {
+        if (argc==1)
+        {
+            usage();
+            return 2;
+        }
         try
         {
             agentTopology.setown(loadConfiguration(defaultYaml, argv, "hthor", "ECLAGENT", "agentexec.xml", nullptr));
@@ -3394,7 +3394,7 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
         }
     }
     else
-        agentTopology.setown(createPTree("hthor")); // MORE - this needs thought!
+        agentTopology.setown(loadConfiguration(defaultYaml, argv, "hthor", "ECLAGENT", nullptr, nullptr));
 
     //Build log file specification
     StringBuffer logfilespec;

ecl/eclcc/eclcc.cpp

@@ -856,7 +856,7 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
                         if (optTargetClusterType==RoxieCluster)
                             generator->addLibrary("ccd");
                         else
-                            generator->addLibrary("hthor");
+                            generator->addLibrary("hthorlib");
 
 
                         compileOk = generator->generateExe(compiler);

ecl/hql/hqlgram.hpp

@@ -514,7 +514,7 @@ public:
     IHqlExpression * createListFromExpressionList(attribute & attr);
     IHqlExpression * createListIndex(attribute & list, attribute & which, IHqlExpression * attr);
     IHqlExpression * createNullPattern();
-    IHqlExpression * createLibraryInstance(const attribute & errpos, IHqlExpression * name, IHqlExpression * func, HqlExprArray & actuals);
+    IHqlExpression * createLibraryInstance(const attribute & errpos, IHqlExpression * name, IHqlExpression * func, HqlExprArray & actuals, IHqlExpression * attrs);
     IHqlExpression * createLocationAttr(const attribute & errpos);
     IHqlExpression * createSortExpr(node_operator op, attribute & dsAttr, const attribute & orderAttr, HqlExprArray & args);
     IHqlExpression * createIffDataset(IHqlExpression * record, IHqlExpression * value);

ecl/hql/hqlgram.y

@@ -3183,6 +3183,11 @@ commonAttribute
     | orderAttribute
     ;
 
+optHintAttribute
+    :                   { $$.setNullExpr(); }
+    | ',' hintAttribute { $$.inherit($2); }
+    ;
+
 hintAttribute
     : HINT '(' hintList ')'
                         {
@@ -7365,10 +7370,10 @@ abstractModule
                             OwnedHqlExpr func = $5.getExpr();
                             HqlExprArray actuals;
                             $8.unwindCommaList(actuals);
-                            $$.setExpr(parser->createLibraryInstance($1, name, func, actuals));
+                            $$.setExpr(parser->createLibraryInstance($1, name, func, actuals, nullptr));
                             $$.setPosition($1);
                         }
-    | LIBRARY '(' libraryName ',' scopeFunctionWithParameters ')'
+    | LIBRARY '(' libraryName ',' scopeFunctionWithParameters optHintAttribute ')'
                         {
                             OwnedHqlExpr value = $5.getExpr();
                             IHqlExpression * func;
@@ -7384,7 +7389,8 @@ abstractModule
                             //Need to create a library definition from referenced attribute, adding the name/internal attribute
                             //and then bind it to create the library instance.
                             OwnedHqlExpr name = $3.getExpr();
-                            $$.setExpr(parser->createLibraryInstance($1, name, func, actuals));
+                            OwnedHqlExpr attrs = $6.getExpr();
+                            $$.setExpr(parser->createLibraryInstance($1, name, func, actuals, attrs));
                             $$.setPosition($1);
                         }
     | startCompoundExpression beginInlineFunctionToken optDefinitions RETURN abstractModule ';' endInlineFunctionToken

ecl/hql/hqlgram2.cpp

@@ -10670,7 +10670,7 @@ void HqlGram::checkNonGlobalModule(const attribute & errpos, IHqlExpression * sc
 }
 
 
-IHqlExpression * HqlGram::createLibraryInstance(const attribute & errpos, IHqlExpression * name, IHqlExpression * func, HqlExprArray & actuals)
+IHqlExpression * HqlGram::createLibraryInstance(const attribute & errpos, IHqlExpression * name, IHqlExpression * func, HqlExprArray & actuals, IHqlExpression * attrs)
 {
     if (!checkParameters(func, actuals, errpos))
         return createNullScope();
@@ -10740,6 +10740,9 @@ IHqlExpression * HqlGram::createLibraryInstance(const attribute & errpos, IHqlEx
 
     HqlExprArray realActuals;
     inputMapper.mapLogicalToReal(realActuals, actuals);
+    if (attrs)
+        attrs->unwindList(realActuals, no_comma);
+
     OwnedHqlExpr bound = bindParameters(errpos, newFunction, realActuals);
     if (!needToMapOutputs)
         return bound.getClear();

esp/services/ws_machine/ws_machineService.cpp

@@ -2956,16 +2956,16 @@ IPropertyTree* Cws_machineEx::getTargetClusterUsageReq(IEspGetTargetClusterUsage
         if (roxie.length())
             readRoxieUsageReq(roxie.str(), constEnv, targetClusterTree);
 
-        SCMStringBuffer eclAgent, eclServer, eclScheduler;
+        SCMStringBuffer eclAgent, eclScheduler;
         targetClusterInfo->getAgentName(eclAgent);
         if (eclAgent.length())
             readOtherComponentUsageReq(eclAgent.str(), eqEclAgent, constEnv, targetClusterTree);
-        targetClusterInfo->getECLServerName(eclServer);
-        if (eclServer.length())
-            readOtherComponentUsageReq(eclServer.str(), targetClusterInfo->isLegacyEclServer() ? eqEclServer : eqEclCCServer, constEnv, targetClusterTree);
         targetClusterInfo->getECLSchedulerName(eclScheduler);
         if (eclScheduler.length())
             readOtherComponentUsageReq(eclScheduler.str(), eqEclScheduler, constEnv, targetClusterTree);
+        const StringArray& eclServers = targetClusterInfo->getECLServerNames();
+        ForEachItemIn(j, eclServers)
+            readOtherComponentUsageReq(eclServers.item(j), targetClusterInfo->isLegacyEclServer() ? eqEclServer : eqEclCCServer, constEnv, targetClusterTree);
 
         usageReq->addPropTree(targetClusterTree->queryName(), LINK(targetClusterTree));
     }

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -726,12 +726,17 @@ public:
     void init(IEspContext &context, bool allowForeignFiles)
     {
         files.setown(createReferencedFileList(context.queryUserId(), context.queryPassword(), allowForeignFiles, false));
+#ifndef _CONTAINERIZED
         clusterInfo.setown(getTargetClusterInfo(target));
         StringBufferAdaptor sba(process);
         if (clusterInfo && clusterInfo->getPlatform()==RoxieCluster)
             clusterInfo->getRoxieProcess(sba);
         if (!process.length())
             return;
+#else
+        StringBuffer process(target);
+#endif
+
         ps.setown(createPackageSet(process.str()));
         if (ps)
             pm = ps->queryActiveMap(target);
@@ -746,7 +751,9 @@ public:
         StringBuffer defReplicateFolder;
         getConfigurationDirectory(NULL, "data2", "roxie", process.str(), defReplicateFolder);
         Owned<IDFUhelper> helper = createIDFUhelper();
+#ifndef _CONTAINERIZED
         files->cloneAllInfo(updateFlags, helper, true, true, clusterInfo->getRoxieRedundancy(), clusterInfo->getChannelsPerNode(), clusterInfo->getRoxieReplicateOffset(), defReplicateFolder);
+#endif
     }
 
     void gatherFileErrors(IArrayOf<IConstLogicalFileError> &errors)
@@ -755,7 +762,9 @@ public:
     }
 
 private:
+#ifndef _CONTAINERIZED
     Owned <IConstWUClusterInfo> clusterInfo;
+#endif
     Owned<IHpccPackageSet> ps;
     const IHpccPackageMap *pm = nullptr;
     StringAttr target;

roxie/ccd/ccd.hpp

@@ -297,7 +297,11 @@ extern bool simpleLocalKeyedJoins;
 extern bool enableKeyDiff;
 extern PTreeReaderOptions defaultXmlReadFlags;
 extern bool mergeSlaveStatistics;
+#ifdef _CONTAINERIZED
+static constexpr bool roxieMulticastEnabled = false;
+#else
 extern bool roxieMulticastEnabled;   // enable use of multicast for sending requests to slaves
+#endif
 extern bool preloadOnceData;
 extern bool reloadRetriesFailed;
 extern bool selfTestMode;

roxie/ccd/ccdmain.cpp

@@ -95,7 +95,9 @@ PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
 
 unsigned udpMulticastBufferSize = 262142;
+#ifndef _CONTAINERIZED
 bool roxieMulticastEnabled = true;
+#endif
 
 IPropertyTree *topology;
 MapStringTo<int> *preferredClusters;
@@ -410,9 +412,9 @@ public:
 };
 
 static std::vector<RoxieEndpointInfo> myRoles;
-static std::vector<unsigned> farmerPorts;
 static std::vector<std::pair<unsigned, unsigned>> slaveChannels;
 
+#ifndef _CONTAINERIZED
 void readStaticTopology()
 {
     // If dynamicServers not set, we read a list of all servers form the topology file, and deduce which ones are on which channel
@@ -523,6 +525,7 @@ void readStaticTopology()
     numChannels = calcNumChannels;
     createStaticTopology(allRoles, traceLevel);
 }
+#endif
 
 int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
 {
@@ -654,16 +657,6 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         StringArray topoValues;
         if (topos)
             topoValues.appendList(topos, ",", true);
-        const char *serverPorts = topology->queryProp("@serverPorts");
-        if (serverPorts)
-        {
-            StringArray values;
-            values.appendList(serverPorts, ",", true);
-            ForEachItemIn(idx, values)
-            {
-                farmerPorts.push_back(atoi(values.item(idx)));
-            }
-        }
 
         if (topology->hasProp("PreferredCluster"))
         {
@@ -700,13 +693,11 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         }
         if (standAloneDll || wuid)
         {
-            unsigned port = topology->getPropInt("@port", 0);
-            runOnce = port == 0;
-            if (port)
-            {
-                farmerPorts.clear();
-                farmerPorts.push_back(port);
-            }
+            IPropertyTree *services = topology->queryPropTree("services");
+            if (topology->getPropBool("@server", false) && services && services->hasChildren())
+                runOnce = false;
+            else
+                runOnce = true;
         }
 
         if (!topology->hasProp("@resolveLocally"))
@@ -715,6 +706,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         traceLevel = topology->getPropInt("@traceLevel", runOnce ? 0 : 1);
         if (traceLevel > MAXTRACELEVEL)
             traceLevel = MAXTRACELEVEL;
+        if (traceLevel && topology->hasProp("logging/@disabled"))
+            topology->setPropBool("logging/@disabled", false);
         udpTraceLevel = topology->getPropInt("@udpTraceLevel", runOnce ? 0 : 1);
         roxiemem::setMemTraceLevel(topology->getPropInt("@memTraceLevel", runOnce ? 0 : 1));
         soapTraceLevel = topology->getPropInt("@soapTraceLevel", runOnce ? 0 : 1);
@@ -912,8 +905,9 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         udpMulticastBufferSize = topology->getPropInt("@udpMulticastBufferSize", 262142);
         udpFlowSocketsSize = topology->getPropInt("@udpFlowSocketsSize", 131072);
         udpLocalWriteSocketSize = topology->getPropInt("@udpLocalWriteSocketSize", 1024000);
-        
+#ifndef _CONTAINERIZED
         roxieMulticastEnabled = topology->getPropBool("@roxieMulticastEnabled", true) && !useAeron;   // enable use of multicast for sending requests to slaves
+#endif
         if (udpSnifferEnabled && !roxieMulticastEnabled)
         {
             DBGLOG("WARNING: ignoring udpSnifferEnabled setting as multicast not enabled");
@@ -1091,9 +1085,39 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
 #else
         topology->addPropBool("@linuxOS", true);
 #endif
+        if (useAeron)
+            setAeronProperties(topology);
+
+#ifdef _CONTAINERIZED
+        allQuerySetNames.append(roxieName);
+#else
         allQuerySetNames.appendListUniq(topology->queryProp("@querySets"), ",");
-        // Set multicast base addresses - must be done before generating slave channels
+#endif
 
+#ifdef _CONTAINERIZED
+        if (!numChannels)
+            throw makeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels not set");
+        IpAddress myIP(".");
+        myNode.setIp(myIP);
+        if (topology->getPropBool("@server", true))
+        {
+            Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./services");
+            ForEach(*roxieFarms)
+            {
+                IPropertyTree &roxieFarm = roxieFarms->query();
+                unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
+                RoxieEndpointInfo me = {RoxieEndpointInfo::RoxieServer, 0, { (unsigned short) port, myIP }, 0};
+                myRoles.push_back(me);
+            }
+        }
+        for (std::pair<unsigned, unsigned> channel: slaveChannels)
+        {
+            mySlaveEP.set(ccdMulticastPort, myIP);
+            RoxieEndpointInfo me = { RoxieEndpointInfo::RoxieSlave, channel.first, mySlaveEP, channel.second };
+            myRoles.push_back(me);
+        }
+#else
+        // Set multicast base addresses - must be done before generating slave channels
         if (roxieMulticastEnabled && !localSlave)
         {
             if (topology->queryProp("@multicastBase"))
@@ -1105,34 +1129,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
             else
                 throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - multicastLast not set");
         }
-        if (useAeron)
-            setAeronProperties(topology);
-
-        if (useDynamicServers)
-        {
-            if (!numChannels)
-                throw makeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels not set");
-            IpAddress myIP(".");
-            myNode.setIp(myIP);
-            for (unsigned port: farmerPorts)
-            {
-                VStringBuffer xpath("RoxieFarmProcess[@port='%u']", port);
-                if (!topology->hasProp(xpath))
-                    topology->addPropTree("RoxieFarmProcess")->setPropInt("@port", port);
-                RoxieEndpointInfo me = {RoxieEndpointInfo::RoxieServer, 0, { (unsigned short) port, myIP }, 0};
-                myRoles.push_back(me);
-            }
-            for (std::pair<unsigned, unsigned> channel: slaveChannels)
-            {
-                mySlaveEP.set(ccdMulticastPort, myIP);
-                RoxieEndpointInfo me = { RoxieEndpointInfo::RoxieSlave, channel.first, mySlaveEP, channel.second };
-                myRoles.push_back(me);
-            }
-        }
-        else
-        {
-            readStaticTopology();
-        }
+        readStaticTopology();
+#endif
         // Now we know all the channels, we can open and subscribe the multicast channels
         if (!localSlave)
         {
@@ -1213,7 +1211,11 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         {
             try
             {
+#ifdef _CONTAINERIZED
+                Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./services");
+#else
                 Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./RoxieFarmProcess");
+#endif
                 ForEach(*roxieFarms)
                 {
                     IPropertyTree &roxieFarm = roxieFarms->query();
@@ -1222,11 +1224,6 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                     if (!numThreads)
                         numThreads = numServerThreads;
                     unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
-                    if (useDynamicServers)
-                    {
-                        if (std::find(std::begin(farmerPorts), std::end(farmerPorts), port) == std::end(farmerPorts))
-                            continue;
-                    }
                     //unsigned requestArrayThreads = roxieFarm.getPropInt("@requestArrayThreads", 5);
                     // NOTE: farmer name [@name=] is not copied into topology
                     const IpAddress &ip = myNode.getNodeAddress();
@@ -1395,18 +1392,11 @@ roxie:
   allFilesDynamic: true
   localSlave: true
   numChannels: 1
-  numServerThreads: 30
   queueNames: roxie.roxie
-  roxieMulticastEnabled: false
-  useAeron: false
-  RoxieFarmProcess:
-    - name: default
-      port: 9876
-      listenQueue: 200
-      numThreads: 0
-    - name: workunit
-      port: 0
-      numThreads: 0
+  traceLevel: 0
+  server: false
+  logging:
+    disabled: true
 )!!";
 
 int STARTQUERY_API start_query(int argc, const char *argv[])

roxie/ccd/ccdserver.cpp

@@ -16657,6 +16657,13 @@ public:
         extra.set(_extra);
         extra.calcUnused();
         setNumOutputs(extra.outputs.ordinality());
+        // Try to resolve library now so that we can detect interface mismatches early. You could argue it should only be a warning - as it is the query
+        // will be suspended and will remain suspended even if a suitable library is later published
+        if (!extra.embedded && _graphNode.getPropBool("hint[@name='required']/@value", false))
+        {
+            StringContextLogger logctx;
+            Owned<IQueryFactory> libraryQuery = _queryFactory.lookupLibrary(extra.libraryName, extra.interfaceHash, logctx);
+        }
     }
 
     virtual IRoxieServerActivity *createActivity(IRoxieSlaveContext *_ctx, IProbeManager *_probeManager) const

roxie/roxie/roxie.cpp

@@ -50,21 +50,14 @@ roxie:
   allFilesDynamic: true
   localSlave: true
   numChannels: 1
-  numServerThreads: 30
   queueNames: roxie.roxie
-  serverPorts: "9876,0"
-  roxieMulticastEnabled: false
-  useAeron: false
-  RoxieFarmProcess:
-    - name: default
+  services:
+    - name: query
       port: 9876
-      listenQueue: 200
-      numThreads: 0
     - name: workunit
       port: 0
-      numThreads: 0
   logging:
-      detail: 100
+    detail: 100
 )!!";
 
 int main(int argc, const char *argv[])

roxie/udplib/udptopo.cpp

@@ -345,10 +345,12 @@ extern UDPLIB_API unsigned getNumSlaves(unsigned channel)
     return topology->querySlaves(channel).ordinality();
 }
 
+#ifndef _CONTAINERIZED
 extern UDPLIB_API void createStaticTopology(const std::vector<RoxieEndpointInfo> &allRoles, unsigned traceLevel)
 {
     topologyManager.setRoles(allRoles);
 }
+#endif
 
 static std::thread topoThread;
 static Semaphore abortTopo;

roxie/udplib/udptopo.hpp

@@ -116,6 +116,7 @@ struct RoxieEndpointInfo
 };
 
 extern UDPLIB_API void startTopoThread(const StringArray &topoServers, const std::vector<RoxieEndpointInfo> &myRoles, unsigned traceLevel);
+#ifndef _CONTAINERIZED
 extern UDPLIB_API void createStaticTopology(const std::vector<RoxieEndpointInfo> &allRoles, unsigned traceLevel);
-
+#endif
 #endif

system/jlib/jlog.cpp

@@ -2333,6 +2333,7 @@ static constexpr const char * logMsgClassesAtt = "@classes";
 static constexpr const char * useLogQueueAtt = "@useLogQueue";
 static constexpr const char * logQueueLenAtt = "@queueLen";
 static constexpr const char * logQueueDropAtt = "@queueDrop";
+static constexpr const char * logQueueDisabledAtt = "@disabled";
 static constexpr const char * useSysLogpAtt ="@enableSysLog";
 
 #ifdef _DEBUG
@@ -2350,6 +2351,11 @@ void setupContainerizedLogMsgHandler()
     IPropertyTree * logConfig = queryComponentConfig().queryPropTree("logging");
     if (logConfig)
     {
+        if (logConfig->getPropBool(logQueueDisabledAtt, false))
+        {
+            removeLog();
+            return;
+        }
         if (logConfig->hasProp(logFieldsAtt))
         {
             //Supported logging fields: AUD,CLS,DET,MID,TIM,DAT,PID,TID,NOD,JOB,USE,SES,COD,MLT,MCT,NNT,COM,QUO,PFX,ALL,STD
@@ -2390,8 +2396,6 @@ void setupContainerizedLogMsgHandler()
         if (logConfig->getPropBool(useSysLogpAtt, useSysLogDefault))
             UseSysLogForOperatorMessages();
     }
-    else
-        removeLog();
 }
 #endif
 

system/jlib/jptree.cpp

@@ -8046,12 +8046,6 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
         applyEnvironmentConfig(*config, envPrefix, *cur);
     }
 
-#ifdef _DEBUG
-    // NB: don't re-hold, if CLI --hold already held.
-    if (!held && config->getPropBool("@hold"))
-        holdLoop();
-#endif
-
     if (outputConfig)
     {
         loadArgsIntoConfiguration(config, argv, { "config", "outputconfig" });
@@ -8069,6 +8063,12 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
     else
         loadArgsIntoConfiguration(config, argv);
 
+#ifdef _DEBUG
+    // NB: don't re-hold, if CLI --hold already held.
+    if (!held && config->getPropBool("@hold"))
+        holdLoop();
+#endif
+
     if (!globalConfiguration)
         globalConfiguration.setown(createPTree("global"));
 

testing/regress/ecl/key/superfile11.xml

@@ -0,0 +1,17 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><i>1</i><id>A</id></Row>
+ <Row><i>1</i><id>B</id></Row>
+ <Row><i>1</i><id>C</id></Row>
+ <Row><i>2</i><id>D</id></Row>
+ <Row><i>2</i><id>E</id></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><i>2</i><id>D</id></Row>
+ <Row><i>2</i><id>E</id></Row>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>

testing/regress/ecl/library2.ecl

@@ -33,7 +33,7 @@ end;
 
 empty := DATASET([], namesRecord);
 
-filterDataset(dataset(namesRecord) ds, string search, boolean onlyOldies) := library('aaaLibrary2',FilterDatasetInterface(ds,empty,search,onlyOldies));
+filterDataset(dataset(namesRecord) ds, string search, boolean onlyOldies) := library('aaaLibrary2',FilterDatasetInterface(ds,empty,search,onlyOldies),hint(required(true)));
 
 namesTable := dataset([
         {'Halliday','Gavin',31},

testing/regress/ecl/superfile11.ecl

@@ -0,0 +1,51 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//NB: testing superfile logic/dfs here, no need to run in all engines
+//nothor
+//noroxie
+
+#onwarning (5401, ignore); // suppress info re. optional missing file
+#onwarning (0, ignore); // suppress info re. DeleteLogicalFile
+
+import Std.File AS FileServices;
+import $.setup;
+
+prefix := setup.Files(false, false).QueryFilePrefix;
+
+rec := RECORD
+ integer i;
+ string1 id;
+END;
+
+ds1 := DATASET([{1,'A'}, {1,'B'}, {1,'C'}], rec);
+ds2 := DATASET([{2,'D'}, {2,'E'}], rec);
+
+inlinesuper := '{'+prefix+'subfile1,'+prefix+'subfile2}';
+isuper := DATASET(inlinesuper, rec, FLAT);
+isupero := DATASET(inlinesuper, rec, FLAT, OPT);
+
+SEQUENTIAL(
+  OUTPUT(ds1,,prefix + 'subfile1', OVERWRITE),
+  OUTPUT(ds2,,prefix + 'subfile2', OVERWRITE),
+  OUTPUT(isuper);
+  FileServices.DeleteLogicalFile(prefix + 'subfile1');
+  OUTPUT(isuper);
+  FileServices.DeleteLogicalFile(prefix + 'subfile2');
+  OUTPUT(isupero);
+);
+