Merge branch 'candidate-7.6.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/defvalue.cpp

@@ -382,7 +382,7 @@ void MemoryValue::toMem(void *target)
 {
     size32_t size = getSize();
     assertThrow(val.length()>=size);
-    memcpy(target, val.get(), size);
+    memcpy_iflen(target, val.get(), size);
 }
 
 unsigned MemoryValue::getHash(unsigned initval)
@@ -460,7 +460,7 @@ StringValue::StringValue(const char *v, ITypeInfo *_type) : MemoryValue(_type)
     unsigned len = _type->getSize();
     assertex(len != UNKNOWN_LENGTH);
     char * temp = (char *)val.allocate(len+1);
-    memcpy(temp, v, len);
+    memcpy_iflen(temp, v, len);
     temp[len] = 0;
 }
 
@@ -603,7 +603,7 @@ IValue *createStringValue(const char *val, ITypeInfo *type, size32_t srcLength,
     else if (tgtLength > srcLength)
     {
         char * extended = (char *)checked_malloc(tgtLength, DEFVALUE_MALLOC_FAILED);
-        memcpy(extended, val, srcLength);
+        memcpy_iflen(extended, val, srcLength);
         memset(extended+srcLength, type->queryCharset()->queryFillChar(), tgtLength-srcLength);
         IValue * ret = new StringValue(extended, type);
         free(extended);
@@ -720,7 +720,7 @@ void * UnicodeValue::getUCharStringValue(unsigned len, void * out)
     size_t vallen = val.length()/2;
     if(vallen > len)
         vallen = len;
-    memcpy(out, val.get(), vallen*2);
+    memcpy_iflen(out, val.get(), vallen*2);
     if(len > vallen)
         ((UChar *)out)[vallen] = 0x0000;
     return out;
@@ -831,7 +831,7 @@ void UnicodeAttr::set(UChar const * _text, unsigned _len)
 {
     free(text);
     text = (UChar *) checked_malloc((_len+1)*2, DEFVALUE_MALLOC_FAILED);
-    memcpy(text, _text, _len*2);
+    memcpy_iflen(text, _text, _len*2);
     text[_len] = 0x0000;
 }
 
@@ -850,7 +850,7 @@ VarUnicodeValue::VarUnicodeValue(unsigned len, const UChar * v, ITypeInfo * _typ
     else
     {
         UChar * temp = (UChar *)checked_malloc((typeLen+1)*2, DEFVALUE_MALLOC_FAILED);
-        memcpy(temp, v, len*2);
+        memcpy_iflen(temp, v, len*2);
         temp[len] = 0;
         val.set(temp, typeLen);
         free(temp);
@@ -945,7 +945,7 @@ void * VarUnicodeValue::getUCharStringValue(unsigned len, void * out)
     unsigned vallen = val.length();
     if(vallen > len)
         vallen = len;
-    memcpy(out, val.get(), vallen*2);
+    memcpy_iflen(out, val.get(), vallen*2);
     if(len > vallen)
         ((UChar *)out)[vallen] = 0x0000;
     return out;
@@ -1255,7 +1255,7 @@ IValue *DataValue::castTo(ITypeInfo *t)
         else
         {
             char *newstr = (char *) checked_malloc(nsize, DEFVALUE_MALLOC_FAILED);
-            memcpy(newstr, val.get(), osize);
+            memcpy_iflen(newstr, val.get(), osize);
             memset(newstr+osize, 0, nsize-osize);
             IValue * ret = new DataValue(newstr, LINK(t));
             free(newstr);
@@ -1275,7 +1275,7 @@ IValue *DataValue::castTo(ITypeInfo *t)
         else
         {
             char *newstr = (char *) checked_malloc(nsize, DEFVALUE_MALLOC_FAILED);
-            memcpy(newstr, val.get(), osize);
+            memcpy_iflen(newstr, val.get(), osize);
             memset(newstr+osize, t->queryCharset()->queryFillChar(), nsize-osize);
             IValue * ret = new StringValue(newstr, t);
             free(newstr);
@@ -1359,7 +1359,7 @@ QStringValue::QStringValue(const char *v, ITypeInfo *_type) : MemoryValue(_type)
 {
     unsigned newSize = _type->getSize();
     char * temp = (char *)val.allocate(newSize);
-    memcpy(temp, v, newSize);
+    memcpy_iflen(temp, v, newSize);
 }
 
 const char *QStringValue::generateECL(StringBuffer &out)
@@ -1679,7 +1679,7 @@ void IntValue::toMem(void *target)
     if (type->isSwappedEndian())
         _cpyrevn(target, data, size);
     else
-        memcpy(target, data, size);
+        memcpy_iflen(target, data, size);
 }
 
 unsigned IntValue::getHash(unsigned initval)
@@ -2530,7 +2530,7 @@ IValue * addValues(IValue * left, IValue * right)
     case type_int:
     case type_swapint:
     case type_packedint:
-        ret = createTruncIntValue(left->getIntValue() + right->getIntValue(), pnt);
+        ret = createTruncIntValue((__int64)((__uint64)left->getIntValue() + (__uint64)right->getIntValue()), pnt);
         break;
     case type_real:
         ret = createRealValue(left->getRealValue() + right->getRealValue(), pnt);
@@ -2572,7 +2572,7 @@ IValue * subtractValues(IValue * left, IValue * right)
     case type_int:
     case type_swapint:
     case type_packedint:
-        ret = createTruncIntValue(left->getIntValue() - right->getIntValue(), pnt);
+        ret = createTruncIntValue((__int64)((__uint64)left->getIntValue() - (__uint64)right->getIntValue()), pnt);
         break;
     case type_real:
         ret = createRealValue(left->getRealValue() - right->getRealValue(), pnt);
@@ -2782,7 +2782,10 @@ IValue * negateValue(IValue * v)
     case type_int:
     case type_swapint:
     case type_packedint:
-        return createTruncIntValue(-(v->getIntValue()), v->getType());
+        {
+            __uint64 value = - (__uint64)v->getIntValue(); // avoid undefined behaviour if value = int_min
+            return createTruncIntValue((__int64)value, v->getType());
+        }
     case type_real:     
         return createRealValue(-(v->getRealValue()), v->getSize());
     case type_decimal:
@@ -3354,7 +3357,10 @@ IValue * shiftLeftValues(IValue * left, IValue * right)
     case type_int:
     case type_swapint:
     case type_packedint:
-        return createTruncIntValue(left->getIntValue() << right->getIntValue(), retType);
+        {
+            __uint64 value = (__uint64)left->getIntValue() << (__uint64)right->getIntValue();
+            return createTruncIntValue((__int64)value, retType);
+        }
     default:
         UNIMPLEMENTED;
     }

common/thorhelper/thorrparse.cpp

@@ -2537,7 +2537,7 @@ void AsciiDfaBuilder::finished()
     unsigned numAccepts = accepts.ordinality();
     dfa.numAccepts = numAccepts;
     dfa.accepts = (unsigned *)malloc(sizeof(unsigned)*numAccepts);
-    memcpy(dfa.accepts, accepts.getArray(), numAccepts*sizeof(unsigned));
+    memcpy_iflen(dfa.accepts, accepts.getArray(), numAccepts*sizeof(unsigned));
 }
 //---------------------------------------------------------------------------
 

common/thorhelper/thorsoapcall.cpp

@@ -884,7 +884,7 @@ public:
         if (wscType == SThttp)
         {
             service.toUpperCase();  //GET/PUT/POST
-            if (strcmp(service.str(), "GET"))
+            if (strcmp(service.str(), "GET") != 0)
                 throw MakeStringException(0, "HTTPCALL Only 'GET' http method currently supported");
             OwnedRoxieString acceptTypeSupplied(helper->getAcceptType()); // text/html, text/xml, etc
             acceptType.set(acceptTypeSupplied);

common/thorhelper/thortalgo.cpp

@@ -1130,7 +1130,7 @@ void LRTableBuilder::finished(unsigned rootId)
     table.rootState = rootId;
     table.numExtraActions = extraActions.ordinality();
     table.extraActions = new LRAction[table.numExtraActions];
-    memcpy(table.extraActions, extraActions.getArray(), sizeof(unsigned) * table.numExtraActions);
+    memcpy_iflen(table.extraActions, extraActions.getArray(), sizeof(unsigned) * table.numExtraActions);
 }
 
 

common/workunit/pkgimpl.hpp

@@ -57,7 +57,7 @@ public:
     {
         ForEachItemIn(idx, subFiles)
         {
-            if (stricmp(subFiles.item(idx).queryProp("@value"), subname))
+            if (strieq(subFiles.item(idx).queryProp("@value"), subname))
                 return idx;
         }
         return NotFound;

dali/base/dafdesc.cpp

@@ -82,7 +82,7 @@ bool getCrcFromPartProps(IPropertyTree &fileattr,IPropertyTree &props, unsigned
     }
     // NB: old @crc keys and compressed were not crc of file but of data within.
     const char *kind = props.queryProp("@kind");
-    if (kind&&strcmp(kind,"key")) // key part
+    if (strsame(kind,"key")) // key part
         return false;
     bool blocked;
     if (isCompressed(fileattr,&blocked)) {

dali/ft/filecopy.cpp

@@ -1757,7 +1757,21 @@ void FileSprayer::derivePartitionExtra()
 void FileSprayer::displayPartition()
 {
     ForEachItemIn(idx, partition)
+    {
         partition.item(idx).display();
+
+#ifdef _DEBUG
+        if ((partition.item(idx).whichInput >= 0) && (partition.item(idx).whichInput < sources.ordinality()) )
+            LOG(MCdebugInfoDetail, unknownJob,
+                     "   Header size: %" I64F "u, XML header size: %" I64F "u, XML footer size: %" I64F "u",
+                     sources.item(partition.item(idx).whichInput).headerSize,
+                     sources.item(partition.item(idx).whichInput).xmlHeaderLength,
+                     sources.item(partition.item(idx).whichInput).xmlFooterLength
+            );
+        else
+            LOG(MCdebugInfoDetail, unknownJob,"   No source file for this partition");
+#endif
+    }
 }
 
 
@@ -3001,27 +3015,19 @@ void FileSprayer::spray()
 bool FileSprayer::isSameSizeHeaderFooter()
 {
     bool retVal = true;
-    unsigned whichHeaderInput = 0;
-    bool isEmpty = true;
-    headerSize = 0;
-    footerSize = 0;
 
     if (sources.ordinality() == 0)
         return retVal;
 
+    unsigned whichHeaderInput = 0;
+    headerSize = sources.item(whichHeaderInput).xmlHeaderLength;
+    footerSize = sources.item(whichHeaderInput).xmlFooterLength;
+
     ForEachItemIn(idx, partition)
     {
         PartitionPoint & cur = partition.item(idx);
         if (cur.inputLength && (idx+1 == partition.ordinality() || partition.item(idx+1).whichOutput != cur.whichOutput))
         {
-            if (isEmpty)
-            {
-                headerSize = sources.item(whichHeaderInput).xmlHeaderLength;
-                footerSize = sources.item(cur.whichInput).xmlFooterLength;
-                isEmpty = false;
-                continue;
-            }
-
             if (headerSize != sources.item(whichHeaderInput).xmlHeaderLength)
             {
                 retVal = false;
@@ -3037,7 +3043,6 @@ bool FileSprayer::isSameSizeHeaderFooter()
             if ( idx+1 != partition.ordinality() )
                 whichHeaderInput = partition.item(idx+1).whichInput;
         }
-
     }
     return retVal;
 }

dali/ft/filecopy.ipp

@@ -117,7 +117,7 @@ public:
     offset_t                offset;
     offset_t                size;               // expanded size
     offset_t                psize;              // physical (compressed) size
-    unsigned                headerSize;
+    offset_t                headerSize;
     offset_t                xmlHeaderLength;
     offset_t                xmlFooterLength;
     unsigned                crc;

deployment/envgen2/EnvGen.cpp

@@ -489,7 +489,7 @@ void CEnvGen::addUpdateAttributesFromString(IPropertyTree *updateTree, const cha
      sbValue.replaceString("[equal]", "=");
 
      StringArray newOldValues;
-     if (strcmp(keyValues[1], ""))
+     if (strcmp(keyValues[1], "") != 0)
      {
         newOldValues.appendList(sbValue.str(), ATTR_V_SEP);
         pAttr->addProp("@value", newOldValues[0]);

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/SetColumnMapping.xml

@@ -38,7 +38,7 @@
   </informaltable>
 
   <para>The <emphasis role="bold">SetColumnMapping </emphasis>function defines
-  how the data in the fields of the <emphasis>file</emphasis> mist be
+  how the data in the fields of the <emphasis>file</emphasis> must be
   transformed between the actual data storage format and the input format used
   to query that data.</para>
 

ecl/eclcc/eclcc.cpp

@@ -1664,6 +1664,9 @@ void EclCC::processFile(EclCompileInstance & instance)
         instance.archive.setown(createAttributeArchive());
 
     instance.wu.setown(createLocalWorkUnit(NULL));
+    //Record the version of the compiler in the workunit, but not when regression testing (to avoid spurious differences)
+    if (!optBatchMode)
+        instance.wu->setDebugValue("eclcc_compiler_version", LANGUAGE_VERSION, true);
     if (optSaveQueryText)
     {
         Owned<IWUQuery> q = instance.wu->updateQuery();

ecl/hql/hqlfold.cpp

@@ -4981,8 +4981,11 @@ static IHqlExpression * getLowerCaseConstant(IHqlExpression * expr)
         return LINK(expr);
     }
 
-    const void * data = value->queryValue();
     unsigned size = type->getSize();
+    if (size == 0)
+        return LINK(expr);
+
+    const void * data = value->queryValue();
     unsigned stringLen = type->getStringLen();
 
     MemoryAttr lower(size);

ecl/hql/hqlstack.cpp

@@ -140,7 +140,7 @@ int FuncCallStack::push(ITypeInfo* argType, IHqlExpression* curParam)
             unsigned argSize = castParam->getSize();
             const void * text = castParam->queryValue();
             str = (char *)malloc(argSize);
-            memcpy(str, text, argSize);
+            memcpy_iflen(str, text, argSize);
 
             // For STRINGn, len doens't need to be passed in.
             if(argType->getSize() == UNKNOWN_LENGTH)

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9598,7 +9598,7 @@ IHqlExpression * HqlCppTranslator::optimizeGraphPostResource(IHqlExpression * ex
     if (options.optimizeResourcedProjects)
     {
         OwnedHqlExpr optimized = insertImplicitProjects(*this, resourced.get(), projectBeforeSpill);
-        traceExpression("AfterResourcedImplicit", resourced);
+        traceExpression("AfterResourcedImplicit", optimized);
         checkNormalized(optimized);
 
         if (optimized != resourced)

esp/esdllib/esdl_transformer2.cpp

@@ -222,7 +222,7 @@ void Esdl2Base::serialize_attributes(StringBuffer &out)
         const char *key = piter->getPropKey();
         if (key && *key)
         {
-            if (stricmp(key, "base_type"))
+            if (stricmp(key, "base_type") != 0)
             {
                 const char *value = m_def->queryProp(key);
                 if (value && *value)

esp/services/WsDeploy/WsDeployService.cpp

@@ -339,7 +339,7 @@ bool CWsDeployFileInfo::navMenuEvent(IEspContext &context,
     if (!strcmp(sbName.str(), m_userWithLock.str()) && !strcmp(sbUserIp.str(), m_userIp.str()))
       throw MakeStringException(-1, "Another browser window already has write access on machine '%s'. Please use that window.", sbUserIp.str());
   }  
-  else if (strcmp(cmd, "SaveEnvironmentAs"))
+  else if (strcmp(cmd, "SaveEnvironmentAs") != 0)
     checkForRefresh(context, &req.getReqInfo(), true);
 
   if (!stricmp(cmd, "Deploy"))

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -1076,8 +1076,11 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
                     Owned<IPTree> gws = createPTree("gateways", 0);
                     Owned<IPTree> soapTree = createPTreeFromXMLString(soapmsg.append("</soap:Envelope>"), ipt_ordered);
                     StringBuffer xpath(baseXpath);
+                    StringBuffer rowName(cfgGateways->queryProp("@legacyRowName"));
 
-                    EsdlBindingImpl::transformGatewaysConfig(tgtcfg, gws, "row");
+                    if (rowName.isEmpty())
+                        rowName.append("row");
+                    EsdlBindingImpl::transformGatewaysConfig(tgtcfg, gws, rowName);
                     xpath.replaceString("{$query}", tgtQueryName);
                     xpath.replaceString("{$method}", mthdef.queryMethodName());
                     xpath.replaceString("{$service}", srvdef.queryName());

esp/services/ws_dfu/ws_dfuService.cpp

@@ -516,7 +516,7 @@ bool CWsDfuEx::onDFUSpace(IEspContext &context, IEspDFUSpaceRequest & req, IEspD
         else
         {
             Owned<IEspSpaceItem> item64 = createSpaceItem();
-            if (stricmp(countby, COUNTBY_OWNER))
+            if (!strieq(countby, COUNTBY_OWNER))
             {
                 if (scopeName)
                     item64->setName(scopeName);

esp/smc/SMCLib/TpWrapper.cpp

@@ -1818,7 +1818,16 @@ void CTpWrapper::appendTpMachine(double clientVersion, IConstEnvironment* constE
 
     Owned<IEspTpMachine> machine = createTpMachine();
     machine->setName(name.str());
-    machine->setNetaddress(networkAddress.str());
+
+    if (networkAddress.length() > 0)
+    {
+        IpAddress ipAddr;
+        ipAddr.ipset(networkAddress.str());
+
+        StringBuffer networkAddressStr;
+        ipAddr.getIpText(networkAddressStr);
+        machine->setNetaddress(networkAddressStr);
+    }
     machine->setPort(instanceInfo.getPort());
     machine->setOS(machineInfo->getOS());
     machine->setDirectory(directory.str());

esp/src/eclwatch/ClusterProcessesQueryWidget.js

@@ -46,22 +46,9 @@ define([
             this.inherited(arguments);
             this.openButton = registry.byId(this.id + "Open");
             this.refreshButton = registry.byId(this.id + "Refresh");
-            this.configurationButton = registry.byId(this.id + "Configuration");
 
             this.machineFilter = new MachineInformationWidget({});
-
-            this.configurationButton = new Button({
-                label: this.i18n.OpenConfiguration,
-                onClick: function(event) {
-                    context._onOpenConfiguration()
-                }
-            });
-
             this.machineFilter.placeAt(this.openButton.domNode, "after");
-            this.configurationButton.placeAt(this.openButton.domNode, "after");
-
-            new ToolbarSeparator().placeAt(this.machineFilter.domNode, "before");
-
             this.machineFilter.machineForm.set("style", "width:500px;");
             this.machineFilter.disable(true);
 
@@ -218,10 +205,7 @@ define([
             });
 
             retVal.on(".dgrid-row:dblclick", function (evt) {
-                if (context._onRowDblClick) {
-                    var item = retVal.row(evt).data;
-                    context._onRowDblClick(item);
-                }
+                event.preventDefault();
             });
 
             retVal.on(".dgrid-cell:click", function(evt){
@@ -284,29 +268,13 @@ define([
         refreshActionState: function () {
             var selection = this.grid.getSelected();
             var isTarget = false;
-            var isProcess = false;
 
             for (var i = 0; i < selection.length; ++i) {
-                if (selection.length > 1) {
-                    if (selection[i].type) {
-                        isTarget = false;
-                        isProcess = false;
-                    } else if (!selection[i].type) {
-                        isTarget = true;
-                        isProcess = false;
-                    }    
-                } else {
-                    if (selection[i] && selection[i].type === "targetClusterProcess") {
-                        isTarget = true;
-                        isProcess = false;
-                    } else {
-                        isTarget = false;
-                        isProcess = true;
-                    }
+                if (selection) {
+                    isTarget = true;
                 }
             }
             this.machineFilter.disable(!isTarget);
-            this.configurationButton.set("disabled", !isProcess);
         },
 
         ensureConfigurationPane: function (id, params) {

esp/src/eclwatch/PreflightDetailsWidget.js

@@ -31,7 +31,6 @@ define([
             init: function (params, route) {
                 if (this.inherited(arguments))
                     return;
-                this.initalized = false;
                 this.params = params;
                 this.setColumns(params);
                 this.refresh(params, route);
@@ -42,7 +41,6 @@ define([
             },
 
             createGrid: function (domID) {
-                var context = this;
                 dojo.destroy(this.id + "Open");
                 dojo.destroy(this.id + "RemovableSeperator2");
 
@@ -67,10 +65,53 @@ define([
 
             setColumns: function (params) {
                 var context = this;
+                var finalColumns;
                 var dynamicColumns = {
                     Location: {label: this.i18n.Location, id: this.i18n.Location, width: 350},
                     Component: {label: this.i18n.Component, id: this.i18n.Component, width: 275},
-                    ComputerUpTime: { label: this.i18n.ComputerUpTime, width: 75 }
+                    Condition: {label: this.i18n.Condition,
+                        renderCell: function(object, value, node, options) {
+                            var conditionConversion = value;
+                            switch (conditionConversion) {
+                                case 'Unknown':
+                                case 'Warning':
+                                case 'Minor':
+                                case 'Major':
+                                case 'Critical':
+                                case 'Fatal':
+                                    domClass.add(node, "ErrorCell");
+                                break;
+                            }
+                            node.innerText = conditionConversion || "";
+                        }
+                    },
+                    State: {label: this.i18n.State,
+                        renderCell: function(object, value, node, options) {
+                            var stateConversion = value;
+                            switch (stateConversion) {
+                                case 'Unknown':
+                                case 'Starting':
+                                case 'Stopping':
+                                case 'Suspended':
+                                case 'Recycling':
+                                case 'Busy':
+                                case 'NA':
+                                    domClass.add(node, "ErrorCell");
+                                break;
+                            }
+                            node.innerText = stateConversion || "";
+                        }
+                    },
+                    ProcessesDown: {label: this.i18n.ProcessesDown,
+                        renderCell: function(object, value, node, options) {
+                            var checkProcess = value;
+                            if (checkProcess) {
+                                domClass.add(node, "ErrorCell");
+                            }
+                            node.innerText = checkProcess || "";
+                        }
+                    },
+                    ComputerUpTime: {label: this.i18n.ComputerUpTime}
                 }
 
                 function handleResponse(response, dynamicColumns) {
@@ -86,7 +127,7 @@ define([
                                     renderCell: function(object, value, node, options) {
                                         switch (request > value && value !== 0) {
                                             case true:
-                                                domClass.add(node, "WarningCell");
+                                                domClass.add(node, "ErrorCell");
                                             break;
                                         }
                                         if (swap && value !== 0) {
@@ -113,52 +154,10 @@ define([
                     }
 
                     var finalColumns = params.Columns.Item;
+
                     for (var index in finalColumns) {
                         var clean = this.cleanColumn(finalColumns[index]);
 
-                        if (clean === "Condition") {
-                            lang.mixin(dynamicColumns, {
-                                "Condition": {
-                                    label: this.i18n.Condition,
-                                    renderCell: function(object, value, node, options) {
-                                        switch (value) {
-                                            case 'Unknown':
-                                            case 'Warning':
-                                            case 'Minor':
-                                            case 'Major':
-                                            case 'Critical':
-                                            case 'Fatal':
-                                                domClass.add(node, "WarningCell");
-                                            break;
-                                        }
-                                        node.innerText = value || "";
-                                    }
-                                }
-                            });
-                        }
-
-                        if (clean === "State") {
-                            lang.mixin(dynamicColumns, {
-                                "State": {
-                                    label: this.i18n.State,
-                                    renderCell: function(object, value, node, options) {
-                                        switch (value) {
-                                            case 'Unknown':
-                                            case 'Starting':
-                                            case 'Stopping':
-                                            case 'Suspended':
-                                            case 'Recycling':
-                                            case 'Busy':
-                                            case 'NA':
-                                                domClass.add(node, "WarningCell");
-                                            break;
-                                        }
-                                        node.innerText = value || "";
-                                    }
-                                }
-                            });
-                        }
-
                         if (clean === "CPULoad") {
                             var request = params.RequestInfo.CpuThreshold;
                             lang.mixin(dynamicColumns, {
@@ -167,7 +166,7 @@ define([
                                     renderCell: function (object, value, node, options) {
                                         switch (request < value) {
                                             case true:
-                                                domClass.add(node, "WarningCell");
+                                                domClass.add(node, "ErrorCell");
                                             break;
                                         }
                                         node.innerText = value + "%";
@@ -186,7 +185,7 @@ define([
                                             case "Not attached to DALI...":
                                             case "empty state hash ...":
                                             case "Node State: not ok ...":
-                                                domClass.add(node, "WarningCell");
+                                                domClass.add(node, "ErrorCell");
                                             break;
                                         }
                                         node.innerText = value || "N/A";
@@ -218,8 +217,11 @@ define([
                         }) : "";
                         lang.mixin(dynamicRowsObj, {
                             Location: row.Address + " " + row.ComponentPath,
-                            Component: row.DisplayType + "[" + row.ComponentName + "]",
-                            ComputerUpTime: row.UpTime
+                            Component: row.DisplayType === "null" ? row.ComponentName  + "[" + row.ComponentName + "]" : row.DisplayType + "[" + row.ComponentName + "]",
+                            ComputerUpTime: row.UpTime,
+                            Condition: ESPPreflight.getCondition(row.ComponentInfo.Condition),
+                            State: ESPPreflight.getState(row.ComponentInfo.State),
+                            ProcessesDown: row.ComponentInfo.Condition === 2 ? row.ComponentName : ""
                         });
                         if (row.Processors) {
                             arrayUtil.forEach(row.Processors.ProcessorInfo, function (processor, idx) {
@@ -266,7 +268,8 @@ define([
                                 lang.mixin(dynamicRowsObj, {
                                     Condition: ESPPreflight.getCondition(setRows.ComponentInfo.Condition),
                                     State: ESPPreflight.getState(setRows.ComponentInfo.State),
-                                    UpTime: setRows.ComponentInfo.UpTime
+                                    UpTime: setRows.ComponentInfo.UpTime,
+                                    ProcessesDown: setRows.Running ? setRows.Running.SWRunInfo[0].Name : ""
                                 });
                             }
                             if (setRows.Processors) {

esp/src/eclwatch/SystemServersQueryWidget.js

@@ -136,22 +136,8 @@ define([
             this.inherited(arguments);
             this.openButton = registry.byId(this.id + "Open");
             this.refreshButton = registry.byId(this.id + "Refresh");
-            this.configurationButton = registry.byId(this.id + "Configuration");
-
             this.machineFilter = new MachineInformationWidget({});
-
-            this.configurationButton = new Button({
-                label: "Open Configuration",
-                onClick: function(event) {
-                    context._onOpenConfiguration();
-                }
-            });
-
             this.machineFilter.placeAt(this.openButton.domNode, "after");
-            this.configurationButton.placeAt(this.openButton.domNode, "after");
-
-            new ToolbarSeparator().placeAt(this.machineFilter.domNode, "before");
-
             this.machineFilter.machineForm.set("style", "width:500px;");
             dojo.destroy(this.id + "Open");
 
@@ -172,7 +158,11 @@ define([
                         width: 20,
                         selectorType: 'checkbox',
                         disabled: function (item) {
-                            return !item.Configuration;
+                            if (!item.Configuration || item.Type === "LDAPServerProcess") {
+                                return true;
+                            } else {
+                                return false;
+                            }
                         },
                     }),
                     Configuration: {
@@ -186,7 +176,7 @@ define([
                         renderCell: function (object, value, node, options) {
                             if (object.Directory) {
                                 domClass.add(node, "centerInCell");
-                                node.innerHTML = "<a href='#' />" + Utility.getImageHTML("configuration.png", context.i18n.Configuration) + "</a>";
+                                node.innerHTML = "<a href='#' class='gridClick'/>" + Utility.getImageHTML("configuration.png", context.i18n.Configuration) + "</a>";
                             }
                         },
                     },
@@ -294,10 +284,7 @@ define([
             });
 
             retVal.on(".dgrid-row:dblclick", function (evt) {
-                if (context._onRowDblClick) {
-                    var item = retVal.row(evt).data;
-                    context._onRowDblClick(item);
-                }
+                event.preventDefault();
             });
 
             retVal.on(".dgrid-cell .gridClick:click", function (evt) {
@@ -403,16 +390,13 @@ define([
 
             for (var i = 0; i < selection.length; ++i) {
                 if (selection[i] && selection[i].type === "clusterProcess") {
-                    isCluster = true;
                     isNode = false;
                 } else {
-                    isCluster = false;
                     isNode = true;
                 }
             }
 
             this.openButton.set("disabled", !isNode);
-            this.configurationButton.set("disabled", !isCluster);
         },
 
         ensureConfigurationPane: function (id, params) {

esp/src/eclwatch/TargetClustersQueryWidget.js

@@ -86,22 +86,9 @@ define([
             this.inherited(arguments);
             this.openButton = registry.byId(this.id + "Open");
             this.refreshButton = registry.byId(this.id + "Refresh");
-            this.configurationButton = registry.byId(this.id + "Configuration");
 
             this.machineFilter = new MachineInformationWidget({});
-
-            this.configurationButton = new Button({
-                label: this.i18n.OpenConfiguration,
-                onClick: function(event) {
-                    context._onOpenConfiguration()
-                }
-            });
-
             this.machineFilter.placeAt(this.openButton.domNode, "after");
-            this.configurationButton.placeAt(this.openButton.domNode, "after");
-
-            new ToolbarSeparator().placeAt(this.machineFilter.domNode, "before");
-
             this.machineFilter.machineForm.set("style", "width:500px;");
             this.machineFilter.disable(true);
             dojo.destroy(this.id + "Open");
@@ -206,10 +193,7 @@ define([
             });
 
             retVal.on(".dgrid-row:dblclick", function (evt) {
-                if (context._onRowDblClick) {
-                    var item = retVal.row(evt).data;
-                    context._onRowDblClick(item);
-                }
+                event.preventDefault();
             });
 
             retVal.on("dgrid-select", function (event) {
@@ -278,29 +262,13 @@ define([
         refreshActionState: function () {
             var selection = this.grid.getSelected();
             var isTarget = false;
-            var isProcess = false;
 
             for (var i = 0; i < selection.length; ++i) {
-                if (selection.length > 1) {
-                    if (!selection[i].type) { // is a process
-                        isTarget = false;
-                        isProcess = false;
-                    } else if (selection[i].type) {
-                        isTarget = true;
-                        isProcess = false;
-                    }    
-                } else {
-                    if (selection[i] && selection[i].type === "clusterProcess") {
-                        isTarget = true;
-                        isProcess = false;
-                    } else {
-                        isTarget = false;
-                        isProcess = true;
-                    }
+                if (selection) {
+                    isTarget = true;
                 }
             }
             this.machineFilter.disable(!isTarget);
-            this.configurationButton.set("disabled", !isProcess);
         },
 
         ensureConfigurationPane: function (id, params) {

esp/src/eclwatch/nls/hpcc.js

@@ -531,6 +531,7 @@ define({root:
     Permission: "Permission",
     Permissions: "Permissions",
     PhysicalFiles: "Physical Files",
+    PhysicalMemory: "Physical Memory",
     PlaceholderFindText: "Wuid, User, (ecl:*, file:*, dfu:*, query:*)...",
     PlaceholderFirstName: "John",
     PlaceholderLastName: "Smith",
@@ -561,6 +562,7 @@ define({root:
     Priority: "Priority",
     Process: "Process",
     Processes: "Processes",
+    ProcessesDown: "Processes Down",
     ProcessFilter: "Process Filter",
     ProcessorInformation: "Processor Information",
     ProgressMessage: "Progress Message",

esp/src/eclwatch/templates/MachineInformationWidget.html

@@ -17,12 +17,12 @@
                     <input id="${id}AutoRefresh" title="${i18n.AutoRefreshIncrement}:" name="AutoRefresh" placeholder="${i18n.AutoRefreshEvery}" colspan="2" value="5" checked data-dojo-type="dijit.form.NumberTextBox" />
                     <input id="${id}CpuThreshold" title="${i18n.WarnIfCPUUsageIsOver}:" name="CpuThreshold"  placeholder="${i18n.EnterAPercentage}" colspan="2" value="95" checked data-dojo-type="dijit.form.ValidationTextBox" />
                     <input id="${id}MemThreshold" title="${i18n.WarnIfAvailableMemoryIsUnder}:" name="MemThreshold"  placeholder="${i18n.EnterAPercentageOrMB}" colspan="2" value="95" checked data-dojo-type="dijit.form.ValidationTextBox" />
-                    <select data-dojo-type="dijit.form.Select" id="${id}MemThresholdType" class="miniSelect">
+                    <select data-dojo-type="dijit.form.Select" id="${id}MemThresholdType" name="MemThresholdType" class="miniSelect">
                         <option colspan="1" value="0" selected="true">%</option>
                         <option colspan="1" value="1" selected="true">MB</option>
                     </select>
                     <input id="${id}DiskThreshold" title="${i18n.WarnIfAvailableDiskSpaceIsUnder}:" name="DiskThreshold"  placeholder="${i18n.EnterAPercentageOrMB}" colspan="2" value="95" checked data-dojo-type="dijit.form.ValidationTextBox" />
-                    <select data-dojo-type="dijit.form.Select" id="${id}DiskThresholdType" class="miniSelect">
+                    <select data-dojo-type="dijit.form.Select" id="${id}DiskThresholdType" name="DiskThresholdType" class="miniSelect">
                         <option colspan="1" value="0" selected="true">%</option>
                         <option colspan="1" value="1" selected="true">MB</option>
                     </select>

esp/src/src/ESPPreflight.ts

@@ -244,7 +244,7 @@ var ClusterProcessesList = declare([ESPRequest.Store], {
     preProcessRow: function (row) {
         lang.mixin(row, {
             Platform: this.getOS(row.OS),
-            hpcc_id: row.Name + "_" + row.ProcessNumber,
+            hpcc_id: row.Name + "_" + row.Netaddress + "_" + row.Directory,
             displayName: row.Name,
             type: "machine",
             Component: row.Type,

plugins/javaembed/java.ecllib

@@ -27,5 +27,5 @@ EXPORT syntaxCheck := Language.syntaxCheck;
 EXPORT checkImport := Language.checkImport;
 EXPORT boolean supportsImport := true;
 EXPORT boolean supportsScript := true;
-EXPORT boolean threadlocal := true;
+EXPORT boolean threadlocal := false;
 EXPORT boolean singletonEmbedContext := true;

plugins/javaembed/javaembed.cpp

@@ -635,6 +635,9 @@ public:
     inline void DeleteGlobalRef(jobject val)
     {
         JNIEnv::DeleteGlobalRef(val);
+#ifdef FORCE_GC
+        forceGC(this);
+#endif
     }
     inline jobject NewGlobalRef(jobject val, const char *)
     {
@@ -822,6 +825,16 @@ class PersistedObject : public MappingBase
 {
 public:
     PersistedObject(const char *_name) : name(_name) {}
+    ~PersistedObject()
+    {
+        if (instance)
+        {
+#ifdef TRACE_GLOBALREF
+            DBGLOG("DeleteGlobalRef(singleton): %p", instance);
+#endif
+            queryJNIEnv()->DeleteGlobalRef(instance);
+        }
+    }
     CriticalSection crit;
     jobject instance = nullptr;
     StringAttr name;
@@ -1015,14 +1028,6 @@ public:
     void doUnregister(const char *key)
     {
         CriticalBlock b(hashCrit);
-        PersistedObject *p = persistedObjects.find(key);
-        if (p && p->instance)
-        {
-            queryJNIEnv()->DeleteGlobalRef(p->instance);
-#ifdef FORCE_GC
-            forceGC(queryJNIEnv());
-#endif
-        }
         persistedObjects.remove(key);
     }
     static void unregister(const char *key);
@@ -2327,9 +2332,11 @@ public:
     }
     ~JavaThreadContext()
     {
-        // Make sure all thread-local function contexts are destroyed before we detach from
+        // Make sure all thread-local function contexts and saved objects are destroyed before we detach from
         // the Java thread
         contexts.kill();
+        persistedObjects.kill();
+        loaders.kill();
         // According to the Java VM 1.7 docs, "A native thread attached to
         // the VM must call DetachCurrentThread() to detach itself before
         // exiting."
@@ -2368,8 +2375,55 @@ public:
         // Note - this object is thread-local so no need for a critsec
         contexts.append(*ctx);
     }
+
+    PersistedObject *getLocalObject(CheckedJNIEnv *JNIenv, const char *name)
+    {
+        // Note - this object is thread-local so no need for a critsec
+        PersistedObject *p;
+        p = persistedObjects.find(name);
+        if (!p)
+        {
+            p = new PersistedObject(name);
+            persistedObjects.replaceOwn(*p);
+        }
+        p->crit.enter();  // needed to keep code common between local/global cases
+        return p;
+    }
+
+    jobject createThreadClassLoader(const char *classPath, const char *classname, size32_t bytecodeLen, const byte *bytecode)
+    {
+        if (bytecodeLen || (classPath && *classPath))
+        {
+            jstring jClassPath = (classPath && *classPath) ? JNIenv->NewStringUTF(classPath) : nullptr;
+            jobject helperName = JNIenv->NewStringUTF(helperLibraryName);
+            jobject contextClassLoaderObj = JNIenv->CallStaticObjectMethod(customLoaderClass, clc_newInstance, jClassPath, getSystemClassLoader(), bytecodeLen, (uint64_t) bytecode, helperName);
+            assertex(contextClassLoaderObj);
+            return contextClassLoaderObj;
+        }
+        else
+        {
+            return getSystemClassLoader();
+        }
+    }
+    jobject getThreadClassLoader(const char *classPath, const char *classname, size32_t bytecodeLen, const byte *bytecode)
+    {
+        StringBuffer key(classname);
+        if (classPath && *classPath)
+            key.append('!').append(classPath);
+        PersistedObject *p;
+        p = loaders.find(key);
+        if (!p)
+        {
+            p = new PersistedObject(key);
+            p->instance = JNIenv->NewGlobalRef(createThreadClassLoader(classPath, classname, bytecodeLen, bytecode), "cachedClassLoader");
+            loaders.replaceOwn(*p);
+        }
+        return p->instance;
+    }
 private:
     IArrayOf<IEmbedFunctionContext> contexts;
+    StringMapOf<PersistedObject> persistedObjects = { false };
+    StringMapOf<PersistedObject> loaders = { false };
 };
 
 class JavaXmlBuilder : implements IXmlWriterExt, public CInterface
@@ -3212,12 +3266,6 @@ public:
     }
     ~JavaEmbedImportContext()
     {
-        if (persistMode == persistThread)
-        {
-            StringBuffer scopeKey;
-            getScopeKey(scopeKey);
-            JavaGlobalState::unregister(scopeKey);
-        }
         if (javaClass)
             JNIenv->DeleteGlobalRef(javaClass);
         if (classLoader)
@@ -3677,7 +3725,7 @@ public:
                 if (classLoader)
                 {
                     JNIenv->DeleteGlobalRef(classLoader);
-                    javaClass = nullptr;
+                    classLoader = nullptr;
                 }
                 loadFunction(classpath, 0, nullptr);
             }
@@ -4110,7 +4158,7 @@ public:
                         StringBuffer scopeKey;
                         getScopeKey(scopeKey);
                         PersistedObjectCriticalBlock persistBlock;
-                        persistBlock.enter(globalState->getGlobalObject(JNIenv, scopeKey));
+                        persistBlock.enter(persistMode==persistThread ? sharedCtx->getLocalObject(JNIenv, scopeKey) : globalState->getGlobalObject(JNIenv, scopeKey));
                         instance = persistBlock.getInstance();
                         if (instance)
                             persistBlock.leave();
@@ -4125,7 +4173,7 @@ public:
                             if (persistMode==persistQuery || persistMode==persistWorkunit || persistMode==persistChannel)
                             {
                                 assertex(engine);
-                                engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode!=persistQuery);
+                                engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode==persistWorkunit);
                             }
                             persistBlock.leave(instance);
                         }
@@ -4299,22 +4347,6 @@ protected:
         return s.append(methodName);
     }
 
-    jobject createThreadClassLoader(const char *classPath, size32_t bytecodeLen, const byte *bytecode)
-    {
-        if (bytecodeLen || (classPath && *classPath))
-        {
-            jstring jClassPath = (classPath && *classPath) ? JNIenv->NewStringUTF(classPath) : nullptr;
-            jobject helperName = JNIenv->NewStringUTF(helperLibraryName);
-            jobject contextClassLoaderObj = JNIenv->CallStaticObjectMethod(customLoaderClass, clc_newInstance, jClassPath, sharedCtx->getSystemClassLoader(), bytecodeLen, (uint64_t) bytecode, helperName);
-            assertex(contextClassLoaderObj);
-            return contextClassLoaderObj;
-        }
-        else
-        {
-            return sharedCtx->getSystemClassLoader();
-        }
-    }
-
     jobject createInstance()
     {
         jmethodID constructor;
@@ -4377,7 +4409,7 @@ protected:
                     // If a persist scope is specified, we may want to use a pre-existing object. If we do we share its classloader, class, etc.
                     assertex(classname.length());  // MORE - what does this imply?
                     getScopeKey(scopeKey);
-                    persistBlock.enter(globalState->getGlobalObject(JNIenv, scopeKey));
+                    persistBlock.enter(persistMode==persistThread ? sharedCtx->getLocalObject(JNIenv, scopeKey) : globalState->getGlobalObject(JNIenv, scopeKey));
                     instance = persistBlock.getInstance();
                     if (instance)
                         persistBlock.leave();
@@ -4392,7 +4424,7 @@ protected:
                 {
                     if (!classname)
                         throw MakeStringException(MSGAUD_user, 0, "Invalid import name - Expected classname.methodname:signature");
-                    classLoader = JNIenv->NewGlobalRef(createThreadClassLoader(classpath, bytecodeLen, bytecode), "classLoader");
+                    classLoader = JNIenv->NewGlobalRef(sharedCtx->getThreadClassLoader(classpath, classname, bytecodeLen, bytecode), "classLoader");
                     sharedCtx->setThreadClassLoader(classLoader);
 
                     jmethodID loadClassMethod = JNIenv->GetMethodID(JNIenv->GetObjectClass(classLoader), "loadClass","(Ljava/lang/String;)Ljava/lang/Class;");
@@ -4421,7 +4453,7 @@ protected:
                         if (persistMode==persistQuery || persistMode==persistWorkunit || persistMode==persistChannel)
                         {
                             assertex(engine);
-                            engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode!=persistQuery);
+                            engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode==persistWorkunit);
                         }
                         persistBlock.leave(instance);
                     }
@@ -4510,12 +4542,12 @@ protected:
         case persistGlobal:
             ret.append("global");
             break;
-        case persistChannel:
-            ret.append(nodeNum).append('.');
             // Fall into
         case persistWorkunit:
             engine->getQueryId(ret, true);
             break;
+        case persistChannel:
+            ret.append(nodeNum).append('.');
         case persistQuery:
             engine->getQueryId(ret, false);
             break;

plugins/stringlib/stringlib.cpp

@@ -421,7 +421,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringSubsOut(unsigned & tgtLen, char * & tg
     }
     else
     {
-        memcpy(tgt, src, srcLen);
+        memcpy_iflen(tgt, src, srcLen);
     }
 
     tgtLen = srcLen;
@@ -452,7 +452,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringSubs(unsigned & tgtLen, char * & tgt,
     }
     else
     {
-        memcpy(tgt, src, srcLen);
+        memcpy_iflen(tgt, src, srcLen);
     }
 
     tgtLen = srcLen;
@@ -503,7 +503,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringRepad(unsigned & tgtLen, char * & tgt,
         if (!tgt)
             rtlThrowOutOfMemory(0, "In StringLib.StringRepad");
         tgtLen = tLen;
-        memcpy(tgt,base,srcLen);
+        memcpy_iflen(tgt,base,srcLen);
         memset(tgt+srcLen,' ',tLen-srcLen);
     }
     else
@@ -610,7 +610,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringExtract(unsigned & tgtLen, char * & tg
         if ( finger[len] == ',' )
             break;
     tgt = (char *)CTXMALLOC(parentCtx, len);
-    memcpy(tgt,finger,len);
+    memcpy_iflen(tgt,finger,len);
     tgtLen = len;
 }
 
@@ -847,7 +847,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringFindReplace (unsigned & tgtLen, char *
     if ( srcLen < stokLen || stokLen == 0)
     {
         tgt = (char *) CTXMALLOC(parentCtx, srcLen);
-        memcpy(tgt, src, srcLen);
+        memcpy_iflen(tgt, src, srcLen);
         tgtLen = srcLen;
     }
     else
@@ -1129,8 +1129,8 @@ STRINGLIB_API void STRINGLIB_CALL slStringExcludeNthWord(unsigned & tgtLen, char
     if (len)
     {
         tgt = (char *)CTXMALLOC(parentCtx, len);
-        memcpy(tgt,src,startLast);
-        memcpy(tgt+startLast,src+idx,(srcLen - idx));
+        memcpy_iflen(tgt,src,startLast);
+        memcpy_iflen(tgt+startLast,src+idx,(srcLen - idx));
     }
     else
         tgt = NULL;
@@ -1234,7 +1234,7 @@ STRINGLIB_API void STRINGLIB_CALL slSplitWords(bool & __isAllResult, size32_t &
     if ((lenSeparator == 0) || (lenSrc < lenSeparator))
     {
         *((size32_t *)result) = lenSrc;
-        memcpy(result+sizeof(size32_t), src, lenSrc);
+        memcpy_iflen(result+sizeof(size32_t), src, lenSrc);
         return;
     }
 
@@ -1253,7 +1253,7 @@ STRINGLIB_API void STRINGLIB_CALL slSplitWords(bool & __isAllResult, size32_t &
             {
                 size32_t len = startWord ? (cur - startWord) : 0;
                 memcpy(target, &len, sizeof(len));
-                memcpy(target+sizeof(size32_t), startWord, len);
+                memcpy_iflen(target+sizeof(size32_t), startWord, len);
                 target += sizeof(size32_t) + len;
                 startWord = NULL;
             }
@@ -1273,7 +1273,7 @@ STRINGLIB_API void STRINGLIB_CALL slSplitWords(bool & __isAllResult, size32_t &
             startWord = cur;
         size32_t len = (end - startWord);
         memcpy(target, &len, sizeof(len));
-        memcpy(target+sizeof(size32_t), startWord, len);
+        memcpy_iflen(target+sizeof(size32_t), startWord, len);
         target += sizeof(size32_t) + len;
     }
     assert(target == result + sizeRequired);
@@ -1324,7 +1324,7 @@ STRINGLIB_API void STRINGLIB_CALL slCombineWords(size32_t & __lenResult, void *
         size32_t len;
         memcpy(&len, src+offset, sizeof(len));
         offset += sizeof(len);
-        memcpy(target, src+offset, len);
+        memcpy_iflen(target, src+offset, len);
         target += len;
         offset += len;
     }
@@ -1429,7 +1429,7 @@ STRINGLIB_API void STRINGLIB_CALL slFormatDate(size32_t & __lenResult, char * &
 #endif
         len = strlen(buf);
         out = static_cast<char *>(CTXMALLOC(parentCtx, len));
-        memcpy(out, buf, len);
+        memcpy_iflen(out, buf, len);
     }
 
     __lenResult = len;
@@ -1454,7 +1454,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringExtract50(char *tgt, unsigned srcLen,
         memcpy(tgt,resret,50);
     else
     {
-        memcpy(tgt,resret,lenret);
+        memcpy_iflen(tgt,resret,lenret);
         memset(tgt+lenret,' ',50-lenret);
     }
     CTXFREE(parentCtx, resret);
@@ -1505,7 +1505,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringFindReplace80(char * tgt, unsigned src
     {
         if (srcLen > 80)
             srcLen = 80;
-        memcpy(tgt, src, srcLen);
+        memcpy_iflen(tgt, src, srcLen);
         if (srcLen < 80)
             memset(tgt+srcLen, ' ', 80 - srcLen);
     }
@@ -1520,7 +1520,7 @@ STRINGLIB_API void STRINGLIB_CALL slStringFindReplace80(char * tgt, unsigned src
             {
                 if (rtokLen > lim)
                     rtokLen = lim;
-                memcpy(tgt, rtok, rtokLen);
+                memcpy_iflen(tgt, rtok, rtokLen);
                 tgt += rtokLen;
                 i += stokLen;
                 lim -= rtokLen;

roxie/ccd/ccdcontext.cpp

@@ -2660,13 +2660,13 @@ public:
         if (running)
             throw MakeStringException(ROXIE_DEBUG_ERROR, "Command not available while query is running");
         output->outputBeginNested("Variables", true);
-        if (!type || stricmp(type, "temporary"))
+        if (!type || strieq(type, "temporary"))
         {
             output->outputBeginNested("Temporary", true);
             ctx->printResults(output, name, (unsigned) ResultSequenceInternal);
             output->outputEndNested("Temporary");
         }
-        if (!type || stricmp(type, "global"))
+        if (!type || strieq(type, "global"))
         {
             output->outputBeginNested("Global", true);
             ctx->printResults(output, name, (unsigned) ResultSequenceStored);
@@ -3136,7 +3136,11 @@ public:
     virtual StringBuffer &getQueryId(StringBuffer &result, bool isShared) const
     {
         if (workUnit)
-            result.append(workUnit->queryWuid()); // In workunit mode, this works for both shared and non-shared variants
+        {
+            if (isShared)
+                result.append('Q');
+            result.append(workUnit->queryWuid());
+        }
         else if (isShared)
             result.append('Q').append(factory->queryHash());
         else

roxie/ccd/ccdfile.cpp

@@ -1984,7 +1984,7 @@ public:
     {
         ForEachItemIn(idx, subNames)
         {
-            if (stricmp(subNames.item(idx), subname))
+            if (strieq(subNames.item(idx), subname))
                 return idx;
         }
         return NotFound;

roxie/ccd/ccdstate.cpp

@@ -271,7 +271,7 @@ public:
     {
         ForEachItemIn(idx, subFiles)
         {
-            if (stricmp(subFiles.item(idx).queryProp("@value"), subname))
+            if (strieq(subFiles.item(idx).queryProp("@value"), subname))
                 return idx;
         }
         return NotFound;

rtl/eclrtl/CMakeLists.txt

@@ -91,6 +91,10 @@ if (USE_BOOST_REGEX)
     endif ()
 endif ()
 
+if ((CMAKE_COMPILER_IS_GNUCXX AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0.0))
+  set_source_files_properties(eclrtl.cpp PROPERTIES COMPILE_FLAGS -fno-tree-vectorize)
+endif ()
+
 HPCC_ADD_LIBRARY( eclrtl SHARED ${SRCS} )
 
 if(CENTOS_6_BOOST)

rtl/eclrtl/eclregex.cpp

@@ -106,7 +106,7 @@ public:
         {
             outlen = subs[n].second - subs[n].first;
             out = (char *)rtlMalloc(outlen);
-            memcpy(out, subs[n].first, outlen);
+            memcpy_iflen(out, subs[n].first, outlen);
         }
         else
         {
@@ -200,7 +200,7 @@ public:
         }
         outlen = tgt.length();
         out = (char *)rtlMalloc(outlen);
-        memcpy(out, tgt.data(), outlen);
+        memcpy_iflen(out, tgt.data(), outlen);
     }
 
     IStrRegExprFindInstance * find(const char * str, size32_t from, size32_t len, bool needToKeepSearchString) const

rtl/eclrtl/eclrtl.cpp

@@ -378,11 +378,14 @@ bool vunicodeNeedsNormalize(UChar * in, UErrorCode * err)
 
 void unicodeReplaceNormalized(unsigned inlen, UChar * in, UErrorCode * err)
 {
-    UChar * buff = (UChar *)rtlMalloc(inlen*sizeof(UChar));
-    unsigned len = unorm_normalize(in, inlen, UNORM_NFC, 0, buff, inlen, err);
-    while(len<inlen) buff[len++] = 0x0020;
-    memcpy(in, buff, inlen * sizeof(UChar));
-    free(buff);
+    if (inlen)
+    {
+        UChar * buff = (UChar *)rtlMalloc(inlen*sizeof(UChar));
+        unsigned len = unorm_normalize(in, inlen, UNORM_NFC, 0, buff, inlen, err);
+        while(len<inlen) buff[len++] = 0x0020;
+        memcpy(in, buff, inlen * sizeof(UChar));
+        free(buff);
+    }
 }
 
 void vunicodeReplaceNormalized(unsigned inlen, UChar * in, UErrorCode * err)
@@ -452,7 +455,7 @@ void unicodeNormalizedCopy(UChar * out, UChar * in, unsigned len)
     if(unicodeNeedsNormalize(len, in, &err))
         unorm_normalize(in, len, UNORM_NFC, 0, out, len, &err);
     else
-        memcpy(out, in, len);
+        memcpy_iflen(out, in, len);
 }
 
 void normalizeUnicodeString(UnicodeString const & in, UnicodeString & out)
@@ -495,6 +498,9 @@ UChar unicodeSpace = 0x0020;
 
 void codepageBlankFill(char const * codepage, char * out, size_t len)
 {
+    if (len == 0)
+        return;
+
     CriticalBlock b(ubcCrit);
     MemoryAttr * cached = unicodeBlankCache->getValue(codepage);
     if(cached)
@@ -598,11 +604,13 @@ NO_SANITIZE("undefined") __int64 rtlRoundUp(double x)
 #define intToStringNBody() \
     unsigned len = numtostr(temp, val); \
     if (len > l) \
-        memset(t,'*',l); \
+    { \
+        memset_iflen(t,'*',l); \
+    } \
     else \
     { \
-        memcpy(t,temp,len); \
-        memset(t+len, ' ', l-len); \
+        memcpy_iflen(t,temp,len); \
+        memset_iflen(t+len, ' ', l-len); \
     }
 
 
@@ -674,11 +682,13 @@ void rtlInt8ToStrX(size32_t & l, char * & t, __int64 val)
     unsigned len = numtostr(astr, val); \
     rtlStrToEStr(sizeof(estr),estr,len,astr); \
     if (len > l) \
-        memset(t,0x2A,l); \
+    { \
+        memset_iflen(t,0x2A,l); \
+    } \
     else \
     { \
-        memcpy(t,estr,len); \
-        memset(t+len, '@', l-len); \
+        memcpy_iflen(t,estr,len); \
+        memset_iflen(t+len, '@', l-len); \
     }
 
 void rtl_l42en(size32_t l, char * t, unsigned val)
@@ -824,7 +834,7 @@ double rtlStrToReal(size32_t l, const char * t)
 {
     MemoryAttr heapMem;
     char * temp = (char *)CONDSTACKALLOC(heapMem, l+1);
-    memcpy(temp, t, l);
+    memcpy_iflen(temp, t, l);
     temp[l] = 0;
     return rtlVStrToReal(temp);
 }
@@ -863,6 +873,9 @@ double rtlUnicodeToReal(size32_t l, UChar const * t)
 
 static void truncFixedReal(size32_t l, char * t, StringBuffer & temp)
 {
+    if (l == 0)
+        return;
+
     const char * str = temp.str();
     unsigned len = temp.length();
     if (len > l)
@@ -1190,6 +1203,9 @@ bool rtlVStrToBool(const char * t)
 
 void holeIntFormat(size32_t maxlen, char * target, __int64 value, unsigned width, unsigned flags)
 {
+    if (maxlen == 0)
+        return;
+
     StringBuffer result;
     if (flags & 1)
         result.appendf("%0*" I64F "d", width, value);
@@ -1207,7 +1223,7 @@ void holeIntFormat(size32_t maxlen, char * target, __int64 value, unsigned width
 
 void holeRealFormat(size32_t maxlen, char * target, double value, unsigned width, unsigned places)
 {
-    if ((int) width <= 0)
+    if (((int) width <= 0) || (maxlen == 0))
         return;
 
     const unsigned tempSize = 500;
@@ -1287,16 +1303,22 @@ bool rtlDataToBool(unsigned len, const void * _src)
 
 void rtlBoolToData(unsigned tlen, void * tgt, bool src)
 {
-    memset(tgt, 0, tlen);
-    if (src)
-        ((char *)tgt)[tlen-1] = 1;
+    if (likely(tlen))
+    {
+        memset(tgt, 0, tlen);
+        if (src)
+            ((char *)tgt)[tlen-1] = 1;
+    }
 }
 
 void rtlBoolToStr(unsigned tlen, void * tgt, bool src)
 {
-    memset(tgt, ' ', tlen);
-    if (src)
-        ((char *)tgt)[tlen-1] = '1';
+    if (likely(tlen))
+    {
+        memset(tgt, ' ', tlen);
+        if (src)
+            ((char *)tgt)[tlen-1] = '1';
+    }
 }
 
 void rtlBoolToVStr(char * tgt, bool src)
@@ -1337,7 +1359,7 @@ void rtlDataToData(unsigned tlen, void * tgt, unsigned slen, const void * src)
 {
     if (slen > tlen)
         slen = tlen;
-    memcpy(tgt, src, slen);
+    memcpy_iflen(tgt, src, slen);
     if (tlen > slen)
         memset((char *)tgt+slen, 0, tlen-slen);
 }
@@ -1346,7 +1368,7 @@ void rtlStrToData(unsigned tlen, void * tgt, unsigned slen, const void * src)
 {
     if (slen > tlen)
         slen = tlen;
-    memcpy(tgt, src, slen);
+    memcpy_iflen(tgt, src, slen);
     if (tlen > slen)
         memset((char *)tgt+slen, 0, tlen-slen);
 }
@@ -1355,7 +1377,7 @@ void rtlStrToStr(unsigned tlen, void * tgt, unsigned slen, const void * src)
 {
     if (slen > tlen)
         slen = tlen;
-    memcpy(tgt, src, slen);
+    memcpy_iflen(tgt, src, slen);
     if (tlen > slen)
         memset((char *)tgt+slen, ' ', tlen-slen);
 }
@@ -1364,7 +1386,7 @@ void rtlStrToVStr(unsigned tlen, void * tgt, unsigned slen, const void * src)
 {
     if ((slen >= tlen) && (tlen != 0))
         slen = tlen-1;
-    memcpy(tgt, src, slen);
+    memcpy_iflen(tgt, src, slen);
     *((char *)tgt+slen)=0;
 }
 
@@ -1399,7 +1421,7 @@ void rtlEStrToEStr(unsigned tlen, void * tgt, unsigned slen, const void * src)
 {
     if (slen > tlen)
         slen = tlen;
-    memcpy(tgt, src, slen);
+    memcpy_iflen(tgt, src, slen);
     if (tlen > slen)
         memset((char *)tgt+slen, '@', tlen-slen);
 }
@@ -1429,7 +1451,7 @@ char *rtlCreateQuotedString(unsigned _len_tgt,char * tgt)
     // Add ' at start and end. MORE! also needs to handle embedded quotes
     char * result = (char *)rtlMalloc(_len_tgt + 3);
     result[0] = '\'';
-    memcpy(result+1, tgt, _len_tgt);
+    memcpy_iflen(result+1, tgt, _len_tgt);
     result[_len_tgt+1] = '\'';
     result[_len_tgt+2] = 0;
     return result;
@@ -1463,7 +1485,7 @@ void rtlConcat(unsigned & tlen, char * * tgt, ...)
         if (len+1==0)
             break;
         char * str = va_arg(args, char *);
-        memcpy(cur, str, len);
+        memcpy_iflen(cur, str, len);
         cur += len;
     }
     va_end(args);
@@ -1497,7 +1519,7 @@ void rtlConcatVStr(char * * tgt, ...)
         if (len+1==0)
             break;
         char * str = va_arg(args, char *);
-        memcpy(cur, str, len);
+        memcpy_iflen(cur, str, len);
         cur += len;
     }
     va_end(args);
@@ -1603,7 +1625,7 @@ void rtlConcatStrF(unsigned tlen, void * _tgt, int fill, ...)
             break;
         const char * str = va_arg(args, const char *);
         unsigned copyLen = len + offset > tlen ? tlen - offset : len;
-        memcpy(tgt+offset, str, copyLen);
+        memcpy_iflen(tgt+offset, str, copyLen);
         offset += copyLen;
     }
     va_end(args);
@@ -1616,6 +1638,9 @@ void rtlConcatStrF(unsigned tlen, void * _tgt, int fill, ...)
 
 void rtlConcatVStrF(unsigned tlen, char * tgt, ...)
 {
+    if (unlikely(tlen == 0))
+        return;
+
     va_list args;
 
     unsigned offset = 0;
@@ -1627,7 +1652,7 @@ void rtlConcatVStrF(unsigned tlen, char * tgt, ...)
             break;
         const char * str = va_arg(args, const char *);
         unsigned copyLen = len + offset > tlen ? tlen - offset : len;
-        memcpy(tgt+offset, str, copyLen);
+        memcpy_iflen(tgt+offset, str, copyLen);
         offset += copyLen;
     }
     va_end(args);
@@ -1690,7 +1715,7 @@ unsigned rtlConcatStrToStr(unsigned tlen, char * tgt, unsigned idx, unsigned sle
     unsigned len = tlen-idx;
     if (len > slen)
         len = slen;
-    memcpy(tgt+idx, src, len);
+    memcpy_iflen(tgt+idx, src, len);
     return idx+len;
 }
 
@@ -1780,7 +1805,7 @@ void rtlConcatExtend(unsigned & tlen, char * & tgt, unsigned slen, const char *
 {
     unsigned len = tlen + slen;
     tgt = (char *)rtlRealloc(tgt, len);
-    memcpy(tgt+tlen, src, slen);
+    memcpy_iflen(tgt+tlen, src, slen);
     tlen = len;
 }
 
@@ -1788,7 +1813,7 @@ void rtlConcatUnicodeExtend(size32_t & tlen, UChar * & tgt, size32_t slen, const
 {
     unsigned len = tlen + slen;
     tgt = (UChar *)rtlRealloc(tgt, len * sizeof(UChar));
-    memcpy(tgt+tlen, src, slen * sizeof(UChar));
+    memcpy_iflen(tgt+tlen, src, slen * sizeof(UChar));
     tlen = len;
 }
 
@@ -1829,7 +1854,7 @@ void * doSubStrFT(unsigned & tlen, unsigned slen, const void * src, unsigned fro
 
     unsigned copylen = to - from;
     char * buffer = (char *)rtlMalloc(len);
-    memcpy(buffer, (byte *)src+from, copylen);
+    memcpy_iflen(buffer, (byte *)src+from, copylen);
     if (copylen < len)
         memset(buffer+copylen, fillChar, len-copylen);
     tlen = len;
@@ -1842,7 +1867,7 @@ void rtlSubStrFX(unsigned & tlen, char * & tgt, unsigned slen, const char * src,
 
     tlen = slen-from;
     tgt = (char *) rtlMalloc(tlen);
-    memcpy(tgt, src+from, tlen);
+    memcpy_iflen(tgt, src+from, tlen);
 }
 
 void rtlSubStrFTX(unsigned & tlen, char * & tgt, unsigned slen, const char * src, unsigned from, unsigned to)
@@ -1860,7 +1885,7 @@ void rtlSubStrFT(unsigned tlen, char * tgt, unsigned slen, const char * src, uns
     unsigned copylen = to - from;
     if (copylen > tlen)
         copylen = tlen;
-    memcpy(tgt, (const char *)src+from, copylen);
+    memcpy_iflen(tgt, (const char *)src+from, copylen);
     if (copylen < tlen)
         memset(tgt+copylen, fillChar, tlen-copylen);
 }
@@ -1873,7 +1898,7 @@ void rtlSubDataFT(unsigned tlen, void * tgt, unsigned slen, const void * src, un
     unsigned copylen = to - from;
     if (copylen > tlen)
         copylen = tlen;
-    memcpy(tgt, (char *)src+from, copylen);
+    memcpy_iflen(tgt, (char *)src+from, copylen);
     if (copylen < tlen)
         memset((byte*)tgt+copylen, 0, tlen-copylen);
 }
@@ -1889,7 +1914,7 @@ void rtlSubDataFX(unsigned & tlen, void * & tgt, unsigned slen, const void * src
 
     tlen = slen-from;
     tgt = (char *) rtlMalloc(tlen);
-    memcpy(tgt, (const byte *)src+from, tlen);
+    memcpy_iflen(tgt, (const byte *)src+from, tlen);
 }
 
 void rtlUnicodeSubStrFTX(unsigned & tlen, UChar * & tgt, unsigned slen, UChar const * src, unsigned from, unsigned to)
@@ -1900,7 +1925,7 @@ void rtlUnicodeSubStrFTX(unsigned & tlen, UChar * & tgt, unsigned slen, UChar co
 
     tgt = (UChar *)rtlMalloc(tlen*2);
     unsigned copylen = to - from;
-    memcpy(tgt, src+from, copylen*2);
+    memcpy_iflen(tgt, src+from, copylen*2);
     while(copylen<tlen)
         tgt[copylen++] = 0x0020;
 }
@@ -1911,7 +1936,7 @@ void rtlUnicodeSubStrFX(unsigned & tlen, UChar * & tgt, unsigned slen, UChar con
 
     tlen = slen - from;
     tgt = (UChar *)rtlMalloc(tlen*2);
-    memcpy(tgt, src+from, tlen*2);
+    memcpy_iflen(tgt, src+from, tlen*2);
 }
 
 
@@ -2109,7 +2134,7 @@ inline void rtlTrimUtf8Start(unsigned & trimLen, size32_t & trimSize, size32_t l
 inline char * rtlDupSubString(const char * src, unsigned len)
 {
     char * buffer = (char *)rtlMalloc(len + 1);
-    memcpy(buffer, src, len);
+    memcpy_iflen(buffer, src, len);
     buffer[len] = 0;
     return buffer;
 }
@@ -2117,7 +2142,7 @@ inline char * rtlDupSubString(const char * src, unsigned len)
 inline UChar * rtlDupSubUnicode(UChar const * src, unsigned len)
 {
     UChar * buffer = (UChar *)rtlMalloc((len + 1) * 2);
-    memcpy(buffer, src, len*2);
+    memcpy_iflen(buffer, src, len*2);
     buffer[len] = 0x00;
     return buffer;
 }
@@ -2126,16 +2151,19 @@ inline void rtlCopySubStringV(size32_t tlen, char * tgt, unsigned slen, const ch
 {
     if (slen >= tlen)
         slen = tlen-1;
-    memcpy(tgt, src, slen);
+    memcpy_iflen(tgt, src, slen);
     tgt[slen] = 0;
 }
 
 //not yet used, but would be needed for assignment to string rather than vstring
 inline void rtlCopySubString(size32_t tlen, char * tgt, unsigned slen, const char * src, char fill)
 {
+    if (unlikely(tlen == 0))
+        return;
+
     if (slen > tlen)
         slen = tlen;
-    memcpy(tgt, src, slen);
+    memcpy_iflen(tgt, src, slen);
     memset(tgt + slen, fill, tlen-slen);
 }
 
@@ -2606,7 +2634,7 @@ ECLRTL_API void rtlAssignTrimUnicodeLeftV(size32_t tlen, UChar * tgt, unsigned s
     rtlTrimUnicodeLeft(len, str, slen, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2619,7 +2647,7 @@ ECLRTL_API void rtlAssignTrimVUnicodeLeftV(size32_t tlen, UChar * tgt, const UCh
     rtlTrimVUnicodeLeft(len, str, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2632,7 +2660,7 @@ ECLRTL_API void rtlAssignTrimUnicodeRightV(size32_t tlen, UChar * tgt, unsigned
     rtlTrimUnicodeRight(len, str, slen, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2645,7 +2673,7 @@ ECLRTL_API void rtlAssignTrimVUnicodeRightV(size32_t tlen, UChar * tgt, const UC
     rtlTrimVUnicodeRight(len, str, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2658,7 +2686,7 @@ ECLRTL_API void rtlAssignTrimUnicodeBothV(size32_t tlen, UChar * tgt, unsigned s
     rtlTrimUnicodeBoth(len, str, slen, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2671,7 +2699,7 @@ ECLRTL_API void rtlAssignTrimVUnicodeBothV(size32_t tlen, UChar * tgt, const UCh
     rtlTrimVUnicodeBoth(len, str, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2684,7 +2712,7 @@ ECLRTL_API void rtlAssignTrimUnicodeAllV(size32_t tlen, UChar * tgt, unsigned sl
     rtlTrimUnicodeAll(len, str, slen, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2697,7 +2725,7 @@ ECLRTL_API void rtlAssignTrimVUnicodeAllV(size32_t tlen, UChar * tgt, const UCha
     rtlTrimVUnicodeAll(len, str, src);
     if (len >= tlen)
         len = tlen-1;
-    memcpy(tgt, str, len*2);
+    memcpy_iflen(tgt, str, len*2);
     tgt[len] = 0;
     rtlFree(str);
 }
@@ -2710,7 +2738,7 @@ int rtlCompareStrStr(unsigned l1, const char * p1, unsigned l2, const char * p2)
     unsigned len = l1;
     if (len > l2)
         len = l2;
-    int diff = memcmp(p1, p2, len);
+    int diff = memcmp_iflen(p1, p2, len);
     if (diff == 0)
     {
         if (len != l1)
@@ -2748,7 +2776,7 @@ int rtlCompareDataData(unsigned l1, const void * p1, unsigned l2, const void * p
     unsigned len = l1;
     if (len > l2)
         len = l2;
-    int diff = memcmp(p1, p2, len);
+    int diff = memcmp_iflen(p1, p2, len);
     if (diff == 0)
     {
         if (l1 > l2)
@@ -2764,7 +2792,7 @@ int rtlCompareEStrEStr(unsigned l1, const char * p1, unsigned l2, const char * p
     unsigned len = l1;
     if (len > l2)
         len = l2;
-    int diff = memcmp(p1, p2, len);
+    int diff = memcmp_iflen(p1, p2, len);
     if (diff == 0)
     {
         if (len != l1)
@@ -3600,7 +3628,7 @@ void rtlUnicodeToEscapedStrX(unsigned & outlen, char * & out, unsigned inlen, UC
     escapeUnicode(inlen, in, outbuff);
     outlen = outbuff.length();
     out = (char *)rtlMalloc(outlen);
-    memcpy(out, outbuff.str(), outlen);
+    memcpy_iflen(out, outbuff.str(), outlen);
 }
 
 bool rtlCodepageToCodepage(unsigned outlen, char * out, unsigned inlen, char const * in, char const * outcodepage, char const * incodepage)
@@ -3678,7 +3706,7 @@ bool rtlCodepageToCodepage(unsigned outlen, char * out, unsigned inlen, char con
 {
     if (inlen > outlen)
         inlen = outlen;
-    memcpy(out, in, inlen);
+    memcpy_iflen(out, in, inlen);
     if (inlen < outlen)
         memset(out+inlen, ' ', outlen-inlen);
     return true;
@@ -3712,7 +3740,7 @@ bool rtlCodepageToCodepage(StringBuffer & out, unsigned maxoutlen, unsigned inle
 void rtlStrToDataX(unsigned & tlen, void * & tgt, unsigned slen, const void * src)
 {
     void * data  = rtlMalloc(slen);
-    memcpy(data, src, slen);
+    memcpy_iflen(data, src, slen);
 
     tgt = data;
     tlen = slen;
@@ -3721,7 +3749,7 @@ void rtlStrToDataX(unsigned & tlen, void * & tgt, unsigned slen, const void * sr
 void rtlStrToStrX(unsigned & tlen, char * & tgt, unsigned slen, const void * src)
 {
     char * data  = (char *)rtlMalloc(slen);
-    memcpy(data, src, slen);
+    memcpy_iflen(data, src, slen);
 
     tgt = data;
     tlen = slen;
@@ -3730,7 +3758,7 @@ void rtlStrToStrX(unsigned & tlen, char * & tgt, unsigned slen, const void * src
 char * rtlStrToVStrX(unsigned slen, const void * src)
 {
     char * data  = (char *)rtlMalloc(slen+1);
-    memcpy(data, src, slen);
+    memcpy_iflen(data, src, slen);
     data[slen] = 0;
     return data;
 }
@@ -4948,7 +4976,7 @@ unsigned rtlUtf8Char(const void * data)
 void rtlUnicodeToUnicode(size32_t outlen, UChar * out, size32_t inlen, UChar const *in)
 {
     if(inlen>outlen) inlen = outlen;
-    memcpy(out, in, inlen*2);
+    memcpy_iflen(out, in, inlen*2);
     while(inlen<outlen)
         out[inlen++] = 0x0020;
 }
@@ -4956,7 +4984,7 @@ void rtlUnicodeToUnicode(size32_t outlen, UChar * out, size32_t inlen, UChar con
 void rtlUnicodeToVUnicode(size32_t outlen, UChar * out, size32_t inlen, UChar const *in)
 {
     if((inlen>=outlen) && (outlen != 0)) inlen = outlen-1;
-    memcpy(out, in, inlen*2);
+    memcpy_iflen(out, in, inlen*2);
     out[inlen] = 0x0000;
 }
 
@@ -4973,14 +5001,14 @@ void rtlVUnicodeToVUnicode(size32_t outlen, UChar * out, UChar const *in)
 void rtlUnicodeToUnicodeX(unsigned & tlen, UChar * & tgt, unsigned slen, UChar const * src)
 {
     tgt  = (UChar *)rtlMalloc(slen*2);
-    memcpy(tgt, src, slen*2);
+    memcpy_iflen(tgt, src, slen*2);
     tlen = slen;
 }
 
 UChar * rtlUnicodeToVUnicodeX(unsigned slen, UChar const * src)
 {
     UChar * data  = (UChar *)rtlMalloc((slen+1)*2);
-    memcpy(data, src, slen*2);
+    memcpy_iflen(data, src, slen*2);
     data[slen] = 0x0000;
     return data;
 }
@@ -5016,7 +5044,7 @@ void rtlUtf8ToUtf8(size32_t outlen, char * out, size32_t inlen, const char *in)
             break;
         offset += nextSize;
     }
-    memcpy(out, in, offset);
+    memcpy_iflen(out, in, offset);
     if (offset != outsize)
         memset(out+offset, ' ', outsize-offset);
 }
@@ -5025,7 +5053,7 @@ void rtlUtf8ToUtf8X(size32_t & outlen, char * & out, size32_t inlen, const char
 {
     unsigned insize = rtlUtf8Size(inlen, in);
     char * buffer = (char *)rtlMalloc(insize);
-    memcpy(buffer, in, insize);
+    memcpy_iflen(buffer, in, insize);
     outlen = inlen;
     out = buffer;
 }
@@ -5054,6 +5082,9 @@ unsigned rtlUnicodeStrlen(UChar const * str)
 
 void rtlUtf8ToData(size32_t outlen, void * out, size32_t inlen, const char *in)
 {
+    if (unlikely(outlen == 0))
+        return;
+
     unsigned insize = rtlUtf8Size(inlen, in);
     if (insize >= outlen)
         rtlCodepageToCodepage(outlen, (char *)out, insize, in, ASCII_LIKE_CODEPAGE, UTF8_CODEPAGE);
@@ -5262,7 +5293,7 @@ ECLRTL_API void rtlUtf8SubStrFTX(unsigned & tlen, char * & tgt, unsigned slen, c
     unsigned copySize = rtlUtf8Size(copylen, src+startOffset);
 
     char * buffer = (char *)rtlMalloc(copySize + fillSize);
-    memcpy(buffer, (byte *)src+startOffset, copySize);
+    memcpy_iflen(buffer, (byte *)src+startOffset, copySize);
     if (fillSize)
         memset(buffer+copySize, ' ', fillSize);
     tlen = len;
@@ -5277,7 +5308,7 @@ ECLRTL_API void rtlUtf8SubStrFX(unsigned & tlen, char * & tgt, unsigned slen, ch
     unsigned copySize = rtlUtf8Size(len, src+startOffset);
 
     char * buffer = (char *)rtlMalloc(copySize);
-    memcpy(buffer, (byte *)src+startOffset, copySize);
+    memcpy_iflen(buffer, (byte *)src+startOffset, copySize);
     tlen = len;
     tgt = buffer;
 }
@@ -5353,13 +5384,16 @@ ECLRTL_API unsigned rtlConcatUtf8ToUtf8(unsigned tlen, char * tgt, unsigned offs
     //normalization is done in the space filling routine at the end
     unsigned ssize = rtlUtf8Size(slen, src);
     assertex(tlen * UTF8_MAXSIZE >= offset+ssize);
-    memcpy(tgt+offset, src, ssize);
+    memcpy_iflen(tgt+offset, src, ssize);
     return offset + ssize;
 
 }
 
 ECLRTL_API void rtlUtf8SpaceFill(unsigned tlen, char * tgt, unsigned offset)
 {
+    if (unlikely(tlen == 0))
+        return;
+
     const byte * src = (const byte *)tgt;
     for (unsigned i=0; i<offset; i++)
     {
@@ -5489,13 +5523,16 @@ ECLRTL_API void rtlCreateRange(size32_t & outlen, char * & out, unsigned fieldLe
     outlen = fieldLen;
     out = (char *)rtlMalloc(fieldLen);
     if (len >= compareLen)
-        memcpy(out, str, compareLen);
+    {
+        memcpy_iflen(out, str, compareLen);
+    }
     else
     {
-        memcpy(out, str, len);
+        memcpy_iflen(out, str, len);
         memset(out+len, pad, compareLen-len);
     }
-    memset(out + compareLen, fill, fieldLen-compareLen);
+    if (fieldLen > compareLen)
+        memset(out + compareLen, fill, fieldLen-compareLen);
 }
 
 
@@ -5562,14 +5599,17 @@ ECLRTL_API void rtlCreateUnicodeRange(size32_t & outlen, UChar * & out, unsigned
     outlen = fieldLen;
     out = (UChar *)rtlMalloc(fieldLen*sizeof(UChar));
     if (len >= compareLen)
-        memcpy(out, str, compareLen*sizeof(UChar));
+    {
+        memcpy_iflen(out, str, compareLen*sizeof(UChar));
+    }
     else
     {
-        memcpy(out, str, len * sizeof(UChar));
+        memcpy_iflen(out, str, len * sizeof(UChar));
         while (len != compareLen)
             out[len++] = ' ';
     }
-    memset(out + compareLen, fill, (fieldLen-compareLen) * sizeof(UChar));
+    if (fieldLen > compareLen)
+        memset(out + compareLen, fill, (fieldLen-compareLen) * sizeof(UChar));
 }
 
 ECLRTL_API void rtlCreateUnicodeRangeLow(size32_t & outlen, UChar * & out, unsigned fieldLen, unsigned compareLen, size32_t len, const UChar * str)

rtl/eclrtl/rtlint.cpp

@@ -229,7 +229,7 @@ int rtlReadSwapInt2(const void * _data)
 
 int rtlReadSwapInt3(const void * _data)                     
 { 
-    const signed char * scdata = (const signed char *)_data;
+    const unsigned char * scdata = (const unsigned char *)_data;
     int temp = scdata[0] << 16;
     _cpyrev2(&temp, scdata+1);
     return temp;
@@ -244,8 +244,8 @@ int rtlReadSwapInt4(const void * _data)
 
 __int64 rtlReadSwapInt5(const void * _data)                     
 { 
-    const signed char * scdata = (const signed char *)_data;
-    __int64 temp = ((__int64)scdata[0]) << 32;
+    const unsigned char * scdata = (const unsigned char *)_data;
+    __int64 temp = ((__uint64)scdata[0]) << 32;
     _cpyrev4(&temp, scdata+1);
     return temp;
 }
@@ -253,7 +253,7 @@ __int64 rtlReadSwapInt5(const void * _data)
 __int64 rtlReadSwapInt6(const void * _data)                     
 { 
     const signed char * scdata = (const signed char *)_data;
-    __int64 temp = ((__int64)scdata[0]) << 40;
+    __int64 temp = ((__uint64)scdata[0]) << 40;
     _cpyrev5(&temp, scdata+1);
     return temp;
 }
@@ -261,7 +261,7 @@ __int64 rtlReadSwapInt6(const void * _data)
 __int64 rtlReadSwapInt7(const void * _data)                     
 { 
     const signed char * scdata = (const signed char *)_data;
-    __int64 temp = ((__int64)scdata[0]) << 48;
+    __int64 temp = ((__uint64)scdata[0]) << 48;
     _cpyrev6(&temp, scdata+1);
     return temp;
 }

rtl/eclrtl/rtlqstr.cpp

@@ -512,10 +512,12 @@ void rtlQStrToQStr(size32_t outlen, char * out, size32_t inlen, const char * in)
     size32_t inSize = QStrSize(inlen);
     size32_t outSize = QStrSize(outlen);
     if (inSize >= outSize)
-        memcpy(out, in, outSize);
+    {
+        memcpy_iflen(out, in, outSize);
+    }
     else
     {
-        memcpy(out, in, inSize);
+        memcpy_iflen(out, in, inSize);
         memset(out+inSize, 0, outSize-inSize);
     }
 }
@@ -524,7 +526,7 @@ void rtlQStrToQStrX(unsigned & outlen, char * & out, unsigned inlen, const char
 {
     size32_t inSize = QStrSize(inlen);
     char * data  = (char *)malloc(inSize);
-    memcpy(data, in, inSize);
+    memcpy_iflen(data, in, inSize);
 
     outlen = inlen;
     out = data;
@@ -536,7 +538,7 @@ int rtlCompareQStrQStr(size32_t llen, const void * left, size32_t rlen, const vo
     size32_t rsize = QStrSize(rlen);
     if (lsize < rsize)
     {
-        int ret = memcmp(left, right, lsize);
+        int ret = memcmp_iflen(left, right, lsize);
         if (ret == 0)
         {
             const byte * r = (const byte *)right;
@@ -549,7 +551,7 @@ int rtlCompareQStrQStr(size32_t llen, const void * left, size32_t rlen, const vo
         }
         return ret;
     }
-    int ret = memcmp(left, right, rsize);
+    int ret = memcmp_iflen(left, right, rsize);
     if (ret == 0)
     {
         const byte * l = (const byte *)left;

system/include/platform.h

@@ -540,6 +540,10 @@ typedef unsigned __int64 timestamp_type;
  #define NO_SANITIZE_FUNCTION
 #endif
 
-
+//Versions of memcpy etc which are safe to use with null parameters if the size is 0
+inline void memcpy_iflen(void * dest, const void * src, size_t n)   { if (likely(n)) memcpy(dest, src, n); }
+inline void memmove_iflen(void * dest, const void * src, size_t n)  { if (likely(n)) memmove(dest, src, n); }
+inline void memset_iflen(void * dest, int c, size_t n)              { if (likely(n)) memset(dest, c, n); }
+inline int memcmp_iflen(const void * l, const void * r, size_t n)   { return likely(n) ? memcmp(l, r, n) : 0; }
 
 #endif

system/jlib/jbuff.cpp

@@ -174,7 +174,10 @@ MemoryAttr::MemoryAttr(const MemoryAttr & src)
 
 void MemoryAttr::set(size_t _len, const void * _ptr)
 {
-    memcpy(allocate(_len), _ptr, _len);
+    if (_len)
+        memcpy(allocate(_len), _ptr, _len);
+    else
+        clear();
 }
 
 
@@ -525,10 +528,13 @@ MemoryBuffer & MemoryBuffer::append(const unsigned char * value)
 
 MemoryBuffer & MemoryBuffer::append(unsigned len, const void * value)
 {
-    unsigned newLen = checkMemoryBufferOverflow(curLen, len);
-    _realloc(newLen);
-    memcpy(buffer + curLen, value, len);
-    curLen += len;
+    if (likely(len))
+    {
+        unsigned newLen = checkMemoryBufferOverflow(curLen, len);
+        _realloc(newLen);
+        memcpy(buffer + curLen, value, len);
+        curLen += len;
+    }
     return *this;
 }
 

testing/regress/ecl/hthor/javascope.xml

@@ -0,0 +1,90 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>: parallel</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>: sequential</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>thread: parallel</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>channel: sequential</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>query: sequential</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>workunit: sequential</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><a>37</a></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><a>39</a></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><a>42</a></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><a>124</a></Row>
+</Dataset>

testing/regress/ecl/javascope.ecl

@@ -0,0 +1,113 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT Java;
+
+//class=embedded
+//class=3rdparty
+
+//version forceNonThread=false
+//version forceNonThread=true
+
+
+import ^ as root;
+forceNonThread := #IFDEFINED(root.forceNonThread, false);
+
+// Check that implicitly-created objects have appropriate lifetime
+
+implicit(STRING p, STRING s) := MODULE
+  STRING st := ''
+#if (forceNonThread)
+   : STORED('st')
+#end
+  ;
+
+  EXPORT INTEGER accumulate(INTEGER b) := EMBED(Java : PERSIST(p), GLOBALSCOPE(s+st))
+    class x
+    {
+      public x()
+      {
+        synchronized (x.class)
+        { 
+          idx = nextIdx;
+          nextIdx = nextIdx+1;
+        }
+//        System.out.println("created  " + idx + x.class.getName());
+      }
+      public void finalize()
+      {
+//        System.out.println("finalize " + n + " " + idx);
+      }
+      public synchronized int accumulate(int b)
+      {
+        n = n + b;
+        return n;
+      }
+      int n = 0;
+      int idx = 0;
+      static int nextIdx = 0;
+    }
+  ENDEMBED;
+
+  SHARED r := RECORD
+    integer a; 
+  END;
+
+
+  // The parallel test runs all on separate threads (except the last two calls) to ensure that separate threads
+  // are independent when using PERSIST('thread') or PERSIST('none')
+
+  EXPORT ptest := PARALLEL (
+    output(p + ': parallel');
+    output(project(nofold(dataset([{1}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{2}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{3}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{10}], r)), transform(r, self.a := accumulate(LEFT.a)+accumulate(LEFT.a*2))));
+  );
+
+  // The sequential test runs sequentially for ones that are supposed to interact across threads (otherwise results are indeterminate)
+
+  EXPORT stest := ORDERED (
+    output(p + ': sequential');
+    output(project(nofold(dataset([{1}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{2}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{3}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{10}], r)), transform(r, self.a := accumulate(LEFT.a)+accumulate(LEFT.a*2))));
+  );
+
+END;
+
+gc() := EMBED(Java)
+  public static void gc()
+  {
+    System.gc();
+  }
+ENDEMBED;
+
+ORDERED (
+  implicit('','').ptest;
+  implicit('','').stest;
+  implicit('thread','').ptest;
+  implicit('channel','').stest;
+  implicit('query','').stest;
+  implicit('workunit','').stest;
+//  implicit('global','').stest;
+//  gc();
+);
+
+// Check that explicitly-created objects have appropriate lifetime (how?) but are not shared
+

testing/regress/ecl/key/javascope.xml

@@ -0,0 +1,90 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>: parallel</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>: sequential</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>thread: parallel</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><a>40</a></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>channel: sequential</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>query: sequential</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>workunit: sequential</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><a>52</a></Row>
+</Dataset>

testing/regress/ecl/key/spray_test_xml.xml

@@ -0,0 +1,23 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><result>Despray Pass</result></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><result>Spray Pass</result></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>Compare: Pass</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>Prep file header length: OK</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>Prep file footer length: OK</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>Spray file header length: OK</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>Spray file footer length: OK</Result_8></Row>
+</Dataset>

testing/regress/ecl/key/unaligned_unicode.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><b>true</b><u>Hello                                                                                               </u></Row>
+</Dataset>

testing/regress/ecl/spray_test_xml.ecl

@@ -0,0 +1,185 @@
+/*##############################################################################
+
+    Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//noroxie
+
+//class=spray
+
+//version xml_header='none',xml_footer='none'
+//version xml_header='none',xml_footer='short'
+//version xml_header='none',xml_footer='long'
+//version xml_header='short',xml_footer='none'
+//version xml_header='short',xml_footer='short'
+//version xml_header='short',xml_footer='long'
+//version xml_header='long',xml_footer='none'
+//version xml_header='long',xml_footer='short'
+//version xml_header='long',xml_footer='long'
+
+import Std.File AS FileServices;
+import $.setup;
+import ^ as root;
+
+prefix := setup.Files(false, false).QueryFilePrefix;
+
+dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+
+espUrl := FileServices.GetEspURL() + '/FileSpray';
+
+unsigned VERBOSE := 0;
+unsigned CLEANUP := 1;
+
+string xml_header := #IFDEFINED(root.xml_header, 'short');
+#if (xml_header = 'none')
+    string header := '';
+#elseif (xml_header = 'short')
+    string header := '<Header>Head</Header>\n';
+#else
+    string header := '<Header>Head 123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890</Header>\n';
+#end
+unsigned expectedHeaderLength := LENGTH(header);
+
+string xml_footer := #IFDEFINED(root.xml_footer, 'none');
+
+#if (xml_footer = 'none')
+    string footer := '';
+#elseif (xml_footer = 'short')
+    string footer := '<Footer>Foot</Footer>';
+#else
+    string footer := '<Footer>Foot 123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890</Footer>';
+#end
+unsigned expectedFooterLength := LENGTH(footer);
+
+Layout_Person := RECORD
+  STRING3  name;
+  UNSIGNED2 age;
+  BOOLEAN good;
+END;
+
+
+allPeople := DATASET([ {'foo', 10, 1},
+                       {'bar', 12, 0},
+                       {'baz', 32, 1}]
+            ,Layout_Person);
+
+setupXmlPrepFileName := prefix + 'original_xml';
+desprayXmlOutFileName := dropzonePath + prefix + '-desprayed_xml';
+sprayXmlTargetFileName := prefix + 'sprayed_xml';
+dsSetup := DISTRIBUTE(allPeople);
+
+//  Create a small logical file
+setupXmlFile := output(dsSetup, , setupXmlPrepFileName, XML( 'Row', HEADING(header , footer), NOROOT), OVERWRITE);
+
+rec := RECORD
+  string result;
+  string msg;
+end;
+
+
+// Despray it to default drop zone
+rec despray(rec l) := TRANSFORM
+  SELF.msg := FileServices.fDespray(
+                       LOGICALNAME := setupXmlPrepFileName
+                      ,DESTINATIONIP := '.'
+                      ,DESTINATIONPATH := desprayXmlOutFileName
+                      ,ALLOWOVERWRITE := True
+                      );
+  SELF.result := 'Despray Pass';
+end;
+
+dst1 := NOFOLD(DATASET([{'', ''}], rec));
+p1 := NOTHOR(PROJECT(NOFOLD(dst1), despray(LEFT)));
+c1 := CATCH(NOFOLD(p1), ONFAIL(TRANSFORM(rec,
+                                 SELF.result := 'Despray Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+#if (VERBOSE = 1)
+    desprayOut := output(c1);
+#else
+    desprayOut := output(c1, {result});
+#end
+
+
+
+rec spray(rec l) := TRANSFORM
+    SELF.msg := FileServices.fSprayXml(
+                        SOURCEIP := '.',
+                        SOURCEPATH := desprayXmlOutFileName,
+                        SOURCEROWTAG := 'Row',
+                        DESTINATIONGROUP := 'mythor',
+                        DESTINATIONLOGICALNAME := sprayXmlTargetFileName,
+                        TIMEOUT := -1,
+                        ESPSERVERIPPORT := espUrl,
+                        ALLOWOVERWRITE := true
+                        );
+    self.result := 'Spray Pass';
+end;
+
+
+dst2 := NOFOLD(DATASET([{'', ''}], rec));
+p2 := NOTHOR(PROJECT(NOFOLD(dst2), spray(LEFT)));
+c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
+                                 SELF.result := 'Spray Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+#if (VERBOSE = 1)
+    sprayOut := output(c2);
+#else
+    sprayOut := output(c2, {result});
+#end
+
+ds := DATASET(sprayXmlTargetFileName, Layout_Person, XML('Row', NOROOT));
+
+string compareDatasets(dataset(Layout_Person) ds1, dataset(Layout_Person) ds2) := FUNCTION
+   c := COUNT(JOIN(ds1, ds2, left.name=right.name, FULL ONLY));
+   boolean result := (0 = c);
+   #if (VERBOSE = 1)
+    retVal := 'Compare: ' + if(result, 'Pass', 'Fail') + ', Count = ' + intformat(c, 3, 0);
+   #else
+    retVal := 'Compare: ' + if(result, 'Pass', 'Fail');
+   #end
+   RETURN retVal;
+END;
+
+string checkFileHeaderFooterLen(string prefix, string fileName, unsigned expectedLen, boolean isHeader) := FUNCTION
+    len := (INTEGER) if( isHeader, fileservices.GetLogicalFileAttribute(setupXmlPrepFileName, 'headerLength'), fileservices.GetLogicalFileAttribute(setupXmlPrepFileName, 'footerLength'));
+    return ( prefix + ' file ' + if( isHeader, 'header', 'footer')  + ' length: ' +  if (len = expectedLen, 'OK', 'Bad ' + intformat(len, 3, 0) + '/' + intformat(expectedLen, 3, 0)));
+end;
+
+SEQUENTIAL(
+    setupXmlFile,
+    desprayOut,
+    sprayOut,
+    output(compareDatasets(dsSetup,ds)),
+    output(checkFileHeaderFooterLen('Prep', setupXmlPrepFileName, expectedHeaderLength, TRUE));
+    output(checkFileHeaderFooterLen('Prep', setupXmlPrepFileName, expectedFooterLength, FALSE));
+    output(checkFileHeaderFooterLen('Spray', sprayXmlTargetFileName, expectedHeaderLength, TRUE));
+    output(checkFileHeaderFooterLen('Spray', sprayXmlTargetFileName, expectedFooterLength, FALSE));
+
+#if (VERBOSE = 1)
+    output(dsSetup, NAMED('dsSetup')),
+    output(ds, NAMED('ds')),
+    output(JOIN(dsSetup, ds, left.name=right.name, FULL ONLY, LOCAL)),
+#end
+
+#if (CLEANUP = 1)
+    // Clean-up
+    FileServices.DeleteExternalFile('.', desprayXmlOutFileName),
+    FileServices.DeleteLogicalFile(setupXmlPrepFileName),
+    FileServices.DeleteLogicalFile(sprayXmlTargetFileName)
+#end
+);

testing/regress/ecl/thor/javascope.xml

@@ -0,0 +1,90 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>: parallel</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>: sequential</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>thread: parallel</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><a>40</a></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>channel: sequential</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>query: sequential</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>workunit: sequential</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><a>37</a></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><a>39</a></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><a>42</a></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><a>124</a></Row>
+</Dataset>

testing/regress/ecl/unaligned_unicode.ecl

@@ -0,0 +1,30 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+r := RECORD
+  boolean b;
+  UNICODE80 u;
+END;
+
+r2 := RECORD
+  boolean b;
+  UNICODE100 u;
+END;
+
+d := NOFOLD(DATASET([{true, 'Hello'}], r));
+
+OUTPUT(PROJECT(d, TRANSFORM(r2, SELF := LEFT)));

thorlcr/activities/piperead/thprslave.cpp

@@ -38,6 +38,7 @@ protected:
     bool eof, pipeFinished;
     unsigned retcode;
     unsigned flags;
+    Owned<IException> verifyPipeException;
 
 protected:
 
@@ -119,9 +120,10 @@ protected:
                     }
                 }
                 if (START_FAILURE == retcode) // PIPE process didn't start at all, START_FAILURE is our own error code
-                    throw MakeActivityException(this, TE_PipeReturnedFailure, "Process failed to start: %s - PIPE(%s)", stdError.str(), pipeCommand.get());
+                    verifyPipeException.setown(MakeActivityException(this, TE_PipeReturnedFailure, "Process failed to start: %s - PIPE(%s)", stdError.str(), pipeCommand.get()));
                 else
-                    throw MakeActivityException(this, TE_PipeReturnedFailure, "Process returned %d:%s - PIPE(%s)", retcode, stdError.str(), pipeCommand.get());
+                    verifyPipeException.setown(MakeActivityException(this, TE_PipeReturnedFailure, "Process returned %d:%s - PIPE(%s)", retcode, stdError.str(), pipeCommand.get()));
+                throw verifyPipeException.getLink();
             }
         }
     }
@@ -478,7 +480,8 @@ public:
         Owned<IException> wrexc = pipeWriter->joinExc();
         PARENT::stop();
         verifyPipe();
-        if (wrexc)
+        Owned<IException> hadVerifyPipeException = verifyPipeException.getClear(); // ensured cleared in case in CQ
+        if (wrexc && !hadVerifyPipeException)
             throw wrexc.getClear();
         if (retcode!=0 && !(flags & TPFnofail))
             throw MakeActivityException(this, TE_PipeReturnedFailure, "Process returned %d", retcode);

tools/esdlcmd/esdl2ecl.cpp

@@ -71,7 +71,7 @@ public:
             if (typeEntry && *typeEntry)
             {
                 const char *src = (*typeEntry)->src.get();
-                if (stricmp(src, localsrc))
+                if (stricmp(src, localsrc) != 0)
                 {
                     const char *finger = src;
                     if (!strnicmp(finger, "wsm_", 4))