Merge branch 'candidate-5.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

Conflicts:
	ecl/hqlcpp/hqlcpp.cpp

common/fileview2/fvresultset.cpp

@@ -1564,26 +1564,26 @@ void CResultSetCursor::writeXmlText(IXmlWriter &writer, int columnIndex, const c
         {
             __int64 value = getIntFromInt(type, cur, isMappedIndexField(columnIndex));
             if (type.isSigned())
-                writer.outputInt((__int64) value, name);
+                writer.outputInt((__int64) value, type.getSize(), name);
             else
-                writer.outputUInt((unsigned __int64) value, name);
+                writer.outputUInt((unsigned __int64) value, type.getSize(), name);
             break;
         }
     case type_swapint:
         {
             __int64 value = getIntFromSwapInt(type, cur, isMappedIndexField(columnIndex));
             if (type.isSigned())
-                writer.outputInt((__int64) value, name);
+                writer.outputInt((__int64) value, type.getSize(), name);
             else
-                writer.outputUInt((unsigned __int64) value, name);
+                writer.outputUInt((unsigned __int64) value, type.getSize(), name);
             break;
         }
     case type_packedint:
         {
             if (type.isSigned())
-                writer.outputInt(rtlGetPackedSigned(cur), name);
+                writer.outputInt(rtlGetPackedSigned(cur), type.getSize(), name);
             else
-                writer.outputUInt(rtlGetPackedUnsigned(cur), name);
+                writer.outputUInt(rtlGetPackedUnsigned(cur), type.getSize(), name);
             break;
         }
     case type_decimal:
@@ -3306,11 +3306,11 @@ extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const ch
 
             writer.outputBeginNested(getSeverityTagname(severity, flags), false);
             if (code)
-                writer.outputUInt(code, "Code");
+                writer.outputUInt(code, sizeof(unsigned), "Code");
             if (filename.length())
                 writer.outputCString(filename.str(), "Filename");
             if (lineno)
-                writer.outputUInt(lineno, "Line");
+                writer.outputUInt(lineno, sizeof(unsigned), "Line");
 
             writer.outputCString(src.str(), "Source");
             writer.outputCString(msg.str(), "Message");

common/thorhelper/roxiedebug.cpp

@@ -444,14 +444,14 @@ public:
     {
         // MORE
     }
-    virtual void outputInt(__int64 field, const char *fieldname) 
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname)
     {
         if (intValueSet && !matchSeen && checkFieldName(fieldname))
         {
             matchSeen = checkCondition((field == intValue) ? 0 : ((field > intValue) ? 1 : -1));
         }
     }
-    virtual void outputUInt(unsigned __int64 field, const char *fieldname) 
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname)
     {
         // NOTE - "contains" is interpreted as "equals" on numeric fields
         if (uintValueSet && !matchSeen && checkFieldName(fieldname))
@@ -851,7 +851,7 @@ void CBreakpointInfo::toXML(IXmlWriter *output) const
     }
     if (rowCount)
     {
-        output->outputInt(rowCount, "@rowCount"); 
+        output->outputInt(rowCount, sizeof(int), "@rowCount");
         output->outputCString(BreakpointCountModes[rowCountMode], "@rowCountMode"); 
     }
     ForEachItemIn(edgeIdx, activeEdges)
@@ -906,14 +906,14 @@ void DebugActivityRecord::outputProperties(IXmlWriter *output)
     {
         output->outputBeginNested("att", false); 
         output->outputCString("localTime", "@name"); 
-        output->outputUInt((unsigned) (cycle_to_nanosec(localCycles)/1000), "@value"); 
+        output->outputUInt((unsigned) (cycle_to_nanosec(localCycles)/1000), sizeof(unsigned), "@value");
         output->outputEndNested("att");
     }
     if (totalCycles)
     {
         output->outputBeginNested("att", false); 
         output->outputCString("totalTime", "@name"); 
-        output->outputUInt((unsigned) (cycle_to_nanosec(totalCycles)/1000), "@value"); 
+        output->outputUInt((unsigned) (cycle_to_nanosec(totalCycles)/1000), sizeof(unsigned), "@value");
         output->outputEndNested("att");
     }
 }
@@ -1321,7 +1321,7 @@ void CBaseDebugContext::deserialize(MemoryBuffer &buff)
 void CBaseServerDebugContext::doStandardResult(IXmlWriter *output) const
 {
     const char *stateString = running ? "running" : queryStateString(currentState);
-    output->outputInt(sequence, "@sequence");
+    output->outputInt(sequence, sizeof(int), "@sequence");
     output->outputString(strlen(stateString), stateString, "@state");
     if (currentNode)
         currentNode->outputId(output, "@nodeId");
@@ -1331,7 +1331,7 @@ void CBaseServerDebugContext::doStandardResult(IXmlWriter *output) const
         output->outputBool(true, "@skip");
     if (currentGraph)
         output->outputCString(currentGraph->queryGraphName(), "@graphId");
-    output->outputInt(graphChangeSequence, "@graphSequenceNum");
+    output->outputInt(graphChangeSequence, sizeof(int), "@graphSequenceNum");
     if (graphChangeSequence != prevGraphChangeSequence)
         output->outputBool(true, "@graphChanged");
     if (currentBreakpointUID != (unsigned) -1)
@@ -1343,7 +1343,7 @@ void CBaseServerDebugContext::doStandardResult(IXmlWriter *output) const
     {
         output->outputBeginNested("Exception", true);
         output->outputCString("Roxie", "Source");
-        output->outputInt(currentException->errorCode(), "Code");
+        output->outputInt(currentException->errorCode(), sizeof(int), "Code");
         StringBuffer s;
         currentException->errorMessage(s);
         output->outputString(s.length(), s.str(), "Message");
@@ -1356,7 +1356,7 @@ void CBaseServerDebugContext::_listBreakpoint(IXmlWriter *output, IBreakpointInf
     if (bp.queryMode() != BreakpointModeNone)
     {
         output->outputBeginNested("break", true);
-        output->outputInt(idx, "@idx");
+        output->outputInt(idx, sizeof(int), "@idx");
         bp.toXML(output);
         output->outputEndNested("break");
     }
@@ -1699,7 +1699,7 @@ void CBaseServerDebugContext::debugCounts(IXmlWriter *output, unsigned sinceSequ
         {
             output->outputBeginNested("edge", true);
             output->outputCString((const char *) edges.query().getKey(), "@edgeId");
-            output->outputUInt(edge->queryCount(), "@count");
+            output->outputUInt(edge->queryCount(), sizeof(unsigned), "@count");
             output->outputEndNested("edge");
         }
         if (reset)
@@ -1803,7 +1803,7 @@ void CBaseServerDebugContext::debugGetConfig(IXmlWriter *output, const char *nam
     }
     else if (stricmp(name, "stopOnLimits")==0)
     {
-        output->outputInt(stopOnLimits, "@value");
+        output->outputInt(stopOnLimits, sizeof(int), "@value");
     }
     else if (stricmp(name, "historySize")==0)
     {
@@ -1814,11 +1814,11 @@ void CBaseServerDebugContext::debugGetConfig(IXmlWriter *output, const char *nam
             IActivityDebugContext *activityCtx = currentGraph->lookupActivityByEdgeId(id);
             if (!activityCtx)
                 throw MakeStringException(THORHELPER_DEBUG_ERROR, "Edge %s not found in current graph", id);
-            output->outputInt(activityCtx->queryHistoryCapacity(), "@value");
+            output->outputInt(activityCtx->queryHistoryCapacity(), sizeof(int), "@value");
         }
         else
         {
-            output->outputInt(defaultHistoryCapacity, "@value");
+            output->outputInt(defaultHistoryCapacity, sizeof(int), "@value");
         }
     }
     else
@@ -1996,7 +1996,7 @@ void CBaseDebugGraphManager::outputLinksForChildGraph(IXmlWriter *output, const
         StringBuffer idText;
         idText.append(childNode->idText).append('_').append(parentId);
         output->outputCString(idText.str(), "@id"); // MORE - is this guaranteed to be unique?
-        output->outputBeginNested("att", false); output->outputCString("_childGraph", "@name"); output->outputInt(1, "@value"); output->outputEndNested("att");
+        output->outputBeginNested("att", false); output->outputCString("_childGraph", "@name"); output->outputInt(1, sizeof(int), "@value"); output->outputEndNested("att");
         output->outputBeginNested("att", false); output->outputCString("_sourceActivity", "@name"); childNode->outputId(output, "@value"); output->outputEndNested("att"); // MORE!!!
         output->outputBeginNested("att", false); output->outputCString("_targetActivity", "@name"); output->outputCString(parentId, "@value"); output->outputEndNested("att");
         output->outputEndNested("edge");
@@ -2028,20 +2028,20 @@ void CBaseDebugGraphManager::outputChildGraph(IXmlWriter *output, unsigned seque
                 output->outputString(kindStr.length(), kindStr.str(), "@shortlabel");
                 if (node->activity->isPassThrough())
                 {
-                    output->outputBeginNested("att", false); output->outputCString("_isPassthrough", "@name"); output->outputInt(1, "@value"); output->outputEndNested("att");
+                    output->outputBeginNested("att", false); output->outputCString("_isPassthrough", "@name"); output->outputInt(1, sizeof(int), "@value"); output->outputEndNested("att");
                 }
                 if (node->totalCycles)
                 {
                     output->outputBeginNested("att", false); 
                     output->outputCString("totalTime", "@name"); 
-                    output->outputUInt((unsigned) (cycle_to_nanosec(node->totalCycles)/1000), "@value"); 
+                    output->outputUInt((unsigned) (cycle_to_nanosec(node->totalCycles)/1000), sizeof(unsigned), "@value");
                     output->outputEndNested("att");
                 }
                 if (node->localCycles)
                 {
                     output->outputBeginNested("att", false); 
                     output->outputCString("localTime", "@name"); 
-                    output->outputUInt((unsigned) (cycle_to_nanosec(node->localCycles)/1000), "@value"); 
+                    output->outputUInt((unsigned) (cycle_to_nanosec(node->localCycles)/1000), sizeof(unsigned), "@value");
                     output->outputEndNested("att");
                 }
                 node->outputProperties(output);
@@ -2246,7 +2246,7 @@ void CBaseDebugGraphManager::getXGMML(IXmlWriter *output, unsigned sequence, boo
             {
                 output->outputBeginNested("edge", true);
                 output->outputCString(dependency.edgeId, "@id");
-                output->outputBeginNested("att", false); output->outputCString("_dependsOn", "@name"); output->outputInt(1, "@value"); output->outputEndNested("att");
+                output->outputBeginNested("att", false); output->outputCString("_dependsOn", "@name"); output->outputInt(1, sizeof(int), "@value"); output->outputEndNested("att");
                 output->outputBeginNested("att", false); output->outputCString("_sourceActivity", "@name"); source->outputId(output, "@value"); output->outputEndNested("att");
                 output->outputBeginNested("att", false); output->outputCString("_targetActivity", "@name"); target->outputId(output, "@value"); output->outputEndNested("att");
                 output->outputEndNested("edge");

common/thorhelper/roxiehelper.cpp

@@ -1126,6 +1126,17 @@ void FlushingStringBuffer::startScalar(const char *resultName, unsigned sequence
     }
 }
 
+void FlushingStringBuffer::setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size)
+{
+    startScalar(resultName, sequence);
+    s.append(value);
+}
+void FlushingStringBuffer::setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size)
+{
+    startScalar(resultName, sequence);
+    s.append(value);
+}
+
 void FlushingStringBuffer::incrementRowCount()
 {
     CriticalBlock b(crit);
@@ -1190,6 +1201,25 @@ void FlushingJsonBuffer::startScalar(const char *resultName, unsigned sequence)
         tail.set("}]}");
     }
 }
+
+void FlushingJsonBuffer::setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size)
+{
+    startScalar(resultName, sequence);
+    if (size < 7) //JavaScript only supports 53 significant bits
+        s.append(value);
+    else
+        s.append('"').append(value).append('"');
+}
+
+void FlushingJsonBuffer::setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size)
+{
+    startScalar(resultName, sequence);
+    if (size < 6) //JavaScript doesn't support unsigned, and only supports 53 significant bits
+        s.append(value);
+    else
+        s.append('"').append(value).append('"');
+}
+
 //=====================================================================================================
 
 ClusterWriteHandler::ClusterWriteHandler(char const * _logicalName, char const * _activityType)

common/thorhelper/roxiehelper.hpp

@@ -179,6 +179,8 @@ public:
     virtual void *getPayload(size32_t &length);
     virtual void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL);
     virtual void startScalar(const char *resultName, unsigned sequence);
+    virtual void setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size);
+    virtual void setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size);
     virtual void incrementRowCount();
 };
 
@@ -195,6 +197,8 @@ public:
     void encodeData(const void *data, unsigned len);
     void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL);
     void startScalar(const char *resultName, unsigned sequence);
+    virtual void setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size);
+    virtual void setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size);
 };
 
 inline const char *getFormatName(TextMarkupFormat fmt)

common/thorhelper/thorcommon.hpp

@@ -272,9 +272,9 @@ public:
     {
         ctx->setResultDecimal(stepname, sequence, len, precision, isSigned, val);
     }
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 value)
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size)
     {
-        ctx->setResultInt(name, sequence, value);
+        ctx->setResultInt(name, sequence, value, size);
     }
     virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data)
     {
@@ -292,9 +292,9 @@ public:
     {
         ctx->setResultString(name, sequence, len, str);
     }
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value)
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size)
     {
-        ctx->setResultUInt(name, sequence, value);
+        ctx->setResultUInt(name, sequence, value, size);
     }
     virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str)
     {

common/thorhelper/thorxmlwrite.cpp

@@ -136,7 +136,7 @@ void CommonXmlWriter::outputData(unsigned len, const void *field, const char *fi
     }
 }
 
-void CommonXmlWriter::outputInt(__int64 field, const char *fieldname)
+void CommonXmlWriter::outputInt(__int64 field, unsigned size, const char *fieldname)
 {
     if (checkForAttribute(fieldname))
         outputXmlAttrInt(field, fieldname+1, out);
@@ -150,7 +150,7 @@ void CommonXmlWriter::outputInt(__int64 field, const char *fieldname)
     }
 }
 
-void CommonXmlWriter::outputUInt(unsigned __int64 field, const char *fieldname)
+void CommonXmlWriter::outputUInt(unsigned __int64 field, unsigned size, const char *fieldname)
 {
     if (checkForAttribute(fieldname))
         outputXmlAttrUInt(field, fieldname+1, out);
@@ -486,16 +486,31 @@ void CommonJsonWriter::outputData(unsigned len, const void *field, const char *f
     appendJSONDataValue(out, checkItemName(fieldname), len, field);
 }
 
-void CommonJsonWriter::outputInt(__int64 field, const char *fieldname)
+void CommonJsonWriter::outputInt(__int64 field, unsigned size, const char *fieldname)
 {
-    checkDelimit();
-    appendJSONValue(out, checkItemName(fieldname), field);
+    if (size < 7) //JavaScript only supports 53 significant bits
+    {
+        checkDelimit();
+        appendJSONValue(out, checkItemName(fieldname), field);
+    }
+    else
+    {
+        appendJSONNameOrDelimit(out, checkItemName(fieldname));
+        out.append('"').append(field).append('"');
+    }
 }
-
-void CommonJsonWriter::outputUInt(unsigned __int64 field, const char *fieldname)
+void CommonJsonWriter::outputUInt(unsigned __int64 field, unsigned size, const char *fieldname)
 {
-    checkDelimit();
-    appendJSONValue(out, checkItemName(fieldname), field);
+    if (size < 6) //JavaScript doesn't support unsigned, and only supports 53 significant bits
+    {
+        checkDelimit();
+        appendJSONValue(out, checkItemName(fieldname), field);
+    }
+    else
+    {
+        appendJSONNameOrDelimit(out, checkItemName(fieldname));
+        out.append('"').append(field).append('"');
+    }
 }
 
 void CommonJsonWriter::outputReal(double field, const char *fieldname)
@@ -849,7 +864,7 @@ void CommonEncodedXmlWriter::outputData(unsigned len, const void *field, const c
     }
 }
 
-void CommonEncodedXmlWriter::outputInt(__int64 field, const char *fieldname)
+void CommonEncodedXmlWriter::outputInt(__int64 field, unsigned size, const char *fieldname)
 {
     if (checkForAttribute(fieldname))
         outputXmlAttrInt(field, fieldname+1, out);
@@ -863,7 +878,7 @@ void CommonEncodedXmlWriter::outputInt(__int64 field, const char *fieldname)
     }
 }
 
-void CommonEncodedXmlWriter::outputUInt(unsigned __int64 field, const char *fieldname)
+void CommonEncodedXmlWriter::outputUInt(unsigned __int64 field, unsigned size, const char *fieldname)
 {
     if (checkForAttribute(fieldname))
         outputXmlAttrUInt(field, fieldname+1, out);
@@ -1058,13 +1073,13 @@ void SimpleOutputWriter::outputData(unsigned len, const void *field, const char
     outputXmlData(len, field, NULL, out);
 }
 
-void SimpleOutputWriter::outputInt(__int64 field, const char *)
+void SimpleOutputWriter::outputInt(__int64 field, unsigned size, const char *)
 {
     outputFieldSeparator();
     outputXmlInt(field, NULL, out);
 }
 
-void SimpleOutputWriter::outputUInt(unsigned __int64 field, const char *)
+void SimpleOutputWriter::outputUInt(unsigned __int64 field, unsigned size, const char *)
 {
     outputFieldSeparator();
     outputXmlUInt(field, NULL, out);

common/thorhelper/thorxmlwrite.hpp

@@ -62,8 +62,8 @@ public:
     virtual void outputString(unsigned len, const char *field, const char *fieldname);
     virtual void outputBool(bool field, const char *fieldname);
     virtual void outputData(unsigned len, const void *field, const char *fieldname);
-    virtual void outputInt(__int64 field, const char *fieldname);
-    virtual void outputUInt(unsigned __int64 field, const char *fieldname);
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname);
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname);
     virtual void outputReal(double field, const char *fieldname);
     virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
@@ -133,8 +133,8 @@ public:
     virtual void outputString(unsigned len, const char *field, const char *fieldname);
     virtual void outputBool(bool field, const char *fieldname);
     virtual void outputData(unsigned len, const void *field, const char *fieldname);
-    virtual void outputInt(__int64 field, const char *fieldname);
-    virtual void outputUInt(unsigned __int64 field, const char *fieldname);
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname);
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname);
     virtual void outputReal(double field, const char *fieldname);
     virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
@@ -203,8 +203,8 @@ public:
     virtual void outputString(unsigned len, const char *field, const char *fieldname);
     virtual void outputBool(bool field, const char *fieldname);
     virtual void outputData(unsigned len, const void *field, const char *fieldname);
-    virtual void outputInt(__int64 field, const char *fieldname);
-    virtual void outputUInt(unsigned __int64 field, const char *fieldname);
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname);
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname);
     virtual void outputReal(double field, const char *fieldname);
     virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
@@ -242,8 +242,6 @@ public:
     virtual void outputString(unsigned len, const char *field, const char *fieldname);
     virtual void outputBool(bool field, const char *fieldname);
     virtual void outputData(unsigned len, const void *field, const char *fieldname);
-    virtual void outputInt(__int64 field, const char *fieldname);
-    virtual void outputUInt(unsigned __int64 field, const char *fieldname);
     virtual void outputReal(double field, const char *fieldname);
     virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
@@ -259,7 +257,8 @@ public:
     virtual void outputInlineXml(const char *text){} //for appending raw xml content
     virtual void outputXmlns(const char *name, const char *uri){}
 
-
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname);
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname);
 
     void newline();
 protected:

docs/RunningHPCCinaVirtualMachine/RunningHPCCinaVirtualMachine.xml

@@ -256,6 +256,11 @@
                     </listitem>
                   </varlistentry>
                 </variablelist></para>
+
+              <para>Choose the VM Image file for the VMware player. The
+              <emphasis>filename</emphasis>-<emphasis
+              role="bold">vmx</emphasis>.ova file is appropriate for the
+              VMware player.</para>
             </listitem>
 
             <listitem>
@@ -376,6 +381,21 @@
                 </variablelist></para>
             </listitem>
           </orderedlist></para>
+
+        <sect3>
+          <title>Guest Additions</title>
+
+          <para>The HPCC VM Images include Guest Additions. Guest Additions
+          are device drivers and system applications that optimize the guest
+          operating system for better performance and usability. If you would
+          like to use your mouse pointer, and/or other desktop tools with your
+          virtual machine, start the X windows display after you log into to
+          your virtual machine. </para>
+
+          <para>Log in with the credentials provided. (user: hpccdemo,
+          password: hpccdemo) At the command prompt enter: <emphasis
+          role="bold">startx</emphasis>.</para>
+        </sect3>
       </sect2>
     </sect1>
 
@@ -519,6 +539,10 @@
                     </listitem>
                   </varlistentry>
                 </variablelist></para>
+
+              <para>Choose the VM Image file for VirtualBox. The
+              <emphasis>filename</emphasis>.ova file is appropriate for the
+              VirtualBox.</para>
             </listitem>
 
             <listitem>
@@ -693,6 +717,21 @@
                 </variablelist></para>
             </listitem>
           </orderedlist></para>
+
+        <sect3>
+          <title>Guest Additions</title>
+
+          <para>The HPCC VM Images include Guest Additions. Guest Additions
+          are device drivers and system applications that optimize the guest
+          operating system for better performance and usability. If you would
+          like to use your mouse pointer, and/or other desktop tools with your
+          virtual machine, start the X windows display after you log into to
+          your virtual machine.</para>
+
+          <para>Log in with the credentials provided. (user: hpccdemo,
+          password: hpccdemo) At the command prompt enter: <emphasis
+          role="bold">startx</emphasis>.</para>
+        </sect3>
       </sect2>
     </sect1>
   </chapter>

ecl/eclagent/eclagent.cpp

@@ -770,13 +770,13 @@ void EclAgent::outputFormattedResult(const char * name, unsigned sequence, bool
         outputSerializer->close(sequence, false);
 }
 
-void EclAgent::setResultInt(const char * name, unsigned sequence, __int64 val)
+void EclAgent::setResultInt(const char * name, unsigned sequence, __int64 val, unsigned size)
 {
     LOG(MCsetresult, unknownJob, "setResultInt(%s,%d,%"I64F"d)", nullText(name), sequence, val);
     Owned<IWUResult> r = updateResult(name, sequence);
     if (r)
     {
-        r->setResultInt(val);   
+        r->setResultInt(val);
         r->setResultStatus(ResultStatusCalculated);
     }
     else
@@ -793,13 +793,13 @@ void EclAgent::setResultInt(const char * name, unsigned sequence, __int64 val)
     }
 }
 
-void EclAgent::setResultUInt(const char * name, unsigned sequence, unsigned __int64 val)
+void EclAgent::setResultUInt(const char * name, unsigned sequence, unsigned __int64 val, unsigned size)
 {
     LOG(MCsetresult, unknownJob, "setResultUInt(%s,%d,%"I64F"u)", nullText(name), sequence, val);
     Owned<IWUResult> r = updateResult(name, sequence);
     if (r)
     {
-        r->setResultUInt(val);  
+        r->setResultUInt(val);
         r->setResultStatus(ResultStatusCalculated);
     }
     else
@@ -2645,9 +2645,9 @@ void EclAgent::updatePersist(IRemoteConnection *persistLock, const char * logica
     eclName.append(lfn).append("$eclcrc");
     whenName.append(lfn).append("$when");
 
-    setResultInt(crcName,ResultSequencePersist,allCRC);
-    setResultInt(eclName,ResultSequencePersist,eclCRC);
-    setResultInt(whenName,ResultSequencePersist,time(NULL));
+    setResultInt(crcName,ResultSequencePersist,allCRC,sizeof(int));
+    setResultInt(eclName,ResultSequencePersist,eclCRC,sizeof(int));
+    setResultInt(whenName,ResultSequencePersist,time(NULL),sizeof(int));
 
     reportProgress("Convert persist write lock to read lock");
     changePersistLockMode(persistLock, RTM_LOCK_READ, logicalName, true);
@@ -3705,14 +3705,14 @@ class DebugProbe : public InputProbe, implements IActivityDebugContext
     {
         output->outputBeginNested("att", false);
         output->outputCString(name, "@name");
-        output->outputInt(value, "@value");
+        output->outputInt(value, 0, "@value");
         output->outputEndNested("att");
     }
 
     void rowToXML(IXmlWriter *output, const void *row, unsigned sequence, unsigned rowCount, bool skipped, bool limited, bool eof, bool eog) const
     {
         output->outputBeginNested("Row", true);
-        output->outputInt(sequence, "@seq");
+        output->outputInt(sequence, 0, "@seq");
         if (skipped)
             output->outputBool(true, "@skip");
         if (limited)
@@ -3723,9 +3723,9 @@ class DebugProbe : public InputProbe, implements IActivityDebugContext
             output->outputBool(true, "@eog");
         if (row)
         {
-            output->outputInt(rowCount, "@count");
+            output->outputInt(rowCount, 0, "@count");
             IOutputMetaData *meta = queryOutputMeta();
-            output->outputInt(meta->getRecordSize(row), "@size");
+            output->outputInt(meta->getRecordSize(row), 0, "@size");
             meta->toXML((const byte *) row, *output);
         }
         output->outputEndNested("Row");
@@ -3863,8 +3863,8 @@ public:
                         else
                         {
                             output->outputBeginNested("Row", true);
-                            output->outputInt(rowData->querySequence(), "@sequence");
-                            output->outputInt(rowData->queryRowCount(), "@count");
+                            output->outputInt(rowData->querySequence(), 0, "@sequence");
+                            output->outputInt(rowData->queryRowCount(), 0, "@count");
                             output->outputEndNested("Row");
                         }
                     }

ecl/eclagent/eclagent.ipp

@@ -443,7 +443,7 @@ public:
     void setStandAloneOptions(bool _isStandAloneExe, bool _isRemoteWorkunit, bool _resolveFilesLocally, bool _writeResultsToStdout, outputFmts _outputFmt, IUserDescriptor *_standAloneUDesc);
     inline bool needToLockWorkunit() { return !isStandAloneExe; }           //If standalone exe then either no dali, or a unique remote workunit.
 
-    virtual void setResultInt(const char * stepname, unsigned sequence, __int64);
+    virtual void setResultInt(const char * stepname, unsigned sequence, __int64, unsigned size);
     virtual void setResultReal(const char * stepname, unsigned sequence, double);
     virtual void setResultBool(const char * stepname, unsigned sequence, bool);
     virtual void setResultString(const char * stepname, unsigned sequence, int len, const char *);
@@ -451,7 +451,7 @@ public:
     virtual void setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend);
     virtual void setResultRaw(const char * stepname, unsigned sequence, int len, const void *);
     virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer);
-    virtual void setResultUInt(const char * stepname, unsigned sequence, unsigned __int64);
+    virtual void setResultUInt(const char * stepname, unsigned sequence, unsigned __int64, unsigned size);
     virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str);
     virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val);
     virtual void setResultVarString(const char * stepname, unsigned sequence, const char *);

ecl/eclcc/eclcc.cpp

@@ -215,6 +215,7 @@ public:
     FILE * errout;
     Owned<IPropertyTree> srcArchive;
     Owned<IPropertyTree> generatedMeta;
+    Owned<IPropertyTree> globalDependTree;
     bool legacyImport;
     bool legacyWhen;
     bool fromArchive;
@@ -332,13 +333,15 @@ protected:
     StringBuffer cclogFilename;
     StringAttr optLogfile;
     StringAttr optIniFilename;
-    StringAttr optManifestFilename;
     StringAttr optOutputDirectory;
     StringAttr optOutputFilename;
     StringAttr optQueryRepositoryReference;
     StringAttr optComponentName;
     FILE * batchLog;
 
+    StringAttr optManifestFilename;
+    StringArray resourceManifestFiles;
+
     IFileArray inputFiles;
     StringArray inputFileNames;
     StringArray applicationOptions;
@@ -494,6 +497,8 @@ void EclCC::loadManifestOptions()
 {
     if (!optManifestFilename)
         return;
+    resourceManifestFiles.append(optManifestFilename);
+
     Owned<IPropertyTree> mf = createPTreeFromXMLFile(optManifestFilename);
     IPropertyTree *ecl = mf->queryPropTree("ecl");
     if (ecl)
@@ -714,6 +719,26 @@ void EclCC::reportCompileErrors(IErrorReceiver & errorProcessor, const char * pr
 
 //=========================================================================================
 
+void gatherResourceManifestFilenames(EclCompileInstance & instance, StringArray &filenames)
+{
+    IPropertyTree *tree = (instance.archive) ? instance.archive.get() : instance.globalDependTree.get();
+    if (!tree)
+        return;
+    Owned<IPropertyTreeIterator> iter = tree->getElements((instance.archive) ? "Module/Attribute" : "Attribute");
+    ForEach(*iter)
+    {
+        StringBuffer filename(iter->query().queryProp("@sourcePath"));
+        if (filename.length())
+        {
+            getFullFileName(filename, true).append(".manifest");
+            if (filenames.contains(filename))
+                continue;
+            if (checkFileExists(filename))
+                filenames.append(filename);
+        }
+    }
+}
+
 void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char * queryFullName, IErrorReceiver & errorProcessor, const char * outputFile)
 {
     StringBuffer processName(outputFile);
@@ -732,8 +757,9 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
                 if (!optShared)
                     wu->setDebugValueInt("standAloneExe", 1, true);
                 EclGenerateTarget target = optWorkUnit ? EclGenerateNone : (optNoCompile ? EclGenerateCpp : optShared ? EclGenerateDll : EclGenerateExe);
-                if (optManifestFilename)
-                    generator->addManifest(optManifestFilename);
+                gatherResourceManifestFilenames(instance, resourceManifestFiles);
+                ForEachItemIn(i, resourceManifestFiles)
+                    generator->addManifest(resourceManifestFiles.item(i));
                 if (instance.srcArchive)
                 {
                     generator->addManifestFromArchive(instance.srcArchive);
@@ -1067,6 +1093,8 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
         HqlParseContext parseCtx(instance.dataServer, instance.archive);
+        if (!instance.archive)
+            parseCtx.globalDependTree.setown(createPTree(ipt_none)); //to locate associated manifests, keep separate from user specified MetaOptions
         if (optGenerateMeta || optIncludeMeta)
         {
             HqlParseContext::MetaOptions options;
@@ -1172,6 +1200,9 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
             if (optIncludeMeta || optGenerateMeta)
                 instance.generatedMeta.setown(parseCtx.getMetaTree());
 
+            if (parseCtx.globalDependTree)
+                instance.globalDependTree.set(parseCtx.globalDependTree);
+
             if (optEvaluateResult && !errorProcessor.errCount() && instance.query)
                 evaluateResult(instance);
         }
@@ -1599,8 +1630,9 @@ void EclCC::generateOutput(EclCompileInstance & instance)
                 option->setProp("@value", valueStr.str());
                 instance.archive->addPropTree("Option", option.getClear());
             }
-            if (optManifestFilename)
-                addManifestResourcesToArchive(instance.archive, optManifestFilename);
+            gatherResourceManifestFilenames(instance, resourceManifestFiles);
+            ForEachItemIn(i, resourceManifestFiles)
+                addManifestResourcesToArchive(instance.archive, resourceManifestFiles.item(i));
 
             outputXmlToOutputFile(instance, instance.archive);
         }

ecl/hql/hqlexpr.cpp

@@ -711,10 +711,10 @@ void HqlParseContext::noteBeginAttribute(IHqlScope * scope, IFileContents * cont
         setDefinitionText(attr, "", contents);
     }
 
+    ISourcePath * sourcePath = contents->querySourcePath();
+
     if (checkBeginMeta())
     {
-        ISourcePath * sourcePath = contents->querySourcePath();
-
         IPropertyTree * attr = metaTree->addPropTree("Source", createPTree("Source"));
         setFullNameProp(attr, "@name", scope->queryFullName(), name->str());
         attr->setProp("@sourcePath", sourcePath->str());
@@ -726,6 +726,7 @@ void HqlParseContext::noteBeginAttribute(IHqlScope * scope, IFileContents * cont
         IPropertyTree * attr = globalDependTree->addPropTree("Attribute", createPTree("Attribute"));
         attr->setProp("@module", scope->queryFullName());
         attr->setProp("@name", name->str());
+        attr->setProp("@sourcePath", sourcePath->str());
         //attr->setPropInt("@flags", symbol->getObType());  MORE
     }
 }

ecl/hql/hqlmanifest.cpp

@@ -200,8 +200,10 @@ void ResourceManifest::addToArchive(IPropertyTree *archive)
 
     StringBuffer xml;
     toXML(manifest, xml);
-    additionalFiles->setProp("Manifest", xml.str());
-    additionalFiles->setProp("Manifest/@originalFilename", absFilename.str());
+
+    IPropertyTree *manifest = additionalFiles->addPropTree("Manifest", createPTree("Manifest", ipt_none));
+    manifest->setProp("@originalFilename", absFilename.str());
+    manifest->setProp(NULL, xml.str());
 }
 
 void addManifestResourcesToArchive(IPropertyTree *archive, const char *filename)

ecl/hqlcpp/hqlcpp.cpp

@@ -1629,7 +1629,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.resourceMaxSockets, "resourceMaxSockets", 0),
         DebugOption(options.resourceMaxActivities, "resourceMaxActivities", 0),
         DebugOption(options.resourceMaxHeavy, "resourceMaxHeavy", 1),
-        DebugOption(options.resourceMaxDistribute, "resourceMaxDistribute", 2),
+        DebugOption(options.resourceMaxDistribute, "resourceMaxDistribute", 8),
         DebugOption(options.unlimitedResources,"unlimitedResources", false),
         DebugOption(options.filteredReadSpillThreshold, "filteredReadSpillThreshold", 999),
         DebugOption(options.allowThroughSpill,"allowThroughSpill", true),
@@ -1719,7 +1719,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.checkAmbiguousRollupCondition,"checkAmbiguousRollupCondition",true),
         DebugOption(options.matchExistingDistributionForJoin,"matchExistingDistributionForJoin",true),
         DebugOption(options.createImplicitKeyedDistributeForJoin,"createImplicitKeyedDistributeForJoin",false),
-        DebugOption(options.expandHashJoin,"expandHashJoin",false),
+        DebugOption(options.expandHashJoin,"expandHashJoin",true),
         DebugOption(options.traceIR,"traceIR",false),
         DebugOption(options.preserveCaseExternalParameter,"preserveCaseExternalParameter",true),
         DebugOption(options.optimizeParentAccess,"optimizeParentAccess",false),

ecl/hqlcpp/hqlcppsys.ecl

@@ -460,11 +460,11 @@ const char * cppSystemText[]  = {
     "   outputXmlBool(boolean value, const varstring name) :    eclrtl,omethod,entrypoint='outputBool';",
     "   outputXmlData(const data value, const varstring nameo) :    eclrtl,omethod,entrypoint='outputData';",
     "   outputXmlDecimal(const data1 value, unsigned4 size, unsigned4 precision, const varstring name) :    eclrtl,omethod,entrypoint='outputDecimal';",
-    "   outputXmlInt(integer8 value, const varstring name) :    eclrtl,omethod,entrypoint='outputInt';",
+    "   outputXmlInt(integer8 value, unsigned4 size, const varstring name) :    eclrtl,omethod,entrypoint='outputInt';",
     "   outputXmlQString(const qstring value, const varstring name) :   eclrtl,omethod,entrypoint='outputQString';",
     "   outputXmlReal(real value, const varstring name) :   eclrtl,omethod,entrypoint='outputReal';",
     "   outputXmlString(const string value, const varstring name) : eclrtl,omethod,entrypoint='outputString';",
-    "   outputXmlUInt(unsigned8 value, const varstring name) :  eclrtl,omethod,entrypoint='outputUInt';",
+    "   outputXmlUInt(unsigned8 value, unsigned4 size, const varstring name) :  eclrtl,omethod,entrypoint='outputUInt';",
     "   outputXmlUnicode(const unicode value, const varstring name) :   eclrtl,omethod,entrypoint='outputUnicode';",
     "   outputXmlUtf8(const utf8 value, const varstring name) : eclrtl,omethod,entrypoint='outputUtf8';",
     "   outputXmlBeginArray(const varstring name) :  eclrtl,omethod,entrypoint='outputBeginArray';",
@@ -673,8 +673,8 @@ const char * cppSystemText[]  = {
     "   varstring getExpandLogicalName(const varstring lfn) : ctxmethod,entrypoint='getExpandLogicalName';",
     "   deleteFile(const varstring lfn) : gctxmethod,entrypoint='deleteFile';",
 
-    "   setResultInt(const varstring stepname, unsigned4 sequence, integer8 value) : ctxmethod,entrypoint='setResultInt';",
-    "   setResultUInt(const varstring stepname, unsigned4 sequence, unsigned8 value) : ctxmethod,entrypoint='setResultUInt';",
+    "   setResultInt(const varstring stepname, unsigned4 sequence, integer8 value, unsigned4 size) : ctxmethod,entrypoint='setResultInt';",
+    "   setResultUInt(const varstring stepname, unsigned4 sequence, unsigned8 value, unsigned4 size) : ctxmethod,entrypoint='setResultUInt';",
     "   setResultReal(const varstring stepname, unsigned4 sequence, real8 value) : ctxmethod,entrypoint='setResultReal';",
     "   setResultBool(const varstring stepname, unsigned4 sequence, boolean value) : ctxmethod,entrypoint='setResultBool';",
     "   setResultData(const varstring stepname, unsigned4 sequence, const data value) : ctxmethod,entrypoint='setResultData';",

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5107,6 +5107,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
     IIdAtom * func = NULL;
     CHqlBoundExpr valueToSave;
     LinkedHqlExpr castValue = value;
+    LinkedHqlExpr size;
     switch(retType)
     {
     case type_int:
@@ -5115,6 +5116,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
             bool isSigned = schemaType->isSigned();
             func = isSigned ? setResultIntId : setResultUIntId;
             schemaType.setown(makeIntType(8, isSigned));
+            size.setown(getSizetConstant(schemaType->getSize()));
             break;
         }
     case type_boolean:  func = setResultBoolId; break;
@@ -5239,6 +5241,8 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
             s.clear().append("&").append(helper);
             args.append(*createQuoted(s, makeBoolType()));
         }
+        else if (func == setResultIntId || func == setResultUIntId)
+            args.append(*getSizetConstant(schemaType->getSize()));
 
         buildFunctionCall(ctx, func, args);
     }
@@ -10879,6 +10883,7 @@ void HqlCppTranslator::buildXmlSerializeScalar(BuildCtx & ctx, IHqlExpression *
 {
     ITypeInfo * type = selected->queryType()->queryPromotedType();
     LinkedHqlExpr value = selected;
+    LinkedHqlExpr size;
     IIdAtom * func;
     switch (type->getTypeCode())
     {
@@ -10909,6 +10914,7 @@ void HqlCppTranslator::buildXmlSerializeScalar(BuildCtx & ctx, IHqlExpression *
     case type_swapint:
     case type_packedint:
     case type_bitfield:
+        size.setown(getSizetConstant(type->getSize()));
         if (type->isSigned())
             func = outputXmlIntId;
         else
@@ -10925,6 +10931,8 @@ void HqlCppTranslator::buildXmlSerializeScalar(BuildCtx & ctx, IHqlExpression *
     HqlExprArray args;
     args.append(*createVariable("out", makeBoolType()));
     args.append(*value.getLink());
+    if (size)
+        args.append(*LINK(size));
     if (name)
         args.append(*LINK(name));
     else

ecl/hqlcpp/hqlres.cpp

@@ -122,6 +122,50 @@ bool ResourceManager::getDuplicateResourceId(const char *srctype, const char *re
     return false;
 }
 
+void updateManifestResourcePaths(IPropertyTree &resource, const char *dir)
+{
+    StringBuffer filepath;
+    makeAbsolutePath(resource.queryProp("@filename"), dir, filepath);
+    resource.setProp("@originalFilename", filepath.str());
+
+    StringBuffer respath;
+    makePathUniversal(filepath.str(), respath);
+    resource.setProp("@resourcePath", respath.str());
+}
+
+void expandManifestDirectory(IPropertyTree *manifestSrc, IPropertyTree &res, StringBuffer &dir, IDirectoryIterator *it, const char*mask, bool recursive)
+{
+    if (!it)
+        return;
+    ForEach(*it)
+    {
+        if (it->isDir())
+        {
+            if (recursive)
+                expandManifestDirectory(manifestSrc, res, dir, it->query().directoryFiles(mask, false, true), mask, recursive);
+            continue;
+        }
+        StringBuffer reldir;
+        Owned<IPropertyTree> newRes = createPTreeFromIPT(&res);
+        reldir.append(splitRelativePath(it->query().queryFilename(), dir, reldir));
+        VStringBuffer xpath("Resource[@filename='%s']", reldir.str());
+        if (manifestSrc->hasProp(xpath))
+            continue;
+        newRes->setProp("@filename", reldir.str());
+        updateManifestResourcePaths(*newRes, dir.str());
+        if (manifestSrc->hasProp(xpath.setf("resource[@resourcePath='%s']", newRes->queryProp("@resourcePath"))))
+            continue;
+        manifestSrc->addPropTree("Resource", newRes.getClear());
+    }
+}
+
+void expandManifestDirectory(IPropertyTree *manifestSrc, IPropertyTree &res, StringBuffer &dir, const char *path, const char*mask, bool recursive)
+{
+    Owned<IDirectoryIterator> it = createDirectoryIterator(path, mask);
+    expandManifestDirectory(manifestSrc, res, dir, it, mask, recursive);
+}
+
+
 void ResourceManager::addManifestFile(const char *filename)
 {
     Owned<IPropertyTree> manifestSrc = createPTreeFromXMLFile(filename);
@@ -149,24 +193,39 @@ void ResourceManager::addManifestFile(const char *filename)
             item.setProp("@originalFilename", filepath.str());
             item.setProp("@resourcePath", respath.str());
 
-            if (!item.hasProp("@type"))
-                item.setProp("@type", "UNKNOWN");
-            int id;
-            if (getDuplicateResourceId(item.queryProp("@type"), respath.str(), NULL, id))
-            {
-                item.setPropInt("@id", id);
-                manifest->addPropTree("Resource", LINK(&item));
-            }
-            else
+            if (containsFileWildcard(filepath))
             {
-                MemoryBuffer content;
-                loadResource(filepath.str(), content);
-                addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
+                StringBuffer wildpath;
+                const char *tail = splitDirTail(filepath, wildpath);
+                expandManifestDirectory(manifestSrc, item, dir, wildpath, tail, item.getPropBool("@recursive"));
+                manifestSrc->removeTree(&item);
             }
+
         }
         else
             manifest->addPropTree(item.queryName(), LINK(&item));
     }
+
+    Owned<IPropertyTreeIterator> resources = manifestSrc->getElements("Resource[@filename]");
+    ForEach(*resources)
+    {
+        IPropertyTree &item = resources->query();
+
+        if (!item.hasProp("@type"))
+            item.setProp("@type", "UNKNOWN");
+        int id;
+        if (getDuplicateResourceId(item.queryProp("@type"), item.queryProp("@resourcePath"), NULL, id))
+        {
+            item.setPropInt("@id", id);
+            manifest->addPropTree("Resource", LINK(&item));
+        }
+        else
+        {
+            MemoryBuffer content;
+            loadResource(item.queryProp("@originalFilename"), content);
+            addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
+        }
+    }
 }
 
 void ResourceManager::addManifest(const char *filename)

ecl/hqlcpp/hqlresource.cpp

@@ -40,9 +40,8 @@
 //#define VERIFY_RESOURCING
 //#define SPOT_UNCONDITIONAL_CONDITIONS
 
-#define DEFAULT_LARGEMEM_BUFFER_SIZE (0x58000000) // ~ 1.4GB
 #define DEFAULT_MAX_SOCKETS 2000 // configurable by setting max_sockets in .ini
-#define DEFAULT_TOTAL_MEMORY ((1024*1024*1800)-DEFAULT_LARGEMEM_BUFFER_SIZE)
+#define DEFAULT_TOTAL_MEMORY ((1024*1024*1800))
 #define FIXED_CLUSTER_SIZE 400
 #define MEM_Const_Minimal (1*1024*1024)
 #define DEFAULT_MAX_ACTIVITIES  100
@@ -64,7 +63,7 @@ MODULE_EXIT()
 
 static void setHashResources(IHqlExpression * expr, CResources & resources, const CResourceOptions & options)
 {
-    unsigned memneeded = MEM_Const_Minimal+resources.clusterSize*4*DISTRIBUTE_SINGLE_BUFFER_SIZE+DISTRIBUTE_PULL_BUFFER_SIZE;
+    unsigned memneeded = MEM_Const_Minimal+DISTRIBUTE_RESMEM(resources.clusterSize);
     resources.set(RESslavememory, memneeded).set(REShashdist, 1);
 }
 

esp/CMakeLists.txt

@@ -26,6 +26,6 @@ HPCC_ADD_SUBDIRECTORY (services "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (smc "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (test "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (tools "CLIENTTOOLS")
-HPCC_ADD_SUBDIRECTORY (xslt "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (xslt)
 HPCC_ADD_SUBDIRECTORY (esdllib)
 

esp/bindings/SOAP/Platform/soapservice.cpp

@@ -47,6 +47,7 @@ int CSoapService::processHeader(CHeader* header, IEspContext* ctx)
     if(ctx == NULL)
         return 0;
 
+    int returnValue = 0;
     bool authenticated = !ctx->toBeAuthenticated();
     for (int i = 0; i < num; i++)
     {
@@ -101,30 +102,31 @@ int CSoapService::processHeader(CHeader* header, IEspContext* ctx)
                         authenticated = false;
                     }
                     if(!authenticated)
-                    {           
-                        StringBuffer peerStr;
-                        ctx->getPeer(peerStr);
-                        DBGLOG("User authentication failed for soap request from %s", peerStr.str());
-                        return SOAP_AUTHENTICATION_ERROR;
-                    }
-                    return 0;
+                        returnValue = SOAP_AUTHENTICATION_ERROR;
+                    break;
                 }
             }
         }
     }
 
+    if (returnValue == 0)
+    {
+        if (authenticated)
+            return 0;
+        returnValue = SOAP_AUTHENTICATION_REQUIRED;
+    }
+
     StringBuffer peerStr;
     ctx->getPeer(peerStr);
     const char* userId = ctx->queryUserId();
-    DBGLOG("SOAP request from %s@%s", (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown");
-
-    if(!authenticated)
-    {
-        DBGLOG("User authentication required");
-        return SOAP_AUTHENTICATION_REQUIRED;
-    }
+    VStringBuffer msg("SOAP request from %s@%s.", (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown");
+    if (returnValue == SOAP_AUTHENTICATION_ERROR)
+        msg.append(" User authentication failed");
+    else
+        msg.append(" User authentication required");
+    DBGLOG("%s", msg.str());
 
-    return 0;
+    return returnValue;
 }
 
 int CSoapService::processRequest(ISoapMessage &req, ISoapMessage& resp)
@@ -195,6 +197,10 @@ int CSoapService::processRequest(ISoapMessage &req, ISoapMessage& resp)
         }
     }
 
+    StringBuffer peerStr;
+    ctx->getPeer(peerStr);
+    const char* userId = ctx->queryUserId();
+
     CBody* req_body = req_envelope->get_body();
     Owned<CRpcResponse> rpc_response;
     rpc_response.setown(new CRpcResponse);
@@ -209,16 +215,19 @@ int CSoapService::processRequest(ISoapMessage &req, ISoapMessage& resp)
     } catch (XmlPullParserException& e) {
         response.set_status(SOAP_CLIENT_ERROR);
         response.set_err(e.getMessage().c_str());
-        DBGLOG("parsing xml error: %s. Offending XML: [%s]", e.getMessage().c_str(), requeststr.str());
+        DBGLOG("SOAP request from %s@%s. Parsing xml error: %s. Offending XML: [%s]", (userId&&*userId)?userId:"unknown",
+            (peerStr.length()>0)?peerStr.str():"unknown", e.getMessage().c_str(), requeststr.str());
         return 0;
     } catch (...) {
         response.set_status(SOAP_CLIENT_ERROR);
         response.set_err("Unknown error when parsing soap body XML");
-        ERRLOG("unknown error when parsing: %s", requeststr.str());
+        ERRLOG("SOAP request from %s@%s. Unknown error when parsing: %s",  (userId&&*userId)?userId:"unknown",
+            (peerStr.length()>0)?peerStr.str():"unknown", requeststr.str());
         return 0;
     }
 
-    DBGLOG("SOAP method <%s>", rpc_call->get_name());
+    DBGLOG("SOAP method <%s> from %s@%s.", rpc_call->get_name(),  (userId&&*userId)?userId:"unknown",
+        (peerStr.length()>0)?peerStr.str():"unknown");
 
     // call the rpc and set the response
     if(m_soapbinding != NULL)

esp/esdllib/esdl_transformer2.cpp

@@ -397,13 +397,13 @@ void Esdl2Base::output_content(Esdl2TransformerContext &ctx, const char * conten
                     case ESDLT_INT16:
                     case ESDLT_INT32:
                     case ESDLT_INT64:
-                        ctx.writer->outputInt(atoi(content), tagname);
+                        ctx.writer->outputInt(atoi(content), sizeof(int), tagname);
                         break;
                     case ESDLT_UINT8:
                     case ESDLT_UINT16:
                     case ESDLT_UINT32:
                     case ESDLT_UINT64:
-                        ctx.writer->outputUInt(atoi(content), tagname);
+                        ctx.writer->outputUInt(atoi(content), sizeof(unsigned), tagname);
                         break;
                     case ESDLT_BYTE:
                     case ESDLT_UBYTE:

esp/src/Visualization

@@ -1 +1 @@
-Subproject commit 1074723616023d5e04f03576b390f01b51e57231
+Subproject commit 28418fa21158e692a4cd49dde72996d8d04641bf

esp/src/eclwatch/ActivityWidget.js

@@ -193,6 +193,11 @@ define([
             });
 
             this._refreshActionState();
+            ESPUtil.MonitorVisibility(this.gridTab, function (visibility) {
+                if (visibility) {
+                    context.refreshGrid();
+                }
+            });
         },
 
         createGrid: function (domID) {

esp/src/eclwatch/ESPUtil.js

@@ -19,9 +19,11 @@ define([
     "dojo/i18n",
     "dojo/i18n!./nls/hpcc",
     "dojo/_base/array",
+    "dojo/dom-class",
     "dojo/Stateful",
     "dojo/query",
     "dojo/json",
+    "dojo/aspect",
 
     "dijit/registry",
     "dijit/Tooltip",
@@ -35,7 +37,7 @@ define([
     "dgrid/extensions/ColumnReorder",
     "dgrid/extensions/DijitRegistry",
     "dgrid/extensions/Pagination"
-], function (declare, lang, i18n, nlsHPCC, arrayUtil, Stateful, query, json,
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, domClass, Stateful, query, json, aspect,
     registry, Tooltip,
     Grid, OnDemandGrid, Keyboard, Selection, ColumnResizer, ColumnHider, ColumnReorder, DijitRegistry, Pagination) {
 
@@ -281,6 +283,53 @@ define([
             }
             baseClass.push(GridHelper);
             return declare(baseClass, params);
+        },
+
+        MonitorVisibility: function (widget, callback) {
+            //  There are many places that may cause the widget to be hidden, the possible places are calculated by walking the DOM hierarchy upwards. 
+            var watchList = {};
+            var domNode = widget.domNode;
+            while (domNode) {
+                if (domNode.id) {
+                    watchList[domNode.id] = false;
+                }
+                domNode = domNode.parentElement;
+            }
+
+            function isHidden() {
+                for (key in watchList) {
+                    if (watchList[key] === true) {
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            //  Hijack the dojo style class replacement call and monitor for elements in our watchList. 
+            aspect.around(domClass, "replace", function (origFunc) {
+                return function (node, addStyle, removeStyle) {
+                    if (node.firstChild && (node.firstChild.id in watchList)) {
+                        if (addStyle === "dijitHidden" || addStyle === "hpccHidden") {
+                            if (!isHidden()) {
+                                if (callback(false, node)) {
+                                    addStyle = "hpccHidden";
+                                    removeStyle = "hpccVisible";
+                                }
+                            }
+                            watchList[node.firstChild.id] = true;
+                        } else if ((addStyle === "dijitVisible" || addStyle === "hpccVisible") && watchList[node.firstChild.id] === true) {
+                            watchList[node.firstChild.id] = false;
+                            if (!isHidden()) {
+                                if (callback(true, node)) {
+                                    addStyle = "hpccVisible";
+                                    removeStyle = "hpccHidden";
+                                }
+                            }
+                        }
+                    }
+                    return origFunc(node, addStyle, removeStyle);
+                }
+            });
         }
     };
 });

esp/src/eclwatch/FileSpray.js

@@ -372,6 +372,9 @@ define([
                 }
                 return response; 
             });
+        },
+        GetSprayTargets: function(params) {
+            return ESPRequest.send("FileSpray", "GetSprayTargets", params);
         }
     };
 });

esp/src/eclwatch/GetDFUWorkunitsWidget.js

@@ -206,6 +206,11 @@ define([
             topic.subscribe("hpcc/dfu_wu_created", function (topic) {
                 context.refreshGrid();
             });
+            ESPUtil.MonitorVisibility(this.workunitsTab, function (visibility) {
+                if (visibility) {
+                    context.refreshGrid();
+                }
+            });
         },
 
         initTab: function () {

esp/src/eclwatch/GraphWidget.js

@@ -20,7 +20,6 @@ define([
     "dojo/i18n!./nls/hpcc",
     "dojo/_base/array",
     "dojo/_base/Deferred",
-    "dojo/aspect",
     "dojo/has",
     "dojo/dom",
     "dojo/dom-construct",
@@ -36,6 +35,7 @@ define([
     "dijit/layout/ContentPane",
 
     "hpcc/_Widget",
+    "hpcc/ESPUtil",
 
     "dojo/text!../templates/GraphWidget.html",
 
@@ -44,9 +44,9 @@ define([
     "dijit/form/Button",
     "dijit/form/NumberSpinner"
     
-], function (declare, lang, i18n, nlsHPCC, arrayUtil, Deferred, aspect, has, dom, domConstruct, domClass, domStyle, Memory, Observable, QueryResults, Evented,
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, Deferred, has, dom, domConstruct, domClass, domStyle, Memory, Observable, QueryResults, Evented,
             registry, BorderContainer, ContentPane,
-            _Widget,
+            _Widget, ESPUtil,
             template) {
 
     var GraphStore = declare("GraphStore", [Memory], {
@@ -1033,40 +1033,19 @@ define([
             },
 
             watchStyleChange: function () {
-                //  Prevent control from being "hidden" as it gets destroyed on Chrome/FF/(Maybe IE11?)
-                var watchList = [];
-                var context = this;
-                var domNode = this.domNode;
-
-                //  There are many places that may cause the plugin to be hidden, the possible places are calculated by walking the hierarchy upwards. 
-                while (domNode) {
-                    if (domNode.id) {
-                        watchList[domNode.id] = false;
-                    }
-                    domNode = domNode.parentElement;
-                }
-
-                //  Hijack the dojo style class replacement call and monitor for elements in our watchList. 
-                aspect.around(domClass, "replace", function (origFunc) {
-                    return function (node, addStyle, removeStyle) {
-                        if (node.firstChild && (node.firstChild.id in watchList)) {
-                            if (addStyle == "dijitHidden") {
-                                watchList[node.firstChild.id] = true;
-                                dojo.style(node, "width", "1px");
-                                dojo.style(node, "height", "1px");
-                                dojo.style(node.firstChild, "width", "1px");
-                                dojo.style(node.firstChild, "height", "1px");
-                                return;
-                            } else if (addStyle == "dijitVisible" && watchList[node.firstChild.id] == true) {
-                                watchList[node.firstChild.id] = false;
-                                dojo.style(node, "width", "100%");
-                                dojo.style(node, "height", "100%");
-                                dojo.style(node.firstChild, "width", "100%");
-                                dojo.style(node.firstChild, "height", "100%");
-                                return;
-                            }
-                        }
-                        return origFunc(node, addStyle, removeStyle);
+                ESPUtil.MonitorVisibility(this, function (visibility, node) {
+                    if (visibility) {
+                        dojo.style(node, "width", "100%");
+                        dojo.style(node, "height", "100%");
+                        dojo.style(node.firstChild, "width", "100%");
+                        dojo.style(node.firstChild, "height", "100%");
+                        return true;
+                    } else {
+                        dojo.style(node, "width", "1px");
+                        dojo.style(node, "height", "1px");
+                        dojo.style(node.firstChild, "width", "1px");
+                        dojo.style(node.firstChild, "height", "1px");
+                        return true;
                     }
                 });
             },

esp/src/eclwatch/JSGraphWidget.js

@@ -201,6 +201,8 @@ define([
                         if (!subgraph.__widget) {
                             subgraph.__widget = new Shape()
                                 .shape("rect")
+                                .width(0)
+                                .height(0)
                             ;
                             subgraph.__widget.__hpcc_globalID = subgraph.__hpcc_id;
                         }

esp/src/eclwatch/LZBrowseWidget.js

@@ -420,19 +420,19 @@ define([
 
             this.initLandingZonesGrid();
             this.sprayFixedDestinationSelect.init({
-                Groups: true
+                SprayTargets: true
             });
             this.sprayDelimitedDestinationSelect.init({
-                Groups: true
+                SprayTargets: true
             });
             this.sprayXmlDestinationSelect.init({
-                Groups: true
+                SprayTargets: true
             });
             this.sprayVariableDestinationSelect.init({
-                Groups: true
+                SprayTargets: true
             });
             this.sprayBlobDestinationSelect.init({
-                Groups: true
+                SprayTargets: true
             });
             var context = this;
             this.dropZoneTargetSelect.init({

esp/src/eclwatch/TargetSelectWidget.js

@@ -60,6 +60,8 @@ define([
             }
             if (params.Groups === true) {
                 this.loadGroups();
+            } else if (params.SprayTargets === true) {
+                this.loadSprayTargets();
             } else if (params.DropZones === true) {
                 this.loadDropZones();
             } else if (params.DropZoneFolders === true) {
@@ -177,6 +179,24 @@ define([
             });
         },
 
+        loadSprayTargets: function () {
+            var context = this;
+            FileSpray.GetSprayTargets({
+                load: function (response) {
+                    if (lang.exists("GetSprayTargetsResponse.GroupNodes.GroupNode", response)) {
+                        var targetData = response.GetSprayTargetsResponse.GroupNodes.GroupNode;
+                        for (var i = 0; i < targetData.length; ++i) {
+                            context.options.push({
+                                label: targetData[i].Name,
+                                value: targetData[i].Name
+                            });
+                        }
+                        context._postLoad();
+                    }
+                }
+            });
+        },
+
         loadWUState: function() {
             for (var key in WsWorkunits.States) {
                 this.options.push({

esp/src/eclwatch/WUQueryWidget.js

@@ -20,7 +20,6 @@ define([
     "dojo/i18n!./nls/hpcc",
     "dojo/_base/array",
     "dojo/dom",
-    "dojo/dom-class",
     "dojo/dom-form",
     "dojo/date",
     "dojo/on",
@@ -57,7 +56,7 @@ define([
     "dijit/ToolbarSeparator",
     "dijit/TooltipDialog"
 
-], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domClass, domForm, date, on, topic,
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, date, on, topic,
                 registry, Menu, MenuItem, MenuSeparator, PopupMenuItem,
                 selector,
                 _TabContainerWidget, WsWorkunits, ESPUtil, ESPWorkunit, DelayLoadWidget, TargetSelectWidget, FilterDropDownWidget,
@@ -257,6 +256,12 @@ define([
             topic.subscribe("hpcc/ecl_wu_created", function (topic) {
                 context.refreshGrid();
             });
+
+            ESPUtil.MonitorVisibility(this.workunitsTab, function (visibility) {
+                if (visibility) {
+                    context.refreshGrid();
+                }
+            });
         },
 
         initTab: function () {

esp/src/eclwatch/css/hpcc.css

@@ -1025,29 +1025,29 @@ margin-left:-20px;
     height: auto;
 }
 
-.claro .graph .zoom {
+.claro .graph_Graph .zoom {
     fill: white;
 }
 
-.claro .graph .edge .shape {
+.claro .graph_Graph .edge .common_Shape {
     fill: white;
 }
 
-.claro .graph .graphVertex > .shape {
+.claro .graph_Graph .graphVertex > .common_Shape {
     stroke: black;
     fill: none;
 }
 
-.claro .graph .graphVertex > .shape.selected {
+.claro .graph_Graph .graphVertex > .common_Shape.selected {
     stroke: #1f77b4;
 }
 
-.claro .graph .graphVertex .textbox .shape {
+.claro .graph_Graph .graphVertex .common_TextBox .common_Shape {
     stroke: black;
     fill: white;
 }
 
-.claro .graph .graphVertex .textbox.selected .shape {
+.claro .graph_Graph .graphVertex .common_TextBox.selected .common_Shape {
     fill: #dcf1ff;
     stroke: #1f77b4;
 }

roxie/ccd/ccdactivities.cpp

@@ -468,12 +468,12 @@ public:
     virtual void setResultBool(const char *name, unsigned sequence, bool value) { throwUnexpected(); }
     virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
     virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) { throwUnexpected(); } 
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 value) { throwUnexpected(); }
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) { throwUnexpected(); }
     virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
     virtual void setResultReal(const char * stepname, unsigned sequence, double value) { throwUnexpected(); }
     virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) { throwUnexpected(); }
     virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) { throwUnexpected(); }
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value) { throwUnexpected(); }
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) { throwUnexpected(); }
     virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) { throwUnexpected(); }
     virtual void setResultVarString(const char * name, unsigned sequence, const char * value) { throwUnexpected(); }
     virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) { throwUnexpected(); }

roxie/ccd/ccdcontext.cpp

@@ -179,7 +179,7 @@ public:
             {
                 output->outputBeginNested("edge", true);
                 output->outputCString((const char *) edges.query().getKey(), "@edgeId");
-                output->outputUInt(edge->queryCount(), "@count");
+                output->outputUInt(edge->queryCount(), sizeof(unsigned), "@count");
                 output->outputEndNested("edge");
             }
             if (reset)
@@ -332,7 +332,7 @@ private:
         return r->getResultInt();
     }
 
-    void setResultInt(const char * name, unsigned sequence, unsigned __int64 value)
+    void setResultInt(const char * name, unsigned sequence, unsigned __int64 value, unsigned size)
     {
         WorkunitUpdate w(&workunit->lock());
         w->setResultInt(name, sequence, value);
@@ -512,9 +512,9 @@ private:
         eclName.append(lfn).append("$eclcrc");
         whenName.append(lfn).append("$when");
 
-        setResultInt(crcName, ResultSequencePersist, allCRC);
-        setResultInt(eclName, ResultSequencePersist, eclCRC);
-        setResultInt(whenName, ResultSequencePersist, time(NULL));
+        setResultInt(crcName, ResultSequencePersist, allCRC, sizeof(int));
+        setResultInt(eclName, ResultSequencePersist, eclCRC, sizeof(int));
+        setResultInt(whenName, ResultSequencePersist, time(NULL), sizeof(int));
 
         logctx.CTXLOG("Convert persist write lock to read lock");
         changePersistLockMode(persistLock, RTM_LOCK_READ, logicalName, true);
@@ -1517,12 +1517,12 @@ public:
     virtual void setResultBool(const char *name, unsigned sequence, bool value) { throwUnexpected(); }
     virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
     virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) { throwUnexpected(); }
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 value) { throwUnexpected(); }
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) { throwUnexpected(); }
     virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
     virtual void setResultReal(const char * stepname, unsigned sequence, double value) { throwUnexpected(); }
     virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) { throwUnexpected(); }
     virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) { throwUnexpected(); }
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value) { throwUnexpected(); }
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) { throwUnexpected(); }
     virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) { throwUnexpected(); }
     virtual void setResultVarString(const char * name, unsigned sequence, const char * value) { throwUnexpected(); }
     virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) { throwUnexpected(); }
@@ -3227,7 +3227,7 @@ public:
             }
         }
     }
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 value)
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size)
     {
         if (isSpecialResultSequence(sequence))
         {
@@ -3239,11 +3239,13 @@ public:
             FlushingStringBuffer *r = queryResult(sequence);
             if (r)
             {
-                r->startScalar(name, sequence);
                 if (isRaw)
+                {
+                    r->startScalar(name, sequence);
                     r->append(sizeof(value), (char *)&value);
+                }
                 else
-                    r->appendf("%"I64F"d", value);
+                    r->setScalarInt(name, sequence, value, size);
             }
         }
 
@@ -3269,7 +3271,7 @@ public:
         }
     }
 
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value)
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size)
     {
         if (isSpecialResultSequence(sequence))
         {
@@ -3281,11 +3283,13 @@ public:
             FlushingStringBuffer *r = queryResult(sequence);
             if (r)
             {
-                r->startScalar(name, sequence);
                 if (isRaw)
+                {
+                    r->startScalar(name, sequence);
                     r->append(sizeof(value), (char *)&value);
+                }
                 else
-                    r->appendf("%"I64F"u", value);
+                    r->setScalarUInt(name, sequence, value, size);
             }
         }
 

roxie/ccd/ccddebug.cpp

@@ -509,14 +509,14 @@ class DebugProbe : public InputProbe, implements IActivityDebugContext
     {
         output->outputBeginNested("att", false);
         output->outputCString(name, "@name");
-        output->outputInt(value, "@value");
+        output->outputInt(value, sizeof(int), "@value");
         output->outputEndNested("att");
     }
 
     void rowToXML(IXmlWriter *output, const void *row, unsigned sequence, unsigned rowCount, bool skipped, bool limited, bool eof, bool eog) const
     {
         output->outputBeginNested("Row", true);
-        output->outputInt(sequence, "@seq");
+        output->outputInt(sequence, sizeof(int), "@seq");
         if (skipped)
             output->outputBool(true, "@skip");
         if (limited)
@@ -527,9 +527,9 @@ class DebugProbe : public InputProbe, implements IActivityDebugContext
             output->outputBool(true, "@eog");
         if (row)
         {
-            output->outputInt(rowCount, "@count");
+            output->outputInt(rowCount, sizeof(int), "@count");
             IOutputMetaData *meta = queryOutputMeta();
-            output->outputInt(meta->getRecordSize(row), "@size");
+            output->outputInt(meta->getRecordSize(row), sizeof(int), "@size");
             meta->toXML((const byte *) row, *output);
         }
         output->outputEndNested("Row");
@@ -675,8 +675,8 @@ public:
                         else
                         {
                             output->outputBeginNested("Row", true);
-                            output->outputInt(rowData->querySequence(), "@sequence");
-                            output->outputInt(rowData->queryRowCount(), "@count");
+                            output->outputInt(rowData->querySequence(), sizeof(int), "@sequence");
+                            output->outputInt(rowData->queryRowCount(), sizeof(int), "@count");
                             output->outputEndNested("Row");
                         }
                     }
@@ -1270,7 +1270,7 @@ public:
             output->outputBeginNested("Result", true);
             IActivityDebugContext *edge = manager->lookupActivityByEdgeId(id);
             if (edge)
-                output->outputInt(edge->queryProxyId(), "@proxyId");
+                output->outputInt(edge->queryProxyId(), sizeof(int), "@proxyId");
             output->outputEndNested("Result");
         }
         else

rtl/eclrtl/rtlfield.cpp

@@ -201,9 +201,9 @@ size32_t RtlIntTypeInfo::process(const byte * self, const byte * selfrow, const
 size32_t RtlIntTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
 {
     if (isUnsigned())
-        target.outputUInt(rtlReadUInt(self, length), queryXPath(field));
+        target.outputUInt(rtlReadUInt(self, length), length, queryXPath(field));
     else
-        target.outputInt(rtlReadInt(self, length), queryXPath(field));
+        target.outputInt(rtlReadInt(self, length), length, queryXPath(field));
     return length;
 }
 
@@ -231,9 +231,9 @@ size32_t RtlSwapIntTypeInfo::process(const byte * self, const byte * selfrow, co
 size32_t RtlSwapIntTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
 {
     if (isUnsigned())
-        target.outputUInt(rtlReadSwapUInt(self, length), queryXPath(field));
+        target.outputUInt(rtlReadSwapUInt(self, length), length, queryXPath(field));
     else
-        target.outputInt(rtlReadSwapInt(self, length), queryXPath(field));
+        target.outputInt(rtlReadSwapInt(self, length), length, queryXPath(field));
     return length;
 }
 
@@ -265,11 +265,12 @@ size32_t RtlPackedIntTypeInfo::process(const byte * self, const byte * selfrow,
 
 size32_t RtlPackedIntTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
 {
+    size32_t fieldsize = rtlGetPackedSize(self);
     if (isUnsigned())
-        target.outputUInt(rtlGetPackedUnsigned(self), queryXPath(field));
+        target.outputUInt(rtlGetPackedUnsigned(self), fieldsize, queryXPath(field));
     else
-        target.outputInt(rtlGetPackedSigned(self), queryXPath(field));
-    return rtlGetPackedSize(self); 
+        target.outputInt(rtlGetPackedSigned(self), fieldsize, queryXPath(field));
+    return fieldsize;
 }
 
 //-------------------------------------------------------------------------------------------------------------------
@@ -1417,11 +1418,12 @@ size32_t RtlBitfieldTypeInfo::process(const byte * self, const byte * selfrow, c
 
 size32_t RtlBitfieldTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
 {
+    size32_t fieldsize = size(self, selfrow);
     if (isUnsigned())
-        target.outputUInt(unsignedValue(self), queryXPath(field));
+        target.outputUInt(unsignedValue(self), fieldsize, queryXPath(field));
     else
-        target.outputInt(signedValue(self), queryXPath(field));
-    return size(self, selfrow);
+        target.outputInt(signedValue(self), fieldsize, queryXPath(field));
+    return fieldsize;
 }
 
 //-------------------------------------------------------------------------------------------------------------------

rtl/include/eclhelper.hpp

@@ -39,8 +39,8 @@ if the supplied pointer was not from the roxiemem heap. Usually an OwnedRoxieStr
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      156
-#define MIN_ACTIVITY_INTERFACE_VERSION  156             //minimum value that is compatible with current interface - without using selectInterface
+#define ACTIVITY_INTERFACE_VERSION      157
+#define MIN_ACTIVITY_INTERFACE_VERSION  157             //minimum value that is compatible with current interface - without using selectInterface
 
 typedef unsigned char byte;
 
@@ -161,8 +161,8 @@ public:
     virtual void outputString(unsigned len, const char *field, const char *fieldname) = 0;
     virtual void outputBool(bool field, const char *fieldname) = 0;
     virtual void outputData(unsigned len, const void *field, const char *fieldname) = 0;
-    virtual void outputInt(__int64 field, const char *fieldname) = 0;
-    virtual void outputUInt(unsigned __int64 field, const char *fieldname) = 0;
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname) = 0;
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname) = 0;
     virtual void outputReal(double field, const char *fieldname) = 0;
     virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname) = 0;
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname) = 0;
@@ -569,12 +569,12 @@ interface ICodeContext : public IResourceContext
     virtual void setResultBool(const char *name, unsigned sequence, bool value) = 0;
     virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) = 0;
     virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) = 0; 
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 value) = 0;
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) = 0;
     virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) = 0;
     virtual void setResultReal(const char * stepname, unsigned sequence, double value) = 0;
     virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) = 0;
     virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) = 0;
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value) = 0;
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) = 0;
     virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) = 0;
     virtual void setResultVarString(const char * name, unsigned sequence, const char * value) = 0;
     virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) = 0;

system/include/platform.h

@@ -291,7 +291,6 @@ typedef int socklen_t;
 #include <string.h>
 #include <ctype.h>
 #include <limits.h>
-#include <unistd.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <errno.h>

system/jlib/jfile.cpp

@@ -31,8 +31,7 @@
 #include <time.h>
 #include <dirent.h>
 #include <utime.h>
-
-
+#include <sys/syscall.h>
 #include <sys/vfs.h>
 #include <sys/mman.h>
 #include <sys/sendfile.h>

testing/regress/ecl/jsonout.ecl

@@ -40,6 +40,8 @@ personRecord :=
             RECORD
 string20        surname;
 string10        forename;
+integer4        category;
+integer8        uid;
 phoneRecord     homePhone;
 boolean         hasMobile;
                 ifblock(self.hasMobile)
@@ -52,8 +54,8 @@ string2         endmarker := '$$';
             END;
 
 namesTable := dataset([
-        {'Halliday','Gavin','09876',123456,true,'07967',838690, 'n/a','n/a',true,'gavin@edata.com',[{'To kill a mocking bird','Lee'},{'Zen and the art of motorcycle maintainence','Pirsig'}], ALL},
-        {'Halliday','Abigail','09876',654321,false,'','',false,[{'The cat in the hat','Suess'},{'Wolly the sheep',''}], ['Red','Yellow']}
+        {'Halliday','Gavin', 1, 1, '09876',123456,true,'07967',838690, 'n/a','n/a',true,'gavin@edata.com',[{'To kill a mocking bird','Lee'},{'Zen and the art of motorcycle maintainence','Pirsig'}], ALL},
+        {'Halliday','Abigail', 1, 2, '09876',654321,false,'','',false,[{'The cat in the hat','Suess'},{'Wolly the sheep',''}], ['Red','Yellow']}
         ], personRecord);
 
 output(namesTable,,'REGRESS::TEMP::output_object_namedArray.json',overwrite, json);

testing/regress/hpcc/util/ecl/file.py

@@ -23,6 +23,7 @@ import os
 import traceback
 import re
 import tempfile
+import xml.etree.ElementTree as ET
 
 from ...util.util import isPositiveIntNum, getConfig
 
@@ -61,6 +62,7 @@ class ECLFile:
         self.baseEcl = os.path.basename(ecl)
         self.basename = os.path.splitext(self.baseEcl)[0]
         self.baseXml = self.basename + '.xml'
+        self.baseQueryXml = self.basename+'.queryxml'
         self.ecl = self.baseEcl
         self.xml_e = self.baseXml
         self.xml_r = self.baseXml
@@ -83,7 +85,13 @@ class ECLFile:
             self.forcePublish=self.args.publish
 
         self.optX =[]
-        self.optXHash={}
+
+        # The final set of stored variables are the union of queryxml, config and CLI
+        # The values in the queryxml file is the lowest precedence
+        # the relevant value in config file Params array it the middle and
+        # -X in the CLI is the highest.
+        self.optXHash=self.checkQueryxmlFile()
+
         self.config = getConfig()
         try:
             # Process definitions of stored input value(s) from config
@@ -133,6 +141,22 @@ class ECLFile:
         self.mergeHashToStrArray(self.optFHash,  self.optF)
         pass
 
+    def checkQueryxmlFile(self):
+        retHash = {}
+        path = os.path.join(self.dir_ec, self.baseQueryXml)
+        logging.debug("%3d. checkQueryxmlFile() checks path:'%s' ",  self.taskId,  path )
+        if os.path.isfile(path):
+            # we have defaults for stored variables in xml file.
+            tree = ET.parse(path)
+            root = tree.getroot()
+            for child in root:
+                key = '-X'+child.tag
+                val = child.text
+                retHash[key] = val
+            pass
+        logging.debug("%3d. checkQueryxmlFile() returns with %s stored parameter(s) ",  self.taskId,  len(retHash) )
+        return retHash
+
     def processKeyValPairs(self,  optArr,  optHash):
         for optStr in optArr:
             [key,  val] = optStr.split('=')

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -53,8 +53,6 @@
 
 #define NUMSLAVEPORTS       2
 #define DEFAULTCONNECTTIMEOUT 10000
-#define DEFAULT_OUT_BUFFER_SIZE 0x100000        // 1MB
-#define DEFAULT_IN_BUFFER_SIZE  0x100000*32  // 32MB input buffer
 #define DEFAULT_WRITEPOOLSIZE 16
 #define DISK_BUFFER_SIZE 0x10000 // 64K
 #define DEFAULT_TIMEOUT (1000*60*60)
@@ -979,9 +977,9 @@ public:
         iCompare = NULL;
         ihash = NULL;
         fixedEstSize = 0;
-        bucketSendSize = activity->getOptUInt(THOROPT_HDIST_BUCKET_SIZE, DEFAULT_OUT_BUFFER_SIZE);
+        bucketSendSize = activity->getOptUInt(THOROPT_HDIST_BUCKET_SIZE, DISTRIBUTE_DEFAULT_OUT_BUFFER_SIZE);
         istop = _istop;
-        inputBufferSize = activity->getOptUInt(THOROPT_HDIST_BUFFER_SIZE, DEFAULT_IN_BUFFER_SIZE);
+        inputBufferSize = activity->getOptUInt(THOROPT_HDIST_BUFFER_SIZE, DISTRIBUTE_DEFAULT_IN_BUFFER_SIZE);
         pullBufferSize = DISTRIBUTE_PULL_BUFFER_SIZE;
         selfstopped = false;
         pull = false;

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -721,6 +721,16 @@ protected:
         bool stopped;
         SimpleInterThreadQueueOf<CSendItem, true> blockQueue;
         Owned<IException> exception;
+
+        void clearQueue()
+        {
+            loop
+            {
+                Owned<CSendItem> sendItem = blockQueue.dequeueNow();
+                if (NULL == sendItem)
+                    break;
+            }
+        }
     public:
         CRowProcessor(CInMemJoinBase &_owner) : threaded("CRowProcessor", this), owner(_owner)
         {
@@ -730,17 +740,13 @@ protected:
         ~CRowProcessor()
         {
             blockQueue.stop();
-            loop
-            {
-                Owned<CSendItem> sendItem = blockQueue.dequeueNow();
-                if (NULL == sendItem)
-                    break;
-            }
+            clearQueue();
             wait();
         }
         void start()
         {
             stopped = false;
+            clearQueue();
             exception.clear();
             threaded.start();
         }

thorlcr/graph/thgraph.hpp

@@ -449,12 +449,12 @@ class graph_decl CGraphBase : public CInterface, implements IEclGraphResults, im
         virtual void setResultBool(const char *name, unsigned sequence, bool value) { ctx->setResultBool(name, sequence, value); }
         virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) { ctx->setResultData(name, sequence, len, data); }
         virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) { ctx->setResultDecimal(stepname, sequence, len, precision, isSigned, val); }
-        virtual void setResultInt(const char *name, unsigned sequence, __int64 value) { ctx->setResultInt(name, sequence, value); }
+        virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) { ctx->setResultInt(name, sequence, value, size); }
         virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) { ctx->setResultRaw(name, sequence, len, data); }
         virtual void setResultReal(const char * stepname, unsigned sequence, double value) { ctx->setResultReal(stepname, sequence, value); }
         virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) { ctx->setResultSet(name, sequence, isAll, len, data, transformer); }
         virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) { ctx->setResultString(name, sequence, len, str); }
-        virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value) { ctx->setResultUInt(name, sequence, value); }
+        virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) { ctx->setResultUInt(name, sequence, value, size); }
         virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) { ctx->setResultUnicode(name, sequence, len, str); }
         virtual void setResultVarString(const char * name, unsigned sequence, const char * value) { ctx->setResultVarString(name, sequence, value); }
         virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) { ctx->setResultVarUnicode(name, sequence, value); }

thorlcr/graph/thgraphmaster.cpp

@@ -864,12 +864,12 @@ public:
         else
             throw MakeStringException(TE_UnexpectedParameters, "Unexpected parameters to setResultDecimal");
     }
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 result)
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 result, unsigned size)
     {
         Owned<IWUResult> r = updateResult(name, sequence);
         if (r)
         {
-            r->setResultInt(result);    
+            r->setResultInt(result);
             r->setResultStatus(ResultStatusCalculated);
         }
         else
@@ -942,12 +942,12 @@ public:
         else
             throw MakeStringException(TE_UnexpectedParameters, "Unexpected parameters to setResultVarString");
     }
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 result)
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 result, unsigned size)
     {
         Owned<IWUResult> r = updateResult(name, sequence);
         if (r)
         {
-            r->setResultUInt(result);   
+            r->setResultUInt(result);
             r->setResultStatus(ResultStatusCalculated);
         }
         else

thorlcr/graph/thgraphslave.cpp

@@ -927,12 +927,12 @@ public:
     virtual void setResultBool(const char *name, unsigned sequence, bool value) { throwUnexpected(); }
     virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
     virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) { throwUnexpected(); } 
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 value) { throwUnexpected(); }
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) { throwUnexpected(); }
     virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
     virtual void setResultReal(const char * stepname, unsigned sequence, double value) { throwUnexpected(); }
     virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) { throwUnexpected(); }
     virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) { throwUnexpected(); }
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value) { throwUnexpected(); }
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) { throwUnexpected(); }
     virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) { throwUnexpected(); }
     virtual void setResultVarString(const char * name, unsigned sequence, const char * value) { throwUnexpected(); }
     virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) { throwUnexpected(); }

thorlcr/thorutil/thbufdef.hpp

@@ -32,7 +32,7 @@
 #define INDEXWRITE_SMART_BUFFER_SIZE            (0x100000*12)           // 12MB
 #define COUNTPROJECT_SMART_BUFFER_SIZE          (0x100000*12)           // 12MB
 #define ENTH_SMART_BUFFER_SIZE                  (0x100000*12)           // 12MB
-#define JOIN_SMART_BUFFER_SIZE                 (0x100000*12)            // 12MB
+#define JOIN_SMART_BUFFER_SIZE                  (0x100000*12)           // 12MB
 #define LOOKUPJOINL_SMART_BUFFER_SIZE           (0x100000*12)           // 12MB
 #define CATCH_BUFFER_SIZE                       (0x100000*12)           // 12MB
 #define SKIPLIMIT_BUFFER_SIZE                   (0x100000*12)           // 12MB
@@ -42,7 +42,8 @@
 #define NSPLITTER_SPILL_BUFFER_SIZE             (0x100000)              // 1MB
 #define DISTRIBUTE_PULL_BUFFER_SIZE             (0x100000*32)           // 32MB
 #define SORT_BUFFER_TOTAL                       (0x100000*20)           // 20MB (estimate)
-#define DISTRIBUTE_SINGLE_BUFFER_SIZE           (0x10000)               // 64K  - NB per node and multiplied by async send
+#define DISTRIBUTE_DEFAULT_OUT_BUFFER_SIZE      (0x100000)              // 1MB (* targets (numnodes), on each slave)
+#define DISTRIBUTE_DEFAULT_IN_BUFFER_SIZE       (0x100000*32)           // 32MB input buffer (on each slave)
 #define FUNNEL_MIN_BUFF_SIZE                    (0x100000*2)            // 2MB
 #define FUNNEL_MAX_BUFF_SIZE                    (0x100000*20)           // 20MB
 #define COMBINE_MAX_BUFF_SIZE                   (0x100000*20)           // 20MB
@@ -54,6 +55,7 @@
 #define LOOP_SMART_BUFFER_SIZE                  (0x100000*12)           // 12MB
 #define LOCALRESULT_BUFFER_SIZE                 (0x100000*10)           // 10MB
 
+#define DISTRIBUTE_RESMEM(N) ((DISTRIBUTE_DEFAULT_OUT_BUFFER_SIZE * (N)) + DISTRIBUTE_DEFAULT_IN_BUFFER_SIZE)
 
 
 #endif

tools/esdlcomp/esdlcomp.cpp

@@ -1214,7 +1214,8 @@ void ESDLcompiler::Process()
     write_esxdl();
 
     fclose(yyin);
-    close(esxdlo);
+	if (gOutfile > 0)
+		close (gOutfile);
 
     yyCleanupESDLGlobals();
     yylex_destroy();