Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -1723,7 +1723,6 @@ class CLocalWUResult : public CInterface, implements IWUResult
     mutable CriticalSection crit;
     Owned<IPropertyTree> p;
     Owned<IProperties> xmlns;
-    void getSchema(TypeInfoArray &types, StringAttrArray &names, IStringVal * ecl=NULL) const;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -1762,7 +1761,7 @@ public:
     virtual bool        getResultIsAll() const;
     virtual IProperties *queryResultXmlns();
     virtual IStringVal& getResultFieldOpt(const char *name, IStringVal &str) const;
-
+    virtual void getSchema(IArrayOf<ITypeInfo> &types, StringAttrArray &names, IStringVal * ecl=NULL) const;
 
     // interface IWUResult
     virtual void        setResultStatus(WUResultStatus status);
@@ -7545,7 +7544,7 @@ bool findSize(int size, IntArray &sizes)
     return false;
 }
 
-void CLocalWUResult::getSchema(TypeInfoArray &types, StringAttrArray &names, IStringVal * eclText) const
+void CLocalWUResult::getSchema(IArrayOf<ITypeInfo> &types, StringAttrArray &names, IStringVal * eclText) const
 {
     MemoryBuffer schema;
     p->getPropBin("SchemaRaw", schema);
@@ -9470,6 +9469,25 @@ extern WORKUNIT_API bool secDebugWorkunit(const char * wuid, ISecManager &secmgr
     return false;
 }
 
+void getSimpleResultType(IWUResult *r, Owned<ITypeInfo> &type)
+{
+    TypeInfoArray types;
+    StringAttrArray names;
+    r->getSchema(types, names, NULL);
+    if (types.ordinality()==1)
+        type.set(&types.item(0));
+}
+
+bool isSuppliedParamScalar(IWUResult *r, IPropertyTree &curVal, Owned<ITypeInfo> &type)
+{
+    if (!r->isResultScalar())
+        return false;
+    if (!curVal.hasChildren())
+        return true;
+    getSimpleResultType(r, type);
+    return type && type->isScalar();
+}
+
 void updateSuppliedXmlParams(IWorkUnit * w)
 {
     Owned<const IPropertyTree> params = w->getXmlParams();
@@ -9483,8 +9501,9 @@ void updateSuppliedXmlParams(IWorkUnit * w)
         Owned<IWUResult> r = updateWorkUnitResult(w, name, -1);
         if (r)
         {
+            Owned<ITypeInfo> type;
             StringBuffer s;
-            if (r->isResultScalar() && !curVal.hasChildren())
+            if (isSuppliedParamScalar(r, curVal, type))
             {
                 curVal.getProp(".", s);
                 r->setResultXML(s);
@@ -9493,7 +9512,9 @@ void updateSuppliedXmlParams(IWorkUnit * w)
             else
             {
                 toXML(&curVal, s);
-                bool isSet = (curVal.hasProp("Item") || curVal.hasProp("string"));
+                if (!type)
+                    getSimpleResultType(r, type);
+                bool isSet = (type && type->getTypeCode()==type_set);
                 r->setResultRaw(s.length(), s.str(), isSet ? ResultFormatXmlSet : ResultFormatXml);
             }
         }

common/workunit/workunit.hpp

@@ -269,6 +269,7 @@ enum WUResultFormat
 };
 
 
+interface ITypeInfo;
 
 interface IConstWUResult : extends IInterface
 {
@@ -303,6 +304,7 @@ interface IConstWUResult : extends IInterface
     virtual bool getResultIsAll() const = 0;
     virtual const IProperties *queryResultXmlns() = 0;
     virtual IStringVal &getResultFieldOpt(const char *name, IStringVal &str) const = 0;
+    virtual void getSchema(IArrayOf<ITypeInfo> &types, StringAttrArray &names, IStringVal * eclText) const = 0;
 };
 
 

dali/dfu/dfuutil.cpp

@@ -810,11 +810,11 @@ public:
                     else
                         throwError1(DFUERR_DSuperFileDoesntContainSub, subfiles[i1]);
         }
+        if (removesuperfile && toremove.ordinality()!=superfile->numSubFiles())
+            removesuperfile = false;
         // Do we have something to delete?
-        if (toremove.ordinality()) {
+        if (toremove.ordinality() || removesuperfile) {
             transaction->start();
-            if (removesuperfile && toremove.ordinality()!=superfile->numSubFiles())
-                removesuperfile = false;
             ForEachItemIn(i2,toremove)
                 superfile->removeSubFile(toremove.item(i2).text.get(),delsub,false,transaction);
             // Delete superfile if empty

docs/Automation/CMakeLists.txt

@@ -1,47 +0,0 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-SET(XSD_SOURCE_DIR ${HPCC_SOURCE_DIR}/initfiles/componentfiles/configxml)
-SET(XSD_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}/xsd )
-SET(XML_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}/xml )
-
-FILE(MAKE_DIRECTORY ${XSD_TARGET_DIR})
-
-# get all *.xsd.in and *.xsd files
-FILE(GLOB XSD_TEMPLATE_FILES RELATIVE "${XSD_SOURCE_DIR}" "${XSD_SOURCE_DIR}/*.xsd.in")
-FILE(GLOB XSD_FILES RELATIVE "${XSD_SOURCE_DIR}" "${XSD_SOURCE_DIR}/*.xsd")
-
-# Skip some files which cause problem
-LIST(REMOVE_ITEM XSD_FILES "esp_service_wsecl2.xsd")
-
-# process and copy *.xsd.in files
-FOREACH( XSD_TEMPLATE_FILE  ${XSD_TEMPLATE_FILES} )
-   STRING(REGEX REPLACE "(.*).in" "\\1" XSD_FILE "${XSD_TEMPLATE_FILE}")
-   CONFIGURE_FILE(${XSD_SOURCE_DIR}/${XSD_TEMPLATE_FILE} ${XSD_TARGET_DIR}/${XSD_FILE})
-ENDFOREACH()
-
-# copy *.xsd.in files
-FOREACH( XSD_FILE  ${XSD_FILES} )
-   CONFIGURE_FILE(${XSD_SOURCE_DIR}/${XSD_FILE} ${XSD_TARGET_DIR}/${XSD_FILE} COPYONLY)
-ENDFOREACH()
-
-# create complete *.xsd file list
-FOREACH( XSD_TEMPLATE_FILE  ${XSD_TEMPLATE_FILES} )
-   STRING(REGEX REPLACE "(.*).in" "\\1" XSD_FILE "${XSD_TEMPLATE_FILE}")
-   SET(XSD_FILES ${XSD_FILES} ${XSD_FILE})
-ENDFOREACH()
-
-# create MakeFiles to generate xsd file to xml file
-XSD_TO_XML("${XSD_FILES}" ${XSD_TARGET_DIR} ${XML_TARGET_DIR})

docs/CMakeLists.txt

@@ -66,7 +66,6 @@ add_subdirectory(InstantCloud)
 add_subdirectory(Installing_and_RunningTheHPCCPlatform)
 add_subdirectory(RDDERef)
 add_subdirectory(RunningHPCCinaVirtualMachine)
-add_subdirectory(UsingConfigManager)
 add_subdirectory(ECLPlayground)
 add_subdirectory(ECLPluginForEclipse)
 add_subdirectory(ECLScheduler)

docs/ECLStandardLibraryReference/SLR-Mods/Copy.xml

@@ -24,8 +24,9 @@
   <emphasis role="bold">] [</emphasis> <emphasis>,compress</emphasis>
   <emphasis role="bold">] [</emphasis> <emphasis>,forcePush</emphasis>
   <emphasis role="bold">] [</emphasis>
-  <emphasis>,transferBufferSize</emphasis> <emphasis
-  role="bold">]);</emphasis><emphasis></emphasis></para>
+  <emphasis>,transferBufferSize</emphasis> <emphasis role="bold">] 
+  [</emphasis> <emphasis>,preserveCompression</emphasis> <emphasis
+  role="bold">]);</emphasis></para>
 
   <para><emphasis>dfuwuid</emphasis> <emphasis role="bold"> :=
   STD.File.fCopy<indexterm>
@@ -48,7 +49,8 @@
   <emphasis role="bold">] [</emphasis> <emphasis>,compress</emphasis>
   <emphasis role="bold">] [</emphasis> <emphasis>,forcePush</emphasis>
   <emphasis role="bold">] [</emphasis>
-  <emphasis>,transferBufferSize</emphasis> <emphasis
+  <emphasis>,transferBufferSize</emphasis> 
+  <emphasis role="bold">] [</emphasis> <emphasis>,preserveCompression</emphasis> <emphasis
   role="bold">]);</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
@@ -166,6 +168,14 @@
         </row>
 
         <row>
+          <entry><emphasis>preserveCompression </emphasis></entry>
+
+          <entry>Optional. A boolean TRUE or FALSE flag indicating whether to
+          preserve the compression of the old file when copying. If omitted,
+          the default is TRUE.</entry>
+        </row>
+
+        <row>
           <entry><emphasis>dfuwuid</emphasis></entry>
 
           <entry>The attribute name to receive the null-terminated string

docs/HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml

@@ -771,6 +771,14 @@ dfuplus action=dspray srcname=le::imagedb
                   <emphasis>srcdali</emphasis>. If omitted, the General
                   Options <emphasis>password</emphasis> is used.</entry>
                 </row>
+
+                <row>
+                  <entry><emphasis>preservecompression</emphasis></entry>
+
+                  <entry>Optional. A boolean flag (0 | 1) indicating whether
+                  to preserve the compression of the source file. If omitted,
+                  the default is 1.</entry>
+                </row>
               </tbody>
             </tgroup>
           </informaltable>

docs/XMLGeneration/CMakeLists.txt

@@ -25,6 +25,7 @@ FILE(GLOB XSD_FILES RELATIVE "${XSD_SOURCE_DIR}" "${XSD_SOURCE_DIR}/*.xsd")
 
 # Skip some files which cause problem
 LIST(REMOVE_ITEM XSD_FILES "esp_service_wsecl2.xsd")
+LIST(REMOVE_ITEM XSD_FILES "ftslave_linux.xsd")
 
 # process and copy *.xsd.in files
 FOREACH( XSD_TEMPLATE_FILE  ${XSD_TEMPLATE_FILES} )

ecl/ecl-package/ecl-package.cpp

@@ -498,7 +498,7 @@ private:
 class EclCmdPackageAdd : public EclCmdCommon
 {
 public:
-    EclCmdPackageAdd() : optActivate(false), optOverWrite(false), optGlobalScope(false), optAllowForeign(false)
+    EclCmdPackageAdd() : optActivate(false), optOverWrite(false), optGlobalScope(false), optAllowForeign(false), optPreloadAll(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -539,6 +539,8 @@ public:
                 continue;
             if (iter.matchFlag(optAllowForeign, ECLOPT_ALLOW_FOREIGN))
                 continue;
+            if (iter.matchFlag(optPreloadAll, ECLOPT_PRELOAD_ALL_PACKAGES))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -595,6 +597,7 @@ public:
         request->setGlobalScope(optGlobalScope);
         request->setSourceProcess(optSourceProcess);
         request->setAllowForeignFiles(optAllowForeign);
+        request->setPreloadAllPackages(optPreloadAll);
 
         Owned<IClientAddPackageResponse> resp = packageProcessClient->AddPackage(request);
         if (resp->getExceptions().ordinality())
@@ -627,6 +630,7 @@ public:
                     "   --global-scope              The specified packagemap can be shared across multiple targets\n"
                     "   --source-process            Process cluster to copy files from\n"
                     "   --allow-foreign             Do not fail if foreign files are used in packagemap\n"
+                    "   --preload-all               Set preload files option for all packages\n"
                     "   <target>                    Name of target to use when adding package map information\n"
                     "   <filename>                  Name of file containing package map information\n",
                     stdout);
@@ -645,6 +649,7 @@ private:
     bool optOverWrite;
     bool optGlobalScope;
     bool optAllowForeign;
+    bool optPreloadAll;
 };
 
 class EclCmdPackageValidate : public EclCmdCommon

ecl/eclcmd/eclcmd_common.hpp

@@ -109,6 +109,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_COMMENT "--comment"
 
 #define ECLOPT_CHECK_PACKAGEMAPS "--check-packagemaps"
+#define ECLOPT_PRELOAD_ALL_PACKAGES "--preload-all"
 
 #define ECLOPT_EXCEPTION_LEVEL "--exception-level"
 #define ECLOPT_RESULT_LIMIT "--limit"

ecl/hqlcpp/hqlhtcpp.cpp

@@ -15691,7 +15691,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySectionInput(BuildCtx & ctx, I
 ABoundActivity * HqlCppTranslator::doBuildActivityMetaActivity(BuildCtx & ctx, IHqlExpression * expr)
 {
     Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
-    if (!targetThor())
+    if (targetHThor())
         return boundDataset.getClear();
 
     assertex(expr->hasAttribute(pullAtom));

esp/scm/ws_packageprocess.ecm

@@ -37,6 +37,7 @@ ESPrequest AddPackageRequest
     bool GlobalScope(0);
     string SourceProcess;
     bool AllowForeignFiles(true);
+    [min_ver("1.02")] bool PreloadAllPackages(false);
 };
 
 
@@ -232,7 +233,7 @@ ESPresponse [exceptions_inline] GetPackageMapSelectOptionsResponse
     ESParray<string> ProcessFilters;
 };
 
-ESPservice [version("1.01"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
+ESPservice [version("1.02"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
     ESPmethod AddPackage(AddPackageRequest, AddPackageResponse);

esp/scm/ws_topology.ecm

@@ -58,6 +58,7 @@ ESPStruct TpLogicalCluster
     string Queue;
         [min_ver("1.14")] string LanguageVersion;
     [min_ver("1.11")] string Process;
+    [min_ver("1.22")] string Type;
 };
 //  ===========================================================================
 
@@ -583,7 +584,7 @@ ESPresponse [exceptions_inline,encode(0)] TpGetServicePluginsResponse
     ESParray<ESPstruct TpEspServicePlugin, Plugin> Plugins;
 };
 
-ESPservice [noforms, version("1.21"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
+ESPservice [noforms, version("1.22"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
 {
     ESPuses ESPStruct TpBinding;
     ESPuses ESPstruct TpCluster;

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1157,7 +1157,7 @@ void CWsEclBinding::SOAPSectionToXsd(WsEclWuInfo &wuinfo, IPropertyTree *parmTre
             }
 
             schema.appendf("<xsd:element minOccurs=\"0\" maxOccurs=\"1\" name=\"%s\" type=\"%s\"", name, type.str());
-            if (part.hasProp("@width") || part.hasProp("@height"))
+            if (part.hasProp("@width") || part.hasProp("@height") || part.hasProp("@password"))
             {
                 schema.append("><xsd:annotation><xsd:appinfo><form");
                 unsigned rows = part.getPropInt("@height");

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -184,7 +184,7 @@ void makePackageActive(IPropertyTree *pkgSet, IPropertyTree *psEntryNew, const c
 
 //////////////////////////////////////////////////////////
 
-void addPackageMapInfo(const char *xml, StringArray &filesNotFound, const char *process, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, const char *prefix, bool activate, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles)
+void addPackageMapInfo(const char *xml, StringArray &filesNotFound, const char *process, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, const char *prefix, bool activate, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles, bool preloadAll)
 {
     if (!xml || !*xml)
         throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "PackageMap info not provided");
@@ -211,6 +211,8 @@ void addPackageMapInfo(const char *xml, StringArray &filesNotFound, const char *
     ForEach(*iter)
     {
         IPropertyTree &item = iter->query();
+        if (preloadAll)
+            item.setPropBool("@preload", true);
         Owned<IPropertyTreeIterator> superFiles = item.getElements("SuperFile");
         ForEach(*superFiles)
         {
@@ -563,7 +565,7 @@ bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageReque
     buildPkgSetId(pkgSetId, processName.get());
 
     StringArray filesNotFound;
-    addPackageMapInfo(req.getInfo(), filesNotFound, processName, target, pmid, pkgSetId, daliip, srcCluster, prefix, activate, overWrite, userdesc, req.getAllowForeignFiles());
+    addPackageMapInfo(req.getInfo(), filesNotFound, processName, target, pmid, pkgSetId, daliip, srcCluster, prefix, activate, overWrite, userdesc, req.getAllowForeignFiles(), req.getPreloadAllPackages());
     resp.setFilesNotFound(filesNotFound);
 
     StringBuffer msg;

esp/services/ws_topology/ws_topologyService.cpp

@@ -1169,31 +1169,21 @@ bool CWsTopologyEx::onTpLogicalClusterQuery(IEspContext &context, IEspTpLogicalC
         if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
             throw MakeStringException(ECLWATCH_TOPOLOGY_ACCESS_DENIED, "Failed to do Cluster Query. Permission denied.");
 
-        Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
-        Owned<IConstEnvironment> constEnv = factory->openEnvironment();
-        Owned<IPropertyTree> root = &constEnv->getPTree();
-        if (!root)
-            throw MakeStringException(ECLWATCH_CANNOT_GET_ENV_INFO, "Failed to get environment information.");
-
         IArrayOf<IEspTpLogicalCluster> clusters;
-        Owned<IPropertyTreeIterator> clusterIterator = root->getElements("Software/Topology/Cluster");
-        if (clusterIterator->first()) 
+        CConstWUClusterInfoArray wuClusters;
+        getEnvironmentClusterInfo(wuClusters);
+        ForEachItemIn(c, wuClusters)
         {
-            do {
-                IPropertyTree &cluster0 = clusterIterator->query();
-                StringBuffer processName;
-                const char* clusterName0 = cluster0.queryProp("@name");
-                if (!clusterName0 || !*clusterName0)
-                    continue;
-
-                IEspTpLogicalCluster* pService = createTpLogicalCluster("","");
-                pService->setName(clusterName0);
-                pService->setLanguageVersion("3.0.0");
-                clusters.append(*pService);
-
-            } while (clusterIterator->next());
+            IConstWUClusterInfo& wuCluster = wuClusters.item(c);
+            SCMStringBuffer name;
+            wuCluster.getName(name);
+
+            Owned<IEspTpLogicalCluster> cluster = createTpLogicalCluster();
+            cluster->setName(name.str());
+            cluster->setType(clusterTypeString(wuCluster.getPlatform(), false));
+            cluster->setLanguageVersion("3.0.0");
+            clusters.append(*cluster.getClear());
         }
-
         resp.setTpLogicalClusters(clusters);
     }
     catch(IException* e)

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -33,6 +33,7 @@
 
 const unsigned ROXIECONNECTIONTIMEOUT = 1000;   //1 second
 const unsigned ROXIECONTROLQUERYTIMEOUT = 3000; //3 second
+const unsigned ROXIECONTROLQUERIESTIMEOUT = 30000; //30 second
 const unsigned ROXIELOCKCONNECTIONTIMEOUT = 60000; //60 second
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
@@ -1071,7 +1072,7 @@ IPropertyTree* getQueriesOnCluster(const char *target, const char *queryset)
     try
     {
         Owned<ISocket> sock = ISocket::connect_timeout(eps.item(0), ROXIECONNECTIONTIMEOUT);
-        return sendRoxieControlAllNodes(sock, "<control:queries/>", true, ROXIECONTROLQUERYTIMEOUT);
+        return sendRoxieControlAllNodes(sock, "<control:queries/>", false, ROXIECONTROLQUERIESTIMEOUT);
     }
     catch(IException* e)
     {

esp/src/eclwatch/ESPLogicalFile.js

@@ -214,7 +214,7 @@ define([
                     if (lang.exists("DFUFileParts.DFUPart", DFUFilePartsOnCluster)) {
                         arrayUtil.forEach(DFUFilePartsOnCluster.DFUFileParts.DFUPart, function (DFUPart, idx) {
                             DFUFileParts.DFUPart.push(lang.mixin({
-                                __hpcc__id: DFUPart.Id + "--" + DFUPart.Copy,
+                                __hpcc_id: DFUPart.Id + "--" + DFUPart.Copy,
                                 Cluster: DFUFilePartsOnCluster.Cluster
                             }, DFUPart));
                         }, this);

esp/src/eclwatch/HelpersWidget.js

@@ -196,6 +196,7 @@ define([
                             break;
                         case "Workunit XML":
                         case "Archive Query":
+                        case "xml":
                             sourceMode = "xml";
                             break;
                     }
@@ -208,7 +209,7 @@ define([
                             params: {
                                 Wuid: Wuid,
                                 sourceMode: sourceMode,
-                                sourceURL: this._getURL(row)
+                                sourceURL: this._getURL(row, id)
                             }
                         }
                     });

esp/xslt/wsecl3_form.xsl

@@ -669,6 +669,7 @@ function switchInputForm()
                 <xsl:choose>
                     <xsl:when test="count($ems)=1 and ($cpxType/xsd:sequence/@maxOccurs='unbounded' or $ems[1]/@maxOccurs='unbounded')">
                         <xsl:variable name="stype" select="$ems[1]/@type"/>
+                        <xsl:variable name="itemname" select="$ems[1]/@name"/>
                         <xsl:choose>
                             <xsl:when test="$ems/xsd:complexType">
                                 <xsl:variable name="btype" select="$ems/@name"/>
@@ -704,6 +705,8 @@ function switchInputForm()
                                     <xsl:text disable-output-escaping="yes"><![CDATA[[Enter one item per line]<br/>]]></xsl:text>
                                     <xsl:text disable-output-escaping="yes"><![CDATA[<textarea name=']]></xsl:text>
                                     <xsl:value-of select="$fieldId"/>
+                                    <xsl:text disable-output-escaping="yes"><![CDATA[.]]></xsl:text>
+                                    <xsl:value-of select="$itemname"/>
                                     <xsl:text disable-output-escaping="yes"><![CDATA[$' id=']]></xsl:text><!--Note $ at end of name not id marks field as array input via textarea-->
                                     <xsl:value-of select="$fieldId"/>
                                     <xsl:text disable-output-escaping="yes"><![CDATA[' cols=']]></xsl:text>

initfiles/bash/sbin/deb/postrm.in

@@ -17,6 +17,21 @@
 
 ###<REPLACE>###
 
+# Restore install link if it was present
+#######################
+linkf=/tmp/.hpccinslink
+if [[ -f ${linkf} ]] ; then
+   if [[ -n ${INSTALL_DIR} ]] ; then
+      if [[ ! -d ${INSTALL_DIR} ]] ; then
+         linkinfo=$(cat "${linkf}")
+         if [[ -n ${linkinfo} ]] ; then
+            ln -s ${linkinfo}
+         fi
+      fi
+   fi
+   rm -f ${linkf}
+fi
+
 # On some systems, when running an update (as opposed to a remove) the postinstall script does
 # not get run. To work around that issue, we run the post-install actions in the post-remove script
 # (which IS run)

initfiles/componentfiles/configxml/CMakeLists.txt

@@ -31,7 +31,6 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_BINARY_DIR}/eclagent_config.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/esp.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/espsmcservice.xsd
-    ${CMAKE_CURRENT_BINARY_DIR}/ftslave_linux.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/roxie.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/thor.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/buildset.xml

initfiles/sbin/prerm.in

@@ -47,5 +47,15 @@ if [ -f ${CONFIG_DIR}/installed ] ; then
     rm ${CONFIG_DIR}/installed
 fi
 
+# Check if install dir is a link
+#######################
+if [[ -n ${INSTALL_DIR} ]] ; then
+#  trim for readlink
+   ins_dir=$(echo "${INSTALL_DIR}" | sed 's/\/*$//')
+   if [[ -h ${ins_dir} ]] ; then
+       echo "$(readlink "${ins_dir}") ${INSTALL_DIR}" > /tmp/.hpccinslink
+   fi
+fi
+
 exit 0
  

system/jlib/jmisc.hpp

@@ -277,8 +277,8 @@ interface IAbortRequestCallback
 class LocalAbortHandler
 {
 public:
-    LocalAbortHandler(AbortHandler _handler=NULL)   { handler = _handler; shandler = NULL; addAbortHandler(handler); }
-    LocalAbortHandler(SimpleAbortHandler _handler=NULL) { shandler = _handler; handler = NULL; addAbortHandler(shandler); }
+    LocalAbortHandler(AbortHandler _handler)   { handler = _handler; shandler = NULL; addAbortHandler(handler); }
+    LocalAbortHandler(SimpleAbortHandler _handler) { shandler = _handler; handler = NULL; addAbortHandler(shandler); }
     ~LocalAbortHandler()                            { if (handler) { removeAbortHandler(handler); } else removeAbortHandler(shandler); }
 private:
     AbortHandler            handler;

thorlcr/activities/funnel/thfunnelslave.cpp

@@ -715,12 +715,14 @@ public:
     {
         helper = (IHThorNonEmptyArg *) queryHelper();
         sendReceiving = false;
+        masterMpTag = TAG_NULL;
     }
 
 // IThorSlaveActivity overloaded methods
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        masterMpTag = container.queryJob().deserializeMPTag(data);
+        if (!container.queryLocalOrGrouped())
+            masterMpTag = container.queryJob().deserializeMPTag(data);
         appendOutputLinked(this);
     }
     void abort()

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -188,7 +188,7 @@ public:
         StringBuffer partFname;
         getPartFilename(partDesc, 0, partFname);
         bool compress=false;
-        OwnedIFileIO iFileIO = createMultipleWrite(this, partDesc, 0, compress, false, NULL, this, false, true, &abortSoon);
+        OwnedIFileIO iFileIO = createMultipleWrite(this, partDesc, 0, TW_RenameToPrimary, compress, NULL, this, &abortSoon);
         Owned<IFileIOStream> out = createBufferedIOStream(iFileIO);
         ActPrintLog("INDEXWRITE: created fixed output stream %s", partFname.str());
         unsigned flags = COL_PREFIX;

thorlcr/activities/thactivityutil.cpp

@@ -656,14 +656,15 @@ class CWriteHandler : public CSimpleInterface, implements IFileIO
     bool *aborted;
     CActivityBase &activity;
     IPartDescriptor &partDesc;
-    bool remote, direct, renameToPrimary;
+    bool remote;
     CFIPScope fipScope;
+    unsigned twFlags;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CWriteHandler(CActivityBase &_activity, IPartDescriptor &_partDesc, IFile *_primary, IFileIO *_primaryio, ICopyFileProgress *_iProgress, bool _direct, bool _renameToPrimary, bool *_aborted) 
-        : activity(_activity), partDesc(_partDesc), primary(_primary), primaryio(_primaryio), iProgress(_iProgress), direct(_direct), renameToPrimary(_renameToPrimary), aborted(_aborted), fipScope(primary->queryFilename())
+    CWriteHandler(CActivityBase &_activity, IPartDescriptor &_partDesc, IFile *_primary, IFileIO *_primaryio, ICopyFileProgress *_iProgress, unsigned _twFlags, bool *_aborted)
+        : activity(_activity), partDesc(_partDesc), primary(_primary), primaryio(_primaryio), iProgress(_iProgress), twFlags(_twFlags), aborted(_aborted), fipScope(primary->queryFilename())
     {
         RemoteFilename rfn;
         partDesc.getFilename(0, rfn);
@@ -680,11 +681,11 @@ public:
             primary->remove(); // i.e. never completed, so remove partial (temp) primary
             return;
         }
-        if (renameToPrimary)
+        if (twFlags & TW_RenameToPrimary)
         {
             OwnedIFile tmpIFile;
             CFIPScope fipScope;
-            if (remote)
+            if (remote && !(twFlags & TW_External))
             {
                 StringBuffer tmpName(primaryName.str());
                 tmpName.append(".tmp");
@@ -749,7 +750,7 @@ public:
     virtual void close() { primaryio->close(); }
 };
 
-IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, unsigned recordSize, bool &compress, bool extend, ICompressor *ecomp, ICopyFileProgress *iProgress, bool direct, bool renameToPrimary, bool *aborted, StringBuffer *_outLocationName)
+IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, unsigned recordSize, unsigned twFlags, bool &compress, ICompressor *ecomp, ICopyFileProgress *iProgress, bool *aborted, StringBuffer *_outLocationName)
 {
     StringBuffer outLocationNameI;
     StringBuffer &outLocationName = _outLocationName?*_outLocationName:outLocationNameI;
@@ -757,8 +758,8 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc,
     RemoteFilename rfn;
     partDesc.getFilename(0, rfn);
     StringBuffer primaryName;
-    rfn.getPath(primaryName);   
-    if (direct)
+    rfn.getPath(primaryName);
+    if (twFlags & TW_Direct)
     {
         if (0 == outLocationName.length())
             outLocationName.append(primaryName.str());
@@ -767,7 +768,7 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc,
     {
         // use temp name
         GetTempName(outLocationName, "partial");
-        if (rfn.isLocal())
+        if (rfn.isLocal() || (twFlags & TW_External))
         { // ensure local tmp in same directory as target
             StringBuffer dir;
             splitDirTail(primaryName, dir);
@@ -784,21 +785,21 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc,
     {
         if (activity->getOptBool(THOROPT_COMP_FORCELZW, false))
             recordSize = 0; // by default if fixed length (recordSize set), row diff compression is used. This forces LZW
-        fileio.setown(createCompressedFileWriter(file, recordSize, extend, true, ecomp));
+        fileio.setown(createCompressedFileWriter(file, recordSize, 0 != (twFlags & TW_Extend), true, ecomp));
         if (!fileio)
         {
             compress = false;
             Owned<IThorException> e = MakeActivityWarning(activity, TE_LargeBufferWarning, "Could not write file '%s' compressed", outLocationName.str());
             activity->fireException(e);
-            fileio.setown(file->open(extend&&file->exists()?IFOwrite:IFOcreate)); 
+            fileio.setown(file->open((twFlags & TW_Extend)&&file->exists()?IFOwrite:IFOcreate));
         }
     }
     else
-        fileio.setown(file->open(extend&&file->exists()?IFOwrite:IFOcreate)); 
+        fileio.setown(file->open((twFlags & TW_Extend)&&file->exists()?IFOwrite:IFOcreate));
     if (!fileio)
         throw MakeActivityException(activity, TE_FileCreationFailed, "Failed to create file for write (%s) error = %d", outLocationName.str(), GetLastError());
     ActPrintLog(activity, "Writing to file: %s, compress=%s, rdiff=%s", file->queryFilename(), compress ? "true" : "false", (compress && recordSize) ? "true" : "false");
-    return new CWriteHandler(*activity, partDesc, file, fileio, iProgress, direct, renameToPrimary, aborted);
+    return new CWriteHandler(*activity, partDesc, file, fileio, iProgress, twFlags, aborted);
 }
 
 StringBuffer &locateFilePartPath(CActivityBase *activity, const char *logicalFilename, IPartDescriptor &partDesc, StringBuffer &filePath)

thorlcr/activities/thactivityutil.ipp

@@ -190,8 +190,12 @@ StringBuffer &locateFilePartPath(CActivityBase *activity, const char *logicalFil
 void doReplicate(CActivityBase *activity, IPartDescriptor &partDesc, ICopyFileProgress *iProgress=NULL);
 void cancelReplicates(CActivityBase *activity, IPartDescriptor &partDesc);
 
+#define TW_Extend 0x01
+#define TW_Direct 0x02
+#define TW_External 0x04
+#define TW_RenameToPrimary 0x08
 interface IPartDescriptor;
-IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, unsigned recordSize, bool &compress, bool extend, ICompressor *ecomp, ICopyFileProgress *iProgress, bool direct, bool renameToPrimary, bool *aborted, StringBuffer *_locationName=NULL);
+IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, unsigned recordSize, unsigned twFlags, bool &compress, ICompressor *ecomp, ICopyFileProgress *iProgress, bool *aborted, StringBuffer *_locationName=NULL);
 
 
 #endif

thorlcr/activities/thdiskbaseslave.cpp

@@ -334,9 +334,15 @@ void CDiskWriteSlaveActivityBase::open()
     if (query && compress)
         UNIMPLEMENTED;
 
-    bool direct = query || (external && !firstNode());
-    bool rename = !external || (!query && lastNode());
-    Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, diskRowMinSz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
+    unsigned twFlags = external ? TW_External : 0;
+    if (query || (external && !firstNode()))
+        twFlags |= TW_Direct;
+    if (!external || (!query && lastNode()))
+        twFlags |= TW_RenameToPrimary;
+    if (extend||(external&&!query))
+        twFlags |= TW_Extend;
+
+    Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, diskRowMinSz, twFlags, compress, ecomp, this, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
 
     if (compress)
     {

thorlcr/graph/thgraph.cpp

@@ -554,7 +554,13 @@ void CGraphElementBase::onCreate()
     if (!nullAct)
     {
         CGraphElementBase *ownerActivity = owner->queryOwner() ? owner->queryOwner()->queryElement(ownerId) : NULL;
-        baseHelper->onCreate(queryCodeContext(), ownerActivity ? ownerActivity->queryHelper() : NULL, haveCreateCtx?&createCtxMb:NULL);
+        if (ownerActivity)
+        {
+            ownerActivity->onCreate(); // ensure owner created, might not be if this is child query inside another child query.
+            baseHelper->onCreate(queryCodeContext(), ownerActivity->queryHelper(), haveCreateCtx?&createCtxMb:NULL);
+        }
+        else
+            baseHelper->onCreate(queryCodeContext(), NULL, haveCreateCtx?&createCtxMb:NULL);
     }
 }
 
@@ -974,6 +980,7 @@ bool isGlobalActivity(CGraphElementBase &container)
         case TAKchildcount:
         case TAKwhen_dataset:
         case TAKwhen_action:
+        case TAKnonempty:
             if (!container.queryLocalOrGrouped())
                 return true;
             break;
@@ -1064,7 +1071,6 @@ bool isGlobalActivity(CGraphElementBase &container)
         case TAKrowresult:
         case TAKremotegraph:
         case TAKlibrarycall:
-        case TAKnonempty:
         default:
             return true; // if in doubt
     }