Merge remote-tracking branch 'origin/closedown-4.2.x'

Conflicts:
	ecl/eclcc/eclcc.cpp
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/remote/rmtspawn.cpp

@@ -131,6 +131,12 @@ ISocket * spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoin
     }
     cmd.append(' ').append(args);
 
+    if (abort && abort->abortRequested())
+    {
+        LOG(MCdetailDebugInfo, unknownJob, "Action aborted before connecting to slave (%3d)", replyTag);
+        return NULL;
+    }
+
     if (SSHusername.isEmpty())
     {
 #if defined(_WIN32)
@@ -162,9 +168,6 @@ ISocket * spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoin
     ISocket * result = NULL;
     while (!result && attempts)
     {
-        if (abort && abort->abortRequested())
-            break;
-
         try
         {
             StringBuffer tmp;
@@ -238,6 +241,9 @@ ISocket * spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoin
             MilliSleep(rand()%400+100);
             attempts--;
         }
+
+        if (abort && abort->abortRequested())
+            break;
     }
     if (error)
         throw error;

dali/base/dadfs.cpp

@@ -999,18 +999,14 @@ protected:
         // derived's prepare must call this before locking
         lockedFiles.append(*LINK(file));
     }
-    bool lock(bool *dirty=NULL)
+    bool lock()
     {
         // Files most have been acquired already by derived's class prepare
         ForEachItemIn(i,lockedFiles)
         {
             try
             {
-                if (lockedFiles.item(i).lockProperties(SDS_SUB_LOCK_TIMEOUT)) // returns true if needs reload
-                {
-                    if (dirty)
-                        *dirty = true;
-                }
+                lockedFiles.item(i).lockProperties(0);
             }
             catch (ISDSException *e)
             {
@@ -1334,6 +1330,11 @@ class CDistributedFileTransaction: public CInterface, implements IDistributedFil
     unsigned depth;
     unsigned prepared;
 
+    /* 'owner' is set if, transaction object is implicitly created, because none provided
+     * The owner cannot be release or unlocked. The transaction can still retry if other files are locked,
+     * so need to ensure 'owner' remains in tracked file cache.
+     */
+    IDistributedSuperFile *owner;
 
 
     void validateAddSubFile(IDistributedSuperFile *super, IDistributedFile *sub, const char *subName);
@@ -1359,8 +1360,8 @@ class CDistributedFileTransaction: public CInterface, implements IDistributedFil
 
 public:
     IMPLEMENT_IINTERFACE;
-    CDistributedFileTransaction(IUserDescriptor *user)
-        : isactive(false), depth(0), prepared(0)
+    CDistributedFileTransaction(IUserDescriptor *user, IDistributedSuperFile *_owner=NULL)
+        : isactive(false), depth(0), prepared(0), owner(_owner)
     {
         setUserDescriptor(udesc,user);
     }
@@ -1617,6 +1618,8 @@ public:
     {
         trackedFiles.kill();
         trackedFilesByName.kill();
+        if (owner)
+            addFile(owner); // ensure remains tracked
     }
     void clearFile(IDistributedFile *file)
     {
@@ -2587,7 +2590,10 @@ public:
 #endif
                 try
                 {
-                    safeChangeModeWrite(conn,queryLogicalName(),reload,timeoutms);
+                    if (0 == timeoutms)
+                        conn->changeMode(RTM_LOCK_WRITE, 0, true); // 0 timeout, test and fail immediately if contention
+                    else
+                        safeChangeModeWrite(conn,queryLogicalName(),reload,timeoutms);
                 }
                 catch(IException *)
                 {
@@ -4409,25 +4415,14 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             // Try to lock all files
             addFileLock(parent);
             addFileLock(sub);
-            bool dirty=false;
-            if (lock(&dirty))
+            if (lock())
             {
-                if (dirty)
-                {
-                    // in the process of previous attempt to lock for exclusive access, locks were released
-                    // need to reload to ensure position and # of files is correct
-                    CDistributedSuperFile *sf = dynamic_cast<CDistributedSuperFile *>(parent.get());
-                    if (sf)
-                    {
-                        sf->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
-                        // Potentially the subfiles have changed format or file we wanted is already a member
-                        transaction->validateAddSubFile(parent, sub, subfile);
-                    }
-                }
                 transaction->noteAddSubFile(parent, parentlname, sub);
                 return true;
             }
             unlock();
+            parent.clear();
+            sub.clear();
             return false;
         }
         void run()
@@ -4487,25 +4482,8 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             addFileLock(parent);
             if (sub)
                 addFileLock(sub);
-            bool dirty=false;
-            if (lock(&dirty))
+            if (lock())
             {
-                if (dirty)
-                {
-                    // in the process of previous attempt to lock for exclusive access, locks were released
-                    // need to reload to ensure position and # of files is correct
-                    CDistributedSuperFile *sf = dynamic_cast<CDistributedSuperFile *>(parent.get());
-                    if (sf)
-                    {
-                        sf->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
-                        // potentially subfile _was_ a subfile, but isn't anymore, after dirty update
-                        if (!subfile.isEmpty())
-                        {
-                            if (!transaction->isSubFile(parent, subfile, true))
-                                WARNLOG("addSubFile: File %s is not a subfile of %s", subfile.get(), parent->queryLogicalName());
-                        }
-                    }
-                }
                 if (sub)
                     transaction->noteRemoveSubFile(parent, sub);
                 else
@@ -4513,6 +4491,8 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
                 return true;
             }
             unlock();
+            parent.clear();
+            sub.clear();
             return false;
         }
         void run()
@@ -4564,20 +4544,10 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
                 throw MakeStringException(-1,"removeOwnedSubFiles: SuperFile %s cannot be found", parentlname.get());
             // Try to lock all files
             addFileLock(parent);
-            bool dirty=false;
-            if (lock(&dirty))
-            {
-                if (dirty)
-                {
-                    // in the process of previous attempt to lock for exclusive access, locks were released
-                    // need to reload to ensure position and # of files is correct
-                    CDistributedSuperFile *sf = dynamic_cast<CDistributedSuperFile *>(parent.get());
-                    if (sf)
-                        sf->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
-                }
+            if (lock())
                 return true;
-            }
             unlock();
+            parent.clear();
             return false;
         }
         void run()
@@ -4630,33 +4600,6 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
         Linked<IDistributedSuperFile> file;
         StringAttr parentlname;
         StringAttr filelname;
-
-        bool refresh(IDistributedSuperFile *super) // returns true if any changes
-        {
-            if (!super)
-                return false;
-            IArrayOf<IDistributedFile> copyOfSubFiles;
-            unsigned s=0;
-            for (; s<super->numSubFiles(); s++)
-               copyOfSubFiles.append(*LINK(&super->querySubFile(s)));
-            CDistributedSuperFile *_super = dynamic_cast<CDistributedSuperFile *>(super);
-            _super->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
-            if (copyOfSubFiles.ordinality() != super->numSubFiles())
-                return true;
-            for (s=0; s<super->numSubFiles(); s++)
-            {
-                IDistributedFile *file = &(super->querySubFile(s));
-                if (file != &copyOfSubFiles.item(s))
-                    return true;
-            }
-            return false;
-        }
-        bool refresh() // returns true if any changes
-        {
-            bool pChanged = refresh(parent);
-            bool fChanged = refresh(file);
-            return pChanged || fChanged;
-        }
     public:
         cSwapFileAction(const char *_parentlname,const char *_filelname)
             : parentlname(_parentlname), filelname(_filelname)
@@ -4680,17 +4623,11 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             addFileLock(file);
             for (unsigned i=0; i<file->numSubFiles(); i++)
                 addFileLock(&file->querySubFile(i));
-            bool dirty=false;
-            if (lock(&dirty))
-            {
-                if (!dirty)
-                    return true;
-                // in the process of previous attempt to lock for exclusive access, locks were released
-                // need to reload to ensure position and # of files is correct
-                if (!refresh()) // refreshes the supers and checks if any changes, if there are, transaction will unlock and retry
-                    return true;
-            }
+            if (lock())
+                return true;
             unlock();
+            parent.clear();
+            file.clear();
             return false;
         }
         void run()
@@ -5871,7 +5808,7 @@ public:
             localtrans.set(_transaction);
         }
         else
-            localtrans.setown(new CDistributedFileTransaction(udesc));
+            localtrans.setown(new CDistributedFileTransaction(udesc, this));
         localtrans->ensureFile(this);
 
         if (addcontents)
@@ -5918,7 +5855,7 @@ public:
             localtrans.set(_transaction);
         }
         else
-            localtrans.setown(new CDistributedFileTransaction(udesc));
+            localtrans.setown(new CDistributedFileTransaction(udesc, this));
 
         // Make sure this file is in cache (reuse below)
         localtrans->ensureFile(this);
@@ -5975,7 +5912,7 @@ public:
             localtrans.set(_transaction);
         }
         else
-            localtrans.setown(new CDistributedFileTransaction(udesc));
+            localtrans.setown(new CDistributedFileTransaction(udesc, this));
 
         // Make sure this file is in cache (reuse below)
         localtrans->addFile(this);
@@ -6005,7 +5942,7 @@ public:
             localtrans.set(_transaction);
         }
         else
-            localtrans.setown(new CDistributedFileTransaction(udesc));
+            localtrans.setown(new CDistributedFileTransaction(udesc, this));
         // Make sure this file is in cache
         localtrans->ensureFile(this);
 
@@ -7589,6 +7526,7 @@ public:
                 return true;
         }
         unlock();
+        super.clear();
         return false;
     }
     void retry()
@@ -7699,6 +7637,7 @@ public:
             return true;
         }
         unlock();
+        file.clear();
         return false;
     }
     void run()

dali/sasha/sacoalescer.cpp

@@ -264,11 +264,7 @@ public:
                         while (!wait_program(h,runcode,false)) {
                             stopsem.wait(1000*60);
                             if (stopped) {
-#ifdef _WIN32
-                                interrupt_program(h);
-#else
-                                interrupt_program(h,SIGINT);
-#endif
+                                interrupt_program(h, false);
                                 break;
                             }
                             PROGLOG("COALESCER running");

docs/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml

@@ -82,8 +82,8 @@
 
         <para><itemizedlist>
             <listitem>
-              <para>Keyboard and mouse actions are shown in small caps, such
-              as: DOUBLE-CLICK, or press the ENTER key. word.</para>
+              <para>Keyboard and mouse actions are shown in all caps, such as:
+              DOUBLE-CLICK, or press the ENTER keyword.</para>
             </listitem>
 
             <listitem>
@@ -106,7 +106,8 @@
           url="http://hpccsystems.com/download/free-community-edition/client-tools">http://hpccsystems.com/download/free-community-edition/client-tools</ulink></para>
 
           <para>Download the appropriate Client Tools for your Operating
-          System. (available for CentOS, Ubuntu, Mac OSX, or Windows)</para>
+          System. (available for RPM-Based systems, Debian-Based systems, Mac
+          OSX, or Windows)</para>
         </listitem>
 
         <listitem>
@@ -135,11 +136,99 @@
 
           <para><emphasis role="bold">Mac OSX:</emphasis></para>
 
-          <para>Run the installation file, for example:
-          hpccsystems-clienttools_community-4.X.X-XDarwin-x86_64.dmg. Follow
-          the prompts to complete the installation.</para>
+          <para>Open the Apple disk image file (.dmg) and then run the
+          installation package (.pkg). Follow the prompts to complete the
+          installation.</para>
         </listitem>
       </orderedlist>
     </sect2>
+
+    <sect2 id="Other_Installs">
+      <title>Multiple Version Installations</title>
+
+      <para>It is possible to install multiple versions of the client tools if
+      you need to work with multiple versions of the platform.</para>
+
+      <para>To install the client tools, obtain the appropriate installation
+      package for your operating system and the version to match your HPCC
+      Systems server:</para>
+
+      <orderedlist>
+        <listitem>
+          <para>Download the appropriate Client Tools for your Operating
+          System and version.</para>
+
+          <para>Client tools can be found at the HPCC Systems download
+          page:</para>
+
+          <para><ulink
+          url="http://hpccsystems.com/download/free-community-edition/client-tools">http://hpccsystems.com/download/free-community-edition/client-tools</ulink></para>
+
+          <variablelist>
+            <varlistentry>
+              <term>NOTE:</term>
+
+              <listitem>
+                <para>There is a link at the bottom of the list "<emphasis
+                role="blue">view older downloads</emphasis>" if you are
+                looking for previous versions.</para>
+              </listitem>
+            </varlistentry>
+          </variablelist>
+        </listitem>
+
+        <listitem>
+          <para>Install the Client Tools on to your system. Take note of the
+          following considerations:</para>
+        </listitem>
+      </orderedlist>
+
+      <para>Client tool packages starting with 4.2 have built in logic to
+      allow for multiple installations. Prior versions of the client tools
+      package would just overwrite the existing components. The default
+      behavior is that the client tools will use the last one installed,
+      except if you are working directly on the platform. If you are working
+      directly on the platform then it would use the client tools package that
+      gets installed with the platform.</para>
+
+      <para>If you install a version other than the delivered client tools you
+      will have a folder in /opt/HPCCSystems that corresponds to the set of
+      client tools. So you could have a client tools 4.0.x, 4.2.x, 4.4.x,
+      etc.</para>
+
+      <para>For older versions, download the package(s), and install. Install
+      the one you want to use last. Copy to a different folder or Rename the
+      client tools found in /opt/HPCCSystems after installing the older
+      version and before installing the newer version. This is to prevent the
+      newer client tools from overwriting the older one. </para>
+
+      <para>To use the Client tools for the various version number(s)
+      explicitly call the client tool you wish to use, or set up an alias to
+      call the client tool using the proper path or name for the version you
+      intend to use. This would depend on how you chose to save off the older
+      client tools you installed.</para>
+
+      <para><emphasis role="bold">For example</emphasis>, if you wanted to run
+      eclplus:</para>
+
+      <programlisting>eclplus action=view wuid=W12345678</programlisting>
+
+      <para>To run eclplus for an older or another version of client tools,
+      for instance 4.0.x:</para>
+
+      <para><programlisting>/opt/HPCCSystems/4.0.x/clienttools/bin/eclplus action=view wuid=W12345678</programlisting></para>
+
+      <para><emphasis role="bold">Windows</emphasis></para>
+
+      <para>Client tools for Windows installs in a directory such as:
+      C:\Program Files (x86)\HPCCSystems\4.2.0\clientools where the number
+      (4.2.0 for example) corresponds to the version of the client tools.
+      </para>
+
+      <para>The Windows installer will prompt you to delete the previous
+      version during installation. If you want to keep both, decline the offer
+      to uninstall, and choose a different installation directory at the next
+      prompt.</para>
+    </sect2>
   </sect1>
 </chapter>

ecl/eclcc/eclcc.cpp

@@ -724,7 +724,7 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
             bool optSaveCpp = optSaveTemps || optNoCompile || wu->getDebugValueBool("saveCppTempFiles", false);
             //New scope - testing things are linked correctly
             {
-                Owned<IHqlExprDllGenerator> generator = createDllGenerator(&errorProcessor, processName.toCharArray(), NULL, wu, templateDir, optTargetClusterType, this, false);
+                Owned<IHqlExprDllGenerator> generator = createDllGenerator(&errorProcessor, processName.toCharArray(), NULL, wu, templateDir, optTargetClusterType, this, false, false);
 
                 setWorkunitHash(wu, instance.query);
                 if (!optShared)

ecl/hql/hqlfold.cpp

@@ -1553,15 +1553,17 @@ static bool hashElement(node_operator op, IHqlExpression * expr, unsigned __int6
     ITypeInfo * type = value->queryType();
     switch (type->getTypeCode())
     {
+        case type_qstring:
         case type_string:
             {
-                const char * cdata = static_cast<const char *>(value->queryValue());
-                size32_t len = rtlTrimStrLen(type->getStringLen(), cdata);
+                Owned<ITypeInfo> unknownLengthString = makeStringType(UNKNOWN_LENGTH);
+                Owned<IValue> castValue = value->castTo(unknownLengthString);
+                const char * cdata = static_cast<const char *>(castValue->queryValue());
+                size32_t len = rtlTrimStrLen(castValue->queryType()->getStringLen(), cdata);
                 hashCode = (op == no_hash32) ? rtlHash32Data(len, cdata, (unsigned)hashCode) : rtlHash64Data(len, cdata, hashCode);
                 return true;
             }
         case type_data:
-        case type_qstring:
             {
                 size32_t len = type->getSize();
                 const char * cdata = static_cast<const char *>(value->queryValue());

ecl/hqlcpp/hqlckey.cpp

@@ -204,7 +204,7 @@ void HqlCppTranslator::buildJoinMatchFunction(BuildCtx & ctx, const char * name,
         bindTableCursor(matchctx, left, "left", no_left, selSeq);
         bindTableCursor(matchctx, right, "right", no_right, selSeq);
 
-        OwnedHqlExpr cseMatch = options.spotCSE ? spotScalarCSE(match) : LINK(match);
+        OwnedHqlExpr cseMatch = options.spotCSE ? spotScalarCSE(match, NULL, queryOptions().spotCseInIfDatasetConditions) : LINK(match);
         buildReturn(matchctx, cseMatch);
     }
 }
@@ -533,7 +533,7 @@ void KeyedJoinInfo::buildIndexReadMatch(BuildCtx & ctx)
         OwnedHqlExpr fileposVar = createVariable("_filepos", fileposExpr->getType());
 
         if (translator.queryOptions().spotCSE)
-            matchExpr.setown(spotScalarCSE(matchExpr));
+            matchExpr.setown(spotScalarCSE(matchExpr, NULL, translator.queryOptions().spotCseInIfDatasetConditions));
 
         translator.associateBlobHelper(matchctx, rawKey, "blobs");
 

ecl/hqlcpp/hqlcpp.cpp

@@ -1590,6 +1590,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.convertJoinToLookup,"convertJoinToLookup", true),
         DebugOption(options.convertJoinToLookupIfSorted,"convertJoinToLookupIfSorted", false),
         DebugOption(options.spotCSE,"spotCSE", true),
+        DebugOption(options.spotCseInIfDatasetConditions,"spotCseInIfDatasetConditions", false),
         DebugOption(options.optimizeNonEmpty,"optimizeNonEmpty", !targetThor()),                // not sure that it will be conditional resourced correctly for thor
         DebugOption(options.allowVariableRoxieFilenames,"allowVariableRoxieFilenames", false),
         DebugOption(options.foldConstantDatasets,"foldConstantDatasets", true),
@@ -11519,7 +11520,7 @@ void HqlCppTranslator::doBuildUserFunctionReturn(BuildCtx & ctx, ITypeInfo * typ
             //optimize the way that cses are spotted to minimise unnecessary calculations
             OwnedHqlExpr branches = createComma(LINK(value->queryChild(1)), LINK(value->queryChild(2)));
             OwnedHqlExpr cond = LINK(value->queryChild(0));
-            spotScalarCSE(cond, branches, NULL, NULL);
+            spotScalarCSE(cond, branches, NULL, NULL, queryOptions().spotCseInIfDatasetConditions);
             BuildCtx subctx(ctx);
             IHqlStmt * stmt = buildFilterViaExpr(subctx, cond);
             doBuildUserFunctionReturn(subctx, type, branches->queryChild(0));
@@ -11529,7 +11530,7 @@ void HqlCppTranslator::doBuildUserFunctionReturn(BuildCtx & ctx, ITypeInfo * typ
         }
     default:
         {
-            OwnedHqlExpr optimized = spotScalarCSE(value);
+            OwnedHqlExpr optimized = spotScalarCSE(value, NULL, queryOptions().spotCseInIfDatasetConditions);
             if (value->isAction())
                 buildStmt(ctx, value);
             else

ecl/hqlcpp/hqlcpp.ipp

@@ -600,6 +600,7 @@ struct HqlCppOptions
     bool                convertJoinToLookup;
     bool                convertJoinToLookupIfSorted;
     bool                spotCSE;
+    bool                spotCseInIfDatasetConditions;
     bool                noAllToLookupConversion;
     bool                optimizeNonEmpty;
     bool                allowVariableRoxieFilenames;

ecl/hqlcpp/hqlcppds.cpp

@@ -824,7 +824,7 @@ void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoun
         }
     case no_sum:
         {
-            OwnedHqlExpr cseArg = options.spotCSE ? spotScalarCSE(arg) : LINK(arg);
+            OwnedHqlExpr cseArg = options.spotCSE ? spotScalarCSE(arg, NULL, queryOptions().spotCseInIfDatasetConditions) : LINK(arg);
             buildIncrementAssign(loopctx, target, cseArg);
             break;
         }
@@ -834,7 +834,7 @@ void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoun
             BuildCtx maxctx(loopctx);
             OwnedHqlExpr resultExpr = target.getTranslatedExpr();
 
-            OwnedHqlExpr cseArg = options.spotCSE ? spotScalarCSE(arg) : LINK(arg);
+            OwnedHqlExpr cseArg = options.spotCSE ? spotScalarCSE(arg, NULL, queryOptions().spotCseInIfDatasetConditions) : LINK(arg);
             OwnedHqlExpr simpleArg = buildSimplifyExpr(loopctx, cseArg);
             OwnedHqlExpr test = createBoolExpr((op == no_min) ? no_lt : no_gt, LINK(simpleArg), LINK(resultExpr));
             if (doneFirstVar)
@@ -3255,7 +3255,7 @@ void HqlCppTranslator::buildDatasetAssignJoin(BuildCtx & ctx, IHqlCppDatasetBuil
     BoundRow * rightCursor = buildDatasetIterate(rightIterCtx, right, false);
     bindTableCursor(rightIterCtx, right, rightCursor->queryBound(), no_right, selSeq);
 
-    OwnedHqlExpr cseCond = options.spotCSE ? spotScalarCSE(cond) : LINK(cond);
+    OwnedHqlExpr cseCond = options.spotCSE ? spotScalarCSE(cond, NULL, queryOptions().spotCseInIfDatasetConditions) : LINK(cond);
     buildFilter(rightIterCtx, cseCond);
     if (!expr->hasAttribute(leftonlyAtom))
     {

ecl/hqlcpp/hqlcse.cpp

@@ -207,8 +207,8 @@ bool CseSpotterInfo::useInverseForAlias()
 
 
 static HqlTransformerInfo cseSpotterInfo("CseSpotter");
-CseSpotter::CseSpotter() 
-: NewHqlTransformer(cseSpotterInfo)
+CseSpotter::CseSpotter(bool _spotCseInIfDatasetConditions)
+: NewHqlTransformer(cseSpotterInfo), spotCseInIfDatasetConditions(_spotCseInIfDatasetConditions)
 {
     canAlias = true;
     isAssociated = false;
@@ -288,7 +288,7 @@ void CseSpotter::analyseExpr(IHqlExpression * expr)
         extra->canAlias = true;
 
     bool savedCanAlias = canAlias;
-    if (expr->isDataset() && (op != no_select))// && (op != no_if))
+    if (expr->isDataset() && (op != no_select) && (!spotCseInIfDatasetConditions || (op != no_if)))
     {
         //There is little point looking for CSEs within dataset expressions, because only a very small
         //minority which would correctly cse, and it can cause lots of problems - e.g., join conditions.
@@ -1108,7 +1108,7 @@ ANewTransformInfo * CseScopeTransformer::createTransformInfo(IHqlExpression * ex
 #endif
 
 
-IHqlExpression * spotScalarCSE(IHqlExpression * expr, IHqlExpression * limit)
+IHqlExpression * spotScalarCSE(IHqlExpression * expr, IHqlExpression * limit, bool spotCseInIfDatasetConditions)
 {
     if (expr->isConstant())
         return LINK(expr);
@@ -1125,7 +1125,7 @@ IHqlExpression * spotScalarCSE(IHqlExpression * expr, IHqlExpression * limit)
     bool addedAliases = false;
     //First spot the aliases - so that restructuring the ands doesn't lose any existing aliases.
     {
-        CseSpotter spotter;
+        CseSpotter spotter(spotCseInIfDatasetConditions);
         spotter.analyse(transformed, 0);
         if (spotter.foundCandidates())
         {
@@ -1162,9 +1162,9 @@ IHqlExpression * spotScalarCSE(IHqlExpression * expr, IHqlExpression * limit)
 }
 
 
-void spotScalarCSE(SharedHqlExpr & expr, SharedHqlExpr & associated, IHqlExpression * limit, IHqlExpression * invariantSelector)
+void spotScalarCSE(SharedHqlExpr & expr, SharedHqlExpr & associated, IHqlExpression * limit, IHqlExpression * invariantSelector, bool spotCseInIfDatasetConditions)
 {
-    CseSpotter spotter;
+    CseSpotter spotter(spotCseInIfDatasetConditions);
     spotter.analyse(expr, 0);
     if (associated)
         spotter.analyseAssociated(associated, 0);
@@ -1179,9 +1179,9 @@ void spotScalarCSE(SharedHqlExpr & expr, SharedHqlExpr & associated, IHqlExpress
 }
 
 
-void spotScalarCSE(HqlExprArray & exprs, HqlExprArray & associated, IHqlExpression * limit, IHqlExpression * invariantSelector)
+void spotScalarCSE(HqlExprArray & exprs, HqlExprArray & associated, IHqlExpression * limit, IHqlExpression * invariantSelector, bool spotCseInIfDatasetConditions)
 {
-    CseSpotter spotter;
+    CseSpotter spotter(spotCseInIfDatasetConditions);
     spotter.analyseArray(exprs, 0);
     ForEachItemIn(ia, associated)
         spotter.analyseAssociated(&associated.item(ia), 0);

ecl/hqlcpp/hqlcse.ipp

@@ -52,7 +52,7 @@ class CseSpotter : public NewHqlTransformer
 {
     typedef NewHqlTransformer PARENT;
 public:
-    CseSpotter();
+    CseSpotter(bool _spotCseInIfDatasetConditions);
 
     void analyseAssociated(IHqlExpression * expr, unsigned pass);
     bool foundCandidates() const                            { return spottedCandidate; }
@@ -84,6 +84,7 @@ protected:
     bool spottedCandidate;
     bool createLocalAliases;
     bool createdAlias;
+    bool spotCseInIfDatasetConditions;
 };
 
 class ConjunctionTransformer : public NewHqlTransformer
@@ -212,9 +213,9 @@ protected:
 #endif
 
 
-IHqlExpression * spotScalarCSE(IHqlExpression * expr, IHqlExpression * limit = NULL);
-void spotScalarCSE(SharedHqlExpr & expr, SharedHqlExpr & associated, IHqlExpression * limit, IHqlExpression * invariantSelector);
-void spotScalarCSE(HqlExprArray & exprs, HqlExprArray & associated, IHqlExpression * limit, IHqlExpression * invariantSelector);
+IHqlExpression * spotScalarCSE(IHqlExpression * expr, IHqlExpression * limit, bool spotCseInIfDatasetConditions);
+void spotScalarCSE(SharedHqlExpr & expr, SharedHqlExpr & associated, IHqlExpression * limit, IHqlExpression * invariantSelector, bool spotCseInIfDatasetConditions);
+void spotScalarCSE(HqlExprArray & exprs, HqlExprArray & associated, IHqlExpression * limit, IHqlExpression * invariantSelector, bool spotCseInIfDatasetConditions);
 
 //---------------------------------------------------------------------------
 

ecl/hqlcpp/hqlecl.cpp

@@ -60,8 +60,8 @@ class NullContextCallback : public CInterface, implements ICodegenContextCallbac
 class HqlDllGenerator : public CInterface, implements IHqlExprDllGenerator, implements IAbortRequestCallback
 {
 public:
-    HqlDllGenerator(IErrorReceiver * _errs, const char * _wuname, const char * _targetdir, IWorkUnit * _wu, const char * _template_dir, ClusterType _targetClusterType, ICodegenContextCallback * _ctxCallback, bool _checkForLocalFileUploads) :
-        errs(_errs), wuname(_wuname), targetDir(_targetdir), wu(_wu), template_dir(_template_dir), targetClusterType(_targetClusterType), ctxCallback(_ctxCallback), checkForLocalFileUploads(_checkForLocalFileUploads)
+    HqlDllGenerator(IErrorReceiver * _errs, const char * _wuname, const char * _targetdir, IWorkUnit * _wu, const char * _template_dir, ClusterType _targetClusterType, ICodegenContextCallback * _ctxCallback, bool _checkForLocalFileUploads, bool _okToAbort) :
+        errs(_errs), wuname(_wuname), targetDir(_targetdir), wu(_wu), template_dir(_template_dir), targetClusterType(_targetClusterType), ctxCallback(_ctxCallback), checkForLocalFileUploads(_checkForLocalFileUploads), okToAbort(_okToAbort)
     {
         if (!ctxCallback)
             ctxCallback.setown(new NullContextCallback);
@@ -121,6 +121,7 @@ protected:
     bool noOutput;
     EclGenerateTarget generateTarget;
     bool deleteGenerated;
+    bool okToAbort;
 };
 
 
@@ -537,7 +538,8 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
     wu->getDebugValue("compileOptions", optionAdaptor);
     compiler->addCompileOption(options.str());
 
-    compiler->setAbortChecker(this);
+    if (okToAbort)
+        compiler->setAbortChecker(this);
 
     MTIME_SECTION (timer, "Compile_code");
     unsigned time = msTick();
@@ -613,14 +615,14 @@ offset_t HqlDllGenerator::getGeneratedSize() const
 
 extern HQLCPP_API double getECLcomplexity(IHqlExpression * exprs, IErrorReceiver * errs, IWorkUnit *wu, ClusterType targetClusterType)
 {
-    HqlDllGenerator generator(errs, "unknown", NULL, wu, NULL, targetClusterType, NULL, false);
+    HqlDllGenerator generator(errs, "unknown", NULL, wu, NULL, targetClusterType, NULL, false, false);
     return generator.getECLcomplexity(exprs);
 }
 
 
-extern HQLCPP_API IHqlExprDllGenerator * createDllGenerator(IErrorReceiver * errs, const char *wuname, const char * targetdir, IWorkUnit *wu, const char * template_dir, ClusterType targetClusterType, ICodegenContextCallback *ctxCallback, bool checkForLocalFileUploads)
+extern HQLCPP_API IHqlExprDllGenerator * createDllGenerator(IErrorReceiver * errs, const char *wuname, const char * targetdir, IWorkUnit *wu, const char * template_dir, ClusterType targetClusterType, ICodegenContextCallback *ctxCallback, bool checkForLocalFileUploads, bool okToAbort)
 {
-    return new HqlDllGenerator(errs, wuname, targetdir, wu, template_dir, targetClusterType, ctxCallback, checkForLocalFileUploads);
+    return new HqlDllGenerator(errs, wuname, targetdir, wu, template_dir, targetClusterType, ctxCallback, checkForLocalFileUploads, okToAbort);
 }
 
 /*

ecl/hqlcpp/hqlecl.hpp

@@ -51,7 +51,7 @@ public:
     virtual void setSaveGeneratedFiles(bool value) = 0;
 };
 
-extern HQLCPP_API IHqlExprDllGenerator * createDllGenerator(IErrorReceiver * errs, const char *wuname, const char * targetdir, IWorkUnit *wu, const char * template_dir, ClusterType targetClusterType, ICodegenContextCallback * ctxCallback, bool checkForLocalFileUploads);
+extern HQLCPP_API IHqlExprDllGenerator * createDllGenerator(IErrorReceiver * errs, const char *wuname, const char * targetdir, IWorkUnit *wu, const char * template_dir, ClusterType targetClusterType, ICodegenContextCallback * ctxCallback, bool checkForLocalFileUploads, bool okToAbort);
 
 
 //Extract a single level of external libraries.

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1320,7 +1320,7 @@ void HqlCppTranslator::filterExpandAssignments(BuildCtx & ctx, TransformBuilder
     LinkedHqlExpr expr = rawExpr;
 
     if (options.spotCSE)
-        expr.setown(spotScalarCSE(expr));
+        expr.setown(spotScalarCSE(expr, NULL, queryOptions().spotCseInIfDatasetConditions));
     traceExpression("transform cse", expr);
 
 //  expandAliases(ctx, expr);
@@ -3113,7 +3113,7 @@ void HqlCppTranslator::doBuildFunction(BuildCtx & ctx, ITypeInfo * type, const c
     {
         LinkedHqlExpr cseValue = value;
         if (options.spotCSE)
-            cseValue.setown(spotScalarCSE(cseValue));
+            cseValue.setown(spotScalarCSE(cseValue, NULL, queryOptions().spotCseInIfDatasetConditions));
 
         BuildCtx funcctx(ctx);
         if (false)
@@ -5121,7 +5121,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
     {
         LinkedHqlExpr cseValue = castValue;
         if (options.spotCSE)
-            cseValue.setown(spotScalarCSE(cseValue));
+            cseValue.setown(spotScalarCSE(cseValue, NULL, queryOptions().spotCseInIfDatasetConditions));
 
         if ((retType == type_set) && isComplexSet(resultType, false) && castValue->getOperator() == no_list && !isNullList(castValue))
         {
@@ -12327,7 +12327,7 @@ void HqlCppTranslator::buildProcessTransformFunction(BuildCtx & ctx, IHqlExpress
         //self won't clash, so can generate efficient code.
         //Perform cse on both transforms
         OwnedHqlExpr comma = createComma(LINK(transformRow), LINK(transformRight));
-        comma.setown(spotScalarCSE(comma));
+        comma.setown(spotScalarCSE(comma, NULL, queryOptions().spotCseInIfDatasetConditions));
         if (comma->getOperator() == no_alias_scope)
             comma.set(comma->queryChild(0));
 
@@ -14003,7 +14003,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityFirstN(BuildCtx & ctx, IHqlExp
     funcctx.addQuotedCompound("virtual __int64 getLimit()");
     OwnedHqlExpr newLimit = ensurePositiveOrZeroInt64(limit);
     if (options.spotCSE)
-        newLimit.setown(spotScalarCSE(newLimit));
+        newLimit.setown(spotScalarCSE(newLimit, NULL, queryOptions().spotCseInIfDatasetConditions));
     buildReturn(funcctx, newLimit);
 
     if (queryRealChild(expr, 2))
@@ -14679,7 +14679,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityProject(BuildCtx & ctx, IHqlEx
             case no_filter:
                 {
                     LinkedHqlExpr invariant;
-                    OwnedHqlExpr cond = extractFilterConditions(invariant, dataset, normalized, false);
+                    OwnedHqlExpr cond = extractFilterConditions(invariant, dataset, normalized, false, false);
                     //A dataset invariant filter is only worth combining if the engine supports a filtered project operation.
                     if (!options.supportFilterProject && invariant)
                     {
@@ -14902,7 +14902,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityExecuteWhen(BuildCtx & ctx, IH
 
 //---------------------------------------------------------------------------
 
-IHqlExpression * extractFilterConditions(HqlExprAttr & invariant, IHqlExpression * expr, IHqlExpression * dataset, bool spotCSE)
+IHqlExpression * extractFilterConditions(HqlExprAttr & invariant, IHqlExpression * expr, IHqlExpression * dataset, bool spotCSE, bool spotCseInIfDatasetConditions)
 {
     unsigned num = expr->numChildren();
     assertex(num > 1);
@@ -14919,7 +14919,7 @@ IHqlExpression * extractFilterConditions(HqlExprAttr & invariant, IHqlExpression
         return NULL;
 
     if (spotCSE)
-        cond.setown(spotScalarCSE(cond));
+        cond.setown(spotScalarCSE(cond, NULL, spotCseInIfDatasetConditions));
 
     HqlExprArray tests;
     cond->unwindList(tests, no_and);
@@ -14953,7 +14953,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityFilter(BuildCtx & ctx, IHqlExp
     buildInstancePrefix(instance);
 
     HqlExprAttr invariant;
-    OwnedHqlExpr cond = extractFilterConditions(invariant, expr, dataset, options.spotCSE);
+    OwnedHqlExpr cond = extractFilterConditions(invariant, expr, dataset, options.spotCSE, queryOptions().spotCseInIfDatasetConditions);
 
     //Base class returns true, so only generate if no non-invariant conditions
     if (cond)
@@ -14992,7 +14992,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityFilterGroup(BuildCtx & ctx, IH
 
     HqlExprAttr invariant;
     OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
-    OwnedHqlExpr cond = extractFilterConditions(invariant, expr, left, options.spotCSE);
+    OwnedHqlExpr cond = extractFilterConditions(invariant, expr, left, options.spotCSE, options.spotCseInIfDatasetConditions);
 
     //Base class returns true, so only generate if no non-invariant conditions
     if (cond)
@@ -15328,7 +15328,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityCatch(BuildCtx & ctx, IHqlExpr
         BuildCtx isMatchCtx(instance->startctx);
         isMatchCtx.addQuotedCompound("virtual bool isMatch(IException * except)");
         associateLocalFailure(isMatchCtx, "except");
-        OwnedHqlExpr cseFilter = spotScalarCSE(filter);
+        OwnedHqlExpr cseFilter = spotScalarCSE(filter, NULL, queryOptions().spotCseInIfDatasetConditions);
         buildReturn(isMatchCtx, cseFilter, queryBoolType());
     }
 
@@ -15766,7 +15766,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityIf(BuildCtx & ctx, IHqlExpress
     }
 
 
-    OwnedHqlExpr cseCond = options.spotCSE ? spotScalarCSE(cond) : LINK(cond);
+    OwnedHqlExpr cseCond = options.spotCSE ? spotScalarCSE(cond, NULL, queryOptions().spotCseInIfDatasetConditions) : LINK(cond);
     bool isChild = (insideChildOrLoopGraph(ctx) || insideRemoteGraph(ctx) || insideLibrary());
     IHqlExpression * activeGraph = queryActiveSubGraph(ctx)->graphTag;
 
@@ -15887,7 +15887,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityChoose(BuildCtx & ctx, IHqlExp
     funcctx.addQuotedCompound("virtual unsigned getBranch()");
     OwnedHqlExpr fullCond(foldHqlExpression(cond));
     if (options.spotCSE)
-        fullCond.setown(spotScalarCSE(fullCond));
+        fullCond.setown(spotScalarCSE(fullCond, NULL, queryOptions().spotCseInIfDatasetConditions));
     buildReturn(funcctx, fullCond);
 
     StringBuffer label;
@@ -15980,7 +15980,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityCase(BuildCtx & ctx, IHqlExpre
     OwnedHqlExpr fullCond = createValue(op, LINK(unsignedType), args);
     fullCond.setown(foldHqlExpression(fullCond));
     if (options.spotCSE)
-        fullCond.setown(spotScalarCSE(fullCond));
+        fullCond.setown(spotScalarCSE(fullCond, NULL, queryOptions().spotCseInIfDatasetConditions));
     buildReturn(funcctx, fullCond);
 
     bool graphIndependent = isGraphIndependent(fullCond, activeGraph);

ecl/hqlcpp/hqlhtcpp.ipp

@@ -294,7 +294,7 @@ protected:
     bool            matchedDataset;
 };
 
-IHqlExpression * extractFilterConditions(HqlExprAttr & invariant, IHqlExpression * expr, IHqlExpression * dataset, bool spotCSE);
+IHqlExpression * extractFilterConditions(HqlExprAttr & invariant, IHqlExpression * expr, IHqlExpression * dataset, bool spotCSE, bool spotCseInIfDatasetConditions);
 bool isLibraryScope(IHqlExpression * expr);
 extern IHqlExpression * constantMemberMarkerExpr;
 

ecl/hqlcpp/hqliter.cpp

@@ -149,7 +149,7 @@ void TransformSequenceBuilder::buildSequence(BuildCtx & ctx, BuildCtx * declarec
             if (translator.queryOptions().foldFilter)
                 test.setown(foldScopedHqlExpression(translator.queryErrorProcessor(), expr->queryChild(0)->queryNormalizedSelector(), test));
             if (translator.queryOptions().spotCSE)
-                test.setown(spotScalarCSE(test));
+                test.setown(spotScalarCSE(test, NULL, translator.queryOptions().spotCseInIfDatasetConditions));
             translator.buildFilteredReturn(ctx, test, failedFilterValue);
         }
         break;

ecl/hqlcpp/hqlnlp.cpp

@@ -259,7 +259,7 @@ void NlpParseContext::buildValidators(HqlCppTranslator & translator, BuildCtx &
             validctx.associateExpr(activeValidateMarkerExpr, activeValidateMarkerExpr);
             translator.bindTableCursor(validctx, queryNlpParsePseudoTable(), queryNlpParsePseudoTable());
             if (translator.queryOptions().spotCSE)
-                validateExpr.setown(spotScalarCSE(validateExpr));
+                validateExpr.setown(spotScalarCSE(validateExpr, NULL, translator.queryOptions().spotCseInIfDatasetConditions));
             translator.buildReturn(validctx, validateExpr);
             translator.endNestedClass();
 

ecl/hqlcpp/hqlresource.cpp

@@ -3024,7 +3024,7 @@ void EclResourcer::deriveUsageCounts(IHqlExpression * expr)
             else
             {
                 LinkedHqlExpr invariant;
-                OwnedHqlExpr cond = extractFilterConditions(invariant, expr, expr->queryNormalizedSelector(), false);
+                OwnedHqlExpr cond = extractFilterConditions(invariant, expr, expr->queryNormalizedSelector(), false, false);
                 if (invariant)
                     info->isConditionalFilter = true;
             }

ecl/hqlcpp/hqlsource.cpp

@@ -1382,7 +1382,7 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
                         test.setown(foldScopedHqlExpression(translator.queryErrorProcessor(), ds->queryNormalizedSelector(), test));
 
                     if (translator.options.spotCSE)
-                        test.setown(spotScalarCSE(test, ds));
+                        test.setown(spotScalarCSE(test, ds, translator.queryOptions().spotCseInIfDatasetConditions));
 
                     if (!returnIfFilterFails)
                         translator.buildFilter(ctx, test);
@@ -4742,7 +4742,7 @@ void MonitorExtractor::spotSegmentCSE(BuildCtx & ctx)
     HqlExprArray associated;
     IHqlExpression * selector = tableExpr->queryNormalizedSelector();
     translator.traceExpressions("before seg spot", conditions);
-    spotScalarCSE(conditions, associated, NULL, selector);
+    spotScalarCSE(conditions, associated, NULL, selector, translator.queryOptions().spotCseInIfDatasetConditions);
     translator.traceExpressions("after seg spot", conditions);
 
     unsigned curCond = 0;

initfiles/bash/etc/init.d/hpcc_common.in

@@ -379,7 +379,8 @@ createRuntime() {
     # setting up ulimit for thor and other component which needs it.
     ulimit -n 32768
     ulimit -c unlimited
-
+    # for rtprio in Roxie
+    ulimit -Hr 4
 }
 
 start_dafilesrv() {

roxie/ccd/ccdserver.cpp

@@ -1176,6 +1176,15 @@ public:
         }
     }
 
+    void stopDependencies(unsigned parentExtractSize, const byte *parentExtract, unsigned controlId)
+    {
+        ForEachItemIn(idx, dependencies)
+        {
+            if (dependencyControlIds.item(idx) == controlId)
+                dependencies.item(idx).stop(false);
+        }
+    }
+
     virtual unsigned __int64 queryTotalCycles() const
     {
         return totalCycles;
@@ -19067,12 +19076,13 @@ public:
 
     virtual void doExecuteAction(unsigned parentExtractSize, const byte * parentExtract) 
     {
-        int controlId;
+        bool cond;
         {
             ActivityTimer t(totalCycles, timeActivities, ctx->queryDebugContext());
-            controlId = helper.getCondition() ? 1 : 2;
+            cond = helper.getCondition();
         }
-        executeDependencies(parentExtractSize, parentExtract, controlId);
+        stopDependencies(parentExtractSize, parentExtract, cond ? 2 : 1);
+        executeDependencies(parentExtractSize, parentExtract, cond ? 1 : 2);
     }
 
 };
@@ -19222,7 +19232,10 @@ public:
     virtual void stop(bool aborting)
     {
         if (state != STATEstopped)
-            executeDependencies(savedExtractSize, savedExtract, aborting ? WhenFailureId : WhenSuccessId);
+        {
+            stopDependencies(savedExtractSize, savedExtract, aborting ? WhenSuccessId : WhenFailureId);  // These ones don't get executed
+            executeDependencies(savedExtractSize, savedExtract, aborting ? WhenFailureId : WhenSuccessId); // These ones do
+        }
         CRoxieServerActivity::stop(aborting);
     }
 
@@ -19282,7 +19295,10 @@ public:
     virtual void stop(bool aborting)
     {
         if (state != STATEstopped)
+        {
+            stopDependencies(savedExtractSize, savedExtract, aborting ? WhenSuccessId : WhenFailureId);  // these are NOT going to execute
             executeDependencies(savedExtractSize, savedExtract, aborting ? WhenFailureId : WhenSuccessId);
+        }
         CRoxieServerActionBaseActivity::stop(aborting);
     }
 

system/jlib/jcomp.cpp

@@ -737,18 +737,32 @@ class CCompilerWorker : public CInterface, implements IPooledThread
 public:
     IMPLEMENT_IINTERFACE;
 
-    CCompilerWorker(CppCompiler * _compiler) : compiler(_compiler)  {}
+    CCompilerWorker(CppCompiler * _compiler, bool _okToAbort) : compiler(_compiler), okToAbort(_okToAbort)
+    {
+        handle = 0;
+        aborted = false;
+    }
     bool canReuse()             { return true; }
-    bool stop()                 { return true; }
+    bool stop()
+    {
+        if (okToAbort)
+            interrupt_program(handle, true);
+        aborted = true;
+        return true;
+    }
     void init(void *_params)    { params.set((CCompilerThreadParam *)_params); }
 
     void main()
     {
         DWORD runcode = 0;
         bool success;
+        aborted = false;
+        handle = 0;
         try
         {
-            success = invoke_program(params->cmdline, runcode, true, params->logfile) && (runcode == 0);
+            success = invoke_program(params->cmdline, runcode, false, params->logfile, &handle, false, okToAbort);
+            if (success)
+                wait_program(handle, runcode, true);
         }
         catch(IException* e)
         {
@@ -759,8 +773,9 @@ public:
                 PrintLog("%s", sb.str());
             success = false;
         }
+        handle = 0;
 
-        if (!success)
+        if (!success || aborted)
             atomic_inc(&compiler->numFailed);
         params->finishedCompiling.signal();
         return;
@@ -769,9 +784,12 @@ public:
 private:
     CppCompiler * compiler;
     Owned<CCompilerThreadParam> params;
+    HANDLE handle;
+    bool aborted;
+    bool okToAbort;
 };
 
 IPooledThread *CppCompiler::createNew()
 {
-    return new CCompilerWorker(this);
+    return new CCompilerWorker(this, (abortChecker != NULL));
 }

system/jlib/jmisc.cpp

@@ -404,7 +404,7 @@ jlib_decl char* readarg(char*& curptr)
 }
 
 #ifdef _WIN32
-bool invoke_program(const char *command_line, DWORD &runcode, bool wait, const char *outfile, HANDLE *rethandle, bool throwException)
+bool invoke_program(const char *command_line, DWORD &runcode, bool wait, const char *outfile, HANDLE *rethandle, bool throwException, bool newProcessGroup)
 {
     runcode = 0;
     if (rethandle)
@@ -489,9 +489,9 @@ bool wait_program(HANDLE handle,DWORD &runcode,bool block)
     return false;
 }
 
-jlib_decl bool interrupt_program(HANDLE handle,int signum)
+jlib_decl bool interrupt_program(HANDLE handle, bool stopChildren, int signum)
 {
-    if (signum==-9) 
+    if (signum==0)
         return TerminateProcess(handle,1)!=FALSE;
     ERRLOG("interrupt_program signal %d not supported in windows",signum);
     return false;
@@ -504,7 +504,7 @@ void close_program(HANDLE handle)
 
 
 #else
-bool invoke_program(const char *command_line, DWORD &runcode, bool wait, const char *outfile, HANDLE *rethandle, bool throwException)
+bool invoke_program(const char *command_line, DWORD &runcode, bool wait, const char *outfile, HANDLE *rethandle, bool throwException, bool newProcessGroup)
 {
     runcode = 0;
     if (rethandle)
@@ -515,6 +515,9 @@ bool invoke_program(const char *command_line, DWORD &runcode, bool wait, const c
     pid_t pid = fork();
     if (pid == 0) 
     {
+        //Force the child process into its own process group, so we can terminate it and its children.
+        if (newProcessGroup)
+            setpgid(0,0);
         if (outfile&&*outfile) {
             int outh = open(outfile, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
             if(outh >= 0)
@@ -622,11 +625,20 @@ bool wait_program(HANDLE handle,DWORD &runcode,bool block)
 }
 
 
-bool interrupt_program(HANDLE handle,int signum)
+bool interrupt_program(HANDLE handle, bool stopChildren, int signum)
 {
+    if (signum == 0)
+        signum = SIGINT;
+
     pid_t pid = (pid_t)handle;
     if ((int)pid<=0)
         return false;
+
+    //If we need to also stop child processes then kill the process group (same as the pid)
+    //Note: This will not apply to grand-children started by the children by calling invoke_program()
+    //since they will have a different process group
+    if (stopChildren)
+        pid = -pid;
     return (kill(pid, signum)==0);
 }
 

system/jlib/jmisc.hpp

@@ -84,9 +84,9 @@ jlib_decl FILE *xfopen(const char *path, const char *mode);
 jlib_decl const char * queryCcLogName();
 jlib_decl StringBuffer& queryCcLogName(const char* wuid, StringBuffer& logname);
 jlib_decl char* readarg(char*& curptr);
-jlib_decl bool invoke_program(const char *command_line, DWORD &runcode, bool wait=true, const char *outfile=NULL, HANDLE *rethandle=NULL, bool throwException = false);
+jlib_decl bool invoke_program(const char *command_line, DWORD &runcode, bool wait=true, const char *outfile=NULL, HANDLE *rethandle=NULL, bool throwException = false, bool newProcessGroup = false);
 jlib_decl bool wait_program(HANDLE handle,DWORD &runcode,bool block=true);
-jlib_decl bool interrupt_program(HANDLE handle,int signum=-9);
+jlib_decl bool interrupt_program(HANDLE handle, bool killChildren, int signum=0); // no signum means use default
 
 #ifndef _WIN32
 jlib_decl bool CopyFile(const char *file, const char *newfile, bool fail);

system/mp/mpcomm.cpp

@@ -1842,6 +1842,7 @@ bool CMPServer::recv(CMessageBuffer &mbuf, const SocketEndpoint *ep, mptag_t tag
         bool aborted;
         CMessageBuffer *result;
         const SocketEndpoint *ep;
+        SocketEndpoint closedEp; // used if receiving on RANK_ALL
         mptag_t tag;
         Cnfy(const SocketEndpoint *_ep,mptag_t _tag) { ep = _ep; tag = _tag; result = NULL; aborted=false; }
         bool notify(CMessageBuffer *msg)
@@ -1858,9 +1859,15 @@ bool CMPServer::recv(CMessageBuffer &mbuf, const SocketEndpoint *ep, mptag_t tag
             }
             return false;
         }
-        bool notifyClosed(SocketEndpoint &closedep) // called when connection closed
+        bool notifyClosed(SocketEndpoint &_closedEp) // called when connection closed
         {
-            if (ep&&ep->equals(closedep)) {
+            if (NULL == ep) { // ep is NULL if receiving on RANK_ALL
+                closedEp = _closedEp;
+                ep = &closedEp; // used for abort info
+                aborted = true;
+                return true;
+            }
+            else if (ep->equals(_closedEp)) {
                 aborted = true;
                 return true;
             }
@@ -1877,7 +1884,7 @@ bool CMPServer::recv(CMessageBuffer &mbuf, const SocketEndpoint *ep, mptag_t tag
         LOG(MCdebugInfo(100), unknownJob, "CMPserver::recv closed on notify");
         PrintStackReport();
 #endif
-        IMP_Exception *e=new CMPException(MPERR_link_closed,*ep);
+        IMP_Exception *e=new CMPException(MPERR_link_closed,*nfy.ep);
         throw e;
     }
     return false;

testing/regress/ecl/issue10921.ecl

@@ -0,0 +1,66 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+do(func, value) := MACRO
+ PARALLEL(
+    output(func + '(' + #TEXT(value) + ')'),
+    #EXPAND(func)((string)value),
+    #EXPAND(func)((qstring)value),
+    #EXPAND(func)((data)value)
+    )
+ENDMACRO;
+
+doU(func, value) := MACRO
+ PARALLEL(
+    output(func + '(' + #TEXT(value) + ')'),
+    #EXPAND(func)((unicode)value),
+    #EXPAND(func)((utf8)value),
+    #EXPAND(func)((data)value)
+    )
+ENDMACRO;
+
+s1 := 'ABCD123 5' : stored('s1');
+s2 := 'ABCD123 5 ' : stored('s2');
+
+do('HASH', 'ABCD123 5');
+do('HASH', s1);
+do('HASH', 'ABCD123 5 ');
+do('HASH', s2);
+do('HASH32', 'ABCD123 5');
+do('HASH32', s1);
+do('HASH32', 'ABCD123 5 ');
+do('HASH32', s2);
+do('HASH64', 'ABCD123 5');
+do('HASH64', s1);
+do('HASH64', 'ABCD123 5 ');
+do('HASH64', s2);
+
+u1 := U'ABCD123 5' : stored('u1');
+u2 := U'ABCD123 5 ' : stored('u2');
+
+do('HASH', U'ABCD123 5');
+do('HASH', u1);
+do('HASH', U'ABCD123 5 ');
+do('HASH', u2);
+do('HASH32', U'ABCD123 5');
+do('HASH32', u1);
+do('HASH32', U'ABCD123 5 ');
+do('HASH32', u2);
+do('HASH64', U'ABCD123 5');
+do('HASH64', u1);
+do('HASH64', U'ABCD123 5 ');
+do('HASH64', u2);

testing/regress/ecl/key/issue10921.xml

@@ -0,0 +1,288 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>HASH(&apos;ABCD123 5&apos;)</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>3536986230</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>3536986230</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>3536986230</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>HASH(s1)</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>3536986230</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>3536986230</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>3536986230</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>HASH(&apos;ABCD123 5 &apos;)</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>3536986230</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>3536986230</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>3770318389</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>HASH(s2)</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>3536986230</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>3536986230</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>3770318389</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>HASH32(&apos;ABCD123 5&apos;)</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>3612652982</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>3612652982</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>3612652982</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>HASH32(s1)</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>3612652982</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><Result_23>3612652982</Result_23></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><Result_24>3612652982</Result_24></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><Result_25>HASH32(&apos;ABCD123 5 &apos;)</Result_25></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>3612652982</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><Result_27>3612652982</Result_27></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><Result_28>2958691746</Result_28></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><Result_29>HASH32(s2)</Result_29></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><Result_30>3612652982</Result_30></Row>
+</Dataset>
+<Dataset name='Result 31'>
+ <Row><Result_31>3612652982</Result_31></Row>
+</Dataset>
+<Dataset name='Result 32'>
+ <Row><Result_32>2958691746</Result_32></Row>
+</Dataset>
+<Dataset name='Result 33'>
+ <Row><Result_33>HASH64(&apos;ABCD123 5&apos;)</Result_33></Row>
+</Dataset>
+<Dataset name='Result 34'>
+ <Row><Result_34>11165648424516303926</Result_34></Row>
+</Dataset>
+<Dataset name='Result 35'>
+ <Row><Result_35>11165648424516303926</Result_35></Row>
+</Dataset>
+<Dataset name='Result 36'>
+ <Row><Result_36>11165648424516303926</Result_36></Row>
+</Dataset>
+<Dataset name='Result 37'>
+ <Row><Result_37>HASH64(s1)</Result_37></Row>
+</Dataset>
+<Dataset name='Result 38'>
+ <Row><Result_38>11165648424516303926</Result_38></Row>
+</Dataset>
+<Dataset name='Result 39'>
+ <Row><Result_39>11165648424516303926</Result_39></Row>
+</Dataset>
+<Dataset name='Result 40'>
+ <Row><Result_40>11165648424516303926</Result_40></Row>
+</Dataset>
+<Dataset name='Result 41'>
+ <Row><Result_41>HASH64(&apos;ABCD123 5 &apos;)</Result_41></Row>
+</Dataset>
+<Dataset name='Result 42'>
+ <Row><Result_42>11165648424516303926</Result_42></Row>
+</Dataset>
+<Dataset name='Result 43'>
+ <Row><Result_43>11165648424516303926</Result_43></Row>
+</Dataset>
+<Dataset name='Result 44'>
+ <Row><Result_44>9147172056088104930</Result_44></Row>
+</Dataset>
+<Dataset name='Result 45'>
+ <Row><Result_45>HASH64(s2)</Result_45></Row>
+</Dataset>
+<Dataset name='Result 46'>
+ <Row><Result_46>11165648424516303926</Result_46></Row>
+</Dataset>
+<Dataset name='Result 47'>
+ <Row><Result_47>11165648424516303926</Result_47></Row>
+</Dataset>
+<Dataset name='Result 48'>
+ <Row><Result_48>9147172056088104930</Result_48></Row>
+</Dataset>
+<Dataset name='Result 49'>
+ <Row><Result_49>HASH(U&apos;ABCD123 5&apos;)</Result_49></Row>
+</Dataset>
+<Dataset name='Result 50'>
+ <Row><Result_50>3536986230</Result_50></Row>
+</Dataset>
+<Dataset name='Result 51'>
+ <Row><Result_51>3536986230</Result_51></Row>
+</Dataset>
+<Dataset name='Result 52'>
+ <Row><Result_52>3536986230</Result_52></Row>
+</Dataset>
+<Dataset name='Result 53'>
+ <Row><Result_53>HASH(u1)</Result_53></Row>
+</Dataset>
+<Dataset name='Result 54'>
+ <Row><Result_54>3536986230</Result_54></Row>
+</Dataset>
+<Dataset name='Result 55'>
+ <Row><Result_55>3536986230</Result_55></Row>
+</Dataset>
+<Dataset name='Result 56'>
+ <Row><Result_56>3536986230</Result_56></Row>
+</Dataset>
+<Dataset name='Result 57'>
+ <Row><Result_57>HASH(U&apos;ABCD123 5 &apos;)</Result_57></Row>
+</Dataset>
+<Dataset name='Result 58'>
+ <Row><Result_58>3536986230</Result_58></Row>
+</Dataset>
+<Dataset name='Result 59'>
+ <Row><Result_59>3536986230</Result_59></Row>
+</Dataset>
+<Dataset name='Result 60'>
+ <Row><Result_60>3770318389</Result_60></Row>
+</Dataset>
+<Dataset name='Result 61'>
+ <Row><Result_61>HASH(u2)</Result_61></Row>
+</Dataset>
+<Dataset name='Result 62'>
+ <Row><Result_62>3536986230</Result_62></Row>
+</Dataset>
+<Dataset name='Result 63'>
+ <Row><Result_63>3536986230</Result_63></Row>
+</Dataset>
+<Dataset name='Result 64'>
+ <Row><Result_64>3770318389</Result_64></Row>
+</Dataset>
+<Dataset name='Result 65'>
+ <Row><Result_65>HASH32(U&apos;ABCD123 5&apos;)</Result_65></Row>
+</Dataset>
+<Dataset name='Result 66'>
+ <Row><Result_66>3612652982</Result_66></Row>
+</Dataset>
+<Dataset name='Result 67'>
+ <Row><Result_67>3612652982</Result_67></Row>
+</Dataset>
+<Dataset name='Result 68'>
+ <Row><Result_68>3612652982</Result_68></Row>
+</Dataset>
+<Dataset name='Result 69'>
+ <Row><Result_69>HASH32(u1)</Result_69></Row>
+</Dataset>
+<Dataset name='Result 70'>
+ <Row><Result_70>3612652982</Result_70></Row>
+</Dataset>
+<Dataset name='Result 71'>
+ <Row><Result_71>3612652982</Result_71></Row>
+</Dataset>
+<Dataset name='Result 72'>
+ <Row><Result_72>3612652982</Result_72></Row>
+</Dataset>
+<Dataset name='Result 73'>
+ <Row><Result_73>HASH32(U&apos;ABCD123 5 &apos;)</Result_73></Row>
+</Dataset>
+<Dataset name='Result 74'>
+ <Row><Result_74>3612652982</Result_74></Row>
+</Dataset>
+<Dataset name='Result 75'>
+ <Row><Result_75>3612652982</Result_75></Row>
+</Dataset>
+<Dataset name='Result 76'>
+ <Row><Result_76>2958691746</Result_76></Row>
+</Dataset>
+<Dataset name='Result 77'>
+ <Row><Result_77>HASH32(u2)</Result_77></Row>
+</Dataset>
+<Dataset name='Result 78'>
+ <Row><Result_78>3612652982</Result_78></Row>
+</Dataset>
+<Dataset name='Result 79'>
+ <Row><Result_79>3612652982</Result_79></Row>
+</Dataset>
+<Dataset name='Result 80'>
+ <Row><Result_80>2958691746</Result_80></Row>
+</Dataset>
+<Dataset name='Result 81'>
+ <Row><Result_81>HASH64(U&apos;ABCD123 5&apos;)</Result_81></Row>
+</Dataset>
+<Dataset name='Result 82'>
+ <Row><Result_82>11165648424516303926</Result_82></Row>
+</Dataset>
+<Dataset name='Result 83'>
+ <Row><Result_83>11165648424516303926</Result_83></Row>
+</Dataset>
+<Dataset name='Result 84'>
+ <Row><Result_84>11165648424516303926</Result_84></Row>
+</Dataset>
+<Dataset name='Result 85'>
+ <Row><Result_85>HASH64(u1)</Result_85></Row>
+</Dataset>
+<Dataset name='Result 86'>
+ <Row><Result_86>11165648424516303926</Result_86></Row>
+</Dataset>
+<Dataset name='Result 87'>
+ <Row><Result_87>11165648424516303926</Result_87></Row>
+</Dataset>
+<Dataset name='Result 88'>
+ <Row><Result_88>11165648424516303926</Result_88></Row>
+</Dataset>
+<Dataset name='Result 89'>
+ <Row><Result_89>HASH64(U&apos;ABCD123 5 &apos;)</Result_89></Row>
+</Dataset>
+<Dataset name='Result 90'>
+ <Row><Result_90>11165648424516303926</Result_90></Row>
+</Dataset>
+<Dataset name='Result 91'>
+ <Row><Result_91>11165648424516303926</Result_91></Row>
+</Dataset>
+<Dataset name='Result 92'>
+ <Row><Result_92>9147172056088104930</Result_92></Row>
+</Dataset>
+<Dataset name='Result 93'>
+ <Row><Result_93>HASH64(u2)</Result_93></Row>
+</Dataset>
+<Dataset name='Result 94'>
+ <Row><Result_94>11165648424516303926</Result_94></Row>
+</Dataset>
+<Dataset name='Result 95'>
+ <Row><Result_95>11165648424516303926</Result_95></Row>
+</Dataset>
+<Dataset name='Result 96'>
+ <Row><Result_96>9147172056088104930</Result_96></Row>
+</Dataset>

testing/unittests/dalitests.cpp

@@ -467,6 +467,7 @@ class DaliTests : public CppUnit::TestFixture
 // This test requires access to an external IP with dafilesrv running
 //        CPPUNIT_TEST(testDFSRename3);
         CPPUNIT_TEST(testDFSAddFailReAdd);
+        CPPUNIT_TEST(testDFSRetrySuperLock);
         CPPUNIT_TEST(testDFSHammer);
     CPPUNIT_TEST_SUITE_END();
 
@@ -1663,6 +1664,77 @@ public:
         ASSERT(0 == sfile->numSubFiles() && "regress::addreadd::super1 should contain 0 subfiles");
     }
 
+    void testDFSRetrySuperLock()
+    {
+        setupDFS("retrysuperlock");
+
+        logctx.CTXLOG("Creating regress::retrysuperlock::super1 and regress::retrysuperlock::sub1");
+        Owned<IDistributedSuperFile> sfile = dir.createSuperFile("regress::retrysuperlock::super1", user, false, false);
+        sfile->addSubFile("regress::retrysuperlock::sub1", false, NULL, false);
+        sfile.clear();
+
+        class CShortLock : implements IThreaded
+        {
+            StringAttr fileName;
+            unsigned secDelay;
+            CThreaded threaded;
+        public:
+            CShortLock(const char *_fileName, unsigned _secDelay) : fileName(_fileName), secDelay(_secDelay), threaded("CShortLock", this) { }
+            ~CShortLock()
+            {
+                threaded.join();
+            }
+            virtual void main()
+            {
+                Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(fileName, NULL);
+
+                if (!file)
+                {
+                    PROGLOG("File %s not found", fileName.get());
+                    return;
+                }
+                PROGLOG("Locked file: %s, sleeping (before unlock) for %d secs", fileName.get(), secDelay);
+
+                MilliSleep(secDelay * 1000);
+
+                PROGLOG("Unlocking file: %s", fileName.get());
+            }
+            void start() { threaded.start(); }
+        };
+
+        /* Tests transaction failing, due to lock and retrying after having partial success */
+
+        CShortLock sL("regress::retrysuperlock::super1", 15);
+        sL.start();
+
+        sfile.setown(dir.lookupSuperFile("regress::retrysuperlock::super1", user));
+        if (sfile)
+        {
+            logctx.CTXLOG("Removing subfiles from regress::retrysuperlock::super1");
+            sfile->removeSubFile(NULL, false, false);
+            logctx.CTXLOG("SUCCEEDED");
+        }
+        // put it back, for next test
+        sfile->addSubFile("regress::retrysuperlock::sub1", false, NULL, false);
+        sfile.clear();
+
+        // try again, this time in transaction
+        Owned<IDistributedFileTransaction> transaction = createDistributedFileTransaction(user); // disabled, auto-commit
+        logctx.CTXLOG("Starting transaction");
+        transaction->start();
+
+        sfile.setown(transaction->lookupSuperFile("regress::retrysuperlock::super1"));
+        if (sfile)
+        {
+            logctx.CTXLOG("Removing subfiles from regress::retrysuperlock::super1 with transaction");
+            sfile->removeSubFile(NULL, false, false, transaction);
+            logctx.CTXLOG("SUCCEEDED");
+        }
+        sfile.clear();
+        logctx.CTXLOG("Committing transaction");
+        transaction->commit();
+    }
+
     void testDFSRename2()
     {
         setupDFS("rename2");

thorlcr/activities/loop/thloop.cpp

@@ -25,6 +25,8 @@
 
 #include "thloop.ipp"
 
+#define SYNC_TIMEOUT (5*60*1000)
+
 class CLoopActivityMasterBase : public CMasterActivity
 {
 protected:
@@ -44,9 +46,15 @@ protected:
         CMessageBuffer msg;
         while (n--) // a barrier really
         {
-            rank_t sender;
-            if (!receiveMsg(msg, RANK_ALL, mpTag, &sender, LONGTIMEOUT))
-                return false;
+            loop
+            {
+                rank_t sender;
+                if (receiveMsg(msg, RANK_ALL, mpTag, &sender, SYNC_TIMEOUT))
+                    break;
+                if (abortSoon)
+                    return true; // NB: returning true, denotes end of loop
+                ActPrintLog("Still waiting for %d slaves to synchronize global loop", n+1);
+            }
             unsigned slaveLoopCounterReq, slaveEmptyIterations;
             msg.read(slaveLoopCounterReq);
             msg.read(slaveEmptyIterations);