Merge branch 'candidate-7.0.4' into candidate-7.2.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/remote/rmtssh.cpp

@@ -393,6 +393,8 @@ public:
             }
             if (cmdline.length()==0) {
                 // ssh
+                // NB: -o LogLevel=ERROR might be better choice because it would echo ssh error text
+                // whereas QUIET does not, but leaving it as QUIET for now for compatibility.
                 cmdline.appendf("%s -n -o LogLevel=QUIET -o StrictHostKeyChecking=%s ",usepssh?"pssh":"ssh",strict?"yes":"no -o UserKnownHostsFile=/dev/null");
                 if (!usepssh)
                     cmdline.append("-o BatchMode=yes ");
@@ -453,6 +455,8 @@ public:
         }
         CriticalBlock block(sect);
         done.append(i);
+        if ((retcode == 255) && outbuf.isEmpty())
+            outbuf.setf("SSH failed to connect %s: %s", slaves.item(i), cmd.str());
         replytext.append(outbuf.str());
         reply.append((unsigned)retcode);
     }

common/remote/sockfile.cpp

@@ -4766,6 +4766,13 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
             size32_t avail = (size32_t)socket->avail_read();
             if (avail)
                 touch();
+            else if (left)
+            {
+                WARNLOG("notifySelected: Closing mid packet, %d remaining", left);
+                msg.clear();
+                parent->notify(this, msg); // notifying of graceful close
+                return false;
+            }
             if (left==0)
             {
                 try
@@ -4829,12 +4836,6 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
             }
             if (TF_TRACE_FULL)
                 PROGLOG("notifySelected %d,%d",toread,left);
-            if ((left!=0)&&(avail==0))
-            {
-                WARNLOG("notifySelected: Closing mid packet, %d remaining", left);
-                toread = left;
-                msg.clear();
-            }
             left -= toread;
             if (left==0)
             {

common/workunit/workunit.cpp

@@ -11568,7 +11568,7 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
         return str.append("Unrecognized workunit format");
 }
 
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool splitStats)
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool regressionTest)
 {
     const IExtendedWUInterface *ewu = queryExtendedWU(wu);
     if (ewu)
@@ -11576,20 +11576,37 @@ extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const
         Linked<IPropertyTree> p;
         if (unpack||includeProgress)
             p.setown(ewu->getUnpackedTree(includeProgress));
+        else if (regressionTest)
+            p.setown(createPTreeFromIPT(ewu->queryPTree()));
         else
             p.set(ewu->queryPTree());
         if (hidePasswords)
             return exportWorkUnitToXMLFileWithHiddenPasswords(p, filename, extraXmlFlags);
-        if (splitStats)
+
+        if (regressionTest)
         {
-            StringBuffer statsFilename;
-            statsFilename.append(filename).append(".stats");
+            //This removes any items from the xml that will vary from run to run, so they can be binary compared from run to run
+            //The following attributes change with the build.  Simpler to remove rather than needing to updated each build
+            p->removeProp("@buildVersion");
+            p->removeProp("@eclVersion");
+            p->removeProp("@hash");
+
+            //Remove statistics, and extract them to a separate file
             IPropertyTree * stats = p->queryPropTree("Statistics");
             if (stats)
             {
+                StringBuffer statsFilename;
+                statsFilename.append(filename).append(".stats");
                 saveXML(statsFilename, stats, 0, (XML_Format|XML_SortTags|extraXmlFlags) & ~XML_LineBreakAttributes);
                 p->removeProp("Statistics");
             }
+            //Now remove timestamps from exceptions
+            Owned<IPropertyTreeIterator> elems = p->getElements("Exceptions/Exception", iptiter_sort);
+            ForEach(*elems)
+            {
+                IPropertyTree &elem = elems->query();
+                elem.removeProp("@time");
+            }
         }
         saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
     }

common/workunit/workunit.ipp

@@ -189,7 +189,7 @@ template <>  struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
 class WORKUNIT_API CLocalWorkUnit : implements IWorkUnit , implements IExtendedWUInterface, public CInterface
 {
     friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs, bool includeProgress, bool hidePasswords);
-    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress, bool hidePasswords, bool splitStats);
+    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress, bool hidePasswords, bool regressionTest);
 
 protected:
     Owned<IPropertyTree> p;

dali/base/dasess.cpp

@@ -947,9 +947,11 @@ public:
             {
                 CDateTime now;
                 StringBuffer b64sig;
-                createDaliSignature(obj, udesc, now, b64sig);
-                mb.append(b64sig.str());
-                now.serialize(mb);
+                if (createDaliSignature(obj, udesc, now, b64sig))
+                {
+                    mb.append(b64sig.str());
+                    now.serialize(mb);
+                }
             }
             else
             {

dali/server/CMakeLists.txt

@@ -39,7 +39,8 @@ include_directories (
          ./../../system/security/shared 
          ./../../system/security/LdapSecurity 
          ./../../system/security/cryptohelper
-         ./../../system/jlib 
+         ./../../system/jlib
+         ./../../common/workunit
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )

dali/server/daldap.cpp

@@ -26,6 +26,7 @@
 #include "mpbase.hpp"
 #include "dautils.hpp"
 #include "digisign.hpp"
+#include "workunit.hpp"
 
 using namespace cryptohelper;
 
@@ -196,7 +197,7 @@ public:
                 ERRLOG("LDAP: getPermissions(%s) scope=%s user=%s digital signature support not available",key?key:"NULL",obj?obj:"NULL",username.str());
         }
 
-        if (!isEmptyString(user->credentials().getPassword()))
+        if (!isEmptyString(user->credentials().getPassword()) && !isWorkunitDAToken(user->credentials().getPassword()))
         {
             if (!ldapsecurity->authenticateUser(*user, NULL))
             {

docs/EN_US/HPCCSpark/SparkHPCC.xml

@@ -123,7 +123,7 @@
               This can be done using the install-cluster.sh script which is
               provided with HPCC. Use the following command:</para>
 
-              <programlisting>/opt/HPCCSystems/sbin/install-cluster.sh &lt;hpccsystems-plugin-spark&gt;</programlisting>
+              <programlisting>sudo /opt/HPCCSystems/sbin/install-cluster.sh &lt;hpccsystems-plugin-spark&gt;</programlisting>
 
               <para>More details including other options that may be used with
               this command are included in the appendix of Installing and
@@ -476,11 +476,10 @@
       <para>There are several additional artifacts of some interest. The
       <emphasis>org.hpccsystems.spark.ColumnPruner</emphasis> class is
       provided to enable retrieving only the columns of interest from the HPCC
-      Cluster. The <emphasis>targetClusterList</emphasis> parameter on the
-      HpccFile constructor allows you to provide a string of comma delimited
-      field paths for this same purpose. The
+      Cluster. The <emphasis>targetCluster</emphasis> artifact allows you to
+      specify the HPCC cluster on which the target file exists. The
       <emphasis>org.hpccsystems.spark.thor.FileFilter</emphasis> class is
-      provided to enable retrieving only records of interest from the HPCC
+      provided to facilitate filtering records of interest from the HPCC
       Cluster.</para>
 
       <para>The git repository includes two examples under the
@@ -554,8 +553,8 @@
       <programlisting>public HpccFileWriter(String connectionString, String user, String pass) throws Exception { </programlisting>
 
       <para>The first parameter <emphasis>connectionString</emphasis> contains
-      the same information as <emphasis>HpccFile</emphasis> only using a
-      single connection string. It should be in the following format:
+      the same information as <emphasis>HpccFile</emphasis>. It should be in
+      the following format:
       {http|https}://{ECLWATCHHOST}:{ECLWATCHPORT}</para>
 
       <para>The constructor will attempt to connect to HPCC. This connection
@@ -674,8 +673,11 @@
         value is the <emphasis>SparkContext</emphasis> object provided by the
         shell.</para>
 
-        <programlisting> val hpcc = new HpccFile("myfile", "http", "myeclwatchhost", "8010", "myuser", "mypass", "")
- val myRDD = hpcc.getRDD(sc)</programlisting>
+        <programlisting> val espcon = new Connection("http", "myeclwatchhost", "8010");
+ espcon.setUserName("myuser");
+ espcon.setPassword("mypass");
+ val file = new HpccFile("myfile",espcon);
+</programlisting>
 
         <para>Now we have an RDD of the data. Nothing has actually happened at
         this point because Spark performs lazy evaluation and there is nothing
@@ -745,8 +747,11 @@
         and is used instead of the <emphasis>SparkContext</emphasis>
         object.</para>
 
-        <programlisting> val hpcc = new HpccFile("myfile", "http", "myeclwatchhost", "8010", "myuser", "mypass", "")
- val mt_df = hpcc.getDataframe(spark)</programlisting>
+        <programlisting> val espcon = new Connection("http", "myeclwatchhost", "8010");
+ espcon.setUserName("myuser");
+ espcon.setPassword("mypass");
+ val file = new HpccFile("myfile",espcon);
+</programlisting>
 
         <para>The Spark <emphasis>ml</emphasis> Machine Learning classes use
         different data container classes. In the case of Logistic Regression,

ecl/eclcc/eclcc.cpp

@@ -1329,6 +1329,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
                 updateWorkunitStat(instance.wu, SSTcompilestage, "compile:cache", StNumAttribsSimplified, NULL, parseCtx.numAttribsSimplified);
                 updateWorkunitStat(instance.wu, SSTcompilestage, "compile:cache", StNumAttribsProcessed, NULL, parseCtx.numAttribsProcessed);
                 updateWorkunitStat(instance.wu, SSTcompilestage, "compile:cache", StNumAttribsFromCache, NULL, parseCtx.numAttribsFromCache);
+                updateWorkunitStat(instance.wu, SSTcompilestage, "compile:cache", StNumAttribsSimplifiedTooComplex, NULL, parseCtx.numSimplifiedTooComplex);
             }
 
             if (exportDependencies)

ecl/hql/hqlcache.cpp

@@ -133,7 +133,6 @@ protected:
         return EclCachedDefinition::calcUpToDate(optionHash);
     }
 
-
     const char * queryName() const { return cacheTree ? cacheTree->queryProp("@name") : nullptr; }
 
 private:

ecl/hql/hqlerrors.hpp

@@ -507,6 +507,7 @@
 #define HQLERR_CacheMissingOriginal             3152
 #define HQLERR_CacheMissingEntry                3153
 #define HQLERR_PotentialAmbiguity               3154
+#define HQLERR_CannotDefineFunctionFunction     3155
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contains an illegal cycle/recursive definition %s"
@@ -555,6 +556,7 @@
 #define HQLERR_CacheMissingOriginal_Text        "Cannot find original for cache entry '%s'"
 #define HQLERR_CacheMissingEntry_Text           "Cannot process cache entry '%s'"
 #define HQLERR_PotentialAmbiguity_Text          "INTERNAL: Mapping introduces potential ambiguity into expression - please report issue"
+#define HQLERR_CannotDefineFunctionFunction_Text "Cannot define a function that returns a function"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hql/hqlexpr.cpp

@@ -1071,7 +1071,7 @@ bool HqlParseContext::checkEndMeta()
     return wasGathering;
 }
 
-bool HqlParseContext::createCache(const char *simplifiedEcl, bool isMacro)
+bool HqlParseContext::createCache(const char * simplifiedEcl, bool isMacro)
 {
     StringBuffer fullName;
     StringBuffer baseFilename;

ecl/hql/hqlexpr.hpp

@@ -921,7 +921,7 @@ public:
     void beginMetaScope() { metaStack.append(*new FileParseMeta); }
     void beginMetaScope(FileParseMeta & active) { metaStack.append(OLINK(active)); }
     void endMetaScope() { metaStack.pop(); }
-    bool createCache(const char *simplifiedEcl, bool isMacro);
+    bool createCache(const char * simplifiedEcl, bool isMacro);
     inline FileParseMeta & curMeta() { return metaStack.tos(); }
     inline bool hasCacheLocation( ) const { return !metaOptions.cacheLocation.isEmpty();}
 public:
@@ -954,6 +954,7 @@ public:
     unsigned numAttribsSimplified = 0;
     unsigned numAttribsProcessed = 0;
     unsigned numAttribsFromCache = 0;
+    unsigned numSimplifiedTooComplex = 0;
 
 private:
     void createDependencyEntry(IHqlScope * scope, IIdAtom * name);
@@ -1032,6 +1033,7 @@ public:
     inline void incrementAttribsSimplified() { ++parseCtx.numAttribsSimplified; }
     inline void incrementAttribsProcessed() { ++parseCtx.numAttribsProcessed; }
     inline void incrementAttribsFromCache() { ++parseCtx.numAttribsFromCache; }
+    inline void incrementSimplifiedTooComplex() { ++parseCtx.numSimplifiedTooComplex; }
     inline bool neverSimplify(const char *fullname) { return parseCtx.neverSimplify(fullname); }
 protected:
 

ecl/hql/hqlgram2.cpp

@@ -9655,6 +9655,12 @@ void HqlGram::doDefineSymbol(DefineIdSt * defineid, IHqlExpression * _expr, IHql
             targetScope = activeScope.privateScope;
         }
     }
+    if (isParametered && expr->isFunction())
+    {
+        expr.setown(createNullExpr(expr->queryType()->queryChildType()));
+        reportError(HQLERR_CannotDefineFunctionFunction, idattr.pos, HQLERR_CannotDefineFunctionFunction_Text);
+    }
+
     OwnedHqlExpr normalized = normalizeFunctionExpression(defineid, expr, failure, isParametered, activeScope.activeParameters, activeScope.createDefaults(), modifiers);
     defineSymbolInScope(targetScope, defineid, normalized, idattr, assignPos, semiColonPos);
 
@@ -12294,25 +12300,31 @@ bool parseForwardModuleMember(HqlGramCtx & _parent, IHqlScope *scope, IHqlExpres
 // if ctx.checkSimpleDef() then simplified definitions are created and checked - even if there is no cache configured.
 void parseAttribute(IHqlScope * scope, IFileContents * contents, HqlLookupContext & ctx, IIdAtom * name, const char * fullName)
 {
-    bool alreadySimplified = false;
+    bool usingSimplifiedFromCache = false;
     bool cacheUptoDate = false;
     HqlLookupContext attrCtx(ctx);
     attrCtx.noteBeginAttribute(scope, contents, name);
 
-    // Check if cache is up to date and obtain simplified definition from cache
+    // Set cacheUptoDate to true if cache is upto date
+    // Set usingSimplified to true if cache is upto date and option to ignoreSimplified & ignoreCache are both false
+    // Use simplified expression from cache if usingSimplified is true & syntaxChecking
+    // * None of these are set of ctx.regenerateCache==true, as the simplified expression & cache will need to regenerated.
     if (ctx.hasCacheLocation() && !ctx.regenerateCache())
     {
         HqlParseContext & parseContext = ctx.queryParseContext();
         Owned<IEclCachedDefinition> cached = parseContext.cache->getDefinition(fullName);
         cacheUptoDate = cached->isUpToDate(parseContext.optionHash);
-        if (cacheUptoDate && ctx.syntaxChecking() && !ctx.ignoreSimplified() && !ctx.ignoreCache())
+        if (cacheUptoDate && !ctx.ignoreSimplified() && !ctx.ignoreCache())
         {
-            IFileContents * cachecontents = cached->querySimplifiedEcl();
-            if (cachecontents)
+            usingSimplifiedFromCache = true;
+            if (ctx.syntaxChecking())
             {
-                contents = cachecontents;
-                alreadySimplified = true;
-                ctx.incrementAttribsFromCache();
+                IFileContents * cachecontents = cached->querySimplifiedEcl();
+                if (cachecontents)
+                {
+                    contents = cachecontents;
+                    ctx.incrementAttribsFromCache();
+                }
             }
         }
     }
@@ -12338,32 +12350,50 @@ void parseAttribute(IHqlScope * scope, IFileContents * contents, HqlLookupContex
     OwnedHqlExpr parsed = scope->lookupSymbol(name, LSFsharedOK|LSFnoreport, ctx);
     ctx.incrementAttribsProcessed();
 
-    if (parsed && !alreadySimplified)
+    if (parsed && !usingSimplifiedFromCache)
     {
         //Forward scopes cannot be cached because the dependencies are not known as it is parsed
         const bool canCache = parsed->getOperator() != no_forwardscope;
         if (canCache)
         {
             const bool isMacro = parsed->isMacro();
-            bool updateCache = ctx.hasCacheLocation() && (!cacheUptoDate || ctx.regenerateCache());
-            bool useSimplified = ctx.syntaxChecking() && !ctx.ignoreSimplified();
+            const bool updateCache = ctx.hasCacheLocation() && (!cacheUptoDate || ctx.regenerateCache());
+            const bool useSimplified = ctx.syntaxChecking() && !ctx.ignoreSimplified();
 
             OwnedHqlExpr simplified;
             StringBuffer simplifiedEcl;
+            // Simplified expression will be generated when
+            // - Not a macro and ignoreSimplified == false
+            // - And simplifiedExpression is required because cache is no up to date, verify option is used or syntaxChecking
+            // - And attribute has not been specifically excluded with the neverSimplify option
             if (!isMacro && !ctx.ignoreSimplified() &&
                 (updateCache || ctx.checkSimpleDef() || useSimplified) && !ctx.neverSimplify(fullName))
                 simplified.setown(createSimplifiedDefinition(parsed));
 
-            // create plain text ecl representation of the simplified expression (if it will be needed later)
+            // If plain text ecl representation of the simplified expression is needed for some reason
+            // (i.e. to update cache or verify simplified expression)
+            // And the simplified expression is less complex than the original expression
+            // Then a plain text ecl representation of the simplified expression is produced here
             if (simplified && (updateCache||ctx.checkSimpleDef()))
             {
                 regenerateDefinition(simplified, simplifiedEcl);
-                if (ctx.checkSimpleDef())
+                // Ensure the simplified expression is less complex
+                if (simplifiedEcl.length()<contents->length())
+                {
+                    if (ctx.checkSimpleDef())
+                    {
+                        simplifiedEcl.append("\n/* Simplified expression expression tree:\n");
+                        EclIR::getIRText(simplifiedEcl, 0, queryLocationIndependent(simplified));
+                        simplifiedEcl.append("*/\n");
+                    }
+                }
+                else
                 {
-                    // Dump of simplified expression in an ecl comment for diagnostics
-                    simplifiedEcl.append("\n/* Simplified expression IR:\n");
-                    EclIR::getIRText(simplifiedEcl, 0, queryLocationIndependent(simplified));
-                    simplifiedEcl.append("*/\n");
+                    // Simplified expr is more complex, so blank out plain text ecl string
+                    // to ensure it's not written to cache or used for verifying
+                    simplifiedEcl.clear();
+                    simplified.clear();
+                    ctx.incrementSimplifiedTooComplex();
                 }
             }
             if (updateCache)

ecl/hql/hqlthql.cpp

@@ -734,6 +734,8 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             s.append(']');
 #endif
         }
+        if (containsImplicitNormalize(expr))
+            s.append('.');
         if (containsAssertKeyed(expr))
             s.append('K');
         if (containsAliasLocally(expr))

ecl/hql/hqltrans.cpp

@@ -3926,7 +3926,6 @@ IHqlExpression * ScopedTransformer::createTransformed(IHqlExpression * expr)
     case no_extractresult:
     case no_createdictionary:
         {
-            unsigned __int64 savedCount = getNestedUsageCount();
             IHqlExpression * dataset = expr->queryChild(0);
             pushScope(expr);
             IHqlExpression * transformedDs = transform(dataset);
@@ -3936,11 +3935,6 @@ IHqlExpression * ScopedTransformer::createTransformed(IHqlExpression * expr)
                 children.append(*transform(expr->queryChild(idx)));
             clearDataset(nested);
             popScope();
-
-            //There were no references to outer datasets in the contained expressions => it will always be transformed
-            //the same way as long as there are no instances of globalDataset.childDataset
-            if (!containsImplicitNormalize(expr) && (savedCount == getNestedUsageCount()))
-                noteTransformedOnce(expr);
             break;
         }
     case no_projectrow:
@@ -4402,6 +4396,17 @@ bool ScopedTransformer::isDatasetRelatedToScope(IHqlExpression * search)
     return false;
 }
 
+bool ScopedTransformer::isWithinRootScope() const
+{
+    ForEachItemIn(idx, scopeStack)
+    {
+        ScopeInfo & cur = scopeStack.item(idx);
+        if (cur.dataset || cur.left)
+            return false;
+    }
+    return true;
+}
+
 bool ScopedTransformer::checkInScope(IHqlExpression * selector, bool allowCreate)
 {
     switch (selector->getOperator())
@@ -4445,16 +4450,10 @@ bool ScopedTransformer::checkInScope(IHqlExpression * selector, bool allowCreate
     {
         ScopeInfo & cur = scopeStack.item(idx);
         if (cur.dataset && cur.dataset->queryNormalizedSelector(false) == normalized)
-        {
-            cur.usageCount++;
             return true;
-        }
 
         if (isInImplictScope(cur.dataset, normalized))
-        {
-            cur.usageCount++;
             return true;
-        }
     }
     return false;
 }
@@ -4535,15 +4534,6 @@ unsigned ScopedTransformer::tableNesting()
     return numTables;
 }
 
-unsigned __int64 ScopedTransformer::getNestedUsageCount()
-{
-    unsigned __int64 total = 0;
-    ForEachItemIn(i, scopeStack)
-        total += scopeStack.item(i).usageCount;
-    return total;
-}
-
-
 bool ScopedTransformer::insideActivity()
 {
     return (scopeStack.ordinality() != 0) || (savedStack.ordinality() != 0);

ecl/hql/hqltrans.ipp

@@ -1035,7 +1035,6 @@ public:
 
 public:
     IHqlExpression * context;
-    unsigned __int64 usageCount = 0;
     HqlExprAttr     dataset;
     HqlExprAttr     transformedDataset;
     HqlExprAttr     left;
@@ -1069,10 +1068,10 @@ protected:
     bool isDatasetActive(IHqlExpression * expr);                // is it in an active dataset?
     bool isDatasetARow(IHqlExpression * expr);                  // can it be used as a row?
     bool isDatasetRelatedToScope(IHqlExpression * dataset);
+    bool isWithinRootScope() const;
     bool isNewDataset()                                             { return innerScope && innerScope->isEmpty(); }
     bool insideActivity();
     unsigned tableNesting();
-    unsigned __int64 getNestedUsageCount();
 
     IHqlExpression * getScopeState();
 

ecl/hqlcpp/hqlcerrors.hpp

@@ -170,7 +170,7 @@
 #define HQLERR_CouldNotFindDataset              4158
 #define HQLERR_CouldNotAnyDatasetX              4159
 #define HQLERR_NestedThorNodes                  4160
-#define HQLERR_MissingTransformAssignXX         4161
+#define HQLERR_MissingTransformAssignX          4161
 #define HQLERR_JoinXTooComplex                  4162
 #define HQLERR_GlobalDedupFuzzy                 4163
 #define HQLERR_GlobalDedupNoEquality            4164
@@ -480,7 +480,7 @@
 #define HQLERR_CouldNotFindDataset_Text         "Could not find dataset %s"
 #define HQLERR_CouldNotAnyDatasetX_Text         "Could not find dataset %s (no tables in scope)"
 #define HQLERR_NestedThorNodes_Text             "INTERNAL: Thor nodes should not be nested"
-#define HQLERR_MissingTransformAssignXX_Text    "INTERNAL: Missing assignment from transform to %s[%p]"
+#define HQLERR_MissingTransformAssignX_Text     "INTERNAL: Missing assignment from transform to %s"
 #define HQLERR_JoinXTooComplex_Text             "JOIN%s contains no equality conditions - use ,ALL to allow"
 #define HQLERR_GlobalDedupFuzzy_Text            "A global DEDUP(ALL) or local hash dedup cannot include comparisons in the dedup criteria"
 #define HQLERR_GlobalDedupNoEquality_Text       "Global dedup,ALL must have a field to partition"

ecl/hqlcpp/hqlcse.cpp

@@ -1510,6 +1510,7 @@ IHqlExpression * TableInvariantTransformer::createTransformed(IHqlExpression * e
     }
 
     OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
+    updateOrphanedSelectors(transformed, expr);
     if (queryBodyExtra(expr)->createAlias)
     {
         if (!isTrivialAlias(expr))

ecl/hqlcpp/hqlhtcpp.cpp

@@ -863,9 +863,9 @@ protected:
 
     virtual void onMissingAssignment(IHqlExpression * expr)
     {
-            StringBuffer s;
-            expr->toString(s);
-            throwError2(HQLERR_MissingTransformAssignXX, s.str(), expr);
+        StringBuffer s;
+        expr->toString(s);
+        throwError1(HQLERR_MissingTransformAssignX, s.str());
     }
 
     void pushCondition(IHqlExpression * cond)
@@ -11408,7 +11408,7 @@ void HqlCppTranslator::buildXmlSerialize(BuildCtx & subctx, IHqlExpression * exp
                     {
                         StringBuffer s;
                         expr->toString(s);
-                        throwError2(HQLERR_MissingTransformAssignXX, s.str(), expr);
+                        throwError1(HQLERR_MissingTransformAssignX, s.str());
                     }
 
                     selected.set(match->queryChild(0));

ecl/hqlcpp/hqlttcpp.cpp

@@ -758,6 +758,8 @@ IHqlExpression * HqlThorBoundaryTransformer::createTransformed(IHqlExpression *
     case no_sizeof:
     case no_offsetof:
         return getTransformedChildren(expr);
+    case no_mapto:
+        return NewHqlTransformer::createTransformed(expr);
     }
 
     //Unusually, wrap the expression in a thor node before processing annotations.
@@ -8391,7 +8393,7 @@ IHqlExpression * NewScopeMigrateTransformer::createTransformed(IHqlExpression *
                     if ((rootOp == no_select) || (rootOp == no_field))
                         break;
 
-                    if (isIndependentOfScope(datasetExpr) && !isContextDependent(expr))
+                    if (isIndependentOfScope(expr) && !isContextDependent(expr))
                     {
                         return hoist(expr, transformed);
                     }
@@ -9921,7 +9923,7 @@ IHqlExpression * HqlLinkedChildRowTransformer::createTransformedBody(IHqlExpress
 
 HqlScopeTaggerInfo::HqlScopeTaggerInfo(IHqlExpression * _expr) : MergingTransformInfo(_expr)
 {
-    if (!onlyTransformOnce() && isIndependentOfScope(_expr))
+    if (!onlyTransformOnce() && (!containsImplicitNormalize(_expr) || _expr->isIndependentOfScope()))
     {
         //If the node doesn't have any active selectors then it isn't going to be context dependent
         setOnlyTransformOnce(true);
@@ -9943,11 +9945,24 @@ ANewTransformInfo * HqlScopeTagger::createTransformInfo(IHqlExpression * expr)
  *
  *   SELF.x := ds1
  *      This either means the entire ds1, or the current row in ds1 = depending on whether dataset is active or not
- *      (e.g, due to a TABLE() statement.)  This code reports a warning in this case - active(ds1) should be used for
+ *      (e.g, due to a TABLE() statement.)  This code reports an error in this case - active(ds1) should be used for
  *      the second case.
  *
- * In order to achieve this the entire expression tree needs to be transformed differently within each potentital dataset
+ * The following contexts are situations where the datasets are potentially ambiguous:
+ *    o left hand side of a no_select (case detailed above)
+ *    o parameters
+ *    o right hand sizes of assignments
+ *    o sizeof()
+ *
+ * For the first the ambiguity is resolved.  For all the others, it is treated as a new dataset, and a warning/error
+ * is reported if the dataset is in scope.  (?How would you suppress this?)
+ * reference to an active dataset should be done with ROW(dataset)
+ *
+ * In order to achieve this the entire expression tree needs to be transformed differently within each potential dataset
  * scope combination.
+ * This means that an expression that does not contain a normalized select (a.ds.x) must be transformed the same way each
+ * time.
+ *
 
 Details of the no_select representation:
 
@@ -9969,6 +9984,49 @@ Note:
 
   */
 
+
+//Recursively search for an expression which refers to the first selector that is unresolved.
+static IHqlExpression * findOriginalReference(IHqlExpression * * location, IHqlExpression * expr, IHqlExpression * transformed, IHqlExpression * search)
+{
+    for(;;)
+    {
+        if (transformed == search)
+            return expr;
+
+        //If this is a no_select expression that is not resolving from a row then we have a match
+        node_operator transformOp = transformed->getOperator();
+        if ((transformOp == no_select) && !isNewSelector(transformed))
+            return expr;
+
+        //If tree has been transformed to a different structure then do not continue
+        if (transformOp != expr->getOperator())
+            return nullptr;
+
+        //Keep track of the best location that we have seen so far
+        IHqlExpression * symbol = queryLocation(expr);
+        if (symbol && location && (symbol->getStartLine() != 0))
+            *location = symbol;
+
+        //Find the first child expression that is also dependent on that selector
+        bool matched = false;
+        ForEachChild(i, transformed)
+        {
+            IHqlExpression * cur = transformed->queryChild(i);
+            if (cur->usesSelector(search))
+            {
+                transformed = cur;
+                expr = expr->queryChild(i);
+                matched = true;
+                break;
+            }
+        }
+
+        if (!matched || !expr)
+            return nullptr;
+    }
+}
+
+
 static HqlTransformerInfo hqlScopeTaggerInfo("HqlScopeTagger");
 HqlScopeTagger::HqlScopeTagger(IErrorReceiver & _errors, ErrorSeverityMapper & _errorMapper)
 : ScopedDependentTransformer(hqlScopeTaggerInfo), errors(_errors), errorMapper(_errorMapper)
@@ -10331,6 +10389,7 @@ IHqlExpression * HqlScopeTagger::createTransformed(IHqlExpression * expr)
             }
             break;
         case annotate_symbol:
+        case annotate_location:
             {
                 ErrorSeverityMapper::SymbolScope saved(errorMapper, expr);
                 OwnedHqlExpr transformedBody = transform(body);
@@ -10338,10 +10397,6 @@ IHqlExpression * HqlScopeTagger::createTransformed(IHqlExpression * expr)
                     return LINK(expr);
                 return expr->cloneAnnotation(transformedBody);
             }
-        case annotate_location:
-            {
-                break;
-            }
         }
         OwnedHqlExpr transformedBody = transform(body);
         if (body == transformedBody)
@@ -10463,15 +10518,38 @@ IHqlExpression * HqlScopeTagger::createTransformed(IHqlExpression * expr)
             }
             return expr->clone(children);
         }
+    case no_colon:
+    {
+        OwnedHqlExpr transformed = Parent::createTransformed(expr);
+        IHqlExpression * child = transformed->queryChild(0);
+        if (!child->isIndependentOfScope())
+            reportRootSelectorError(expr->queryChild(0), child);
+        return transformed.getClear();
+    }
+    case no_apply:
+    case no_output:
+    case no_outputscalar:
+    case no_setresult:
+    case no_buildindex:
+        {
+            OwnedHqlExpr transformed = Parent::createTransformed(expr);
+            if (isWithinRootScope() && !transformed->isIndependentOfScope())
+                reportRootSelectorError(expr, transformed);
+            return transformed.getClear();
+        }
     }
 
     return Parent::createTransformed(expr);
 }
 
-
 void HqlScopeTagger::reportError(WarnErrorCategory category, const char * msg)
 {
     IHqlExpression * location = errorMapper.queryActiveSymbol();
+    reportError(category, msg, location);
+}
+
+void HqlScopeTagger::reportError(WarnErrorCategory category, const char * msg, IHqlExpression * location)
+{
     //Make this an error when we are confident...
     int startLine= location ? location->getStartLine() : 0;
     int startColumn = location ? location->getStartColumn() : 0;
@@ -10481,6 +10559,38 @@ void HqlScopeTagger::reportError(WarnErrorCategory category, const char * msg)
     errors.report(err);        // will throw immediately if it is an error.
 }
 
+void HqlScopeTagger::reportRootSelectorError(IHqlExpression * expr, IHqlExpression * transformed)
+{
+    HqlExprCopyArray inScope;
+    transformed->gatherTablesUsed(nullptr, &inScope);
+    assertex(inScope.ordinality());
+
+    //Recursively search for an expression which refers to the first selector that is unresolved.
+    IHqlExpression * selector = &inScope.item(0);
+    IHqlExpression * location = nullptr;
+    IHqlExpression * original = findOriginalReference(&location, expr, transformed, selector);
+    if (original)
+    {
+        StringBuffer exprText;
+        getECL(original->queryBody(), exprText);
+        elideString(exprText, 100); // very unlikely to be long, but limit just in case
+
+        VStringBuffer msg("expression '%s' is used within expression, but is not in scope", exprText.str());
+        if (!location)
+            location = errorMapper.queryActiveSymbol();
+        reportError(CategoryError, msg, location);
+    }
+    else
+    {
+        StringBuffer exprText;
+        getECL(expr, exprText);
+        elideString(exprText, 30);
+
+        VStringBuffer msg("Global expression '%s' uses fields from a dataset that is not in scope", exprText.str());
+        reportError(CategoryError, msg);
+    }
+}
+
 
 //---------------------------------------------------------------------------------------------------------------------
 

ecl/hqlcpp/hqlttcpp.ipp

@@ -1100,7 +1100,9 @@ protected:
     IHqlExpression * transformWithin(IHqlExpression * dataset, IHqlExpression * scope);
 
     void reportError(WarnErrorCategory category, const char * msg);
+    void reportError(WarnErrorCategory category, const char * msg, IHqlExpression * location);
     void reportSelectorError(IHqlExpression * selector, IHqlExpression * expr);
+    void reportRootSelectorError(IHqlExpression * expr, IHqlExpression * transformed);
 
 protected:
     IErrorReceiver & errors;

ecl/regress/dlingle1.ecl

@@ -0,0 +1,17 @@
+EXPORT Functions := MODULE
+
+    EXPORT fn_AccountStatus_sort_order(STRING status) := FUNCTION
+           fn_AccountStatus_sort_order :=   CASE(status,
+                                                 'OVERDUE'         => 1,
+                                                 'CURRENT'          => 2,
+                                                 'CLOSED'  => 3,
+                                                 999);
+        RETURN fn_AccountStatus_sort_order;
+    END;
+
+    EXPORT fn_AccountStatus_sort_order2(STRING status) := FUNCTION
+        RETURN fn_AccountStatus_sort_order;
+    END;
+
+END; // module.
+

ecl/regress/regress.sh

@@ -30,7 +30,7 @@ syntax="syntax: $0 [-t target_dir] [-c compare_dir] [-I include_dir ...] [-e ecl
 
 ## Default arguments
 target_dir=run_$$
-compare_dir=
+compare_dir=-
 include_dir=
 compare_only=0
 userflags=
@@ -164,7 +164,7 @@ if [[ $eclcc != '' ]]; then
 fi
 
 ## Compare to golden standard (ignore obvious differences)
-if [[ $compare_dir ]]; then
+if [[ $compare_dir != '-' ]]; then
     if [[ ! -d $target_dir ]]; then
         echo " ++ No target dir to compare"
         exit 1

esp/bindings/http/client/httpclient.cpp

@@ -50,7 +50,7 @@ CHttpClientContext::CHttpClientContext(IPropertyTree* config) : m_config(config)
 
 void CHttpClientContext::initPersistentHandler()
 {
-    m_persistentHandler.setown(createPersistentHandler(nullptr, DEFAULT_MAX_PERSISTENT_IDLE_TIME, DEFAULT_MAX_PERSISTENT_REQUESTS, static_cast<PersistentLogLevel>(getEspLogLevel())));
+    m_persistentHandler.setown(createPersistentHandler(nullptr, DEFAULT_MAX_PERSISTENT_IDLE_TIME, DEFAULT_MAX_PERSISTENT_REQUESTS, static_cast<PersistentLogLevel>(getEspLogLevel()), true));
 }
 
 CHttpClientContext::~CHttpClientContext()
@@ -249,6 +249,10 @@ int CHttpClient::connect(StringBuffer& errmsg, bool forceNewConnection)
             ep.port=80;
     }
     m_ep = ep;
+
+    if(m_persistentHandler->inDoNotReuseList(&ep))
+        m_disableKeepAlive = true;
+
     Linked<ISocket> pSock = (m_disableKeepAlive || forceNewConnection)?nullptr:m_persistentHandler->getAvailable(&ep);
     if(pSock)
     {

esp/platform/persistent.cpp

@@ -31,6 +31,8 @@ public:
     CPersistentInfo(bool _inUse, unsigned _timeUsed, unsigned _useCount, SocketEndpoint* _ep)
         : inUse(_inUse), timeUsed(_timeUsed), useCount(_useCount), ep(_ep?(new SocketEndpoint(*_ep)):nullptr)
     {
+        if(_ep)
+            _ep->getUrlStr(epstr);
     }
     virtual ~CPersistentInfo() { } //TODO remove trace
 protected:
@@ -38,9 +40,11 @@ protected:
     unsigned timeUsed;
     unsigned useCount;
     std::unique_ptr<SocketEndpoint> ep;
+    StringBuffer epstr;
 };
 
 using SockInfoMap = MapBetween<Linked<ISocket>, ISocket*, Owned<CPersistentInfo>, CPersistentInfo*>;
+using StringIntMap = MapStringTo<int, int>;
 
 class CPersistentHandler : implements IPersistentHandler, implements ISocketSelectNotify, public Thread
 {
@@ -57,10 +61,13 @@ private:
     PersistentLogLevel m_loglevel = PersistentLogLevel::PLogNormal;
     static int CurID;
     int m_id = 0;
+    bool m_enableDoNotReuseList = false;
+    StringIntMap m_instantCloseCounts;
+    StringIntMap m_doNotReuseList;
 public:
     IMPLEMENT_IINTERFACE;
-    CPersistentHandler(IPersistentSelectNotify* notify, int maxIdleTime, int maxReqs, PersistentLogLevel loglevel)
-                        : m_stop(false), m_notify(notify), m_maxIdleTime(maxIdleTime), m_maxReqs(maxReqs), m_loglevel(loglevel)
+    CPersistentHandler(IPersistentSelectNotify* notify, int maxIdleTime, int maxReqs, PersistentLogLevel loglevel, bool enableDoNotReuseList)
+                        : m_stop(false), m_notify(notify), m_maxIdleTime(maxIdleTime), m_maxReqs(maxReqs), m_loglevel(loglevel), m_enableDoNotReuseList(enableDoNotReuseList)
     {
         m_id = ++CurID;
         m_selectHandler.setown(createSocketSelectHandler());
@@ -74,8 +81,20 @@ public:
     {
         if (!sock)
             return;
-        PERSILOG(PersistentLogLevel::PLogNormal, "PERSISTENT: adding socket %d to handler %d", sock->OShandle(), m_id);
         synchronized block(m_mutex);
+        PERSILOG(PersistentLogLevel::PLogNormal, "PERSISTENT: adding socket %d to handler %d", sock->OShandle(), m_id);
+        if (m_enableDoNotReuseList && ep != nullptr)
+        {
+            StringBuffer epstr;
+            ep->getUrlStr(epstr);
+            if(m_doNotReuseList.getValue(epstr.str()) != nullptr)
+            {
+                PERSILOG(PersistentLogLevel::PLogNormal, "PERSISTENT: socket %d's target endpoint %s is in DoNotReuseList, will not add it.", sock->OShandle(), epstr.str());
+                sock->shutdown();
+                sock->close();
+                return;
+            }
+        }
         m_selectHandler->add(sock, SELECTMODE_READ, this);
         m_infomap.setValue(sock, new CPersistentInfo(false, usTick()/1000, 0, ep));
     }
@@ -90,10 +109,11 @@ public:
         CPersistentInfo* info = nullptr;
         if (val)
             info = *val;
-        if (!info || !info->inUse) //If inUse sock was already removed from select handler
-            m_selectHandler->remove(sock);
+        bool removedFromSelectHandler = info && info->inUse; //If inUse sock was already removed from select handler
         if (info)
             m_infomap.remove(sock);
+        if (!removedFromSelectHandler)
+            m_selectHandler->remove(sock);
     }
 
     virtual void doneUsing(ISocket* sock, bool keep, unsigned usesOverOne) override
@@ -154,6 +174,34 @@ public:
         if (x == 0)
         {
             PERSILOG(PersistentLogLevel::PLogMin, "PERSISTENT: Detected closing of connection %d from the other end", sock->OShandle());
+            if (m_enableDoNotReuseList)
+            {
+                synchronized block(m_mutex);
+                Owned<CPersistentInfo>* val = m_infomap.getValue(sock);
+                CPersistentInfo* info = nullptr;
+                if (val)
+                    info = *val;
+                if (info && info->epstr.length() > 0)
+                {
+                    int* countptr = m_instantCloseCounts.getValue(info->epstr.str());
+                    if (info->useCount == 0)
+                    {
+                        const static int MAX_INSTANT_CLOSES = 5;
+                        int count = 1;
+                        if (countptr)
+                            count = (*countptr)+1;
+                        if (count < MAX_INSTANT_CLOSES)
+                            m_instantCloseCounts.setValue(info->epstr.str(), count);
+                        else if (m_doNotReuseList.getValue(info->epstr.str()) == nullptr)
+                        {
+                            PERSILOG(PersistentLogLevel::PLogMin, "PERSISTENT: Endpoint %s has instantly closed connection for %d times in a row, adding it to DoNotReuseList", info->epstr.str(), MAX_INSTANT_CLOSES);
+                            m_doNotReuseList.setValue(info->epstr.str(), 1);
+                        }
+                    }
+                    else if (countptr)
+                        m_instantCloseCounts.remove(info->epstr.str());
+                }
+            }
             remove(sock);
         }
         else if (m_notify != nullptr)
@@ -238,13 +286,25 @@ public:
             join(1000);
         PERSILOG(PersistentLogLevel::PLogNormal, "PERSISTENT: Handler %d stopped", m_id);
     }
+
+    virtual bool inDoNotReuseList(SocketEndpoint* ep)
+    {
+        if(!ep)
+            return false;
+        StringBuffer epstr;
+        ep->getUrlStr(epstr);
+        if(epstr.length()> 0 && m_doNotReuseList.getValue(epstr.str()) != nullptr)
+            return true;
+        return false;
+    }
+
 };
 
 int CPersistentHandler::CurID = 0;
 
-IPersistentHandler* createPersistentHandler(IPersistentSelectNotify* notify, int maxIdleTime, int maxReqs, PersistentLogLevel loglevel)
+IPersistentHandler* createPersistentHandler(IPersistentSelectNotify* notify, int maxIdleTime, int maxReqs, PersistentLogLevel loglevel, bool enableDoNotReuseList)
 {
-    Owned<CPersistentHandler> handler = new CPersistentHandler(notify, maxIdleTime, maxReqs, loglevel);
+    Owned<CPersistentHandler> handler = new CPersistentHandler(notify, maxIdleTime, maxReqs, loglevel, enableDoNotReuseList);
     handler->start();
     return handler.getClear();
 }

esp/platform/persistent.hpp

@@ -33,6 +33,7 @@ interface IPersistentHandler : implements IInterface
     virtual void doneUsing(ISocket* sock, bool keep, unsigned usesOverOne = 0) = 0;
     virtual Linked<ISocket> getAvailable(SocketEndpoint* ep = nullptr) = 0;
     virtual void stop(bool wait) = 0;
+    virtual bool inDoNotReuseList(SocketEndpoint* ep) = 0;
 };
 
 interface IPersistentSelectNotify
@@ -40,6 +41,6 @@ interface IPersistentSelectNotify
     virtual bool notifySelected(ISocket *sock,unsigned selected, IPersistentHandler* handler) = 0;
 };
 
-IPersistentHandler* createPersistentHandler(IPersistentSelectNotify* notify, int maxIdleTime = DEFAULT_MAX_PERSISTENT_IDLE_TIME, int maxReqs = DEFAULT_MAX_PERSISTENT_REQUESTS, PersistentLogLevel loglevel=PersistentLogLevel::PLogMin);
+IPersistentHandler* createPersistentHandler(IPersistentSelectNotify* notify, int maxIdleTime = DEFAULT_MAX_PERSISTENT_IDLE_TIME, int maxReqs = DEFAULT_MAX_PERSISTENT_REQUESTS, PersistentLogLevel loglevel=PersistentLogLevel::PLogMin, bool enableDoNotReuseList=false);
 
 #endif //__PERSISTENT_HPP__

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -660,14 +660,7 @@ void EsdlServiceImpl::handleServiceRequest(IEspContext &context,
             reqcontent.set(reqWriter->str());
             context.addTraceSummaryTimeStamp(LogNormal, "serialized-xmlreq");
 
-            if (crt)
-            {
-                context.addTraceSummaryTimeStamp(LogNormal, "srt-custreqtrans");
-                crt->processTransform(&context, reqcontent, m_oEspBindingCfg.get());
-                context.addTraceSummaryTimeStamp(LogNormal, "end-custreqtrans");
-            }
-
-            handleFinalRequest(context, tgtcfg, tgtctx, srvdef, mthdef, ns, reqcontent, origResp, isPublishedQuery(implType), implType==EsdlMethodImplProxy);
+            handleFinalRequest(context, crt, tgtcfg, tgtctx, srvdef, mthdef, ns, reqcontent, origResp, isPublishedQuery(implType), implType==EsdlMethodImplProxy);
             context.addTraceSummaryTimeStamp(LogNormal, "end-HFReq");
 
             if (isPublishedQuery(implType))
@@ -831,6 +824,7 @@ void EsdlServiceImpl::getSoapError( StringBuffer& out,
 }
 
 void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
+                                         IEsdlCustomTransform *crt,
                                          Owned<IPropertyTree> &tgtcfg,
                                          Owned<IPropertyTree> &tgtctx,
                                          IEsdlDefService &srvdef,
@@ -846,7 +840,6 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
         "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
         "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\">");
 
-
     if(isroxie)
     {
         const char *tgtQueryName = tgtcfg->queryProp("@queryname");
@@ -879,7 +872,16 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
 
     const char *tgtUrl = tgtcfg->queryProp("@url");
     if (tgtUrl && *tgtUrl)
+    {
+        if (crt)
+        {
+            context.addTraceSummaryTimeStamp(LogNormal, "srt-custreqtrans");
+            crt->processTransform(&context, tgtcfg.get(), tgtctx.get(), srvdef, mthdef, soapmsg, m_oEspBindingCfg.get());
+            context.addTraceSummaryTimeStamp(LogNormal, "end-custreqtrans");
+        }
+
         sendTargetSOAP(context, tgtcfg.get(), soapmsg.str(), out, isproxy, NULL);
+    }
     else
     {
         ESPLOG(LogMax,"No target URL configured for %s",mthdef.queryMethodName());
@@ -888,7 +890,6 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
     }
 
     processResponse(context,srvdef,mthdef,ns,out);
-
 }
 
 void EsdlServiceImpl::handleEchoTest(const char *mthName,
@@ -2280,6 +2281,45 @@ int EsdlBindingImpl::getQualifiedNames(IEspContext& ctx, MethodInfoArray & metho
     return methods.ordinality();
 }
 
+int EsdlBindingImpl::getMethodProperty(IEspContext &context, const char *serv, const char *method, StringBuffer &page, const char *propname, const char *dfault)
+{
+    if (!serv || !*serv || !method || !*method)
+        return 0;
+
+    if (!m_esdl)
+    {
+        ESPLOG(LogMax,"EsdlBindingImpl::getMethodProperty - ESDL definition for service not loaded");
+        return 0;
+    }
+
+    IEsdlDefService *srv = m_esdl->queryService(serv);
+    if (!srv)
+    {
+        ESPLOG(LogMax,"EsdlBindingImpl::getMethodProperty - service (%s) not found", serv);
+        return 0;
+    }
+
+    IEsdlDefMethod *mth = srv->queryMethodByName(method);
+    if (!mth)
+    {
+        ESPLOG(LogMax,"EsdlBindingImpl::getMethodProperty - service (%s) method (%s) not found", serv, method);
+        return 0;
+    }
+
+    const char *value = mth->queryProp(propname);
+    page.append(value ? value : dfault);
+    return 0;
+}
+
+int EsdlBindingImpl::getMethodDescription(IEspContext &context, const char *serv, const char *method, StringBuffer &page)
+{
+    return getMethodProperty(context, serv, method, page, ESDL_METHOD_DESCRIPTION, "No description available");
+}
+int EsdlBindingImpl::getMethodHelp(IEspContext &context, const char *serv, const char *method, StringBuffer &page)
+{
+    return getMethodProperty(context, serv, method, page, ESDL_METHOD_HELP, "No Help available");
+}
+
 bool EsdlBindingImpl::qualifyServiceName(IEspContext &context,
                                         const char *servname,
                                         const char *methname,
@@ -2305,11 +2345,7 @@ bool EsdlBindingImpl::qualifyServiceName(IEspContext &context,
         methQName->clear();
         IEsdlDefMethod *mth = srv->queryMethodByName(methname);
         if (mth)
-        {
             methQName->append(mth->queryName());
-            addMethodDescription(methQName->str(), mth->queryProp(ESDL_METHOD_DESCRIPTION));
-            addMethodHelp(methQName->str(), mth->queryProp(ESDL_METHOD_HELP));
-        }
     }
     else if (methQName != NULL)
         methQName->clear();

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -173,7 +173,7 @@ public:
     virtual bool handleResultLogging(IEspContext &espcontext, IPropertyTree * reqcontext, IPropertyTree * request,  const char * rawresp, const char * finalresp, const char * logdata);
     void handleEchoTest(const char *mthName, IPropertyTree *req, StringBuffer &soapResp, ESPSerializationFormat format);
     void handlePingRequest(const char *mthName,StringBuffer &out,ESPSerializationFormat format);
-    virtual void handleFinalRequest(IEspContext &context, Owned<IPropertyTree> &tgtcfg, Owned<IPropertyTree> &tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, const char *ns, StringBuffer& req, StringBuffer &out, bool isroxie, bool isproxy);
+    virtual void handleFinalRequest(IEspContext &context, IEsdlCustomTransform *crt, Owned<IPropertyTree> &tgtcfg, Owned<IPropertyTree> &tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, const char *ns, StringBuffer& req, StringBuffer &out, bool isroxie, bool isproxy);
     void getSoapBody(StringBuffer& out,StringBuffer& soapresp);
     void getSoapError(StringBuffer& out,StringBuffer& soapresp,const char *,const char *);
 
@@ -263,6 +263,11 @@ public:
     void getRequestContent(IEspContext &context, StringBuffer & req, CHttpRequest* request, const char * servicename, const char * methodname, const char *ns, unsigned flags);
     void setXslProcessor(IInterface *xslp){}
 
+    int getMethodProperty(IEspContext &context, const char *serv, const char *method, StringBuffer &page, const char *propname, const char *dfault);
+
+    virtual int getMethodDescription(IEspContext &context, const char *serv, const char *method, StringBuffer &page) override;
+    virtual int getMethodHelp(IEspContext &context, const char *serv, const char *method, StringBuffer &page) override;
+
     int getQualifiedNames(IEspContext& ctx, MethodInfoArray & methods);
 
     StringBuffer & getServiceName(StringBuffer & resp)

esp/services/esdl_svc_engine/esdl_svc_custom.cpp

@@ -270,6 +270,7 @@ void CEsdlCustomTransformChoose::toDBGLog ()
 CEsdlCustomTransform::CEsdlCustomTransform(IPropertyTree &currentTransform)
 {
     m_name.set(currentTransform.queryProp("@name"));
+    m_target.set(currentTransform.queryProp("@target"));
     DBGLOG("Compiling custom ESDL Transform: '%s'", m_name.str());
 
     Owned<IPropertyTreeIterator> conditionalIterator = currentTransform.getElements("xsdl:choose");
@@ -285,7 +286,7 @@ CEsdlCustomTransform::CEsdlCustomTransform(IPropertyTree &currentTransform)
 #endif
 }
 
-void CEsdlCustomTransform::processTransform(IEspContext * context, StringBuffer & request, IPropertyTree * bindingCfg)
+void CEsdlCustomTransform::processTransform(IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg)
 {
     if (request.length()!=0)
     {
@@ -302,10 +303,31 @@ void CEsdlCustomTransform::processTransform(IEspContext * context, StringBuffer
         VStringBuffer ver("%g", context->getClientVersion());
         if(!xpathContext->addVariable("clientversion", ver.str()))
             ERRLOG("Could not set custom transform variable: clientversion:'%s'", ver.str());
+        //in case transform wants to make use of these values:
+        xpathContext->addVariable("query", tgtcfg->queryProp("@queryname"));
+        xpathContext->addVariable("method", mthdef.queryMethodName());
+        xpathContext->addVariable("service", srvdef.queryName());
 
         auto user = context->queryUser();
         if (user)
         {
+            static const std::map<SecUserStatus, const char*> statusLabels =
+            {
+#define STATUS_LABEL_NODE(s) { s, #s }
+                STATUS_LABEL_NODE(SecUserStatus_Inhouse),
+                STATUS_LABEL_NODE(SecUserStatus_Active),
+                STATUS_LABEL_NODE(SecUserStatus_Exempt),
+                STATUS_LABEL_NODE(SecUserStatus_FreeTrial),
+                STATUS_LABEL_NODE(SecUserStatus_csdemo),
+                STATUS_LABEL_NODE(SecUserStatus_Rollover),
+                STATUS_LABEL_NODE(SecUserStatus_Suspended),
+                STATUS_LABEL_NODE(SecUserStatus_Terminated),
+                STATUS_LABEL_NODE(SecUserStatus_TrialExpired),
+                STATUS_LABEL_NODE(SecUserStatus_Status_Hold),
+                STATUS_LABEL_NODE(SecUserStatus_Unknown),
+#undef STATUS_LABEL_NODE
+            };
+
             Owned<IPropertyIterator> userPropIt = user->getPropertyIterator();
             ForEach(*userPropIt)
             {
@@ -313,6 +335,22 @@ void CEsdlCustomTransform::processTransform(IEspContext * context, StringBuffer
                 if (name && *name)
                     xpathContext->addVariable(name, user->getProperty(name));
             }
+
+            auto it = statusLabels.find(user->getStatus());
+
+            xpathContext->addVariable("espUserName", user->getName());
+            xpathContext->addVariable("espUserRealm", user->getRealm() ? user->getRealm() : "");
+            xpathContext->addVariable("espUserPeer", user->getPeer() ? user->getPeer() : "");
+            xpathContext->addVariable("espUserStatus", VStringBuffer("%d", int(user->getStatus())));
+            if (it != statusLabels.end())
+                xpathContext->addVariable("espUserStatusString", it->second);
+            else
+                throw MakeStringException(-1, "encountered unexpected secure user status (%d) while processing transform", int(user->getStatus()));
+        }
+        else
+        {
+            // enable transforms to distinguish secure versus insecure requests
+            xpathContext->addVariable("espUserName", "");
         }
 
         Owned<IPropertyTreeIterator> configParams = bindingCfg->getElements("Transform/Param");
@@ -324,12 +362,24 @@ void CEsdlCustomTransform::processTransform(IEspContext * context, StringBuffer
             }
         }
 
-        Owned<IPropertyTree> thereq = createPTreeFromXMLString(request.str());
+        Owned<IPropertyTree> theroot = createPTreeFromXMLString(request.str());
+        StringBuffer xpath = m_target.str();
+        if (!xpath.length())
+        {
+            //This default gives us backward compatibility with only being able to write to the actual request
+            const char *tgtQueryName = tgtcfg->queryProp("@queryname");
+            xpath.setf("soap:Body/%s/%s", tgtQueryName ? tgtQueryName : mthdef.queryMethodName(), mthdef.queryRequestType());
+        }
+        //we can use real xpath processing in the future, for now simple substitution is fine
+        xpath.replaceString("{$query}", tgtcfg->queryProp("@queryname"));
+        xpath.replaceString("{$method}", mthdef.queryMethodName());
+        xpath.replaceString("{$service}", srvdef.queryName());
+        IPropertyTree *thereq = theroot->queryPropTree(xpath.str());  //get pointer to the write-able area
         ForEachItemIn(currConditionalIndex, m_customTransformClauses)
         {
             m_customTransformClauses.item(currConditionalIndex).process(context, thereq, xpathContext);
         }
-        toXML(thereq, request.clear());
+        toXML(theroot, request.clear());
 
         ESPLOG(LogMax,"MODIFIED REQUEST: %s", request.str());
     }

esp/services/esdl_svc_engine/esdl_svc_custom.hpp

@@ -30,6 +30,8 @@
 #include "jlog.hpp"
 #include "esp.hpp"
 
+#include "esdl_def.hpp"
+
 #include <map>
 #include <mutex>
 #include <thread>
@@ -127,7 +129,7 @@ private:
 
 interface IEsdlCustomTransform : extends IInterface
 {
-       virtual void processTransform(IEspContext * context,  StringBuffer & request, IPropertyTree * bindingCfg)=0;
+       virtual void processTransform(IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg)=0;
 };
 
 class CEsdlCustomTransform : implements IEsdlCustomTransform, public CInterface
@@ -135,6 +137,7 @@ class CEsdlCustomTransform : implements IEsdlCustomTransform, public CInterface
 private:
     CIArrayOf<CEsdlCustomTransformChoose> m_customTransformClauses;
     StringAttr m_name;
+    StringAttr m_target;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -161,7 +164,7 @@ public:
 #endif
     }
 
-    void processTransform(IEspContext * context,  StringBuffer & request, IPropertyTree * bindingCfg);
+    void processTransform(IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg) override;
 };
 
 #endif /* ESDL_SVC_CUSTOM_HPP_ */

esp/services/ws_machine/ws_machineService.cpp

@@ -1183,7 +1183,7 @@ int Cws_machineEx::runCommand(IEspContext& context, const char* sAddress, const
         IFRunSSH * connection = createFRunSSH();
         connection->init(command.str(),NULL,NULL,NULL,m_SSHConnectTimeoutSeconds,0);
         // executed as single connection
-        connection->exec(IpAddress(sAddress),NULL,true);
+        connection->exec(IpAddress(sAddress),NULL,false);
         response.append(connection->getReplyText()[0]);
         exitCode = connection->getReply()[0];
         int len = response.length();

esp/src/eclwatch/LZBrowseWidget.js

@@ -17,6 +17,7 @@ define([
     "dijit/MenuItem",
     "dijit/MenuSeparator",
     "dijit/PopupMenuItem",
+    "dijit/form/TextBox",
     "dijit/form/ValidationTextBox",
 
     "dgrid/tree",
@@ -58,7 +59,7 @@ define([
 
     "hpcc/TableContainer"
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, domClass, iframe, on, topic,
-    registry, Dialog, Menu, MenuItem, MenuSeparator, PopupMenuItem, ValidationTextBox,
+    registry, Dialog, Menu, MenuItem, MenuSeparator, PopupMenuItem, TextBox, ValidationTextBox,
     tree, editor, selector,
     _TabContainerWidget, FileSpray, ESPUtil, ESPRequest, ESPDFUWorkunit, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, SelectionGridWidget, FilterDropDownWidget, Utility,
     template) {
@@ -109,7 +110,8 @@ define([
                 this.fixedSprayReplicateCheckbox = registry.byId(this.id + "FixedSprayReplicate");
                 this.delimitedSprayReplicateCheckbox = registry.byId(this.id + "DelimitedSprayReplicate");
                 this.xmlSprayReplicateCheckbox = registry.byId(this.id + "XMLSprayReplicate");
-                this.sprayXMLButton = registry.byId(this.id + "SprayXMLButton");
+                this.sprayXMLButton = registry.byId(this.id + "SprayFixedButton");
+                this.sprayFixedButton = registry.byId(this.id + "SprayXMLButton");
                 this.jsonSprayReplicate = registry.byId(this.id + "JSONSprayReplicate");
                 this.variableSprayReplicateCheckbox = registry.byId(this.id + "VariableSprayReplicate");
                 this.blobSprayReplicateCheckbox = registry.byId(this.id + "BlobSprayReplicate");
@@ -535,6 +537,7 @@ define([
                     context.refreshGrid();
                 });
                 this.filter.on("apply", function (evt) {
+                    context.landingZonesGrid.clearSelection();
                     context.refreshHRef();
                     context.refreshGrid();
                 });
@@ -735,16 +738,22 @@ define([
                     columns: {
                         targetName: editor({
                             label: this.i18n.TargetName,
-                            width: 144,
                             autoSave: true,
-                            editor: "text"
-                        }),
+                            editor: "text",
+                            editorArgs: {
+                                style: "width: 100%;"
+                            }
+                        }, TextBox),
                         targetRecordLength: editor({
+                            editorArgs: {
+                                required: true,
+                                placeholder: this.i18n.RequiredForXML,
+                                promptMessage: this.i18n.RequiredForXML,
+                                style: "width: 100%;"
+                            },
                             label: this.i18n.RecordLength,
-                            width: 72,
                             autoSave: true,
-                            editor: "text"
-                        })
+                        }, ValidationTextBox)
                     }
                 });
 
@@ -772,24 +781,7 @@ define([
                         targetRowTag: editor({
                             label: this.i18n.RowTag,
                             width: 100,
-                            autoSave: true,
-                            editor: "dijit.form.ValidationTextBox",
-                            editorArgs: {
-                                required: true,
-                                placeholder: this.i18n.RequiredForXML,
-                                promptMessage: this.i18n.RequiredForXML,
-                                validator: function (value, constraints) {
-                                    var valid = true;
-                                    if (value !== null && value !== undefined && value !== "") {
-                                        valid = true;
-                                        context.sprayXMLButton.set("disabled", false);
-                                    } else {
-                                        context.sprayXMLButton.set("disabled", true);
-                                        valid = false;
-                                    }
-                                    return valid;
-                                }
-                            }
+                            autoSave: true
                         })
                     }
                 });

esp/src/eclwatch/nls/bs/hpcc.js

@@ -63,6 +63,7 @@
     BannerScroll: "Kretanje Reklamnog Bloka",
     BannerSize: "Veličina Reklamnog Bloka",
     BinaryInstalls: "Binarne Instalacije",
+    Bind: "Vežite",
     Binding: "Vezivanje",
     BindingDeleted: "Vezivanje Obrisano",
     Blob: "BLOB",
@@ -94,6 +95,8 @@
     CollapseAll: "Suzite sve",
     Command: "Komanda",
     Comment: "Komentar",
+    Compiled: "Kompajlirano",
+    Compiling: "U procesu kompajliranja",
     Completed: "Kompletiran",
     ComplexityWarning: "Više od praga {threshold} aktivnosti ({activityCount}) - zaustavite prikaz podataka?",
     Component: "Komponenta",
@@ -102,6 +105,7 @@
     CompressedFileSize: "Komprimirana Veličina Datoteke",
     Condition: "Uslov",
     Configuration: "Konfiguracija",
+    ConfigureService: "Servis za Konfiguraciju",
     ConfirmPassword: "Potvrdite Lozinku",
     ConfirmRemoval: "Jeste li sigurni da to želite učiniti?",
     ContactAdmin: "Ako želite promijeniti naziv ove grupe, kontaktirajte administratora LDAP.",
@@ -116,6 +120,7 @@
     Created: "Proizveden",
     CreatedBy: "Autor",
     CreatedTime: "Vrijeme Kreiranja",
+    Creating: "U procesu kreiranja",
     Critical: "Kritično",
     CSV: "CSV",
     Dali: "Dali",
@@ -128,7 +133,10 @@
     DEF: "DEF",
     Defaults: "Unaprijed Definisane Vrijednosti",
     Definition: "Definicija",
+    DefinitionDeleted: "Definicija je izbrisana",
+    DefinitionID: "ID",
     Definitions: "Definicije",
+    DelayedReplication: "Odložena replikacija",
     Delete: "Obrišite",
     DeleteBinding: "Izbrišite Vezivanje",
     Deleted: "Izbrisan",
@@ -179,10 +187,13 @@
     Downloads: "Preuzimanje",
     DownloadToCSV: "Preuzmite u CSV formatu",
     DropZone: "Zona Prijema",
+    DueToInctivity: "Bićete odjavljeni iz svih ECL Watch sesija za 3 minuta zbog neaktivnosti.",
     Duration: "Trajanje",
     DynamicNoServicesFound: "Servisi Nisu Pronađeni",
     EBCDIC: "EBCDIC",
     ECL: "ECL",
+    ECLWatchRequiresCookies: "ECL Watch zahtijeva da kolačići budu omogućeni za nastavak",
+    ECLWatchSessionManagement: "Upravljanje ECL Watchom",
     ECLWorkunit: "ECL RadnaJedinica",
     Edges: "Ivice",
     Edit: "Editujte",
@@ -205,7 +216,7 @@
     ErrorsStatus: "Greške/Stanje",
     ErrorUploadingFile: "Greška prilikom prenosa datoteke(a). Pokušajte provjeriti dozvole za prenos.",
     ErrorWarnings: "Greška/Upozorenja",
-    Escape: "Escape",
+    Escape: "Eskejp",
     ESPBuildVersion: "ESP Trenutna Verzija",
     ESPNetworkAddress: "ESP Netvork Adresa",
 	ESPProcessName: "Ime ESP procesa",
@@ -215,6 +226,8 @@
     EventText: "Opis Događaja",
     EventTextPH: "Tekst O Događaju",
     Exception: "Neočekivani Problem",
+    Executed: "Izvršeno",
+    Executing: "U procesu izvršavanja",
     ExpandAll: "Proširite sve",
 	ExpireDays: "Ističe za (u danima)",
     Export: "Izvezite",
@@ -248,6 +261,7 @@
     FindNext: "Nađite Slijedeći",
     FindPrevious: "Nađite Prethodni",
     Finished: "Završen",
+    FirstN: "Prvih N",
     FirstName: "Ime",
     FirstNRows: "Prvih N Redova",
     Fixed: "Fiksni",
@@ -294,6 +308,7 @@
     InheritedPermissions: "Naslijeđene Dozvole",
     Inputs: "Unosi",
     InvalidResponse: "(Neispravan Odgovor)",
+    InvalidUsernamePassword: "Neispravno korisničko ime ili lozinka, pokušajte ponovo.",
     IP: "IP",
     IPAddress: "IP Adresa",
     IsCompressed: "Je li Sabijen",
@@ -323,6 +338,7 @@
     LastNRows: "Posljednjih N Redova",
     LastRun: "Zadnji Ran",
     LDAPWarning: "<b>Greška LDAP Servica:</b>  &lsquo;Previše korisnika&rsquo; - Molimo koristite filter.",
+    LearnMore: "Naučite više",
     LegacyForm: "Stari Prevaziđeni Formular",
 	Legend: "Legenda",
     LibrariesUsed: "Biblioteke u Korištenju",
@@ -355,6 +371,7 @@
     Logout: "Odjavite se",
     Logs: "Dnevnici",
     LogVisualization: "Registrujte Vizuelizaciju",
+    LogVisualizationUnconfigured: "Vizuelizacija loga nije konfigurisana, proverite kako je menadžer za konfiguraciju podešen",
     LostFile: "Izgubljeni Fajl",
     LostFile2: "Izgubljeni Fajlovi",
     LostFileMessage: "Logički fajl kome nedostaje bar jedan dio ili na primarnoj ili na repliciranoj lokaciji na disku.  Logički file je još uvijek pod kontrolom Dali servera. Brisanje fajla prekida kontrolu Dali servera nad fajlom kao i nad svim preostalim dijelovima fajla na disku.",
@@ -481,11 +498,13 @@
     Permissions: "Dozvole za Pristup",
     PhysicalFiles: "Fizičke Datoteke",
     PlaceholderFindText: "Wuid, Korisnik, Dalje...",
-    PlaceholderFirstName: "John",
-    PlaceholderLastName: "Smith",
+    PlaceholderFirstName: "Jovan",
+    PlaceholderLastName: "Smit",
     Playground: "Igralište",
+    PleaseEnableCookies: "ECL Watch zahtijeva da kolačići budu omogućeni za nastavak.",
     PleaseEnterANumber: "Unestite Broj 1 -",
 	PleaseLogin: "Molimo prijavite se koristeći svoje korisničko ime i lozinku",
+    PleaseLogIntoECLWatch: "Molimo prijavite se u ECL Watch",
     PleasePickADefinition: "Izaberite Definiciju",
     PleaseSelectADynamicESDLService: "Izaberite Dinamički ESDL Servis",
     PleaseSelectAGroupToAddUser: "Izaberite grupu u koju želite da dodate ovog korisnika",
@@ -785,7 +804,7 @@
     UpdateCloneFrom: "Ažurirajte Klon Koristeći",
 	UpdateDFs: "Ažurirajte DFS",
     UpdateSuperFiles: "Ažurirajte Super Datoteke",
-    Upload: "Upload",
+    Upload: "Učitajte",
     URL: "URL",
     Usage: "Upotreba",
     Used: "Korišten",
@@ -837,6 +856,7 @@
     XML: "XML",
     XRef: "XRef",
     Year: "Godina",
+    YouAreAboutToBeLoggedOut: "Vi ćete biti odjavljeni",
     YouAreAboutToDeleteBinding: "Odabrano vezivanje/binding će biti izbrisano. Jeste li sigurni da želite to učiniti?",
 	YouAreAboutToDeleteDefinition: "Vi ste u procesu brisanja ove definicije. Da li ste sigurni da to želite učiniti?",
     YouAreAboutToDeleteThisFile: "Da li ćete obrisati ovu datoteku",
@@ -845,6 +865,7 @@
     YouAreAboutToDeleteThisWorkunit: "Da li ćete obrisati ovu radnu jedinicu",
 	YouAreAboutToRemoveUserFrom: "Vi ste u procesu uklanjanja korisnika iz ove grupe. Da li želite da nastavite?",
     YourBrowserMayNotSupport: "Vaš pretraživač možda ne podržava datoteku (e) ove veličine",
+    YourScreenWasLocked: "ESP je zaključao vaš ekran. Podatci su zastarjeli.",
     ZAP: "Z.A.P",
     ZeroLogicalFilesCheckFilter: "Nema ni jedane Logičke Datoteke(provjerite filter)",
     Zip: "Zapakujte (Zip)",
@@ -852,6 +873,8 @@
     Zoom: "Zum",
     Zoom100Pct: "Zumirajte 100%",
     ZoomAll: "Zumirajte Sve",
+    ZoomMinus: "Zum-",
+    ZoomPlus: "Zum +",
     ZoomWidth: "Zumirajte Širinu"
 })
 );

esp/src/eclwatch/nls/hr/hpcc.js

@@ -1,5 +1,5 @@
 define(
-({    
+({
     Abort: "Prekinite",
     AbortedBy: "Prekinuto od strane",
     AbortedTime: "Vrijeme prekida",
@@ -63,6 +63,7 @@
     BannerScroll: "Kretanje Reklamnog Bloka",
     BannerSize: "Veličina Reklamnog Bloka",
     BinaryInstalls: "Binarne Instalacije",
+    Bind: "Vežite",
     Binding: "Vezivanje",
     BindingDeleted: "Vezivanje Obrisano",
     Blob: "BLOB",
@@ -94,6 +95,8 @@
     CollapseAll: "Suzite sve",
     Command: "Komanda",
     Comment: "Komentar",
+    Compiled: "Kompajlirano",
+    Compiling: "U procesu kompajliranja",
     Completed: "Kompletiran",
     ComplexityWarning: "Više od praga {threshold} aktivnosti ({activityCount}) - prekinite prikaz podataka?",
     Component: "Komponenta",
@@ -102,6 +105,7 @@
     CompressedFileSize: "Komprimirana Veličina Datoteke",
     Condition: "Uslov",
     Configuration: "Konfiguracija",
+    ConfigureService: "Servis za Konfiguraciju",
     ConfirmPassword: "Potvrdite Lozinku",
     ConfirmRemoval: "Jeste li sigurni da to želite učiniti?",
     ContactAdmin: "Ako želite promijeniti naziv ove grupe, kontaktirajte administratora LDAP.",
@@ -116,6 +120,7 @@
     Created: "Proizveden",
     CreatedBy: "Autor",
     CreatedTime: "Vrijeme Izrade",
+    Creating: "U procesu stvaranja",
     Critical: "Kritično",
     CSV: "CSV",
     Dali: "Dali",
@@ -128,7 +133,10 @@
     DEF: "DEF",
     Defaults: "Unaprijed Definisane Vrijednosti",
     Definition: "Definicija",
+    DefinitionDeleted: "Definicija je izbrisana",
+    DefinitionID: "ID",
     Definitions: "Definicije",
+    DelayedReplication: "Odgođena replikacija",
     Delete: "Obrišite",
     DeleteBinding: "Izbrišite Vezivanje",
     Deleted: "Obrisan",
@@ -179,10 +187,13 @@
     Downloads: "Preuzimanje",
     DownloadToCSV: "Preuzmite u CSV formatu",
     DropZone: "Zona Prijema",
+    DueToInctivity: "Bit ćete odjavljeni iz svih ECL Watch sjednica za 3 minute zbog neaktivnosti.",
     Duration: "Trajanje",
     DynamicNoServicesFound: "Servisi Nisu Pronađeni",
     EBCDIC: "EBCDIC",
     ECL: "ECL",
+    ECLWatchRequiresCookies: "ECL Watch zahtijeva da kolačići budu omogućeni za nastavak",
+    ECLWatchSessionManagement: "Upravljanje ECL Watchom",
     ECLWorkunit: "ECL RadnaJedinica",
     Edges: "Ivice",
     Edit: "Editujte",
@@ -205,7 +216,7 @@
     ErrorsStatus: "Greške/Stanje",
     ErrorUploadingFile: "Pogreška prilikom prenosa datoteke(a). Pokušajte provjeriti dozvole za prenos.",
     ErrorWarnings: "Greška/Upozorenja",
-    Escape: "Escape",
+    Escape: "Eskejp",
     ESPBuildVersion: "ESP Trenutna Verzija",
     ESPNetworkAddress: "ESP Netvork Adresa",
     ESPProcessName: "Naziv ESP procesa",
@@ -215,6 +226,8 @@
     EventText: "Opis Događaja",
     EventTextPH: "Tekst O Događaju",
     Exception: "Neočekivani Problem",
+    Executed: "Izvršeno",
+    Executing: "U procesu izvršenja",
     ExpandAll: "Proširite sve",
     ExpireDays: "Istječe za (u danima)",
     Export: "Izvezite",
@@ -248,6 +261,7 @@
     FindNext: "Nađite Slijedeći",
     FindPrevious: "Nađite Prethodni",
     Finished: "Završen",
+    FirstN: "Prvih N",
     FirstName: "Ime",
     FirstNRows: "Prvih N Redova",
     Fixed: "Fiksni",
@@ -294,6 +308,7 @@
     InheritedPermissions: "Naslijeđene Dozvole",
     Inputs: "Unosi",
     InvalidResponse: "(Neispravan Odgovor)",
+    InvalidUsernamePassword: "Nevažeće korisničko ime ili zaporka, pokušajte ponovo.",
     IP: "IP",
     IPAddress: "IP Adresa",
     IsCompressed: "Je li Komprimiran",
@@ -323,6 +338,7 @@
     LastNRows: "Posljednjih N Redova",
     LastRun: "Zadnji Ran",
     LDAPWarning: "<b>Greška LDAP Servica:</b>  &lsquo;Previše korisnika&rsquo; - Molimo koristite filter.",
+    LearnMore: "Naučite više",
     LegacyForm: "Stari Prevaziđeni Formular",
     Legend: "Legenda",
     LibrariesUsed: "Biblioteke u Korištenju",
@@ -340,8 +356,8 @@
     LocalFileSystemsOnly: "Samo Lokalni File Sistemi",
     Location: "Lokacija",
     Lock: "Zaključaj",
-    log_analysis_1: "log_analysis_1*",
     Log: "Dnevnik (Log)",
+    log_analysis_1: "log_analysis_1*",
     LogFile: "Datoteka Aktivnosti",
     LoggedInAs: "Prijavljen kao",
     LoggingOut: "Odjavljivanje",
@@ -355,6 +371,7 @@
     Logout: "Odjavite se",
     Logs: "Dnevnici",
     LogVisualization: "Registrujte Vizuelizaciju",
+    LogVisualizationUnconfigured: "Vizualizacija logotipa nije konfigurirana, provjerite kako je konfiguriran upravitelj konfiguracije",
     LostFile: "Izgubljena Datoteka",
     LostFile2: "Izgubljene Datoteke",
     LostFileMessage: "Logička datoteka kojoj nedostaje bar jedan dio ili na primarnoj ili na repliciranoj lokaciji na disku.  Logička datoteka je još uvijek pod kontrolom Dali servera. Brisanje datoteke uklanja kontrolu Dali servera nad datotekom kao i nad svim preostalim dijelovima datoteke na disku.",
@@ -481,11 +498,13 @@
     Permissions: "Dozvole za Pristup",
     PhysicalFiles: "Fizičke Datoteke",
     PlaceholderFindText: "Wuid, Korisnik, Dalje...",
-    PlaceholderFirstName: "John",
-    PlaceholderLastName: "Smith",
+    PlaceholderFirstName: "Jovan",
+    PlaceholderLastName: "Smit",
     Playground: "Igralište",
+    PleaseEnableCookies: "ECL Watch zahtijeva da kolačići budu omogućeni za nastavak.",
     PleaseEnterANumber: "Unestite Broj 1 -",
     PleaseLogin: "Molimo prijavite se koristeći svoje korisničko ime i lozinku",
+    PleaseLogIntoECLWatch: "Prijavite se na ECL Watch",
     PleasePickADefinition: "Izaberite Definiciju",
     PleaseSelectADynamicESDLService: "Izaberite Dinamički ESDL Servis",
     PleaseSelectAGroupToAddUser: "Izaberite grupu u koju želite da dodate ovog korisnika",
@@ -785,7 +804,7 @@
     UpdateCloneFrom: "Ažurirajte Klon Koristeći",
     UpdateDFs: "Ažurirajte DFS",
     UpdateSuperFiles: "Ažurirajte Super Datoteke",
-    Upload: "Upload",
+    Upload: "Učitajte",
     URL: "URL",
     Usage: "Upotreba",
     Used: "Korišten",
@@ -837,6 +856,7 @@
     XML: "XML",
     XRef: "XRef",
     Year: "Godina",
+    YouAreAboutToBeLoggedOut: "Vi ćete biti odjavljeni",
     YouAreAboutToDeleteBinding: "Odabrano vezivanje/binding će biti izbrisano. Jeste li sigurni da želite to učiniti?",
     YouAreAboutToDeleteDefinition: "U tijeku je brisanje ove definicije. Jeste li sigurni da želite to učiniti?",
     YouAreAboutToDeleteThisFile: "Hoćete li izbrisati ovu datoteku",
@@ -845,6 +865,7 @@
     YouAreAboutToDeleteThisWorkunit: "Hoćete li izbrisati ovu radnu jedinicu",
     YouAreAboutToRemoveUserFrom: "Tražili ste da uklonite korisnika(e) iz ove grupe. Da li želite nastaviti?",
     YourBrowserMayNotSupport: "Vaš pretraživač možda ne podržava datoteku (e) ove veličine",
+    YourScreenWasLocked: "ESP je zaključao vaš ekran. Podatci su zastarjeli.",
     ZAP: "Z.A.P",
     ZeroLogicalFilesCheckFilter: "Nema ni jedane Logičke Datoteke(provjerite filter)",
     Zip: "Zapakujte (Zip)",
@@ -852,6 +873,8 @@
     Zoom: "Zum",
     Zoom100Pct: "Zumirajte 100%",
     ZoomAll: "Zumirajte Sve",
+    ZoomMinus: "Zum-",
+    ZoomPlus: "Zum +",
     ZoomWidth: "Zumirajte Širinu"
 })
 );

esp/src/eclwatch/nls/sr/hpcc.js

@@ -64,6 +64,7 @@
     BannerScroll: "Кретање Рекламног Блока",
     BannerSize: "Величина Рекламног Блока",
     BinaryInstalls: "Бинарне Инсталације",
+    Bind: "Вежите",
     Binding: "Везивање",
     BindingDeleted: "Везивање Избрисанo",
     Blob: "БЛОБ",
@@ -96,6 +97,8 @@
     CollapseAll: "Сузите све",
     Command: "Команда",
     Comment: "Коментар",
+    Compiled: "Компајлирано",
+    Compiling: "У процесу компајлирања",
     Completed: "Комплетиран",
     ComplexityWarning: "Више од прага {threshold} активности ({activityCount}) - прекините приказ података?",
     Component: "Компонента",
@@ -104,6 +107,7 @@
     CompressedFileSize: "Компримирана Величина Датотеке",
     Condition: "услов",
     Configuration: "Конфигурација",
+    ConfigureService: "Сервис за Конфигурацију",
     ConfirmPassword: "Потврдите Лозинку",
     ConfirmRemoval: "Јесте ли сигурни да то желите учинити?",
     ContactAdmin: "Ако желите променити назив ове групе, контактирајте администратора ЛДАП.",
@@ -118,6 +122,7 @@
     Created: "Произведен",
     CreatedBy: "Аутор",
     CreatedTime: "Време стварања",
+    Creating: "У процесу креирањa",
     Critical: "Критично",
     CSV: "ЦСВ",
     Dali: "Дали",
@@ -130,7 +135,10 @@
     DEF: "ДЕФ",
     Defaults: "Унапред Дефинисане Вредности",
     Definition: "Дефиниција",
+    DefinitionDeleted: "Дефиниција је избрисана",
+    DefinitionID: "ИД",
     Definitions: "Дефиниције",
+    DelayedReplication: "Одложена репликација",
     Delete: "Обришите",
     DeleteBinding: "Избришите Везивање",
     Deleted: "Избрисан",
@@ -181,10 +189,13 @@
     Downloads: "Преузимање",
     DownloadToCSV: "Преузмите у ЦСВ формату",
     DropZone: "Зона Пријема",
+    DueToInctivity: "Бићете одјављени из свих ЕЦЛ Вач сесија за 3 минута због неактивности.",
     Duration: "Трајање",
     DynamicNoServicesFound: "Сервиси Нису Пронађени",
     EBCDIC: "ЕБЦДИK",
     ECL: "ЕЦЛ",
+    ECLWatchRequiresCookies: "ЕЦЛ Вач захтијева да колачићи буду омогућени за наставак",
+    ECLWatchSessionManagement: "Управљање ЕЦЛ  Вачом",
     ECLWorkunit: "ЕЦЛ РаднаЈединица",
     Edges: "Ивице",
     Edit: "Едитујте",
@@ -217,6 +228,8 @@
     EventText: "Опис Догађаја",
     EventTextPH: "Текст О Догађају",
     Exception: "Неочекивани Проблем",
+    Executed: "Извршено",
+    Executing: "У процесу извршавања",
     ExpandAll: "Проширите све",
     ExpireDays: "Истиче за (у данима)",
     Export: "Извезите",
@@ -250,6 +263,7 @@
     FindNext: "Нађите Следећи",
     FindPrevious: "Нађите Претходни",
     Finished: "Завршен",
+    FirstN: "Првих Н",
     FirstName: "Име",
     FirstNRows: "Првих Н Редова",
     Fixed: "Фиксни",
@@ -296,6 +310,7 @@
     InheritedPermissions: "Наслеђене Дозволе",
     Inputs: "Уноси",
     InvalidResponse: "(Неисправан Одговор)",
+    InvalidUsernamePassword: "Неисправно корисничко име или лозинка, покушајте поново.",
     IP: "ИП",
     IPAddress: "ИП Адреса",
     IsCompressed: "Је ли Сабијен",
@@ -325,6 +340,7 @@
     LastNRows: "Последњих Н Редова",
     LastRun: "Задњи Ран",
     LDAPWarning: "<б>Грешка ЛДАП Сервица:</б>  &lsquo;Превише корисника&rsquo; - Молимо користите филтер.",
+    LearnMore: "Научите више",
     LegacyForm: "Стари Превазиђени Формулар",
     Legend: "Легенда",
     LibrariesUsed: "Библиотеке у Кориштењу",
@@ -342,8 +358,8 @@
     LocalFileSystemsOnly: "Само Локални Фајл Системи",
     Location: "Локација",
     Lock: "закључаjтe",
-    log_analysis_1: "лог_аналисис_1*",
     Log: "Дневник (Лог)",
+    log_analysis_1: "лог_аналисис_1*",
     LogFile: "Датотека Активности",
     LoggedInAs: "Пријављен као",
     LoggingOut: "Одјављивањe",
@@ -357,6 +373,7 @@
     Logout: "Одјавитe се",
     Logs: "Дневници",
     LogVisualization: "Региструјте визуализацију",
+    LogVisualizationUnconfigured: "Визуелизација логa није конфигурирана, проверите како је конфигурацијски менаџер подешен",
     LostFile: "Изгубљени Фајл",
     LostFile2: "Изгубљени Фајлови",
     LostFileMessage: "Логички фајл коме недостаје бар један део или на примарној или на реплицираној локацији на диску. Логички фајл је још увек под контролом Дали сервера. Брисање фајлa прекида контролу Дали сервера над фајлом као и над свим преосталим деловима фајлa на диску.",
@@ -486,8 +503,10 @@
     PlaceholderFirstName: "Џон",
     PlaceholderLastName: "Смит",
     Playground: "Игралиште",
+    PleaseEnableCookies: "ЕЦЛ Вач захтијева да колачићи буду омогућени за наставак",
     PleaseEnterANumber: "Унестите Број 1 -",
     PleaseLogin: "Молимо пријавите се користећи своје корисничко име и лозинку",
+    PleaseLogIntoECLWatch: "Молимо пријавите се у ЕЦЛ Вач",
     PleasePickADefinition: "Изаберите дефиницију",
     PleaseSelectADynamicESDLService: "Одаберите Динамички ЕСДЛ сервис",
     PleaseSelectAGroupToAddUser: "Изаберите групу у коју желите да додате корисника",
@@ -834,11 +853,12 @@
     WUID: "PJИД",
     Wuidcannotbeempty: "Pjид Не Може Бити Празан.",
     WUSnapShot: "Тренутна Слика Радне Јединице",
-    XGMML: "XGMML",
-    XLS: "XLS",
-    XML: "XML",
-    XRef: "XRef",
+    XGMML: "XГММЛ",
+    XLS: "XЛС",
+    XML: "XМЛ",
+    XRef: "XРеф",
     Year: "Година",
+    YouAreAboutToBeLoggedOut: "Ви ћете бити одјављени",
     YouAreAboutToDeleteBinding: "Одабрано везивање ће бити обрисано. Да ли сте сигурни да желите да то урадите?",
     YouAreAboutToDeleteDefinition: "У процесу је брисање ове дефиниције. Да ли сте сигурни да желите то учинити?",
     YouAreAboutToDeleteThisFile: "Да ли ћете обрисати ову датотеку",
@@ -847,13 +867,16 @@
     YouAreAboutToDeleteThisWorkunit: "Да ли ћете обрисати ову радну јединицу",
     YouAreAboutToRemoveUserFrom: "тражили сте да уклоните корисника(е) из ове групе. Да ли желите да наставите?",
     YourBrowserMayNotSupport: "Ваш претраживач можда не подржава датотеку (е) ове величине",
-    ZAP: "Z.A.P",
+    YourScreenWasLocked: "ЕСП је закључао ваш екран. Подаци су застарели",
+    ZAP: "З.А.П.",
     ZeroLogicalFilesCheckFilter: "Нема ни једане Логичке Датотеке(проверите филтер)",
     Zip: "Запакујте (Зип)",
     ZippedAnalysisPackage: "Запаковани Пакет са Анализама",
     Zoom: "Зум",
     Zoom100Pct: "Зумирајте 100%",
     ZoomAll: "Зумирајте Све",
+    ZoomMinus: "Зум -",
+    ZoomPlus: "Зум +",
     ZoomWidth: "Зумирајте Ширину"
 })
 );

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -94,7 +94,7 @@
                                             <option value="8">UTF-32LE</option>
                                             <option value="9">UTF-32BE</option>
                                         </select>
-                                        <input id="${id}SprayDelimitedMaxRecordLength" title="${i18n.MaxRecordLength}:" style="width: 95%;" name="sourceMaxRecordSize" value="8192" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input id="${id}SprayDelimitedMaxRecordLength" title="${i18n.MaxRecordLength}:" style="width: 95%;" name="sourceMaxRecordSize" required="false" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input id="${id}SprayDelimitedSeparators" title="${i18n.Separators}:" style="width: 95%;" name="sourceCsvSeparate" value="\," data-dojo-props="trim: true, placeHolder:'\,'" colspan="2" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.OmitSeparator}:" name="NoSourceCsvSeparator" colspan="2" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}SprayDelimitedEscape" title="${i18n.Escape}:" style="width: 95%;" name="sourceCsvEscape" data-dojo-props="trim: true" colspan="2" data-dojo-type="dijit.form.TextBox" />
@@ -144,7 +144,7 @@
                                             <option value="8">UTF-32LE</option>
                                             <option value="9">UTF-32BE</option>
                                         </select>
-                                        <input id="${id}SprayXmlMaxRecordLength" title="${i18n.MaxRecordLength}:" value="8192" style="width: 95%;" name="sourceMaxRecordSize" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input id="${id}SprayXmlMaxRecordLength" title="${i18n.MaxRecordLength}:" style="width: 95%;" name="sourceMaxRecordSize" required="false" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}XMLSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
@@ -187,7 +187,7 @@
                                             <option value="8">UTF-32LE</option>
                                             <option value="9">UTF-32BE</option>
                                         </select>
-                                        <input id="${id}SprayJsonMaxRecordLength" title="${i18n.MaxRecordLength}:" value="8192" style="width: 95%;" name="sourceMaxRecordSize" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input id="${id}SprayJsonMaxRecordLength" title="${i18n.MaxRecordLength}:" value="" style="width: 95%;" name="sourceMaxRecordSize" required="false" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}JSONSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />

esp/src/src/ESPPackageProcess.ts

@@ -10,9 +10,12 @@ var Store = declare([ESPRequest.Store], {
     service: "WsPackageProcess",
     action: "ListPackages",
     responseQualifier: "ListPackagesResponse.PackageMapList.PackageListMapData",
-    responseTotalQualifier: "ListPackagesResponse.PackageMapList.PackageListMapData",
-    idProperty: "__hpcc_id"
+    idProperty: "Id",
+    startProperty: "PageStartFrom",
+    countProperty: "PageSize",
+    SortbyProperty: 'SortBy'
 });
+
 export function CreatePackageMapQueryObjectStore(options) {
     var store = new Store(options);
     return Observable(store);

esp/src/src/FileSpray.ts

@@ -107,16 +107,15 @@ var LandingZonesFilterStore = declare([ESPRequest.Store], {
         }
     },
     preProcessRow: function (row) {
-        var fullPath = row.Path + "/" + row.name;
+        var fullPath = this.dropZone.machine.Directory + row.name;
         lang.mixin(row, {
-            DropZone: {
-                NetAddress: this.dropZone.machine.Netaddress
-            },
+            NetAddress: this.dropZone.machine.Netaddress,
+            Directory: this.dropZone.machine.Directory,
             calculatedID: this.dropZone.machine.Netaddress + fullPath,
             fullPath: fullPath,
             fullFolderPath: row.Path,
-            displayName: row.name,
-            type: row.isDir ? "folder" : "file"
+            displayName: row.Path ? row.Path + row.name : row.name,
+            type: row.isDir ? "filteredFolder" : "file"
         });
     }
 });
@@ -136,7 +135,7 @@ var LandingZonesStore = declare([ESPRequest.Store], {
         if (!query.filter) {
             return inherited(query, options);
         }
-        var landingZonesFilterStore = new LandingZonesFilterStore({ dropZone: query.filter.__dropZone });
+        var landingZonesFilterStore = new LandingZonesFilterStore({ dropZone: query.filter.__dropZone, server: query.filter.Server });
         delete query.filter.__dropZone;
         return landingZonesFilterStore.query(query.filter, options);
     }),

esp/src/stub.htm

@@ -38,9 +38,9 @@
                         switch (responseType) {
                             case 'Mixed':
                             case 'PerSessionOnly':
-                            case 'PerRequestOnly':
                                 document.cookie = ("ESPSessionState=true");
                                 break;
+                            case 'PerRequestOnly':
                             case 'UserNameOnly':
                             case 'None':
                                 document.cookie = ("ESPSessionState=false");
@@ -67,4 +67,4 @@
     <!-- application -->
 </body>
 
-</html>
+</html>

initfiles/examples/EsdlExample/esdl_binding.xml

@@ -11,13 +11,15 @@
        </xsdl:CustomRequestTransform>
     <Method name="JavaEchoPersonInfo" querytype="java" javamethod="EsdlExample.EsdlExampleService.JavaEchoPersonInfo"/>
     <Method name="RoxieEchoPersonInfo" querytype="roxie" url="http://localhost:9876/roxie" queryname="RoxieEchoPersonInfo">
-         <xsdl:CustomRequestTransform>
+         <xsdl:CustomRequestTransform target="soap:Body/{$query}">
             <xsdl:choose>
                <xsdl:when test="$clientversion=1.9">
-                  <xsdl:SetValue target="Row/Name/First"  value="'v1.9'"/>
+                  <xsdl:SetValue target="vertest"  value="'v1.9'"/>
+                  <xsdl:SetValue target="RoxieEchoPersonInfoRequest/Row/Name/First"  value="'v1.9'"/>
                </xsdl:when>
                <xsdl:otherwise>
-                  <xsdl:SetValue target="Row/Name/Last" value="concat('v', $clientversion)"/>
+                  <xsdl:SetValue target="vertest"  value="concat('v', $clientversion)"/>
+                  <xsdl:SetValue target="RoxieEchoPersonInfoRequest/Row/Name/Last" value="concat('v', $clientversion)"/>
                </xsdl:otherwise>
             </xsdl:choose>
          </xsdl:CustomRequestTransform>

roxie/ccd/ccdmain.cpp

@@ -867,14 +867,14 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
 
         enableKeyDiff = topology->getPropBool("@enableKeyDiff", true);
 
+        // NB: these directories will have been setup by topology earlier
+        const char *primaryDirectory = queryBaseDirectory(grp_unknown, 0);
+        const char *secondaryDirectory = queryBaseDirectory(grp_unknown, 1);
+
         // MORE: Get parms from topology after it is populated from Hardware/computer types section in configenv
         //       Then if does not match and based on desired action in topolgy, either warn, or fatal exit or .... etc
-        //       Also get prim path and sec from topology
-#ifdef _WIN32
-        getHardwareInfo(hdwInfo, "C:", "D:");
-#else // linux
-        getHardwareInfo(hdwInfo, "/c$", "/d$");
-#endif
+        getHardwareInfo(hdwInfo, primaryDirectory, secondaryDirectory);
+
         if (traceLevel)
         {
             DBGLOG("Current Hardware Info: CPUs=%i, speed=%i MHz, Mem=%i MB , primDisk=%i GB, primFree=%i GB, secDisk=%i GB, secFree=%i GB, NIC=%i", 

system/jlib/jstatcodes.h

@@ -212,6 +212,7 @@ enum StatisticKind
     StNumAttribsFromCache,
     StNumSmartJoinDegradedToLocal,      // number of times global smart join degraded to local smart join (<=1 unless in loop)
     StNumSmartJoinSlavesDegradedToStd,  // number of times a slave in smart join degraded from local smart join to standard hash join
+    StNumAttribsSimplifiedTooComplex,
     StMax,
 
     //For any quantity there is potentially the following variants.

system/jlib/jstats.cpp

@@ -853,6 +853,7 @@ static const StatisticMeta statsMetaData[StMax] = {
     { NUMSTAT(AttribsFromCache) },
     { NUMSTAT(SmartJoinDegradedToLocal) },
     { NUMSTAT(SmartJoinSlavesDegradedToStd) },
+    { NUMSTAT(AttribsSimplifiedTooComplex) },
 };
 
 

testing/regress/ecl/complexhoist.ecl

@@ -0,0 +1,39 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//HOIST(dataset({unsigned i}) ds) := NOFOLD(SORT(NOFOLD(ds), i));
+HOIST( ds) := MACRO
+//NOFOLD(SORT(NOFOLD(ds), i))
+//NOFOLD(ds)
+ds
+ENDMACRO;
+
+dsOuter := HOIST(DATASET([1,2,3,4,5,6,7,8,9], { unsigned i }));
+dsInner1 := HOIST(DATASET([11,12,13,14,15,16,17,18,19], { unsigned i }));
+dsInner2 := HOIST(DATASET([21,22,23,24,25,26,27,28,29], { unsigned i }));
+
+innerSum1(unsigned x) := SUM(dsInner1, i * x);
+outerSum1(unsigned x) := SUM(dsOuter, x * innerSum1(i));
+innerSum2(unsigned x) := SUM(dsInner2, i * outerSum1(i));
+outerSum2 := SUM(dsOuter, innerSum2(i));
+
+sequential(
+output(innerSum1(1));
+output(outerSum1(1));
+output(innerSum2(1));
+output(outerSum2);
+);

testing/regress/ecl/complexhoist6.ecl

@@ -0,0 +1,44 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+HOIST( ds) := MACRO
+NOFOLD(global(ds,few))          // causes a problem if isInlineTrivialDataset() walks to next item for no_fold
+ENDMACRO;
+
+rec := { unsigned i };
+
+mkRow(unsigned value) := TRANSFORM(rec, SKIP(value = 1000000);   SELF.i := value);
+
+dsOuter  := HOIST(DATASET([1,2,3], rec));
+dsInner := HOIST(DATASET([0,1,2], rec));
+
+
+rec t1(unsigned x, rec l) := TRANSFORM,SKIP(x+l.i not in SET(DATASET(3, transform({ unsigned value}, SELF.value := COUNTER-1+(l.i-1)*2 )), value))
+    SELF := l;
+END;
+
+filteredOuter(unsigned x) := PROJECT(dsOuter, t1(x, LEFT));
+sumFilteredOuter(unsigned x) := AGGREGATE(filteredOuter(x), rec, transform(rec, SELF.i := (1<<(LEFT.i-1)) + RIGHT.i))[1].i;
+sumInnerx() := TABLE(dsInner, {i, sumFilteredOuter(i)});
+sumInner(unsigned x) := SUM(dsInner, x*sumFilteredOuter(i)<<i*3);
+projectOuter() := PROJECT(dsOuter, transform(rec, SELF.i := sumInner(1<<(LEFT.i-1)*16)));
+sumOuter() := SUM(projectOuter(), i);
+
+sequential(
+output(sumInner(1));   // 0b110111011 = 443
+output(sumOuter());    // 0b11011101100000001101110110000000110111011 = 1,902,699,545,019
+);

testing/regress/ecl/key/complexhoist.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>135</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>6075</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>34536375</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>310827375</Result_4></Row>
+</Dataset>

testing/regress/ecl/key/complexhoist6.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>443</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>1902699545019</Result_2></Row>
+</Dataset>

thorlcr/activities/loop/thloop.cpp

@@ -248,7 +248,8 @@ public:
                 if (sync(loopCounter))
                     break;
 
-                boundGraph->execute(*this, condLoopCounter, ownedResults, (IRowWriterMultiReader *)NULL, 0, extractBuilder.size(), extractBuilder.getbytes());
+                boundGraph->queryGraph()->executeChild(extractBuilder.size(), extractBuilder.getbytes(), ownedResults, NULL);
+
                 ++loopCounter;
                 if (barrier) // barrier passed once all slave graphs have completed
                 {
@@ -320,9 +321,17 @@ public:
         unsigned loopCounter = 1;
         for (;;)
         {
+            Owned<IThorGraphResults> results = queryGraph().createThorGraphResults(1);
+            unsigned condLoopCounter = (helper->getFlags() & IHThorGraphLoopArg::GLFcounter) ? loopCounter : 0;
+            IThorBoundLoopGraph *boundGraph = queryContainer().queryLoopGraph();
+            if (condLoopCounter)
+                boundGraph->prepareCounterResult(*this, results, condLoopCounter, 0);
+
             if (sync(loopCounter))
                 break;
-            queryContainer().queryLoopGraph()->execute(*this, (helper->getFlags() & IHThorGraphLoopArg::GLFcounter)?loopCounter:0, loopResults.get(), extractBuilder.size(), extractBuilder.getbytes());
+
+            boundGraph->queryGraph()->executeChild(extractBuilder.size(), extractBuilder.getbytes(), results, loopResults);
+
             ++loopCounter;
         }
     }

thorlcr/activities/loop/thloopslave.cpp

@@ -376,6 +376,10 @@ public:
                 if (loopAgain) // cannot be 0
                     boundGraph->prepareLoopAgainResult(*this, ownedResults, loopAgain);
 
+                loopPending->flush();
+                Owned<IThorResult> inputResult = ownedResults->getResult(1);
+                inputResult->setResultStream(loopPending.getClear(), loopPendingCount);
+
                 // ensure results prepared before graph begins
                 if (syncIterations)
                 {
@@ -394,8 +398,7 @@ public:
                     return NULL;
                 }
 
-                loopPending->flush();
-                boundGraph->execute(*this, condLoopCounter, ownedResults, loopPending.getClear(), loopPendingCount, extractBuilder.size(), extractBuilder.getbytes());
+                boundGraph->queryGraph()->executeChild(extractBuilder.size(), extractBuilder.getbytes(), ownedResults, NULL);
 
                 Owned<IThorResult> result0 = ownedResults->getResult(0);
                 curInput.setown(result0->getRowStream());
@@ -506,10 +509,15 @@ public:
                 resultWriter->putRow(row.getClear());
             }
 
+            IThorBoundLoopGraph *boundGraph = queryContainer().queryLoopGraph();
             for (; loopCounter<=maxIterations; loopCounter++)
             {
+                unsigned condLoopCounter = (helper->getFlags() & IHThorGraphLoopArg::GLFcounter) ? loopCounter : 0;
+                Owned<IThorGraphResults> results = queryGraph().createThorGraphResults(1);
+                if (condLoopCounter)
+                    boundGraph->prepareCounterResult(*this, results, condLoopCounter, 0);
                 sendLoopingCount(loopCounter, 0);
-                queryContainer().queryLoopGraph()->execute(*this, (flags & IHThorGraphLoopArg::GLFcounter)?loopCounter:0, loopResults, extractBuilder.size(), extractBuilder.getbytes());
+                boundGraph->queryGraph()->executeChild(parentExtractSz, parentExtract, results, loopResults);
             }
             int iNumResults = loopResults->count();
             Owned<IThorResult> finalResult = loopResults->getResult(iNumResults-1); //Get the last result, which isnt necessarily 'maxIterations'

thorlcr/graph/thgraph.cpp

@@ -244,6 +244,7 @@ public:
         IThorResult *counterResult = results->createResult(activity, pos, countRowIf, thorgraphresult_nul, SPILL_PRIORITY_DISABLE);
         Owned<IRowWriter> counterResultWriter = counterResult->getWriter();
         counterResultWriter->putRow(counterRowFinal.getClear());
+        graph->setLoopCounter(loopCounter);
     }
     virtual void prepareLoopAgainResult(CActivityBase &activity, IThorGraphResults *results, unsigned pos)
     {
@@ -259,49 +260,6 @@ public:
         IThorResult *loopResult =  activity.queryGraph().createResult(activity, 0, results, resultRowIf, resultType); // loop output
         IThorResult *inputResult = activity.queryGraph().createResult(activity, 1, results, resultRowIf, resultType); // loop input
     }
-    virtual void execute(CActivityBase &activity, unsigned counter, IThorGraphResults *results, IRowWriterMultiReader *inputStream, rowcount_t rowStreamCount, size32_t parentExtractSz, const byte *parentExtract)
-    {
-        if (counter)
-            graph->setLoopCounter(counter);
-        Owned<IThorResult> inputResult = results->getResult(1);
-        if (inputStream)
-            inputResult->setResultStream(inputStream, rowStreamCount);
-        graph->executeChild(parentExtractSz, parentExtract, results, NULL);
-    }
-    virtual void execute(CActivityBase &activity, unsigned counter, IThorGraphResults *graphLoopResults, size32_t parentExtractSz, const byte *parentExtract)
-    {
-        Owned<IThorGraphResults> results = graph->createThorGraphResults(1);
-        if (counter)
-        {
-            prepareCounterResult(activity, results, counter, 0);
-            graph->setLoopCounter(counter);
-        }
-        try
-        {
-            graph->executeChild(parentExtractSz, parentExtract, results, graphLoopResults);
-        }
-        catch (IException *e)
-        {
-            IThorException *te = QUERYINTERFACE(e, IThorException);
-            if (!te)
-            {
-                Owned<IThorException> e2 = MakeActivityException(&activity, e, "Exception running child graphs");
-                e->Release();
-                te = e2.getClear();
-            }
-            else if (!te->queryActivityId())
-                setExceptionActivityInfo(activity.queryContainer(), te);
-            try { graph->abort(te); }
-            catch (IException *abortE)
-            {
-                Owned<IThorException> e2 = MakeActivityException(&activity, abortE, "Exception whilst aborting graph");
-                abortE->Release();
-                EXCLOG(e2, NULL);
-            }
-            graph->queryJobChannel().fireException(te);
-            throw te;
-        }
-    }
     virtual CGraphBase *queryGraph() { return graph; }
 };
 

thorlcr/graph/thgraph.hpp

@@ -28,6 +28,7 @@
 
 #define LONGTIMEOUT (25*60*1000)
 #define MEDIUMTIMEOUT 30000
+#define DEFAULT_MAX_ACTINITWAITTIME_MINS (2*60) // 2hrs
 
 #include "jlib.hpp"
 #include "jarray.hpp"
@@ -158,8 +159,6 @@ interface IThorBoundLoopGraph : extends IInterface
     virtual void prepareLoopResults(CActivityBase &activity, IThorGraphResults *results) = 0;
     virtual void prepareCounterResult(CActivityBase &activity, IThorGraphResults *results, unsigned loopCounter, unsigned pos) = 0;
     virtual void prepareLoopAgainResult(CActivityBase &activity, IThorGraphResults *results, unsigned pos) = 0;
-    virtual void execute(CActivityBase &activity, unsigned counter, IThorGraphResults *results, IRowWriterMultiReader *rowStream, rowcount_t rowStreamCount, size32_t parentExtractSz, const byte * parentExtract) = 0;
-    virtual void execute(CActivityBase &activity, unsigned counter, IThorGraphResults * graphLoopResults, size32_t parentExtractSz, const byte * parentExtract) = 0;
     virtual CGraphBase *queryGraph() = 0;
 };
 

thorlcr/graph/thgraphslave.cpp

@@ -949,8 +949,18 @@ bool CSlaveGraph::recvActivityInitData(size32_t parentExtractSz, const byte *par
 
     if (syncInitData())
     {
-        if (!graphCancelHandler.recv(queryJobChannel().queryJobComm(), msg, 0, mpTag, NULL, LONGTIMEOUT))
-            throw MakeStringException(0, "Error receiving actinit data for graph: %" GIDPF "d", graphId);
+        CTimeMon timer;
+        while (!graphCancelHandler.recv(queryJobChannel().queryJobComm(), msg, 0, mpTag, NULL, MEDIUMTIMEOUT))
+        {
+            if (graphCancelHandler.isCancelled())
+                throw MakeStringException(0, "Aborted whilst waiting to receive actinit data for graph: %" GIDPF "d", graphId);
+            // put an upper limit on time Thor can be stalled here
+            unsigned mins = timer.elapsed()/60000;
+            if (mins >= jobS->queryActInitWaitTimeMins())
+                throw MakeStringException(0, "Timed out after %u minutes, waiting to receive actinit data for graph: %" GIDPF "u", mins, graphId);
+
+            GraphPrintLogEx(this, thorlog_null, MCwarning, "Waited %u minutes for activity initialization message (Master may be blocked on a file lock?).", mins);
+        }
         replyTag = msg.getReplyTag();
         msg.read(len);
     }
@@ -1659,6 +1669,8 @@ CJobSlave::CJobSlave(ISlaveWatchdog *_watchdog, IPropertyTree *_workUnitInfo, co
     getOpt("remoteCompressedOutput", remoteCompressedOutput);
     if (remoteCompressedOutput.length())
         setRemoteOutputCompressionDefault(remoteCompressedOutput);
+
+    actInitWaitTimeMins = getOptInt(THOROPT_ACTINIT_WAITTIME_MINS, DEFAULT_MAX_ACTINITWAITTIME_MINS);
 }
 
 void CJobSlave::addChannel(IMPServer *mpServer)

thorlcr/graph/thgraphslave.hpp

@@ -474,6 +474,7 @@ class graphslave_decl CJobSlave : public CJobBase
     Owned<IPropertyTree> workUnitInfo;
     size32_t oldNodeCacheMem;
     unsigned channelMemoryMB;
+    unsigned actInitWaitTimeMins = DEFAULT_MAX_ACTINITWAITTIME_MINS;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -484,6 +485,7 @@ public:
     virtual void startJob() override;
     virtual void endJob() override;
     const char *queryFindString() const { return key.get(); } // for string HT
+    unsigned queryActInitWaitTimeMins() const { return actInitWaitTimeMins; }
 
     virtual IGraphTempHandler *createTempHandler(bool errorOnMissing);
     ISlaveWatchdog *queryProgressHandler() { return watchdog; }

thorlcr/slave/slavmain.cpp

@@ -2293,13 +2293,8 @@ void slaveMain(bool &jobListenerStopped)
         StringBuffer ipStr;
         ip.getIpText(ipStr);
         PROGLOG("Redirecting local mount to %s", ipStr.str());
-        const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
-        StringBuffer repdir;
-        if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
-            overrideReplicateDirectory = repdir.str();
-        else
-            overrideReplicateDirectory = "/d$";
-        setLocalMountRedirect(ip, overrideReplicateDirectory, "/mnt/mirror");
+        const char *replicateDirectory = queryBaseDirectory(grp_unknown, 1); // default directories configured at start up (see thslavemain.cpp)
+        setLocalMountRedirect(ip, replicateDirectory, "/mnt/mirror");
     }
 
 #endif

thorlcr/thorutil/thormisc.hpp

@@ -95,6 +95,7 @@
 #define THOROPT_KEYLOOKUP_COMPRESS_MESSAGES "keyedJoinCompressMsgs" // compress key and fetch request messages                                   (default = true)
 #define THOROPT_FORCE_REMOTE_DISABLED "forceRemoteDisabled"     // disable remote (via dafilesrv) reads (NB: takes precedence over forceRemoteRead) (default = false)
 #define THOROPT_FORCE_REMOTE_READ     "forceRemoteRead"         // force remote (via dafilesrv) read (NB: takes precedence over environment.conf setting) (default = false)
+#define THOROPT_ACTINIT_WAITTIME_MINS "actInitWaitTimeMins"     // max time to wait for slave activity initialization message from master
 
 
 #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000  // max of row matches before selfjoin emits warning
@@ -140,6 +141,7 @@ public:
     {
         reset();
     }
+    bool isCancelled() const { return cancelled; }
     void reset()
     {
         clear();