Merge branch 'candidate-6.2.0' into candidate-6.4.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -621,6 +621,13 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
         MESSAGE(FATAL_ERROR "You need bison version 2.4.1 or later to build this project (version ${BISON_VERSION} detected)")
       ENDIF()
 
+      IF ("${BISON_VERSION}" VERSION_LESS "2.7.0")
+        #Ignore all warnings - not recommend to develope on this version!
+        SET(bisonopt "-Wnone")
+      ELSE()
+        SET(bisonopt -Werror -Wno-other)
+      ENDIF()
+
       IF ("${FLEX_VERSION}" VERSION_LESS "2.5.35")
         MESSAGE(FATAL_ERROR "You need flex version 2.5.35 or later to build this project (version ${FLEX_VERSION} detected)")
       ENDIF()

common/remote/sockfile.cpp

@@ -1838,7 +1838,18 @@ public:
         initSendBuffer(sendBuffer);
         MemoryBuffer replyBuffer;
         sendBuffer.append((RemoteFileCommandType)RFCsetfileperms).append(filename).append(fPerms);
-        sendRemoteCommand(sendBuffer, replyBuffer);
+        try
+        {
+            sendRemoteCommand(sendBuffer, replyBuffer);
+        }
+        catch (IDAFS_Exception *e)
+        {
+            if (e->errorCode() == RFSERR_InvalidCommand)
+                WARNLOG("umask setFilePermissions (0%o) not supported on remote server", fPerms);
+            else
+                throw e;
+        }
+
     }
 
     offset_t size()

common/thorhelper/roxiedebug.cpp

@@ -1440,8 +1440,8 @@ void CBaseServerDebugContext::waitForDebugger(DebugState state, IActivityDebugCo
     sequence++;
 }
 
-CBaseServerDebugContext::CBaseServerDebugContext(const IContextLogger &_logctx, IPropertyTree *_queryXGMML, SafeSocket &_client) 
-    : CBaseDebugContext(_logctx), queryXGMML(_queryXGMML), client(_client)
+CBaseServerDebugContext::CBaseServerDebugContext(const IContextLogger &_logctx, IPropertyTree *_queryXGMML)
+    : CBaseDebugContext(_logctx), queryXGMML(_queryXGMML)
 {
     sequence = 0;
     previousSequence = 0;

common/thorhelper/roxiedebug.hpp

@@ -302,7 +302,6 @@ class THORHELPER_API CBaseServerDebugContext : public CBaseDebugContext, impleme
 
 protected:
     StringAttr debugId;
-    SafeSocket &client;
 
     Owned<IPropertyTree> queryXGMML;
     Semaphore debugeeSem;
@@ -321,7 +320,7 @@ protected:
     virtual void waitForDebugger(DebugState state, IActivityDebugContext *probe);
 public:
     IMPLEMENT_IINTERFACE;
-    CBaseServerDebugContext(const IContextLogger &_logctx, IPropertyTree *_queryXGMML, SafeSocket &_client) ;
+    CBaseServerDebugContext(const IContextLogger &_logctx, IPropertyTree *_queryXGMML) ;
     void serializeBreakpoints(MemoryBuffer &to);
     virtual void debugInitialize(const char *id, const char *_queryName, bool _breakAtStart);
     virtual void debugTerminate();

common/thorhelper/thorrparse.cpp

@@ -3391,11 +3391,11 @@ bool RegexParser::performMatch(IMatchedAction & action, const void * row, unsign
         const byte * end = endData - algo->minPatternLength;
 
         RegexState state(cache, algo->kind, helper, this, algo->inputFormat, len, start);
+        state.row = row;
+        state.processor = &action;
+        state.best = NULL;
         if (len >= algo->minPatternLength)
         {
-            state.row = row;
-            state.processor = &action;
-            state.best = NULL;
             for (const byte * curScan = start; curScan <= end;)
             {
                 state.cur = curScan;

docs/ECLLanguageReference/ECLR_mods/Expr-LogicalOperators.xml

@@ -58,11 +58,13 @@
 
     <para>When a complex logical expression has multiple OR conditions, you
     should group the OR conditions and order them from least complex to most
-    complex to result in the most efficient processing. If the probability of
-    occurrence is known, you should order them from the most likely to occur
-    to the least likely to occur, because once any part of a compound OR
-    condition evaluates to TRUE, the remainder of the expression is bypassed.
-    This is also true of the order of MAP function conditions.</para>
+    complex to result in the most efficient processing. </para>
+
+    <para>If the probability of occurrence is known, you should order them
+    from the most likely to occur to the least likely to occur, because once
+    any part of a compound OR condition evaluates to TRUE, the remainder of
+    the expression can be bypassed. However, this is not guaranteed. This is
+    also true of the order of MAP function conditions.</para>
 
     <para>Whenever AND and OR logical operations are mixed in the same
     expression, you should use parentheses to group within the expression to

docs/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -2248,7 +2248,9 @@
 
           <para>Right-click in the <emphasis role="bold">Repository Window
           </emphasis>for a Pop-up menu containing options which are available
-          depending on whether you have a folder or ECL file selected.</para>
+          depending on whether you have a folder or ECL file selected. Note,
+          if you have TortoiseGit installed, the menu is extended. See <link
+          linkend="versioncontrol">Version Control</link> for details</para>
 
           <informaltable colsep="1" frame="all" rowsep="1">
             <tgroup cols="2">
@@ -3264,6 +3266,58 @@ ENDMACRO;
         <para><graphic fileref="../../images/CT54.jpg" /></para>
       </sect2>
 
+      <sect2 id="versioncontrol">
+        <title>Version Control</title>
+
+        <para>The ECL IDE does not directly provide version control
+        functionality, but it does support integration with the TortoiseGit
+        Window Shell Interface to Git version control. This open-source
+        toolset provides easy access to Git commands from Windows Explorer.
+        The IDE's integration extends that to the context menu of the
+        Repository toolbox in the IDE.</para>
+
+        <para></para>
+
+        <orderedlist>
+          <listitem>
+            <para>Install and configure TortoiseGit
+            (www.tortoisegit.org)</para>
+
+            <para>Follow the instructions on their site.</para>
+          </listitem>
+
+          <listitem>
+            <para>Clone a Git repository to a folder on your local
+            machine.</para>
+
+            <para>Follow the instructions in your Git repository's
+            site.</para>
+          </listitem>
+
+          <listitem>
+            <para>Add that folder to the list of ECL Folders on the <emphasis
+            role="bold">Compiler</emphasis> tab of the <emphasis
+            role="bold">Preferences</emphasis> dialog.</para>
+
+            <para><graphic fileref="../../images/IDE_eclfolders.jpg" /></para>
+          </listitem>
+
+          <listitem>
+            <para>Press the <emphasis role="bold">Ok</emphasis> button.</para>
+
+            <para>The folder now appears in your Repository toolbox.</para>
+          </listitem>
+
+          <listitem>
+            <para>Right-click on the folder in the Repository toolbox, and
+            choose your Git command from the TortoiseGit menu.</para>
+
+            <para><graphic
+            fileref="../../images/IDE_tortoisegit.jpg" /></para>
+          </listitem>
+        </orderedlist>
+      </sect2>
+
       <sect2 id="Graph_Viewer" role="brk">
         <title><emphasis>Graph Viewer </emphasis></title>
 

ecl/eclagent/eclagent.cpp

@@ -52,6 +52,7 @@
 
 using roxiemem::OwnedRoxieString;
 
+#include <memory>
 #include <new>
 
 #ifdef _USE_CPPUNIT
@@ -432,7 +433,7 @@ IPooledThread *CHThorDebugSocketListener::createNew()
 //=======================================================================================
 
 CHThorDebugContext::CHThorDebugContext(const IContextLogger &_logctx, IPropertyTree *_queryXGMML, EclAgent *_eclAgent) 
-    : CBaseServerDebugContext(_logctx, _queryXGMML, client), eclAgent(_eclAgent)
+    : CBaseServerDebugContext(_logctx, _queryXGMML), eclAgent(_eclAgent)
 {
 }
 
@@ -3373,7 +3374,7 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
     try
     {
 #ifdef MONITOR_ECLAGENT_STATUS  
-        CSDSServerStatus * serverstatus = NULL;
+        std::unique_ptr<CSDSServerStatus> serverstatus;
 #endif
         Owned<ILocalWorkUnit> standAloneWorkUnit;
         if (wuXML)
@@ -3392,7 +3393,7 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
                 initClientProcess(serverGroup, DCR_EclAgent, 0, NULL, NULL, MP_WAIT_FOREVER);
             }
 #ifdef MONITOR_ECLAGENT_STATUS  
-            serverstatus = new CSDSServerStatus("ECLagent");
+            serverstatus.reset(new CSDSServerStatus("ECLagent"));
             serverstatus->queryProperties()->setPropInt("Pid", GetCurrentProcessId());
             serverstatus->commitProperties();
 #endif
@@ -3546,9 +3547,6 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
         catch (RELEASE_CATCH_ALL)
         {
         }
-
-        if (serverstatus)
-            delete serverstatus;
     }
     catch (IException * e)
     {

ecl/hql/CMakeLists.txt

@@ -111,16 +111,16 @@ include_directories (
 
 if (WIN32)
     add_custom_command ( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h
-        #pipe result through findstr to remove warnings that are hard to suppress, and pipe through more to prevent error code from no matches aborting the compile
-        COMMAND ${bisoncmdprefix} ${bisoncmd} --report=state --defines=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h --output=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hqlgram.y 2>&1 | findstr /V "unused value" | more
+        #${bisonopt} is used to force s/r and r/r errors to abort compiling
+        COMMAND ${bisoncmdprefix} ${bisoncmd} ${bisonopt} --report=state --defines=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h --output=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hqlgram.y 2>&1
         DEPENDS hqlgram.y
     )
 else()
     add_custom_command ( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h
         #delete output files before building to force rebuilds with errors to fail rather than continue unnoticed, using the existing output files.
         COMMAND rm -f ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp
-        #pipe result through grep to remove warnings that are hard to suppress, and pipe through cat to prevent error code from no matches aborting the compile
-        COMMAND ${bisoncmdprefix} ${bisoncmd} --report=state --defines=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h --output=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hqlgram.y 2>&1 | grep -v "unused value" | cat
+        #${bisonopt} is used to force s/r and r/r errors to abort compiling
+        COMMAND ${bisoncmdprefix} ${bisoncmd} ${bisonopt} --report=state --defines=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h --output=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hqlgram.y 2>&1
         DEPENDS hqlgram.y
     )
 endif()

ecl/hql/hqlgram.hpp

@@ -374,6 +374,7 @@ public:
     HqlExprArray imports;
     bool inSignedModule;
     bool legacyImport;
+    bool legacyWhen;
 };
 
 typedef const IAtom * const * AtomList;
@@ -875,7 +876,6 @@ protected:
     bool resolveSymbols;
     bool forceResult;
     bool associateWarnings;
-    bool legacyWhenSemantics;
     bool isQuery;
     bool parseConstantText;
     bool expandingMacroPosition;
@@ -1130,10 +1130,15 @@ class HqlLex
         StringBuffer& doGetDataType(StringBuffer & type, const char * text, int lineno, int column);
         void pushText(const char *);
         bool hasLegacyImportSemantics() const;
+        bool hasLegacyWhenSemantics() const;
         void setLegacyImport(bool _legacyImportMode)
         {
             legacyImportMode = _legacyImportMode;
         }
+        void setLegacyWhen(bool _legacyWhenMode)
+        {
+            legacyWhenMode = _legacyWhenMode;
+        }
 
     protected:
         void init(IFileContents * _text);
@@ -1245,6 +1250,7 @@ private:
         UnsignedArray hashendKinds;
         bool hasHashbreak;
         bool legacyImportMode = false;
+        bool legacyWhenMode = false;
         int loopTimes;
 
         bool inComment;

ecl/hql/hqlgram.y

@@ -4093,7 +4093,7 @@ funcRetType
                             $$.setType(makeRowType(expr->getType()));
                             $$.setPosition($1);
                         }
-    | typeDef
+    | userTypedefType
     ;
 
 payloadPart

ecl/hql/hqlgram2.cpp

@@ -348,6 +348,7 @@ HqlGram::HqlGram(IHqlScope * _globalScope, IHqlScope * _containerScope, IFileCon
 
     lexObject = new HqlLex(this, _text, xmlScope, NULL);
     lexObject->setLegacyImport(queryLegacyImportSemantics());
+    lexObject->setLegacyWhen(queryLegacyWhenSemantics());
 
     //MORE: This should be in the parseContext calculated once
     if (lookupCtx.queryRepository() && loadImplicit)
@@ -377,6 +378,7 @@ HqlGram::HqlGram(HqlGramCtx & parent, IHqlScope * _containerScope, IFileContents
     //Clone parseScope
     lexObject = new HqlLex(this, _text, xmlScope, NULL);
     lexObject->setLegacyImport(parent.legacyImport);
+    lexObject->setLegacyWhen(parent.legacyWhen);
     forceResult = true;
     parsingTemplateAttribute = false;
     parseConstantText = _parseConstantText;
@@ -404,6 +406,7 @@ void HqlGram::saveContext(HqlGramCtx & ctx, bool cloneScopes)
     }
 
     ctx.legacyImport = lexObject->hasLegacyImportSemantics();
+    ctx.legacyWhen = lexObject->hasLegacyWhenSemantics();
     ctx.globalScope.set(globalScope);
     appendArray(ctx.defaultScopes, defaultScopes);
     appendArray(ctx.implicitScopes, implicitScopes);
@@ -420,7 +423,6 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
 {
     minimumScopeIndex = 0;
     isQuery = false;
-    legacyWhenSemantics = queryLegacyWhenSemantics();
     current_id = NULL;
     lexObject = NULL;
     expectedAttribute = NULL;
@@ -9237,7 +9239,7 @@ IHqlExpression * HqlGram::associateSideEffects(IHqlExpression * expr, const ECLl
 {
     if (sideEffectsPending())
     {
-        if (legacyWhenSemantics)
+        if (lexObject->hasLegacyWhenSemantics())
         {
             if (okToAddSideEffects(expr))
                 return addSideEffects(expr);
@@ -9316,7 +9318,7 @@ void HqlGram::doDefineSymbol(DefineIdSt * defineid, IHqlExpression * _expr, IHql
             if (isQuery && !insideNestedScope())
             {
                 //If this is a global query, and not inside a nested attribute, then keep any actions on the global list of results
-                if (!legacyWhenSemantics)
+                if (!lexObject->hasLegacyWhenSemantics())
                 {
                     //Should we give a warning here?? export/shared would not be legal if this was within the repository
                 }

ecl/hql/hqlparse.cpp

@@ -316,8 +316,10 @@ void HqlLex::pushText(IFileContents * text, int startLineNo, int startColumn)
     MTIME_SECTION(timer, "HqlLex::pushText");
 #endif
     bool useLegacyImport = hasLegacyImportSemantics();
+    bool useLegacyWhen = hasLegacyWhenSemantics();
     inmacro = new HqlLex(yyParser, text, NULL, NULL);
     inmacro->setLegacyImport(useLegacyImport);
+    inmacro->setLegacyWhen(useLegacyWhen);
     inmacro->set_yyLineNo(startLineNo);
     inmacro->set_yyColumn(startColumn);
 }
@@ -337,8 +339,10 @@ void HqlLex::pushText(const char *s)
 #endif
     Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath, yyParser->inSignedModule, yyParser->gpgSignature);
     bool useLegacyImport = hasLegacyImportSemantics();
+    bool useLegacyWhen = hasLegacyWhenSemantics();
     inmacro = new HqlLex(yyParser, macroContents, NULL, NULL);
     inmacro->setLegacyImport(useLegacyImport);
+    inmacro->setLegacyWhen(useLegacyWhen);
     inmacro->set_yyLineNo(yyLineNo);
     inmacro->set_yyColumn(yyColumn);
 
@@ -354,6 +358,13 @@ bool HqlLex::hasLegacyImportSemantics() const
     return legacyImportMode;
 }
 
+bool HqlLex::hasLegacyWhenSemantics() const
+{
+    if (inmacro)
+        return inmacro->hasLegacyWhenSemantics();
+    return legacyWhenMode;
+}
+
 void HqlLex::setMacroParam(const YYSTYPE & errpos, IHqlExpression* funcdef, StringBuffer& curParam, IIdAtom * argumentId, unsigned& parmno,IProperties *macroParms)
 {
     IHqlExpression * formals = queryFunctionParameters(funcdef);
@@ -573,8 +584,10 @@ void HqlLex::pushMacro(IHqlExpression *expr)
     else
     {
         bool useLegacyImport = hasLegacyImportSemantics();
+        bool useLegacyWhen = hasLegacyWhenSemantics();
         inmacro = new HqlLex(yyParser, macroContents, NULL, LINK(expr));
         inmacro->setLegacyImport(useLegacyImport);
+        inmacro->setLegacyWhen(useLegacyWhen);
 
 #if defined(TRACE_MACRO)
         PrintLog("MACRO>> inmacro %p created for \"%s\" at %d:%d\n",inmacro, s.str(),macroBodyExpr->getStartLine(),macroBodyExpr->getStartColumn());
@@ -700,8 +713,10 @@ void HqlLex::processEncrypted()
     Owned<ISourcePath> sourcePath = createSourcePath("<encrypted>");
     Owned<IFileContents> decryptedContents = createFileContentsFromText((const char *)decrypted.toByteArray(), sourcePath, yyParser->inSignedModule, yyParser->gpgSignature);
     bool useLegacyImport = hasLegacyImportSemantics();
+    bool useLegacyWhen = hasLegacyWhenSemantics();
     inmacro = new HqlLex(yyParser, decryptedContents, NULL, NULL);
     inmacro->setLegacyImport(useLegacyImport);
+    inmacro->setLegacyWhen(useLegacyWhen);
     inmacro->setParentLex(this);
     inmacro->encrypted = true;
 }
@@ -1092,6 +1107,30 @@ void HqlLex::doSlashSlashHash(YYSTYPE const & returnToken, const char * command)
         else
             reportError(returnToken, ERR_EXPECTED_LEFTCURLY, "Expected (");
     }
+    else if (hasPrefix(command, "when", false))
+    {
+        const char * next = skipws(command + 4);
+        if (*next == '(')
+        {
+            next = skipws(next + 1);
+            const char * bra = strchr(next, ')');
+            if (bra)
+            {
+                StringBuffer option(bra - next, next);
+                option.clip();
+                if (strieq(option, "legacy"))
+                    setLegacyWhen(true);
+                else if (strieq(option, "modern"))
+                    setLegacyWhen(false);
+                else
+                    reportError(returnToken, ERR_EXPECTED_ID, "Unknown #import option '%s' - expected legacy or modern", option.str());
+            }
+            else
+                reportError(returnToken, ERR_EXPECTED_RIGHTCURLY, "Expected closing )");
+        }
+        else
+            reportError(returnToken, ERR_EXPECTED_LEFTCURLY, "Expected (");
+    }
     //Ignore any unrecognised commands
 }
 

ecl/hqlcpp/hqlcpp.ipp

@@ -933,6 +933,7 @@ public:
     bool canIterateInline(BuildCtx * ctx, IHqlExpression * expr);
     bool canAssignInline(BuildCtx * ctx, IHqlExpression * expr);
     bool canEvaluateInline(BuildCtx * ctx, IHqlExpression * expr);
+    bool canEvaluateInlineNoSpill(BuildCtx * ctx, IHqlExpression * expr);
 
     void buildAssignChildDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
     void buildChildDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);

ecl/hqlcpp/hqlcppds.cpp

@@ -1942,6 +1942,13 @@ bool HqlCppTranslator::canEvaluateInline(BuildCtx * ctx, IHqlExpression * expr)
     return options.allowInlineSpill ? ::canProcessInline(ctx, expr) : ::canEvaluateInline(ctx, expr);
 }
 
+bool HqlCppTranslator::canEvaluateInlineNoSpill(BuildCtx * ctx, IHqlExpression * expr)
+{
+    if (!isInlineOk())
+        return false;
+    return ::canEvaluateInline(ctx, expr);
+}
+
 bool HqlCppTranslator::canProcessInline(BuildCtx * ctx, IHqlExpression * expr)
 {
     if (!isInlineOk())
@@ -2431,7 +2438,7 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             if (format == FormatLinkedDataset || format == FormatArrayDataset)
             {
                 IHqlExpression * choosenLimit = NULL;
-                if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2) && (canEvaluateInline(&ctx, expr->queryChild(0)) || !canIterateInline(&ctx, expr->queryChild(0))))
+                if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2) && (canEvaluateInlineNoSpill(&ctx, expr->queryChild(0)) || !canIterateInline(&ctx, expr->queryChild(0))))
                 {
                     choosenLimit = expr->queryChild(1);
                     expr = expr->queryChild(0);
@@ -2807,7 +2814,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
         else
         {
             IHqlExpression * choosenLimit = NULL;
-            if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2) && (canEvaluateInline(&ctx, expr->queryChild(0)) || !canIterateInline(&ctx, expr->queryChild(0))))
+            if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2) && (canEvaluateInlineNoSpill(&ctx, expr->queryChild(0)) || !canIterateInline(&ctx, expr->queryChild(0))))
             {
                 choosenLimit = expr->queryChild(1);
                 expr = expr->queryChild(0);

ecl/hqlcpp/hqlinline.cpp

@@ -113,6 +113,8 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     case no_call:
     case no_externalcall:               // no so sure about this - should possibly be assignable only. (also no_call above)
     case no_getresult:
+        if (isStreamed(expr))
+            return RETiterate;
         return RETassign;
     }
 

ecl/regress/hashwhen.ecl

@@ -0,0 +1,22 @@
+
+
+//#when(legacy)
+
+x := function
+
+   output ('ok!');
+   return 3;
+END;
+
+x;
+
+
+//#when(modern)
+
+y := function
+
+   output ('not ok!'); // error
+   return 3;
+END;
+
+y;

ecl/regress/issue16626.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+array_t := set of real8;
+double := real8;
+
+x := SERVICE : library('supermaths')
+   array_t transpose(array_t value, unsigned cols, unsigned rows);
+   double square(double x);
+END;
+
+
+output(x.transpose([1,2,3,4],2,2));
+output(x.square(10.0));

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -36,6 +36,19 @@
 #include "build-config.h"
 
 #include "loggingagentbase.hpp"
+
+bool EsdlBindingImpl::ensureSDSPath(const char * sdsPath)
+{
+    if (!sdsPath)
+        return false;
+
+    Owned<IRemoteConnection> conn = querySDS().connect(sdsPath, myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL);
+    if (!conn)
+        return false;
+
+    return true;
+}
+
 /*
  * trim xpath at first instance of element
  */
@@ -70,7 +83,7 @@ IPropertyTree * fetchESDLDefinitionFromDaliById(const char *id)
 
     DBGLOG("ESDL Binding: Fetching ESDL Definition from Dali: %s ", id);
 
-    Owned<IRemoteConnection> conn = querySDS().connect(ESDL_DEFS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL);
+    Owned<IRemoteConnection> conn = querySDS().connect(ESDL_DEFS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_DESDL);
     if (!conn)
        throw MakeStringException(-1, "Unable to connect to ESDL Service definition information in dali '%s'", ESDL_DEFS_ROOT_PATH);
 
@@ -105,7 +118,7 @@ IPropertyTree * fetchESDLBindingFromDali(const char *process, const char *bindin
     Owned<IRemoteConnection> conn;
     try
     {
-        conn.set(querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL));
+        conn.set(querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_DESDL));
         if (!conn)
         {
             DBGLOG("Unable to connect to ESDL Service binding information in dali %s", ESDL_BINDINGS_ROOT_PATH);
@@ -1040,6 +1053,12 @@ EsdlBindingImpl::EsdlBindingImpl(IPropertyTree* cfg, const char *binding,  const
 
     try
     {
+        if(!ensureSDSPath(ESDL_DEFS_ROOT_PATH))
+            ESPLOG(LogNormal, "ESP Binding '%s' could not ensure %s element on Dali.", binding, ESDL_DEFS_ROOT_PATH);
+
+        if(!ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
+            ESPLOG(LogNormal, "ESP Binding '%s' could not ensure %s element on Dali.", binding, ESDL_BINDINGS_ROOT_PATH);
+
         m_esdlBndCfg.set(fetchESDLBinding(process, binding, m_esdlStateFilesLocation));
 
         if (!m_esdlBndCfg.get())
@@ -1266,6 +1285,7 @@ void EsdlBindingImpl::initEsdlServiceInfo(IEsdlDefService &srvdef)
     xsltpath.append("xslt/esxdl2xsd.xslt");
     m_xsdgen->loadTransform(xsltpath, xsdparams, EsdlXslToXsd );
     m_xsdgen->loadTransform(xsltpath, wsdlparams, EsdlXslToWsdl );
+
 }
 
 void EsdlBindingImpl::getSoapMessage(StringBuffer& soapmsg,

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -370,6 +370,7 @@ public:
     static void splitURLList(const char* urlList, StringBuffer& protocol,StringBuffer& UserName,StringBuffer& Password, StringBuffer& ipportlistbody, StringBuffer& path, StringBuffer& options);
     static void transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyTree* forRoxie );
     static bool makeURL( StringBuffer& url, IPropertyTree& cfg );
+    static bool ensureSDSPath(const char * sdsPath);
 
     bool usesESDLDefinition(const char * name, int version);
     bool usesESDLDefinition(const char * id);

esp/services/ws_esdlconfig/CMakeLists.txt

@@ -67,4 +67,5 @@ target_link_libraries ( ws_esdlconfig
          environment
          dllserver
          SMCLib
+         esdl_svc_engine
     )

esp/services/ws_esdlconfig/ws_esdlconfigservice.cpp

@@ -157,66 +157,6 @@ bool isESDLDefinitionBound(const char * esdldefname, int version)
     return isESDLDefinitionBound(id);
 }
 
-bool checkSDSPathExists(const char * sdsPath)
-{
-    Owned<IRemoteConnection> conn = querySDS().connect(sdsPath, myProcessSession(), RTM_LOCK_READ, 3000);
-    if (conn)
-    {
-        conn->close(false);
-        return true;
-    }
-    return false;
-}
-
-bool ensureSDSSubPath(const char * sdsPath)
-{
-    if (!sdsPath)
-        return false;
-
-    Owned<IRemoteConnection> conn = querySDS().connect(sdsPath, myProcessSession(), RTM_LOCK_READ|RTM_CREATE_QUERY, 4000);
-    if (!conn)
-    {
-        conn.setown(querySDS().connect("/", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL));
-        if (!conn.get())
-            return false;
-
-        IPropertyTree * sdsRoot = conn->queryRoot();
-        if (!sdsRoot)
-            return false;
-
-        sdsRoot->addProp(sdsPath, "");
-        conn->commit();
-    }
-
-    conn->close(false);
-    return true;
-}
-
-bool ensureSDSPath(const char * sdsPath)
-{
-    if (!sdsPath)
-        return false;
-
-    StringArray paths;
-    paths.appendList(sdsPath, PATHSEPSTR);
-
-    StringBuffer fullpath;
-    ForEachItemIn(idx, paths)
-    {
-        if (idx > 0)
-            fullpath.append("/"); //Dali paths aren't os dependent... right?
-        fullpath.append(paths.item(idx));
-
-        if (!checkSDSPathExists(fullpath))
-        {
-            if(!ensureSDSSubPath(fullpath))
-                return false;
-        }
-    }
-
-    return true;
-}
-
 void fetchESDLDefinitionFromDaliById(const char *id, StringBuffer & def)
 {
     if (!id || !*id)
@@ -261,15 +201,16 @@ void CWsESDLConfigEx::init(IPropertyTree *cfg, const char *process, const char *
     if(servicecfg == NULL)
         throw MakeStringException(-1, "config not found for service %s/%s",process, service);
 
-    if(!ensureSDSPath("ESDL"))
-        throw MakeStringException(-1, "Could not ensure '/ESDL' entry in dali configuration");
+    if(!EsdlBindingImpl::ensureSDSPath(ESDL_DEFS_ROOT_PATH))
+        throw MakeStringException(-1, "Could not ensure '%s' element on Dali", ESDL_DEFS_ROOT_PATH);
+
+    if(!EsdlBindingImpl::ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
+        throw MakeStringException(-1, "Could not ensure '%s' element on Dali", ESDL_BINDINGS_ROOT_PATH);
+
 }
 
 IPropertyTree * CWsESDLConfigEx::getESDLDefinitionRegistry(const char * wsEclId, bool readonly)
 {
-    if (!ensureSDSPath(ESDL_DEFS_ROOT_PATH))
-        throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
-
     Owned<IRemoteConnection> conn = querySDS().connect(ESDL_DEFS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL);
     if (!conn)
         throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
@@ -390,29 +331,6 @@ bool CWsESDLConfigEx::existsESDLMethodDef(const char * esdlDefinitionName, unsig
     return found;
 }
 
-void CWsESDLConfigEx::ensureESDLServiceBindingRegistry(const char * espProcName, const char * espBindingName, bool readonly)
-{
-    if (!espProcName || !*espProcName)
-        throw MakeStringException(-1, "Unable to ensure ESDL service binding registry in dali, esp process name not available");
-
-    if (!espBindingName || !*espBindingName)
-        throw MakeStringException(-1, "Unable to ensure ESDL service binding registry in dali, esp binding name not available");
-
-    if (!ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
-        throw MakeStringException(-1, "Unable to connect to ESDL Service binding information in dali %s", ESDL_BINDINGS_ROOT_PATH);
-
-    Owned<IRemoteConnection> globalLock = querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL);
-
-    if (!globalLock)
-        throw MakeStringException(-1, "Unable to connect to ESDL Service binding information in dali %s", ESDL_BINDINGS_ROOT_PATH);
-
-    IPropertyTree * esdlDefinitions = globalLock->queryRoot();
-    if (!esdlDefinitions)
-        throw MakeStringException(-1, "Unable to open ESDL Service binding information in dali %s", ESDL_BINDINGS_ROOT_PATH);
-
-    globalLock->close(false);
-}
-
 IPropertyTree * CWsESDLConfigEx::getEspProcessRegistry(const char * espprocname, const char * espbindingport, const char * servicename)
 {
     if (!espprocname || !*espprocname)
@@ -644,10 +562,7 @@ int CWsESDLConfigEx::publishESDLBinding(const char * bindingName,
         return -1;
     }
 
-    if (!ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
-        throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
-
-    Owned<IRemoteConnection> conn = querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL);
+    Owned<IRemoteConnection> conn = querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL);
     if (!conn)
        throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
 
@@ -1317,22 +1232,6 @@ IPropertyTree * CWsESDLConfigEx::getBindingTree(const char * espProcName, const
         return NULL;
     }
 
-    Owned<IRemoteConnection> globalLock = querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL);
-    if (!globalLock)
-    {
-        msg.set("Unable to connect to ESDL Service binding information in dali");
-        return NULL;
-    }
-
-    IPropertyTree *esdlDefinitions = globalLock->queryRoot();
-    globalLock->close(false);
-
-    if (!esdlDefinitions)
-    {
-        msg.set("Unable to open ESDL Service binding information in dali");
-        return NULL;
-    }
-
     VStringBuffer xpath("%s[@espprocess='%s'][@espbinding='%s']", ESDL_BINDING_PATH, espProcName, espBindingName);
     Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ , SDS_LOCK_TIMEOUT_DESDL);
     if (!conn)

esp/services/ws_esdlconfig/ws_esdlconfigservice.hpp

@@ -42,7 +42,6 @@ public:
     static bool existsESDLMethodDef(const char * esdlDefinitionName, unsigned ver, const char * esdlServiceName, const char * methodName);
     static int publishESDLBinding(const char * bindingName, IPropertyTree * methodsConfig, const char * espProcName, const char * espPort, const char * esdlDefinitionName, int esdlDefinitionVersion, const char * esdlServiceName, StringBuffer & message, bool overwrite);
     static IPropertyTree * getEspProcessRegistry(const char * espprocname, const char * espbingingport, const char * servicename);
-    static void ensureESDLServiceBindingRegistry(const char * espProcName, const char * espBindingName, bool readonly);
     static IPropertyTree * getESDLDefinitionRegistry(const char * wsEclId, bool readonly);
     static void addESDLDefinition(IPropertyTree * queryRegistry, const char * name, IPropertyTree * definitionInfo, StringBuffer &newId, unsigned &newSeq, const char *userid, bool deleteprev);
     static int publishESDLMethod(const char * espProcName, const char * espBindingName, const char * srvDefId, const char * methodName, IPropertyTree * attributesTree, bool readonly, StringBuffer & message);

initfiles/etc/DIR_NAME/environment.conf.in

@@ -22,6 +22,8 @@ interface=*
 use_epoll=true
 # allow kernel pagecache flushing where enabled (true/false)
 allow_pgcache_flush=true
+# report UDP network stats
+udp_stats=true
 mpStart=7101
 mpEnd=7500
 mpSoMaxConn=128

initfiles/etc/DIR_NAME/environment.xml.in

@@ -771,6 +771,7 @@
   <log>${LOG_PATH}</log>
   <logfields>TIM+DAT+MLT+MID+PID+TID+COD+QUO+PFX</logfields>
   <use_epoll>true</use_epoll>
+  <udp_stats>true</udp_stats>
   <runtime>${RUNTIME_PATH}</runtime>
   <lock>${LOCK_PATH}</lock>
   <configs>${CONFIG_DIR}</configs>

lib2/CMakeLists.txt

@@ -41,6 +41,7 @@ elseif (WIN32)
       find_file (XALAN_C_BIN "Xalan-C_1_11.dll" "${EXTERNALS_DIRECTORY}/xalan/xalan-c/win64/bin" NO_DEFAULT_PATH)
       find_file (XALAN_MESSAGES_BIN "XalanMessages_1_11.dll" "${EXTERNALS_DIRECTORY}/xalan/xalan-c/win64/bin" NO_DEFAULT_PATH)
       find_file (XERCES_C_BIN "xerces-c_3_1.dll" "${EXTERNALS_DIRECTORY}/xalan/xerces-c/win64/bin" NO_DEFAULT_PATH)
+      find_file (ZLIB_BIN "zlib.dll" "${EXTERNALS_DIRECTORY}/zlib/1.2.8/lib/Win64" NO_DEFAULT_PATH)
     else()
       find_file (ICU_DT_BIN "icudt57.dll" "${EXTERNALS_DIRECTORY}/icu/bin" NO_DEFAULT_PATH)
       find_file (ICU_IN_BIN "icuin57.dll" "${EXTERNALS_DIRECTORY}/icu/bin" NO_DEFAULT_PATH)
@@ -50,6 +51,7 @@ elseif (WIN32)
       find_file (XALAN_C_BIN "Xalan-C_1_11.dll" "${EXTERNALS_DIRECTORY}/xalan/xalan-c/win32/bin" NO_DEFAULT_PATH)
       find_file (XALAN_MESSAGES_BIN "XalanMessages_1_11.dll" "${EXTERNALS_DIRECTORY}/xalan/xalan-c/win32/bin" NO_DEFAULT_PATH)
       find_file (XERCES_C_BIN "xerces-c_3_1.dll" "${EXTERNALS_DIRECTORY}/xalan/xerces-c/win32/bin" NO_DEFAULT_PATH)
+      find_file (ZLIB_BIN "zlib.dll" "${EXTERNALS_DIRECTORY}/zlib/1.2.8/lib/Win32" NO_DEFAULT_PATH)
     endif()
 
     set(DYLIBS "")
@@ -60,6 +62,7 @@ elseif (WIN32)
     list(APPEND DYLIBS ${OPENSSL_LIB_BIN} ${OPENSSL_SSL_BIN})
     list(APPEND DYLIBS ${XALAN_C_BIN} ${XALAN_MESSAGES_BIN})
     list(APPEND DYLIBS ${XERCES_C_BIN})
+    list(APPEND DYLIBS ${ZLIB_BIN})
 endif()
 
 foreach(dylib ${DYLIBS})

plugins/couchbase/libcouchbase-cxx

@@ -1 +1 @@
-Subproject commit b390d9c0c2e1b0880c00a9836eac54dcec07bb7a
+Subproject commit 6eba9e709181e54e4095f25327c49d343aeabc0d

roxie/ccd/ccdcontext.cpp

@@ -2476,8 +2476,8 @@ class CRoxieServerDebugContext : extends CBaseServerDebugContext
 public:
     IRoxieSlaveContext *ctx;
 
-    CRoxieServerDebugContext(IRoxieSlaveContext *_ctx, const IContextLogger &_logctx, IPropertyTree *_queryXGMML, SafeSocket &_client)
-        : CBaseServerDebugContext(_logctx, _queryXGMML, _client), ctx(_ctx)
+    CRoxieServerDebugContext(IRoxieSlaveContext *_ctx, const IContextLogger &_logctx, IPropertyTree *_queryXGMML)
+        : CBaseServerDebugContext(_logctx, _queryXGMML), ctx(_ctx)
     {
     }
 
@@ -2715,7 +2715,7 @@ protected:
     {
         if (!debugPermitted || !ownEP.port || !nativeProtocol)
             throw MakeStringException(ROXIE_ACCESS_ERROR, "Debug queries are not permitted on this system");
-        debugContext.setown(new CRoxieServerDebugContext(this, logctx, factory->cloneQueryXGMML(), *nativeProtocol->querySafeSocket()));
+        debugContext.setown(new CRoxieServerDebugContext(this, logctx, factory->cloneQueryXGMML()));
         debugContext->debugInitialize(debugUID, factory->queryQueryName(), breakAtStart);
         if (workUnit)
         {

roxie/ccd/ccdserver.cpp

@@ -9341,7 +9341,8 @@ public:
         puller.stop();
         CRoxieServerActivity::stop();
         pipe.clear();
-        readTransformer->setStream(NULL);
+        if (readTransformer)
+            readTransformer->setStream(NULL);
     }
 
     virtual void reset()

system/jlib/jdebug.cpp

@@ -57,6 +57,7 @@
  #include <mach/mach_host.h>
  #include <mach/vm_statistics.h>
 #endif
+#include "build-config.h"
 
 //===========================================================================
 #ifdef _DEBUG
@@ -2339,6 +2340,10 @@ public:
         hook.set(_hook);
         term = false;
         latestCPU = 0;
+        // UDP stats reported unless explicitly disabled
+        Owned<IProperties> conf = createProperties(CONFIG_DIR PATHSEPSTR "environment.conf", true);
+        if (conf->getPropBool("udp_stats", true))
+            traceMode |= PerfMonUDP;
 #ifdef _WIN32
         memset(&liOldIdleTime,0,sizeof(liOldIdleTime));
         memset(&liOldSystemTime,0,sizeof(liOldSystemTime));

system/jlib/jdebug.hpp

@@ -304,7 +304,7 @@ enum
 #ifdef _WIN32
     PerfMonStandard  = PerfMonProcMem
 #else
-    PerfMonStandard  = PerfMonProcMem|PerfMonExtended|PerfMonUDP
+    PerfMonStandard  = PerfMonProcMem|PerfMonExtended
 #endif
 
 };

testing/regress/ecl/issue16322.ecl

@@ -0,0 +1,26 @@
+
+
+r1 := RECORD
+    unsigned id;
+END;
+
+r2 := RECORD
+    unsigned id;
+    unsigned cnt;
+    dataset(r1) child;
+END;
+
+ds := DATASET(10000, TRANSFORM(r2, SELF.id := COUNTER; SELF.cnt := COUNTER % 30; SELF.child := DATASET(COUNTER % 30, TRANSFORM(r1, SELF.id := COUNTER))));
+
+r2 t(r2 l) := TRANSFORM
+    d := DEDUP(l.child, id);
+    d2 := d(id != count(d)+1);
+    SELF.id := l.id;
+    SELF.cnt := l.cnt - count(d2);
+    SELF.child := d2;
+END;
+
+p := PROJECT(ds, t(LEFT),PARALLEL(10));
+
+output(NOFOLD(p)(cnt != 0));
+

testing/regress/ecl/key/issue16322.xml

@@ -0,0 +1,2 @@
+<Dataset name='Result 1'>
+</Dataset>

testing/regress/ecl/key/pat1.xml

@@ -17,3 +17,6 @@
  <Row><_unnamed_1>true</_unnamed_1><_unnamed_2>Gavin</_unnamed_2><_unnamed_3>Gavin</_unnamed_3><_unnamed_4>5</_unnamed_4><_unnamed_5>1</_unnamed_5><_unnamed_6>false</_unnamed_6><_unnamed_7></_unnamed_7><_unnamed_8>0</_unnamed_8><_unnamed_9>0</_unnamed_9></Row>
  <Row><_unnamed_1>true</_unnamed_1><_unnamed_2>I</_unnamed_2><_unnamed_3>I</_unnamed_3><_unnamed_4>1</_unnamed_4><_unnamed_5>1</_unnamed_5><_unnamed_6>false</_unnamed_6><_unnamed_7></_unnamed_7><_unnamed_8>0</_unnamed_8><_unnamed_9>0</_unnamed_9></Row>
 </Dataset>
+<Dataset name='Result 5'>
+ <Row><_unnamed_1>false</_unnamed_1><_unnamed_2></_unnamed_2><_unnamed_3></_unnamed_3><_unnamed_4>0</_unnamed_4><_unnamed_5>0</_unnamed_5><_unnamed_6>false</_unnamed_6><_unnamed_7></_unnamed_7><_unnamed_8>0</_unnamed_8><_unnamed_9>0</_unnamed_9></Row>
+</Dataset>

testing/regress/ecl/pat1.ecl

@@ -61,3 +61,6 @@ output(outfile3);
 //Return all matching sentances, case insignficant.
 outfile4 := PARSE(infile,text,sentance2,results,nocase,scan,skip(ws*),first);
 output(outfile4);
+
+outfile5 := PARSE(infile,text,patWord,results,nocase,not matched only);
+output(outfile5);

testing/regress/environment.xml.in

@@ -761,6 +761,7 @@
   <log>${LOG_PATH}</log>
   <logfields>TIM+DAT+MLT+MID+PID+TID+COD+QUO+PFX</logfields>
   <use_epoll>true</use_epoll>
+  <udp_stats>true</udp_stats>
   <runtime>${RUNTIME_PATH}</runtime>
   <lock>${LOCK_PATH}</lock>
   <configs>${CONFIG_DIR}</configs>