Merge branch 'candidate-6.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -181,7 +181,9 @@ elseif ( NOT MAKE_DOCS_ONLY OR MAKE_CONFIGURATOR )
         HPCC_ADD_SUBDIRECTORY (thorlcr "PLATFORM")
         HPCC_ADD_SUBDIRECTORY (testing)
 
-        if ( NOT WIN32 )
+        if ( WIN32 )
+            HPCC_ADD_SUBDIRECTORY (clienttools/IDEPlugins "CLIENTTOOLS_ONLY")
+        else()
             HPCC_ADD_SUBDIRECTORY (clienttools "CLIENTTOOLS_ONLY")
         endif()
 

clienttools/IDEPlugins/CMakeLists.txt

@@ -0,0 +1,17 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+add_subdirectory(ESDL)

clienttools/IDEPlugins/ESDL/CMakeLists.txt

@@ -0,0 +1,27 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+project(IDEPlugins-ESDL)
+
+set(SRC
+    esdl.bat.in
+    )
+
+foreach(source ${SRC})
+    string(REGEX REPLACE "\\.in$" "" configured_source ${source})
+    configure_file(${source} ${configured_source} @ONLY)
+    install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${configured_source} DESTINATION IDEPlugins/ESDL)
+endforeach()

clienttools/IDEPlugins/ESDL/esdl.bat.in

@@ -0,0 +1,83 @@
+rem ---   ESDL.BAT   ---
+rem %1 = Action:  CalcIncludes, SyntaxCheck, Save, Commit, Submit, Generate, Custom1, Custom2, Custom3, Custom4
+rem %2 = Module Label
+rem %3 = Attribute Label
+rem %4 = Input file path
+rem %5 = Output file path
+rem %6 = Error file path
+rem %7 = User Info
+rem %8 = Ecl Folders
+rem %9 = Server
+rem %10 = Password
+
+set "action=%1%"
+shift
+echo Action:  %action% > %TMP%\log.txt
+echo Module Label:  %1 >> %TMP%\log.txt
+echo Attribute Label:  %2 >> %TMP%\log.txt
+echo Input file path:  %3 >> %TMP%\log.txt
+echo Output file path:  %4 >> %TMP%\log.txt
+echo Error file path:  %5 >> %TMP%\log.txt
+echo User Info:  %6 >> %TMP%\log.txt
+echo Ecl Folders:  %7 >> %TMP%\log.txt
+echo Server:  %8 >> %TMP%\log.txt
+echo Password:  %9 >> %TMP%\log.txt
+
+rem =====================================
+rem  CREATE NEW FILE NAMES
+rem =====================================
+set "newinput=%TMP%\%2%.esdl"
+set "newoutput=%TMP%\%2%.ecl"
+echo New Input file:  %newinput% >> %TMP%\log.txt
+echo New Ouput file:  %newoutput% >> %TMP%\log.txt
+copy /Y %3 %newinput%
+
+:begin
+if %action%==CalcIncludes goto eof
+if %action%==SyntaxCheck goto eof
+if %action%==Save goto genecl
+if %action%==Commit goto genecl
+if %action%==Submit goto publish
+if %action%==Generate goto genecl
+if %action%==Custom1 goto publish
+if %action%==Custom2 goto genecl
+if %action%==Custom3 goto eof
+if %action%==Custom4 goto eof
+
+:publish
+rem =====================================
+rem  PARSE INTO SERVER AND PORT
+rem =====================================
+if [%9] == [] goto passparse
+for /f "tokens=1,2,3 delims=/:" %%a in ("%8") do set http=%%a&set server=%%b&set port=%%c
+echo.server: %server% >> %TMP%\log.txt
+echo.port  : %port% >> %TMP%\log.txt
+
+:publish
+esdl publish %newinput% --server %server%  --port %port% --version 1.0 --username %6 --password %9 -v 2>%5 >>%TMP%\log.txt
+goto end
+
+:genecl
+esdl ecl %newinput% %TMP% --includes --rollup -cde "@ORIG_PREFIX@\@DIR_NAME@\@version@\clienttools\componentfiles" 2>%5 >>%TMP%\log.txt
+goto end
+
+:end
+
+rem =====================================
+rem  MAKE FILE INTO MOD FILE FORMAT
+rem =====================================
+set "newoutputUnique=%2Gen"
+echo //IMPORT:%1.%newoutputUnique% >> %TMP%\modfile.tmp
+type %newoutput% >> %TMP%\modfile.tmp
+del %newoutput%
+copy /Y %TMP%\modfile.tmp %newoutput%
+del %TMP%\modfile.tmp
+
+copy /Y %3 %TMP%\in.txt
+copy /Y %newoutput% %TMP%\%newoutputUnique%.ecl
+copy /Y %TMP%\%newoutputUnique%.ecl %4
+copy /Y %4 %TMP%\out.txt
+del %newoutput%
+copy /Y %5 %TMP%\err.txt
+
+:eof

cmake_modules/optionDefaults.cmake

@@ -10,10 +10,12 @@ if ( WIN32 )
     else()
         set( PREFIX "$ENV{${prog86key}}" )
     endif()
+    set(ORIG_PREFIX ${PREFIX})
     string(REGEX REPLACE "\\\\" "/" PREFIX ${PREFIX})    
 else (WIN32)
     set( PREFIX "${DESTDIR}/opt" )
-endif (WIN32)    
+    set( ORIG_PREFIX ${PREFIX})
+endif (WIN32)
 endif()
 
 if ( NOT EXEC_PREFIX )

common/remote/rmtfile.cpp

@@ -39,7 +39,7 @@ static class CSecuritySettings
 public:
     CSecuritySettings()
     {
-        querySecuritySettings(NULL, &daliServixPort, NULL, NULL);
+        querySecuritySettings(nullptr, &daliServixPort, nullptr, nullptr, nullptr);
     }
 
     unsigned short queryDaliServixPort() { return daliServixPort; }
@@ -315,7 +315,7 @@ bool testDaliServixPresent(const SocketEndpoint &_ep)
     if (ep.isNull())
         return false;
     try {
-        Owned<ISocket> socket = ISocket::connect_timeout(ep,10000);
+        Owned<ISocket> socket = connectDafs(ep, 10000);
         return true;
     }
     catch (IException *e)
@@ -339,7 +339,7 @@ unsigned getDaliServixVersion(const SocketEndpoint &_ep,StringBuffer &ver)
         return 0;
     try
     {
-        Owned<ISocket> socket = ISocket::connect_timeout(ep,10000);
+        Owned<ISocket> socket = connectDafs(ep, 10000);
         return getRemoteVersion(socket,ver);
     }
     catch (IException *e)
@@ -475,7 +475,7 @@ extern REMOTE_API int setDafileSvrTraceFlags(const SocketEndpoint &_ep,byte flag
     if (ep.isNull())
         return -3;
     try {
-        Owned<ISocket> socket = ISocket::connect_wait(ep,5000);
+        Owned<ISocket> socket = connectDafs(ep, 5000);
         return setDafsTrace(socket, flags);
     }
     catch (IException *e)
@@ -493,7 +493,7 @@ extern REMOTE_API int setDafileSvrThrottleLimit(const SocketEndpoint &_ep, Throt
     if (ep.isNull())
         return -3;
     try {
-        Owned<ISocket> socket = ISocket::connect_wait(ep,5000);
+        Owned<ISocket> socket = connectDafs(ep, 5000);
         return setDafsThrottleLimit(socket, throttleClass, throttleLimit, throttleDelayMs, throttleCPULimit, queueLimit, errMsg);
     }
     catch (IException *e)
@@ -511,7 +511,7 @@ extern REMOTE_API int getDafileSvrInfo(const SocketEndpoint &_ep, unsigned level
     if (ep.isNull())
         return false;
     try {
-        Owned<ISocket> socket = ISocket::connect_wait(ep,5000);
+        Owned<ISocket> socket = connectDafs(ep, 5000);
         return getDafsInfo(socket, level, retstr);
     }
     catch (IException *e)

common/remote/sockfile.cpp

@@ -162,7 +162,7 @@ struct dummyReadWrite
 // backward compatible modes
 typedef enum { compatIFSHnone, compatIFSHread, compatIFSHwrite, compatIFSHexec, compatIFSHall} compatIFSHmode;
 
-static const char *VERSTRING= "DS V2.0"       // dont forget FILESRV_VERSION in header
+static const char *VERSTRING= "DS V2.1"       // dont forget FILESRV_VERSION in header
 #ifdef _WIN32
 "Windows ";
 #else
@@ -179,14 +179,16 @@ static unsigned maxReceiveTime = 0;
 static class _securitySettings
 {
 public:
-    bool            useSSL;
+    SSLCfg          useSSL;
     unsigned short  daFileSrvPort;
+    unsigned short  daFileSrvSSLPort;
     const char *    certificate;
     const char *    privateKey;
+    const char *    passPhrase;
 
     _securitySettings()
     {
-        querySecuritySettings(&useSSL, &daFileSrvPort, &certificate, &privateKey);
+        queryDafsSecSettings(&useSSL, &daFileSrvPort, &daFileSrvSSLPort, &certificate, &privateKey, &passPhrase);
     }
 } securitySettings;
 
@@ -202,7 +204,7 @@ static ISecureSocket *createSecureSocket(ISocket *sock,SecureSocketType type)
         if (!secureContext)
         {
             if (securitySettings.certificate)
-                secureContext.setown(createSecureSocketContextEx(securitySettings.certificate,securitySettings.privateKey, NULL, type));
+                secureContext.setown(createSecureSocketContextEx(securitySettings.certificate, securitySettings.privateKey, securitySettings.passPhrase, type));
             else
                 secureContext.setown(createSecureSocketContext(type));
         }
@@ -547,7 +549,10 @@ void setDafsEndpointPort(SocketEndpoint &ep)
     }
     if (ep.port==0)
     {
-        ep.port = securitySettings.daFileSrvPort;
+        if ( (securitySettings.useSSL == SSLNone) || (securitySettings.useSSL == UnsecureFirst) )
+            ep.port = securitySettings.daFileSrvPort;
+        else
+            ep.port = securitySettings.daFileSrvSSLPort;
     }
 }
 
@@ -950,19 +955,32 @@ static Semaphore                 treeCopySem;
 
 class CRemoteBase: public CInterface
 {
-
     Owned<ISocket>          socket;
     static  SocketEndpoint  lastfailep;
     static unsigned         lastfailtime;
-    bool                    useSSL;
-    void connectSocket(SocketEndpoint &ep)
+    SSLCfg                  useSSL;
+
+    void connectSocket(SocketEndpoint &ep, unsigned localConnectTime=0, unsigned localRetries=0)
     {
-        sRFTM tm(maxConnectTime);
-        // called in CConnectionTable::crit
         unsigned retries = 3;
+
+        if (localConnectTime)
+        {
+            if (localRetries)
+                retries = localRetries;
+            if (localConnectTime > maxConnectTime)
+                localConnectTime = maxConnectTime;
+        }
+        else
+            localConnectTime = maxConnectTime;
+
+        sRFTM tm(localConnectTime);
+
+        // called in CConnectionTable::crit
+
         if (ep.equals(lastfailep)) {
             if (msTick()-lastfailtime<DAFS_CONNECT_FAIL_RETRY_TIME) {
-                StringBuffer msg("Failed to connect to dafilesrv/daliservix on ");
+                StringBuffer msg("Failed to connect (host marked down) to dafilesrv/daliservix on ");
                 ep.getUrlStr(msg);
                 throw createDafsException(DAFSERR_connection_failed,msg.str());
             }
@@ -974,7 +992,10 @@ class CRemoteBase: public CInterface
             StringBuffer eps;
             if (TF_TRACE_CLIENT_CONN) {
                 ep.getUrlStr(eps);
-                PROGLOG("Connecting %sto %s",useSSL?"SECURE ":"",eps.str());
+                if (ep.port == securitySettings.daFileSrvSSLPort)
+                    PROGLOG("Connecting SECURE to %s", eps.str());
+                else
+                    PROGLOG("Connecting to %s", eps.str());
                 //PrintStackReport();
             }
             bool ok = true;
@@ -986,7 +1007,7 @@ class CRemoteBase: public CInterface
                 }
                 else
                     socket.setown(ISocket::connect(ep));
-                if (useSSL)
+                if (ep.port == securitySettings.daFileSrvSSLPort)
                 {
 #ifdef _USE_OPENSSL
                     Owned<ISecureSocket> ssock = createSecureSocket(socket.getClear(), ClientSocket);
@@ -1006,7 +1027,7 @@ class CRemoteBase: public CInterface
                         lastfailep.set(ep);
                         lastfailtime = msTick();
                         e->Release();
-                        StringBuffer msg("Failed to connect to dafilesrv/daliservix on ");
+                        StringBuffer msg("Failed to connect (setting host down) to dafilesrv/daliservix on ");
                         ep.getUrlStr(msg);
                         throw createDafsException(DAFSERR_connection_failed,msg.str());
                     }
@@ -1044,7 +1065,10 @@ class CRemoteBase: public CInterface
                     sleeptime = remaining/2;
             }
             Sleep(sleeptime);       // prevent multiple retries beating
-            PROGLOG("Retrying %sconnect",useSSL?"SECURE ":"");
+            if (ep.port == securitySettings.daFileSrvSSLPort)
+                PROGLOG("Retrying SECURE connect");
+            else
+                PROGLOG("Retrying connect");
         }
         if (ConnectionTable)
             ConnectionTable->addLink(ep,socket);
@@ -1131,27 +1155,37 @@ protected: friend class CRemoteFileIO;
                         killSocket(tep);
                 }
             }
-            if (!socket) {
-                bool tryNonSecure = true;
-                if (useSSL) {
-                    try {
-                        connectSocket(tep);//first try secure connect
-                        tryNonSecure = false;
-                    }
-                    catch (IException *e) {
-
-                        StringBuffer s;
-                        e->errorMessage(s);
-                        e->Release();
-                        WARNLOG("Secure connect failed, retrying on legacy port (%s)",s.str());
 
-                        useSSL = false;
-                        tep.port = DAFILESRV_PORT;//retry on nonsecure port
-                        tryNonSecure = true;
+            if (!socket)
+            {
+                bool doConnect = true;
+                if (useSSL == SSLFirst || useSSL == UnsecureFirst)
+                {
+                    // MCK - could maintain a list of 100 or so previous endpoints and if connection failed
+                    // then mark port down for a delay (like 15 min above) to avoid having to try every time ...
+                    try
+                    {
+                        connectSocket(tep, 5000, 1);
+                        doConnect = false;
+                    }
+                    catch (IDAFS_Exception *e)
+                    {
+                        if (e->errorCode() == DAFSERR_connection_failed)
+                        {
+                            unsigned prevPort = tep.port;
+                            if (prevPort == securitySettings.daFileSrvSSLPort)
+                                tep.port = securitySettings.daFileSrvPort;
+                            else
+                                tep.port = securitySettings.daFileSrvSSLPort;
+                            WARNLOG("Connect failed on port %d, retrying on port %d", prevPort, tep.port);
+                            doConnect = true;
+                            e->Release();
+                        }
+                        else
+                            throw e;
                     }
                 }
-
-                if (tryNonSecure)
+                if (doConnect)
                     connectSocket(tep);
             }
         }
@@ -2602,6 +2636,77 @@ void CRemoteFile::copyTo(IFile *dest, size32_t buffersize, ICopyFileProgress *pr
 
 /////////////////////////
 
+ISocket *checkSocketSecure(ISocket *socket)
+{
+    if (securitySettings.useSSL == SSLNone)
+        return LINK(socket);
+
+    char pname[256];
+    pname[0] = 0;
+    int pport = socket->peer_name(pname, sizeof(pname)-1);
+
+    if ( (pport == securitySettings.daFileSrvSSLPort) && (!socket->isSecure()) )
+    {
+        Owned<ISecureSocket> ssock;
+#ifdef _USE_OPENSSL
+        ssock.setown(createSecureSocket(LINK(socket), ClientSocket));
+        int status = ssock->secure_connect();
+        if (status < 0)
+            throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection");
+        return ssock.getClear();
+#else
+        throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection: OpenSSL disabled in build");
+#endif
+    }
+
+    return LINK(socket);
+}
+
+ISocket *connectDafs(SocketEndpoint &ep, unsigned timeoutms)
+{
+    Owned<ISocket> socket;
+
+    if ( (securitySettings.useSSL == SSLNone) || (securitySettings.useSSL == SSLOnly) )
+    {
+        socket.setown(ISocket::connect_timeout(ep, timeoutms));
+        return socket.getClear();
+    }
+
+    unsigned newtimeout = timeoutms;
+    if (newtimeout > 5000)
+        newtimeout = 5000;
+
+    bool tryAgain = false;
+    try
+    {
+        socket.setown(ISocket::connect_timeout(ep, newtimeout));
+    }
+    catch (IJSOCK_Exception *e)
+    {
+        if (e->errorCode() == JSOCKERR_connection_failed)
+        {
+            unsigned prevPort = ep.port;
+            if (prevPort == securitySettings.daFileSrvSSLPort)
+                ep.port = securitySettings.daFileSrvPort;
+            else
+                ep.port = securitySettings.daFileSrvSSLPort;
+            WARNLOG("Connect failed on port %d, retrying on port %d", prevPort, ep.port);
+            tryAgain = true;
+            e->Release();
+        }
+        else
+            throw e;
+    }
+
+    if (tryAgain)
+        socket.setown(ISocket::connect_timeout(ep, timeoutms));
+
+    if (socket)
+        return checkSocketSecure(socket);
+    else
+        return nullptr;
+}
+
 unsigned getRemoteVersion(CRemoteFileIO &remoteFileIO, StringBuffer &ver)
 {
     unsigned ret;
@@ -2640,26 +2745,13 @@ unsigned getRemoteVersion(CRemoteFileIO &remoteFileIO, StringBuffer &ver)
     return ret;
 }
 
-unsigned getRemoteVersion(ISocket * socket, StringBuffer &ver)
+unsigned getRemoteVersion(ISocket *origSock, StringBuffer &ver)
 {
     // used to have a global critical section here
-    if (!socket)
+    if (!origSock)
         return 0;
 
-    Owned<ISecureSocket> ssock;
-
-    if (securitySettings.useSSL && !socket->isSecure())
-    {
-#ifdef _USE_OPENSSL
-        ssock.setown(createSecureSocket(LINK(socket), ClientSocket));
-        int status = ssock->secure_connect();
-        if (status < 0)
-            throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection");
-        socket = ssock;
-#else
-        throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection: OpenSSL disabled in build");
-#endif
-    }
+    Owned<ISocket> socket = checkSocketSecure(origSock);
 
     unsigned ret;
     MemoryBuffer sendbuf;
@@ -4261,7 +4353,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
     int                 lasthandle;
     CriticalSection     sect;
     Owned<ISocket>      acceptsock;
-    Owned<ISocket>      rejectsock;//used to immediately reject nonsecure connection requests when in secure mode
+    Owned<ISocket>      securesock;
     Owned<ISocketSelectHandler> selecthandler;
     Owned<IThreadPool>  threads;    // for commands
     bool stopping;
@@ -5354,8 +5446,8 @@ public:
         stopping = true;
         if (acceptsock) 
             acceptsock->cancel_accept();
-        if (rejectsock)
-            rejectsock->cancel_accept();
+        if (securesock)
+            securesock->cancel_accept();
         reply.append((unsigned)RFEnoerror);
         return false;
     }
@@ -5629,144 +5721,187 @@ public:
         return new cCommandProcessor();
     }
 
-    void run(SocketEndpoint &listenep, bool useSSL)
+    void run(SSLCfg _useSSL, SocketEndpoint &listenep, unsigned sslPort)
     {
-        Owned<ISocket> acceptSocket, secureRejectSocket;
-        if (listenep.isNull())
-            acceptSocket.setown(ISocket::create(listenep.port));
+        SocketEndpoint sslep(listenep);
+        if (sslPort)
+            sslep.port = sslPort;
         else
+            sslep.port = securitySettings.daFileSrvSSLPort;
+        Owned<ISocket> acceptSocket, acceptSSLSocket;
+
+        if (_useSSL != SSLOnly)
         {
-            StringBuffer ips;
-            listenep.getIpText(ips);
-            acceptSocket.setown(ISocket::create_ip(listenep.port,ips.str()));
-        }
-        if (useSSL)
-        {
-            if (!securitySettings.certificate)
-                throw createDafsException(DAFSERR_connection_failed,"SSL Certificate information not found in environment.conf");
-            if (listenep.port <= 0)
-            {
-                assertex(FALSE);//should never get here
-                listenep.port = securitySettings.daFileSrvPort;
-            }
-            //Create unsecure socket to reject non-ssl client requests
+            if (listenep.port == 0)
+                throw createDafsException(DAFSERR_connection_failed, "dafilesrv port not specified");
+
             if (listenep.isNull())
-                secureRejectSocket.setown(ISocket::create(DAFILESRV_PORT));
+                acceptSocket.setown(ISocket::create(listenep.port));
             else
             {
                 StringBuffer ips;
                 listenep.getIpText(ips);
-                secureRejectSocket.setown(ISocket::create_ip(DAFILESRV_PORT,ips.str()));
+                acceptSocket.setown(ISocket::create_ip(listenep.port,ips.str()));
             }
         }
-        run(acceptSocket.getClear(), secureRejectSocket.getClear());
+
+        if (_useSSL)
+        {
+            if (sslep.port == 0)
+                throw createDafsException(DAFSERR_connection_failed, "Secure dafilesrv port not specified");
+
+            if ( (_useSSL != UnsecureFirst) && (!securitySettings.certificate || !securitySettings.privateKey) )
+                throw createDafsException(DAFSERR_connection_failed, "SSL Certificate and/or Key file information not found in environment.conf");
+
+            if (sslep.isNull())
+                acceptSSLSocket.setown(ISocket::create(sslep.port));
+            else
+            {
+                StringBuffer ips;
+                sslep.getIpText(ips);
+                acceptSSLSocket.setown(ISocket::create_ip(sslep.port,ips.str()));
+            }
+        }
+
+        run(_useSSL, acceptSocket.getClear(), acceptSSLSocket.getClear());
     }
-    void run(ISocket *socket, ISocket *secureRejectSocket)
+
+    void run(SSLCfg _useSSL, ISocket *regSocket, ISocket *secureSocket)
     {
-        acceptsock.setown(socket);
-        rejectsock.setown(secureRejectSocket);
-        bool useSSL = secureRejectSocket ? true : false;
-#ifdef _DEBUG
-        SocketEndpoint listenep;
-        socket->getPeerEndpoint(listenep);
-        StringBuffer sb;
-        listenep.getUrlStr(sb);
-        DBGLOG("Server accepting %sfrom %s", useSSL?"SECURE ":"", sb.str());
-#endif
-        selecthandler->start();
+        if (_useSSL != SSLOnly)
+        {
+            if (regSocket)
+                acceptsock.setown(regSocket);
+            else
+                throw createDafsException(DAFSERR_connection_failed, "Invalid non-secure socket");
+        }
 
-        UnsignedArray readSocks;
-        if (useSSL)
+        if (_useSSL)
         {
-            readSocks.append(acceptsock->OShandle());
-            readSocks.append(rejectsock->OShandle());
+            if (secureSocket)
+                securesock.setown(secureSocket);
+            else
+                throw createDafsException(DAFSERR_connection_failed, "Invalid secure socket");
         }
 
+        selecthandler->start();
+
         for (;;)
         {
             Owned<ISocket> sock;
+            Owned<ISocket> sockSSL;
             bool sockavail = false;
-            try
+            bool securesockavail = false;
+            if (_useSSL == SSLNone)
+                sockavail = acceptsock->wait_read(1000*60*1)!=0;
+            else if (_useSSL == SSLOnly)
+                securesockavail = securesock->wait_read(1000*60*1)!=0;
+            else
             {
-                if (!useSSL)
-                    sockavail = acceptsock->wait_read(1000*60*1)!=0;
-                else
+                UnsignedArray readSocks;
+                UnsignedArray waitingSocks;
+                readSocks.append(acceptsock->OShandle());
+                readSocks.append(securesock->OShandle());
+                int numReady = wait_read_multiple(readSocks, 1000*60*1, waitingSocks);
+                if (numReady > 0)
                 {
-                    UnsignedArray waitingSocks;
-                    //SSL Enabled. Listen for non SSL connection on DAFILESRV_PORT and reject them
-                    int numReady = wait_read_multiple(readSocks, 1000*60*1, waitingSocks);
-                    if (numReady)
+                    for (int idx = 0; idx < numReady; idx++)
                     {
-                        for (int idx = 0; idx < numReady; idx++)
-                        {
-                            if (waitingSocks.item(idx) == rejectsock->OShandle())
-                            {
-                                //Unsecure connection attemped, reject !
-                                Owned<ISocket> s;
-                                s.setown(rejectsock->accept(true));
-                                IpAddress ip;
-                                StringBuffer sb;
-                                s->getPeerAddress(ip);
-                                ip.getIpText(sb);
-                                DBGLOG("Rejecting nonsecure connect from %s",sb.str());
-                                s->close();
-                            }
-                            else
-                            {
-                                sockavail = true;
-                            }
-                        }
+                        if (waitingSocks.item(idx) == acceptsock->OShandle())
+                            sockavail = true;
+                        else if (waitingSocks.item(idx) == securesock->OShandle())
+                            securesockavail = true;
                     }
                 }
-#if 0
-                if (!sockavail)
-                {
-                    JSocketStatistics stats;
-                    getSocketStatistics(stats);
-                    StringBuffer s;
-                    getSocketStatisticsString(stats,s);
-                    PROGLOG( "Socket statistics : \n%s\n",s.str());
-                }
-#endif
             }
-            catch (IException *e)
+#if 0
+            if (!sockavail && !securesockavail)
             {
-                EXCLOG(e,"CRemoteFileServer(1)");
-                e->Release();
-                // not sure what to do so just accept
-                sockavail = true;
+                JSocketStatistics stats;
+                getSocketStatistics(stats);
+                StringBuffer s;
+                getSocketStatisticsString(stats,s);
+                PROGLOG( "Socket statistics : \n%s\n",s.str());
             }
+#endif
+
             if (stopping)
                 break;
-            if (sockavail)
+
+            if (sockavail || securesockavail)
             {
-                try
+                if (sockavail)
                 {
-                    sock.setown(acceptsock->accept(true));
-                    if (useSSL)
+                    try
                     {
-#ifdef _USE_OPENSSL
-                        Owned<ISecureSocket> ssock = createSecureSocket(sock.getClear(), ServerSocket);
-                        int status = ssock->secure_accept();
-                        if (status < 0)
-                            throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection");
-                        sock.setown(ssock.getLink());
-#else
-                        throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection: OpenSSL disabled in build");
+                        sock.setown(acceptsock->accept(true));
+                        if (!sock||stopping)
+                            break;
+#ifdef _DEBUG
+                        SocketEndpoint eps;
+                        sock->getPeerEndpoint(eps);
+                        StringBuffer sb;
+                        eps.getUrlStr(sb);
+                        PROGLOG("Server accepting from %s", sb.str());
 #endif
                     }
-                    if (!sock||stopping)
+                    catch (IException *e)
+                    {
+                        EXCLOG(e,"CRemoteFileServer");
+                        e->Release();
                         break;
-                    runClient(sock.getClear());
+                    }
                 }
-                catch (IException *e)
+
+                if (securesockavail)
                 {
-                    EXCLOG(e,"CRemoteFileServer");
-                    e->Release();
-                    sock.clear();
-                    if (!QUERYINTERFACE(e, IJSOCK_Exception))
+                    try
+                    {
+                        sockSSL.setown(securesock->accept(true));
+                        if (!sockSSL||stopping)
+                            break;
+
+                        if ( (_useSSL == UnsecureFirst) && (!securitySettings.certificate || !securitySettings.privateKey) )
+                        {
+                            // for client secure_connect() to fail quickly ...
+                            sockSSL->shutdown();
+                            sockSSL->close();
+                            sockSSL.clear();
+                            securesockavail = false;
+                        }
+                        else
+                        {
+#ifdef _USE_OPENSSL
+                            Owned<ISecureSocket> ssock = createSecureSocket(sockSSL.getClear(), ServerSocket);
+                            int status = ssock->secure_accept();
+                            if (status < 0)
+                                throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection");
+                            sockSSL.setown(ssock.getLink());
+#else
+                            throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection: OpenSSL disabled in build");
+#endif
+#ifdef _DEBUG
+                            SocketEndpoint eps;
+                            sockSSL->getPeerEndpoint(eps);
+                            StringBuffer sb;
+                            eps.getUrlStr(sb);
+                            PROGLOG("Server accepting SECURE from %s", sb.str());
+#endif
+                        }
+                    }
+                    catch (IException *e)
+                    {
+                        EXCLOG(e,"CRemoteFileServer (secure)");
+                        e->Release();
                         break;
+                    }
                 }
+
+                if (sockavail)
+                    runClient(sock.getClear());
+
+                if (securesockavail)
+                    runClient(sockSSL.getClear());
             }
             else
                 checkTimeout();
@@ -5890,8 +6025,8 @@ public:
             PROGLOG("CRemoteFileServer::stop");
         if (acceptsock)
             acceptsock->cancel_accept();
-        if (rejectsock)
-            rejectsock->cancel_accept();
+        if (securesock)
+            securesock->cancel_accept();
         threads->stopAll();
         threads->joinAll(true,60*1000);
     }
@@ -6135,7 +6270,8 @@ protected:
         // IThreaded
             virtual void main()
             {
-                server->run(socket, nullptr);
+                SSLCfg sslCfg = SSLNone;
+                server->run(sslCfg, socket, nullptr);
             }
         };
         enableDafsAuthentication(false);

common/remote/sockfile.hpp

@@ -53,14 +53,14 @@ enum ThrottleClass
 
 interface IRemoteFileServer : extends IInterface
 {
-    virtual void run(SocketEndpoint &listenep, bool useSSL = false) = 0;
+    virtual void run(SSLCfg useSSL, SocketEndpoint &listenep, unsigned sslPort=0) = 0;
     virtual void stop() = 0;
     virtual unsigned idleTime() = 0; // in ms
     virtual void setThrottle(ThrottleClass throttleClass, unsigned limit, unsigned delayMs=DEFAULT_STDCMD_THROTTLEDELAYMS, unsigned cpuThreshold=DEFAULT_STDCMD_THROTTLECPULIMIT, unsigned queueLimit=DEFAULT_STDCMD_THROTTLEQUEUELIMIT) = 0;
     virtual StringBuffer &getStats(StringBuffer &stats, bool reset) = 0;
 };
 
-#define FILESRV_VERSION 20 // don't forget VERSTRING in sockfile.cpp
+#define FILESRV_VERSION 21 // don't forget VERSTRING in sockfile.cpp
 
 interface IKeyManager;
 interface IDelayedFile;
@@ -78,6 +78,8 @@ extern void remoteExtractBlobElements(const SocketEndpoint &ep, const char * pre
 extern int getDafsInfo(ISocket * socket, unsigned level, StringBuffer &retstr);
 extern void setDafsEndpointPort(SocketEndpoint &ep);
 extern void setDafsLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir);
+extern REMOTE_API ISocket *connectDafs(SocketEndpoint &ep, unsigned timeoutms);
+extern REMOTE_API ISocket *checkSocketSecure(ISocket *socket);
 
 // client only
 extern void clientSetDaliServixSocketCaching(bool set);

common/workunit/workunit.cpp

@@ -6070,9 +6070,18 @@ void CLocalWorkUnit::clearExceptions()
 {
     CriticalBlock block(crit);
     // For this to be legally called, we must have the write-able interface. So we are already locked for write.
-    exceptions.kill();
-    exceptionsCached = true;
-    p->removeProp("Exceptions");
+    loadExceptions();
+    ForEachItemInRev(idx, exceptions)
+    {
+        IWUException &e = exceptions.item(idx);
+        SCMStringBuffer s;
+        e.getExceptionSource(s);
+        if (strieq(s.s, "eclcc") || strieq(s.s, "eclccserver") || strieq(s.s, "eclserver") )
+            break;
+        VStringBuffer xpath("Exceptions/Exception[@sequence='%d']", e.getSequence());
+        p->removeProp(xpath);
+        exceptions.remove(idx);
+    }
 }
 
 

dali/base/dautils.cpp

@@ -59,6 +59,23 @@ extern da_decl const char *queryDfsXmlBranchName(DfsXmlBranchKind kind)
     return "UNKNOWN";
 }
 
+extern da_decl DfsXmlBranchKind queryDfsXmlBranchType(const char *typeStr)
+{
+    if (isEmptyString(typeStr))
+        throw makeStringException(0, "Blank DFS xml branch type");
+    if (strieq(typeStr, "File"))
+        return DXB_File;
+    else if (strieq(typeStr, "SuperFile"))
+        return DXB_SuperFile;
+    else if (strieq(typeStr, "Collection"))
+        return DXB_Collection;
+    else if (strieq(typeStr, "Scope"))
+        return DXB_Scope;
+    else if (strieq(typeStr, "HpccInternal"))
+        return DXB_Internal;
+    else
+        throw makeStringExceptionV(0, "Unknown DFS xml Branch type: %s", typeStr);
+}
 
 
 static const char *toLower(const char *s,StringBuffer &str)

dali/base/dautils.hpp

@@ -329,6 +329,7 @@ public:
 };
 
 extern da_decl const char *queryDfsXmlBranchName(DfsXmlBranchKind kind);
+extern da_decl DfsXmlBranchKind queryDfsXmlBranchType(const char *typeStr);
 extern da_decl unsigned getFileGroups(IPropertyTree *pt,StringArray &groups,bool checkclusters=false);
 extern da_decl unsigned getFileGroups(const char *grplist,StringArray &groups); // actually returns labels not groups
 extern da_decl bool isAnonCluster(const char *grp);

dali/dafilesrv/dafilesrv.cpp

@@ -37,19 +37,21 @@
 void usage()
 {
     printf("dafilesrv usage:\n");
-    printf("    dafilesrv -T<n> <port> <-NOSSL> [<send-buff-size-kb> <recv-buff-size-kb>]\n");
+    printf("    dafilesrv [-T<n>] [...] [<port>] [<send-buff-size-kb> <recv-buff-size-kb>]\n");
     printf("                                                  -- run test local\n");
     printf("    dafilesrv -D [ -L <log-dir> ] [ -LOCAL ]      -- run as linux daemon\n");
     printf("    dafilesrv -R                                  -- run remote (linux daemon, windows standalone)\n");
     printf("    dafilesrv -install                            -- install windows service\n");
     printf("    dafilesrv -remove                             -- remove windows service\n\n");
-    
-    printf("add -A to enable authentication to the above \n\n");
-    printf("add -I <instance name>  to specify an instance name\n\n");
-    printf("add -NOSSL to disable SSL sockets, even when specified in configuration\n\n");
-    printf("Standard port is %d\n",DAFILESRV_PORT);
-    printf("Standard SSL port is %d (certificate specs required in environment.conf)\n",SECURE_DAFILESRV_PORT);
-    printf("Version:  %s\n\n",remoteServerVersionString());
+    printf("    add -A to enable authentication to the above\n");
+    printf("    add -I <instance name> to specify an instance name\n");
+    printf("    add -NOSSL to disable SSL sockets, even when specified in configuration\n\n");
+    printf("    additional optional args:\n");
+    printf("        [-p <port>] [-sslp <ssl-port>] [-sbsize <send-buff-size-kb>] [-rbsize <recv-buff-size-kb>]\n");
+    printf("        [-addr <ip>:<port>]\n\n");
+    printf("    Standard port is %d\n",DAFILESRV_PORT);
+    printf("    Standard SSL port is %d (certificate and key required in environment.conf)\n",SECURE_DAFILESRV_PORT);
+    printf("    Version:  %s\n\n",remoteServerVersionString());
 }
 
 static Owned<IRemoteFileServer> server;
@@ -349,11 +351,13 @@ int main(int argc,char **argv)
     StringBuffer logDir;
     StringBuffer instanceName;
 
-   //Get SSL Settings
+    // Get SSL Settings
+    SSLCfg          useSSL;
+    unsigned short  port;
+    unsigned short  sslport;
     const char *    sslCertFile;
-    bool            useSSL;
-    unsigned short  dafsPort;//DAFILESRV_PORT or SECURE_DAFILESRV_PORT
-    querySecuritySettings(&useSSL, &dafsPort, &sslCertFile, NULL);
+    const char *    sslKeyFile;
+    queryDafsSecSettings(&useSSL, &port, &sslport, &sslCertFile, &sslKeyFile, nullptr);
 
     unsigned maxThreads = DEFAULT_THREADLIMIT;
     unsigned maxThreadsDelayMs = DEFAULT_THREADLIMITDELAYMS;
@@ -417,6 +421,9 @@ int main(int argc,char **argv)
         }
     }
 
+    // these should really be in env, but currently they are not ...
+    listenep.port = port;
+
     while (argc>i) {
         if (stricmp(argv[i],"-D")==0) {
             i++;
@@ -448,27 +455,67 @@ int main(int argc,char **argv)
             i++;
             instanceName.clear().append(argv[i++]);
         }
+        else if ((argc>i+1)&&(stricmp(argv[i],"-p")==0)) {
+            i++;
+            listenep.port = atoi(argv[i++]);
+        }
+        else if ((argc>i+1)&&(stricmp(argv[i],"-addr")==0)) {
+            i++;
+            if (strchr(argv[i],'.')||!isdigit(argv[i][0]))
+                listenep.set(argv[i], listenep.port);
+            else
+                listenep.port = atoi(argv[i]);
+            i++;
+        }
+        else if ((argc>i+1)&&(stricmp(argv[i],"-sslp")==0)) {
+            i++;
+            sslport = atoi(argv[i++]);
+        }
+        else if ((argc>i+1)&&(stricmp(argv[i],"-sbsize")==0)) {
+            i++;
+            sendbufsize = atoi(argv[i++]);
+        }
+        else if ((argc>i+1)&&(stricmp(argv[i],"-rbsize")==0)) {
+            i++;
+            recvbufsize = atoi(argv[i++]);
+        }
+        else if (stricmp(argv[i],"-h")==0) {
+            usage();
+            exit(0);
+        }
         else if (stricmp(argv[i],"-LOCAL")==0) { 
             i++;
             locallisten = true;
         }
-        else if (stricmp(argv[i],"-NOSSL")==0) {//overrides config setting
+        else if (stricmp(argv[i],"-NOSSL")==0) { // overrides config setting
             i++;
             if (useSSL)
             {
                 PROGLOG("DaFileSrv SSL specified in config but overridden by -NOSSL in command line");
-                useSSL = false;
-                dafsPort = DAFILESRV_PORT;
+                useSSL = SSLNone;
             }
         }
         else
             break;
     }
 
-    if (useSSL && !sslCertFile)
+    if ( (useSSL == SSLOnly) || (useSSL == SSLFirst) )
     {
-        ERRLOG("DaFileSrv SSL specified but certificate file information missing from environment.conf");
-        exit(-1);
+        if ( !sslCertFile || !sslKeyFile )
+        {
+            ERRLOG("DaFileSrv SSL specified but certificate and/or key file information missing from environment.conf");
+            exit(-1);
+        }
+        if ( !checkFileExists(sslCertFile) )
+        {
+            ERRLOG("DaFileSrv SSL specified but certificate file (%s) not found", sslCertFile);
+            exit(-1);
+        }
+        if ( !checkFileExists(sslKeyFile) )
+        {
+            ERRLOG("DaFileSrv SSL specified but key file (%s) not found", sslKeyFile);
+            exit(-1);
+        }
     }
 
     if (0 == logDir.length())
@@ -499,28 +546,42 @@ int main(int argc,char **argv)
         return 1;
     }
 #endif
-    if (argc == i)
-        listenep.port = dafsPort;
-    else {
+    if (argc > i) {
         if (strchr(argv[i],'.')||!isdigit(argv[i][0]))
-            listenep.set(argv[i], dafsPort);
+            listenep.set(argv[i], listenep.port);
         else
             listenep.port = atoi(argv[i]);
-        if (listenep.port==0) {
-            usage();
-            exit(-1);
-        }
         sendbufsize = (argc>i+1)?(atoi(argv[i+1])*1024):0;
         recvbufsize = (argc>i+2)?(atoi(argv[i+2])*1024):0;
     }
+
+    if ( (useSSL == SSLNone) && (listenep.port == 0) )
+    {
+        printf("\nError, port must not be 0\n");
+        usage();
+        exit(-1);
+    }
+    else if ( (useSSL == SSLOnly) && (sslport == 0) )
+    {
+        printf("\nError, secure port must not be 0\n");
+        usage();
+        exit(-1);
+    }
+    else if ( ((useSSL == SSLFirst) || (useSSL == UnsecureFirst)) && ((listenep.port == 0) || (sslport == 0)) )
+    {
+        printf("\nError, both port and secure port must not be 0\n");
+        usage();
+        exit(-1);
+    }
+
     if (isdaemon) {
 #ifdef _WIN32
         class cserv: public CService
         {
             bool stopped;
             bool started;
+            SSLCfg useSSL;
             SocketEndpoint listenep;
-            bool useSSL;
             bool requireauthenticate;
             unsigned maxThreads;
             unsigned maxThreadsDelayMs;
@@ -531,10 +592,9 @@ int main(int argc,char **argv)
             unsigned parallelSlowRequestLimit;
             unsigned throttleSlowDelayMs;
             unsigned throttleSlowCPULimit;
-
+            unsigned sslport;
             
             class cpollthread: public Thread
-                
             {
                 cserv *parent;
             public:
@@ -553,14 +613,16 @@ int main(int argc,char **argv)
 
         public:
 
-            cserv(SocketEndpoint _listenep, bool _useSSL,
+            cserv(SSLCfg _useSSL, SocketEndpoint _listenep,
                         unsigned _maxThreads, unsigned _maxThreadsDelayMs, unsigned _maxAsyncCopy,
                         unsigned _parallelRequestLimit, unsigned _throttleDelayMs, unsigned _throttleCPULimit,
-                        unsigned _parallelSlowRequestLimit, unsigned _throttleSlowDelayMs, unsigned _throttleSlowCPULimit)
-            : listenep(_listenep),useSSL(_useSSL),pollthread(this),
+                        unsigned _parallelSlowRequestLimit, unsigned _throttleSlowDelayMs, unsigned _throttleSlowCPULimit,
+                        unsigned _sslport)
+            : useSSL(_useSSL), listenep(_listenep), pollthread(this),
                   maxThreads(_maxThreads), maxThreadsDelayMs(_maxThreadsDelayMs), maxAsyncCopy(_maxAsyncCopy),
                   parallelRequestLimit(_parallelRequestLimit), throttleDelayMs(_throttleDelayMs), throttleCPULimit(_throttleCPULimit),
-                  parallelSlowRequestLimit(_parallelSlowRequestLimit), throttleSlowDelayMs(_throttleSlowDelayMs), throttleSlowCPULimit(_throttleSlowCPULimit)
+                  parallelSlowRequestLimit(_parallelSlowRequestLimit), throttleSlowDelayMs(_throttleSlowDelayMs), throttleSlowCPULimit(_throttleSlowCPULimit),
+                  sslport(_sslport)
             {
                 stopped = false;
                 started = false;
@@ -615,13 +677,40 @@ int main(int argc,char **argv)
                                     &dwSize);
                     RegCloseKey(hkey);
                 }
+
+                enableDafsAuthentication(requireauthenticate!=0);
+
                 StringBuffer eps;
                 if (listenep.isNull())
                     eps.append(listenep.port);
                 else
                     listenep.getUrlStr(eps);
-                enableDafsAuthentication(requireauthenticate!=0);
-                PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on %s%s", useSSL?"SECURE ":"",eps.str());
+
+                if (useSSL != SSLOnly)
+                    PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on %s", eps.str());
+                if (useSSL)
+                {
+                    SocketEndpoint sslep(listenep);
+                    sslep.port = sslport;
+                    eps.kill();
+                    if (sslep.isNull())
+                        eps.append(sslep.port);
+                    else
+                        sslep.getUrlStr(eps);
+                    PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on SECURE %s", eps.str());
+                }
+
+                StringBuffer secMethod;
+                if (useSSL == SSLNone)
+                    secMethod.append("SSLNone");
+                else if (useSSL == SSLOnly)
+                    secMethod.append("SSLOnly");
+                else if (useSSL == SSLFirst)
+                    secMethod.append("SSLFirst");
+                else if (useSSL == UnsecureFirst)
+                    secMethod.append("UnsecureFirst");
+                PROGLOG("Dali File Server socket security model: %s", secMethod.str());
+
                 const char * verstring = remoteServerVersionString();
                 PROGLOG("Version: %s", verstring);
                 PROGLOG("Authentication:%s required",requireauthenticate?"":" not");
@@ -630,7 +719,7 @@ int main(int argc,char **argv)
                 server->setThrottle(ThrottleStd, parallelRequestLimit, throttleDelayMs, throttleCPULimit);
                 server->setThrottle(ThrottleSlow, parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit);
                 try {
-                    server->run(listenep, useSSL);
+                    server->run(useSSL, listenep, sslport);
                 }
                 catch (IException *e) {
                     EXCLOG(e,DAFS_SERVICE_NAME);
@@ -639,10 +728,10 @@ int main(int argc,char **argv)
                 PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopped");
                 stopped = true;
             }
-        } service(listenep, useSSL,
+        } service(useSSL, listenep,
                 maxThreads, maxThreadsDelayMs, maxAsyncCopy,
                 parallelRequestLimit, throttleDelayMs, throttleCPULimit,
-                parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit);
+                parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit, sslport);
         service.start();
         return 0;
 #else
@@ -662,13 +751,39 @@ int main(int argc,char **argv)
     PROGLOG("Parallel request limit = %d, throttleDelayMs = %d, throttleCPULimit = %d", parallelRequestLimit, throttleDelayMs, throttleCPULimit);
 
     const char * verstring = remoteServerVersionString();
+
+    enableDafsAuthentication(requireauthenticate);
+
     StringBuffer eps;
     if (listenep.isNull())
         eps.append(listenep.port);
     else
         listenep.getUrlStr(eps);
-    enableDafsAuthentication(requireauthenticate);
-    PROGLOG("Opening Dali File Server on %s%s", useSSL?"SECURE ":"",eps.str());
+    if (useSSL != SSLOnly)
+        PROGLOG("Opening Dali File Server on %s", eps.str());
+    if (useSSL)
+    {
+        SocketEndpoint sslep(listenep);
+        sslep.port = sslport;
+        eps.kill();
+        if (sslep.isNull())
+            eps.append(sslep.port);
+        else
+            sslep.getUrlStr(eps);
+        PROGLOG("Opening Dali File Server on SECURE %s", eps.str());
+    }
+
+    StringBuffer secMethod;
+    if (useSSL == SSLNone)
+        secMethod.append("SSLNone");
+    else if (useSSL == SSLOnly)
+        secMethod.append("SSLOnly");
+    else if (useSSL == SSLFirst)
+        secMethod.append("SSLFirst");
+    else if (useSSL == UnsecureFirst)
+        secMethod.append("UnsecureFirst");
+    PROGLOG("Dali File Server socket security model: %s", secMethod.str());
+
     PROGLOG("Version: %s", verstring);
     PROGLOG("Authentication:%s required",requireauthenticate?"":" not");
     server.setown(createRemoteFileServer(maxThreads, maxThreadsDelayMs, maxAsyncCopy));
@@ -693,7 +808,7 @@ int main(int argc,char **argv)
     writeSentinelFile(sentinelFile);
     try
     {
-        server->run(listenep, useSSL);
+        server->run(useSSL, listenep, sslport);
     }
     catch (IException *e)
     {

dali/daliadmin/daliadmin.cpp

@@ -130,6 +130,7 @@ void usage(const char *exe)
   printf("  wuiddecompress <wildcard> <type> --  scan workunits that match <wildcard> and decompress resources of <type>\n");
   printf("  xmlsize <filename> [<percentage>] --  analyse size usage in xml file, display individual items above 'percentage' \n");
   printf("  migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
+  printf("  translatetoxpath logicalfile [File|SuperFile|Scope]\n");
   printf("\n");
   printf("Common options\n");
   printf("  server=<dali-server-ip>         -- server ip\n");
@@ -2428,6 +2429,14 @@ static void xmlSize(const char *filename, double pc)
     }
 }
 
+static void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType=DXB_File)
+{
+    CDfsLogicalFileName lfn;
+    lfn.set(logicalfile);
+    StringBuffer str;
+    OUTLOG("%s", lfn.makeFullnameQuery(str, tailType).str());
+}
+
 //=============================================================================
 
 static bool begins(const char *&ln,const char *pat)
@@ -3168,7 +3177,7 @@ int main(int argc, char* argv[])
         else if ((i==1)&&(isdigit(*param)||(*param=='.'))&&ep.set(((*param=='.')&&param[1])?(param+1):param,DALI_SERVER_PORT))
             props->setProp("server",ep.getUrlStr(tmps.clear()).str());
         else {
-            if ((0==stricmp(param,"help")) || (0 ==stricmp(param,"-help")) || (0 ==stricmp(param,"--help"))) {
+            if ((strieq(param,"help")) || (strieq(param,"-help")) || (strieq(param,"--help"))) {
                 usage(argv[0]);
                 return -1;
             }
@@ -3211,15 +3220,39 @@ int main(int argc, char* argv[])
         if (!props->getProp("server",daliserv.clear()))
         {
             // external commands
-
-            if (stricmp(cmd,"xmlsize")==0)
+            try
             {
-                CHECKPARAMS(1,2);
-                xmlSize(params.item(1), np>1?atof(params.item(2)):1.0);
+                if (strieq(cmd,"xmlsize"))
+                {
+                    CHECKPARAMS(1,2);
+                    xmlSize(params.item(1), np>1?atof(params.item(2)):1.0);
+                }
+                else if (strieq(cmd,"translatetoxpath"))
+                {
+                    CHECKPARAMS(1,2);
+                    DfsXmlBranchKind branchType;
+                    if (np>1)
+                    {
+                        const char *typeStr = params.item(2);
+                        branchType = queryDfsXmlBranchType(typeStr);
+                    }
+                    else
+                        branchType = DXB_File;
+                    translateToXpath(params.item(1), branchType);
+                }
+                else
+                {
+                    ERRLOG("Unknown command %s",cmd);
+                    ret = 255;
+                }
             }
-            else
-                ERRLOG("Unknown command %s",cmd);
-            return 0;
+            catch (IException *e)
+            {
+                EXCLOG(e,"daliadmin");
+                e->Release();
+                ret = 255;
+            }
+            return ret;
         }
         else
         {
@@ -3248,245 +3281,245 @@ int main(int argc, char* argv[])
                         queryDistributedFileDirectory().setDefaultUser(userDesc);
                     }
                     daliConnectTimeoutMs = 1000 * props->getPropInt("timeout", DEFAULT_DALICONNECT_TIMEOUT);
-                    if (stricmp(cmd,"export")==0) {
+                    if (strieq(cmd,"export")) {
                         CHECKPARAMS(2,2);
                         _export_(params.item(1),params.item(2));
                     }
-                    else if (stricmp(cmd,"import")==0) {
+                    else if (strieq(cmd,"import")) {
                         CHECKPARAMS(2,2);
                         import(params.item(1),params.item(2),false);
                     }
-                    else if (stricmp(cmd,"importadd")==0) {
+                    else if (strieq(cmd,"importadd")) {
                         CHECKPARAMS(2,2);
                         import(params.item(1),params.item(2),true);
                     }
-                    else if (stricmp(cmd,"delete")==0) {
+                    else if (strieq(cmd,"delete")) {
                         CHECKPARAMS(1,1);
                         _delete_(params.item(1),true);
                     }
-                    else if (stricmp(cmd,"set")==0) {
+                    else if (strieq(cmd,"set")) {
                         CHECKPARAMS(2,2);
                         set(params.item(1),params.item(2));
                     }
-                    else if (stricmp(cmd,"get")==0) {
+                    else if (strieq(cmd,"get")) {
                         CHECKPARAMS(1,1);
                         get(params.item(1));
                     }
-                    else if (stricmp(cmd,"bget")==0) {
+                    else if (strieq(cmd,"bget")) {
                         CHECKPARAMS(2,2);
                         bget(params.item(1),params.item(2));
                     }
-                    else if (stricmp(cmd,"wget")==0) {
+                    else if (strieq(cmd,"wget")) {
                         CHECKPARAMS(1,1);
                         wget(params.item(1));
                     }
-                    else if (stricmp(cmd,"xget")==0) {
+                    else if (strieq(cmd,"xget")) {
                         CHECKPARAMS(1,1);
                         wget(params.item(1));
                     }
-                    else if (stricmp(cmd,"add")==0) {
+                    else if (strieq(cmd,"add")) {
                         CHECKPARAMS(1,2);
                         add(params.item(1), (np>1) ? params.item(2) : nullptr);
                     }
-                    else if (stricmp(cmd,"delv")==0) {
+                    else if (strieq(cmd,"delv")) {
                         CHECKPARAMS(1,1);
                         delv(params.item(1));
                     }
-                    else if (stricmp(cmd,"count")==0) {
+                    else if (strieq(cmd,"count")) {
                         CHECKPARAMS(1,1);
                         count(params.item(1));
                     }
-                    else if (stricmp(cmd,"dfsfile")==0) {
+                    else if (strieq(cmd,"dfsfile")) {
                         CHECKPARAMS(1,1);
                         dfsfile(params.item(1),userDesc);
                     }
-                    else if (stricmp(cmd,"dfspart")==0) {
+                    else if (strieq(cmd,"dfspart")) {
                         CHECKPARAMS(2,2);
                         dfspart(params.item(1),userDesc,atoi(params.item(2)));
                     }
-                    else if (stricmp(cmd,"dfscheck")==0) {
+                    else if (strieq(cmd,"dfscheck")) {
                         CHECKPARAMS(0,0);
                         dfsCheck();
                     }
-                    else if (stricmp(cmd,"dfscsv")==0) {
+                    else if (strieq(cmd,"dfscsv")) {
                         CHECKPARAMS(1,1);
                         dfscsv(params.item(1),userDesc);
                     }
-                    else if (stricmp(cmd,"dfsgroup")==0) {
+                    else if (strieq(cmd,"dfsgroup")) {
                         CHECKPARAMS(1,2);
                         dfsGroup(params.item(1),(np>1)?params.item(2):NULL);
                     }
-                    else if (stricmp(cmd,"clusternodes")==0) {
+                    else if (strieq(cmd,"clusternodes")) {
                         CHECKPARAMS(1,2);
                         ret = clusterGroup(params.item(1),(np>1)?params.item(2):NULL);
                     }
-                    else if (stricmp(cmd,"dfsls")==0) {
+                    else if (strieq(cmd,"dfsls")) {
                         CHECKPARAMS(0,2);
                         dfsLs((np>0)?params.item(1):NULL,(np>1)?params.item(2):NULL);
                     }
-                    else if (stricmp(cmd,"dfsmap")==0) {
+                    else if (strieq(cmd,"dfsmap")) {
                         CHECKPARAMS(1,1);
                         dfsmap(params.item(1), userDesc);
                     }
-                    else if (stricmp(cmd,"dfsexist")==0) {
+                    else if (strieq(cmd,"dfsexist")) {
                         CHECKPARAMS(1,1);
                         ret = dfsexists(params.item(1),userDesc);
                     }
-                    else if (stricmp(cmd,"dfsparents")==0) {
+                    else if (strieq(cmd,"dfsparents")) {
                         CHECKPARAMS(1,1);
                         dfsparents(params.item(1),userDesc);
                     }
-                    else if (stricmp(cmd,"dfsunlink")==0) {
+                    else if (strieq(cmd,"dfsunlink")) {
                         CHECKPARAMS(1,1);
                         dfsunlink(params.item(1),userDesc);
                     }
-                    else if (stricmp(cmd,"dfsverify")==0) {
+                    else if (strieq(cmd,"dfsverify")) {
                         CHECKPARAMS(1,1);
                         ret = dfsverify(params.item(1),NULL,userDesc);
                     }
-                    else if (stricmp(cmd,"setprotect")==0) {
+                    else if (strieq(cmd,"setprotect")) {
                         CHECKPARAMS(2,2);
                         setprotect(params.item(1),params.item(2),userDesc);
                     }
-                    else if (stricmp(cmd,"unprotect")==0) {
+                    else if (strieq(cmd,"unprotect")) {
                         CHECKPARAMS(2,2);
                         unprotect(params.item(1),params.item(2),userDesc);
                     }
-                    else if (stricmp(cmd,"listprotect")==0) {
+                    else if (strieq(cmd,"listprotect")) {
                         CHECKPARAMS(0,2);
                         listprotect((np>1)?params.item(1):"*",(np>2)?params.item(2):"*");
 
                     }
-                    else if (stricmp(cmd,"checksuperfile")==0) {
+                    else if (strieq(cmd,"checksuperfile")) {
                         CHECKPARAMS(1,1);
                         bool fix = props->getPropBool("fix");
                         checksuperfile(params.item(1),fix);
                     }
-                    else if (stricmp(cmd,"checksubfile")==0) {
+                    else if (strieq(cmd,"checksubfile")) {
                         CHECKPARAMS(1,1);
                         checksubfile(params.item(1));
                     }
-                    else if (stricmp(cmd,"listexpires")==0) {
+                    else if (strieq(cmd,"listexpires")) {
                         CHECKPARAMS(0,1);
                         listexpires((np>1)?params.item(1):"*",userDesc);
                     }
-                    else if (stricmp(cmd,"listrelationships")==0) {
+                    else if (strieq(cmd,"listrelationships")) {
                         CHECKPARAMS(2,2);
                         listrelationships(params.item(1),params.item(2));
                     }
-                    else if (stricmp(cmd,"dfsperm")==0) {
+                    else if (strieq(cmd,"dfsperm")) {
                         if (!userDesc.get())
                             throw MakeStringException(-1,"dfsperm requires username to be set (user=)");
                         CHECKPARAMS(1,1);
                         ret = dfsperm(params.item(1),userDesc);
                     }
-                    else if (stricmp(cmd,"dfscompratio")==0) {
+                    else if (strieq(cmd,"dfscompratio")) {
                         CHECKPARAMS(1,1);
                         dfscompratio(params.item(1),userDesc);
                     }
-                    else if (stricmp(cmd,"dfsscopes")==0) {
+                    else if (strieq(cmd,"dfsscopes")) {
                         CHECKPARAMS(0,1);
                         dfsscopes((np>1)?params.item(1):"*",userDesc);
                     }
-                    else if (stricmp(cmd,"cleanscopes")==0) {
+                    else if (strieq(cmd,"cleanscopes")) {
                         CHECKPARAMS(0,0);
                         cleanscopes(userDesc);
                     }
-                    else if (stricmp(cmd,"listworkunits")==0) {
+                    else if (strieq(cmd,"listworkunits")) {
                         CHECKPARAMS(0,3);
                         listworkunits((np>0)?params.item(1):NULL,(np>1)?params.item(2):NULL,(np>2)?params.item(3):NULL);
                     }
-                    else if (stricmp(cmd,"listmatches")==0) {
+                    else if (strieq(cmd,"listmatches")) {
                         CHECKPARAMS(0,3);
                         listmatches((np>0)?params.item(1):NULL,(np>1)?params.item(2):NULL,(np>2)?params.item(3):NULL);
                     }
-                    else if (stricmp(cmd,"workunittimings")==0) {
+                    else if (strieq(cmd,"workunittimings")) {
                         CHECKPARAMS(1,1);
                         workunittimings(params.item(1));
                     }
-                    else if (stricmp(cmd,"serverlist")==0) {
+                    else if (strieq(cmd,"serverlist")) {
                         CHECKPARAMS(1,1);
                         serverlist(params.item(1));
                     }
-                    else if (stricmp(cmd,"clusterlist")==0) {
+                    else if (strieq(cmd,"clusterlist")) {
                         CHECKPARAMS(1,1);
                         clusterlist(params.item(1));
                     }
-                    else if (stricmp(cmd,"auditlog")==0) {
+                    else if (strieq(cmd,"auditlog")) {
                         CHECKPARAMS(2,3);
                         auditlog(params.item(1),params.item(2),(np>2)?params.item(3):NULL);
                     }
-                    else if (stricmp(cmd,"coalesce")==0) {
+                    else if (strieq(cmd,"coalesce")) {
                         CHECKPARAMS(0,0);
                         coalesce();
                     }
-                    else if (stricmp(cmd,"mpping")==0) {
+                    else if (strieq(cmd,"mpping")) {
                         CHECKPARAMS(1,1);
                         mpping(params.item(1));
                     }
-                    else if (stricmp(cmd,"daliping")==0) {
+                    else if (strieq(cmd,"daliping")) {
                         CHECKPARAMS(0,1);
                         daliping(daliserv.str(),daliconnectelapsed,(np>0)?atoi(params.item(1)):1);
                     }
-                    else if (stricmp(cmd,"getxref")==0) {
+                    else if (strieq(cmd,"getxref")) {
                         CHECKPARAMS(1,1);
                         getxref(params.item(1));
                     }
-                    else if (stricmp(cmd,"dalilocks")==0) {
+                    else if (strieq(cmd,"dalilocks")) {
                         CHECKPARAMS(0,2);
                         bool filesonly = false;
-                        if (np&&(stricmp(params.item(np),"files")==0)) {
+                        if (np&&(strieq(params.item(np),"files"))) {
                             filesonly = true;
                             np--;
                         }
                         dalilocks(np>0?params.item(1):NULL,filesonly);
                     }
-                    else if (stricmp(cmd,"unlock")==0) {
+                    else if (strieq(cmd,"unlock")) {
                         CHECKPARAMS(2,2);
                         const char *fileOrPath = params.item(2);
-                        if (0 == stricmp("file", fileOrPath))
+                        if (strieq("file", fileOrPath))
                             unlock(params.item(1), true);
-                        else if (0 == stricmp("path", fileOrPath))
+                        else if (strieq("path", fileOrPath))
                             unlock(params.item(1), false);
                         else
                             throw MakeStringException(0, "unknown type [ %s ], must be 'file' or 'path'", fileOrPath);
                     }
-                    else if (stricmp(cmd,"validateStore")==0) {
+                    else if (strieq(cmd,"validateStore")) {
                         CHECKPARAMS(0,2);
                         bool fix = props->getPropBool("fix");
                         bool verbose = props->getPropBool("verbose");
                         bool deleteFiles = props->getPropBool("deletefiles");
                         validateStore(fix, deleteFiles, verbose);
                     }
-                    else if (stricmp(cmd, "workunit") == 0) {
+                    else if (strieq(cmd, "workunit")) {
                         CHECKPARAMS(1,2);
                         bool includeProgress=false;
                         if (np>1)
                             includeProgress = strToBool(params.item(2));
                         dumpWorkunit(params.item(1), includeProgress);
                     }
-                    else if (stricmp(cmd,"wuidCompress")==0) {
+                    else if (strieq(cmd,"wuidCompress")) {
                         CHECKPARAMS(2,2);
                         wuidCompress(params.item(1), params.item(2), true);
                     }
-                    else if (stricmp(cmd,"wuidDecompress")==0) {
+                    else if (strieq(cmd,"wuidDecompress")) {
                         CHECKPARAMS(2,2);
                         wuidCompress(params.item(1), params.item(2), false);
                     }
-                    else if (stricmp(cmd,"dfsreplication")==0) {
+                    else if (strieq(cmd,"dfsreplication")) {
                         CHECKPARAMS(3,4);
                         bool dryRun = np>3 && strieq("dryrun", params.item(4));
                         dfsreplication(params.item(1), params.item(2), atoi(params.item(3)), dryRun);
                     }
-                    else if (stricmp(cmd,"holdlock")==0) {
+                    else if (strieq(cmd,"holdlock")) {
                         CHECKPARAMS(2,2);
                         holdlock(params.item(1), params.item(2), userDesc);
                     }
-                    else if (stricmp(cmd, "progress") == 0) {
+                    else if (strieq(cmd, "progress")) {
                         CHECKPARAMS(2,2);
                         dumpProgress(params.item(1), params.item(2));
                     }
-                    else if (stricmp(cmd, "stats") == 0) {
+                    else if (strieq(cmd, "stats")) {
                         CHECKPARAMS(1, 7);
                         if ((params.ordinality() >= 3) && (strchr(params.item(2), '[')))
                         {
@@ -3501,7 +3534,7 @@ int main(int argc, char* argv[])
                             dumpStats(params.item(1), params.item(2), params.item(3), params.item(4), params.item(5), params.item(6), nullptr, csv);
                         }
                     }
-                    else if (stricmp(cmd, "migratefiles") == 0)
+                    else if (strieq(cmd, "migratefiles"))
                     {
                         CHECKPARAMS(2, 7);
                         const char *srcGroup = params.item(1);
@@ -3522,9 +3555,11 @@ int main(int argc, char* argv[])
                     else
                         ERRLOG("Unknown command %s",cmd);
                 }
-                catch (IException *e) {
+                catch (IException *e)
+                {
                     EXCLOG(e,"daliadmin");
                     e->Release();
+                    ret = 255;
                 }
                 closedownClientProcess();
             }

dali/dfuplus/dfuplus.cpp

@@ -36,15 +36,18 @@
 
 static class CSecuritySettings
 {
-    bool useSSL;
+    SSLCfg useSSL;
     unsigned short daliServixPort;
+    unsigned short daliServixSSLPort;
 public:
     CSecuritySettings()
     {
-        querySecuritySettings(&useSSL, &daliServixPort, nullptr, nullptr);
+        queryDafsSecSettings(&useSSL, &daliServixPort, &daliServixSSLPort, nullptr, nullptr, nullptr);
     }
 
+    SSLCfg querySSLCfg() { return useSSL; }
     unsigned short queryDaliServixPort() { return daliServixPort; }
+    unsigned short queryDaliServixSSLPort() { return daliServixSSLPort; }
 } securitySettings;
 
 class CDafsThread: public Thread
@@ -59,11 +62,13 @@ public:
     {
         if (listenep.port==0)
             listenep.port = securitySettings.queryDaliServixPort();
+#if 0
         StringBuffer eps;
         if (listenep.isNull())
             eps.append(listenep.port);
         else
             listenep.getUrlStr(eps);
+#endif
         enableDafsAuthentication(requireauthenticate);
         server.setown(createRemoteFileServer());
         server->setThrottle(ThrottleStd, 0); // disable throttling
@@ -73,7 +78,7 @@ public:
     int run()
     {
         try {
-            server->run(listenep);
+            server->run(securitySettings.querySSLCfg(), listenep);
         }
         catch (IException *e) {
             EXCLOG(e,"dfuplus(dafilesrv)");
@@ -110,12 +115,51 @@ bool CDfuPlusHelper::runLocalDaFileSvr(SocketEndpoint &listenep,bool requireauth
     Owned<CDafsThread> thr = new CDafsThread(listenep,requireauthenticate);
     if (!thr->ok())
         return false;
-    thr->start();
-    StringBuffer eps;
-    if (listenep.isNull())
-        progress("Started local Dali file server on port %d\n", listenep.port?listenep.port:securitySettings.queryDaliServixPort());
+
+    unsigned port = listenep.port;
+    if (!port)
+        port = securitySettings.queryDaliServixPort();
+
+    unsigned sslport = securitySettings.queryDaliServixSSLPort();
+
+    SSLCfg useSSL = securitySettings.querySSLCfg();
+
+    StringBuffer addlPort;
+    SocketEndpoint printep(listenep);
+    if (printep.isNull())
+    {
+        if (useSSL == SSLNone)
+            addlPort.appendf("%u", port);
+        else if (useSSL == SSLOnly)
+            addlPort.appendf("%u", sslport);
+        else if (useSSL == SSLFirst)
+            addlPort.appendf("%u:%u", sslport, port);
+        else
+            addlPort.appendf("%u:%u", port, sslport);
+        progress("Started local Dali file server on port %s\n", addlPort.str());
+    }
     else
-        progress("Started local Dali file server on %s\n", listenep.getUrlStr(eps).str());
+    {
+        if (useSSL == SSLNone)
+            printep.port = port;
+        else if (useSSL == SSLOnly)
+            printep.port = sslport;
+        else if (useSSL == SSLFirst)
+        {
+            printep.port = sslport;
+            addlPort.appendf(":%u", port);
+        }
+        else
+        {
+            printep.port = port;
+            addlPort.appendf(":%u", sslport);
+        }
+        StringBuffer eps;
+        progress("Started local Dali file server on %s%s\n", printep.getUrlStr(eps).str(), addlPort.str());
+    }
+
+    thr->start();
+
     if (timeout==0) {
         setDafsTrace(nullptr,0); // disable client tracing
         dafsthread.setown(thr.getClear());

docs/DynamicESDL/DESDL-Mods/ESDLESParray.xml

@@ -4,8 +4,10 @@
 <sect2 id="ESParray">
   <title><emphasis role="bold">ESParray</emphasis></title>
 
-  <para>A structure for unbounded arrays. Arrays support inheritance and can
-  be nested.</para>
+  <para><indexterm>
+      <primary>ESParray</primary>
+    </indexterm>A structure for unbounded arrays. Arrays support inheritance
+  and can be nested.</para>
 
   <para><emphasis role="bold">Example:</emphasis></para>
 

docs/DynamicESDL/DESDL-Mods/ESDLESPenum.xml

@@ -4,7 +4,9 @@
 <sect2 id="ESPenum">
   <title>ESPenum</title>
 
-  <para>A structure containing an enumerated value. </para>
+  <para><indexterm>
+      <primary>ESPenum</primary>
+    </indexterm>A structure containing an enumerated value.</para>
 
   <para><emphasis role="bold">Example:</emphasis></para>
 

docs/DynamicESDL/DESDL-Mods/ESDLESPinclude.xml

@@ -4,8 +4,10 @@
 <sect2 id="ESPinclude">
   <title><emphasis role="bold">ESPinclude</emphasis></title>
 
-  <para>ESPinclude allows you to include an external ESDL file. This is
-  similar to the #include statement.</para>
+  <para>ESPinclude <indexterm>
+      <primary>ESPinclude</primary>
+    </indexterm>allows you to include an external ESDL file. This is similar
+  to the #include statement.</para>
 
   <para><emphasis role="bold">Example:</emphasis></para>
 

docs/DynamicESDL/DESDL-Mods/ESDLESPmethod.xml

@@ -4,9 +4,11 @@
 <sect2 id="ESPmethod">
   <title><emphasis role="bold">ESPmethod</emphasis></title>
 
-  <para>This defines a method definition you can reference in an ESPservice
-  structure. The method definition should contain references to a previously
-  defined ESPrequest and ESPresponse.</para>
+  <para><indexterm>
+      <primary>ESPmethod</primary>
+    </indexterm>This defines a method definition you can reference in an
+  ESPservice structure. The method definition should contain references to a
+  previously defined ESPrequest and ESPresponse.</para>
 
   <para><emphasis role="bold">Example:</emphasis></para>
 

docs/DynamicESDL/DESDL-Mods/ESDLESPrequest.xml

@@ -1,16 +1,16 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
-   
+<sect2 id="ESPrequest">
+  <title><emphasis role="bold">ESPrequest</emphasis></title>
 
-    <sect2 id="ESPrequest">
-      <title><emphasis role="bold">ESPrequest</emphasis></title>
+  <para><indexterm>
+      <primary>ESPrequest</primary>
+    </indexterm>The request structure for a method.</para>
 
-      <para>The request structure for a method.</para>
+  <para><emphasis role="bold">Example:</emphasis></para>
 
-      <para><emphasis role="bold">Example:</emphasis></para>
-
-      <programlisting>
+  <programlisting>
 ESPrequest  MyQueryRequest
 
 {
@@ -22,4 +22,4 @@ ESPrequest  MyQueryRequest
     bool Descending(false);
 };
    </programlisting>
-    </sect2>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPservice.xml

@@ -4,7 +4,9 @@
 <sect2 id="EPservice">
   <title>ESPservice</title>
 
-  <para>This defines an ESP web service interface. Once defined, this
+  <para><indexterm>
+      <primary>ESPservice</primary>
+    </indexterm>This defines an ESP web service interface. Once defined, this
   interface definition can be assigned (bound) to a Dynamic ESDL-based ESP
   Service.</para>
 

docs/DynamicESDL/DESDL-Mods/ESDLESPstruct.xml

@@ -1,18 +1,19 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
-   
-   <sect2 id="ESP_Struct">
-      <title>ESP<emphasis role="bold">struct</emphasis></title>
+<sect2 id="ESP_Struct">
+  <title>ESP<emphasis role="bold">struct</emphasis></title>
 
-      <para>ESPstruct is a set of elements grouped together under one name.
-      These elements, known as <emphasis>members</emphasis>, can have
-      different types and different lengths. Structures can be nested and
-      support inheritance.</para>
+  <para>ESPstruct <indexterm>
+      <primary>ESPstruct</primary>
+    </indexterm>is a set of elements grouped together under one name. These
+  elements, known as <emphasis>members</emphasis>, can have different types
+  and different lengths. Structures can be nested and support
+  inheritance.</para>
 
-      <para><emphasis role="bold">Example:</emphasis></para>
+  <para><emphasis role="bold">Example:</emphasis></para>
 
-      <programlisting>
+  <programlisting>
 ESPstruct NameBlock
 
 {
@@ -22,4 +23,4 @@ ESPstruct NameBlock
     int Age;
 };
    </programlisting>
-    </sect2>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLauth_feature.xml

@@ -4,9 +4,11 @@
 <sect2 id="ESDL_Attributes_auth_feature">
   <title><emphasis role="bold">auth_feature</emphasis></title>
 
-  <para>The auth_feature attribute (valid only for an ESPService or ESPMethod)
-  allows you to specify a means to verify a user's permission to execute a
-  method.</para>
+  <para><indexterm>
+      <primary>auth_feature</primary>
+    </indexterm>The auth_feature attribute (valid only for an ESPService or
+  ESPMethod) allows you to specify a means to verify a user's permission to
+  execute a method.</para>
 
   <para>In order to enable this feature, your system must be configured to use
   a form of security that supports feature level authentication, such as LDAP

docs/DynamicESDL/DESDL-Mods/ESDLcounter_and_count_val.xml

@@ -1,26 +1,29 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-   <sect2 id="ESDL_Attributes_counter-count_val">
-      <title>counter and count_val</title>
+<sect2 id="ESDL_Attributes_counter-count_val">
+  <title>counter <indexterm>
+      <primary>counter</primary>
+    </indexterm>and count_val</title>
 
-      <para>These two attributes are used to help ESP calculate the record
-      count of the response.</para>
+  <para><indexterm>
+      <primary>count_val</primary>
+    </indexterm>These two attributes are used to help ESP calculate the record
+  count of the response.</para>
 
-      <para><emphasis>counter</emphasis> counts the number of children of the
-      nodes. When the node is an array, it is the same as the number of items
-      in the array.</para>
+  <para><emphasis>counter</emphasis> counts the number of children of the
+  nodes. When the node is an array, it is the same as the number of items in
+  the array.</para>
 
-      <para><emphasis>count_val</emphasis> will use the value of the node as
-      record count. Field <emphasis role="bold">RecordCount</emphasis> is
-      implicitly marked as <emphasis>count_val</emphasis>.</para>
+  <para><emphasis>count_val</emphasis> will use the value of the node as
+  record count. Field <emphasis role="bold">RecordCount</emphasis> is
+  implicitly marked as <emphasis>count_val</emphasis>.</para>
 
-      <para>When an response has multiple counter, count_val, the sum of the
-      values is returned as record-count.</para>
+  <para>When an response has multiple counter, count_val, the sum of the
+  values is returned as record-count.</para>
 
-      <para><emphasis role="bold">Example:</emphasis></para>
+  <para><emphasis role="bold">Example:</emphasis></para>
 
-      <programlisting>[counter] ESParray&lt;MyRecord, Record&gt; Records;
+  <programlisting>[counter] ESParray&lt;MyRecord, Record&gt; Records;
 [count_val] int TotalFound;   </programlisting>
-    </sect2>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLdepr_ver.xml

@@ -1,16 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-   <sect2 id="ESDL_Attributes_depr_ver">
-      <title><emphasis role="bold">depr_ver</emphasis></title>
+<sect2 id="ESDL_Attributes_depr_ver">
+  <title><emphasis role="bold">depr_ver</emphasis></title>
 
-      <para>The depr_ver attribute allows you to declare a field's end of life
-      version. The field is deprecated at the specified version number.
-      Requests using that version or any subsequent version will not have
-      access to the field.</para>
+  <para><indexterm>
+      <primary>depr_ver</primary>
+    </indexterm>The depr_ver attribute allows you to declare a field's end of
+  life version. The field is deprecated at the specified version number.
+  Requests using that version or any subsequent version will not have access
+  to the field.</para>
 
-      <para><emphasis role="bold">Example:</emphasis></para>
+  <para><emphasis role="bold">Example:</emphasis></para>
 
-      <programlisting>[depr_ver("1.04")] bool IsValid;</programlisting>
-    </sect2>
+  <programlisting>[depr_ver("1.04")] bool IsValid;</programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLdescription.xml

@@ -4,9 +4,11 @@
 <sect2 id="ESDL_Attributes_description">
   <title><emphasis role="bold">description</emphasis></title>
 
-  <para>The description attribute (valid only for an ESPMethod) allows you to
-  specify some additional text to display on the form that is automatically
-  generated to execute a method.</para>
+  <para><indexterm>
+      <primary>description</primary>
+    </indexterm>The description attribute (valid only for an ESPMethod) allows
+  you to specify some additional text to display on the form that is
+  automatically generated to execute a method.</para>
 
   <para><emphasis role="bold">Example:</emphasis></para>
 

docs/DynamicESDL/DESDL-Mods/ESDLecl_hide.xml

@@ -2,7 +2,9 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect2 id="ESDL_Attributes_ecl_hide">
-  <title><emphasis role="bold">ecl_hide</emphasis></title>
+  <title><emphasis role="bold">e<indexterm>
+      <primary>ecl_hide</primary>
+    </indexterm>cl_hide</emphasis></title>
 
   <para>The <emphasis>ecl_hide</emphasis> attribute hides the field from ECL
   (that is, the field is removed when generating the ECL code). This is used

docs/DynamicESDL/DESDL-Mods/ESDLecl_keep.xml

@@ -4,8 +4,10 @@
 <sect2 id="ESDL_Attributes_ecl_keep">
   <title>ecl_keep</title>
 
-  <para>The <emphasis>ecl_keep</emphasis> attribute keeps the field in the
-  generated ECL even though this field would have been hidden without this
+  <para><indexterm>
+      <primary>ecl_keep</primary>
+    </indexterm>The <emphasis>ecl_keep</emphasis> attribute keeps the field in
+  the generated ECL even though this field would have been hidden without this
   attribute.</para>
 
   <para></para>

docs/DynamicESDL/DESDL-Mods/ESDLecl_max_len.xml

@@ -2,7 +2,9 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect2 id="ESDL_Attributes_ecl_max_len">
-  <title>ecl_max_len (<emphasis role="bold">n</emphasis><emphasis
+  <title>ecl_max_len <indexterm>
+      <primary>ecl_max_len</primary>
+    </indexterm>(<emphasis role="bold">n</emphasis><emphasis
   role="bold">)</emphasis></title>
 
   <para>This <emphasis>ecl_max_len</emphasis> attribute tells the ECL

docs/DynamicESDL/DESDL-Mods/ESDLecl_name.xml

@@ -1,24 +1,24 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-    <sect2 id="ESDL_Attributes_ecl_name">
-       <title>ecl_name ("<emphasis role="bold">name</emphasis><emphasis
-       role="bold">")</emphasis></title>
- 
-       <para>The <emphasis>ecl_name</emphasis> attribute specifies the field
-       name in generated ECL code. By default, the field name in ECL is the
-       same as the name defined in ECM. However, in some cases, the name could
-       causes issues in ECL. For example keywords in ECL cannot be used as a
-       field name.</para>
- 
-       <para><emphasis role="bold">Example:</emphasis></para>
- 
-       <programlisting>[ecl_name("_export")] string Export;
+<sect2 id="ESDL_Attributes_ecl_name">
+  <title>ecl_name <indexterm>
+      <primary>ecl_name</primary>
+    </indexterm>("<emphasis role="bold">name</emphasis><emphasis
+  role="bold">")</emphasis></title>
+
+  <para>The <emphasis>ecl_name</emphasis> attribute specifies the field name
+  in generated ECL code. By default, the field name in ECL is the same as the
+  name defined in ECM. However, in some cases, the name could causes issues in
+  ECL. For example keywords in ECL cannot be used as a field name.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>[ecl_name("_export")] string Export;
  [ecl_name("_type")] string Type;   </programlisting>
- 
-       <para>Here, both <emphasis role="bold">EXPORT</emphasis> and <emphasis
-       role="bold">TYPE</emphasis> are ECL keywords and cannot be used as ECL
-       field names. We use <emphasis>ecl_name</emphasis> to tell the esdl2ecl
-       process to generate acceptable names.</para>
-     </sect2>
+
+  <para>Here, both <emphasis role="bold">EXPORT</emphasis> and <emphasis
+  role="bold">TYPE</emphasis> are ECL keywords and cannot be used as ECL field
+  names. We use <emphasis>ecl_name</emphasis> to tell the esdl2ecl process to
+  generate acceptable names.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLecl_null.xml

@@ -2,7 +2,9 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect2 id="ESDL_Attributes_ecl_null">
-  <title>ecl_null (<emphasis role="bold">n</emphasis><emphasis role="bold"> |
+  <title>ecl_null <indexterm>
+      <primary>ecl_null</primary>
+    </indexterm>(<emphasis role="bold">n</emphasis><emphasis role="bold"> |
   </emphasis><emphasis role="bold">string</emphasis><emphasis
   role="bold">)</emphasis></title>
 

docs/DynamicESDL/DESDL-Mods/ESDLecl_type.xml

@@ -1,18 +1,19 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-    <sect2 id="ESDL_Attributes_ecl_type">
-      <title>ecl_type ("<emphasis role="bold">type</emphasis><emphasis
-      role="bold">")</emphasis></title>
+<sect2 id="ESDL_Attributes_ecl_type">
+  <title>ecl_type <indexterm>
+      <primary>ecl_type</primary>
+    </indexterm>("<emphasis role="bold">type</emphasis><emphasis
+  role="bold">")</emphasis></title>
 
-      <para>The <emphasis>ecl_type</emphasis> attribute defines the field type
-      in ECL.</para>
+  <para>The <emphasis>ecl_type</emphasis> attribute defines the field type in
+  ECL.</para>
 
-      <para><emphasis role="bold">Example:</emphasis></para>
+  <para><emphasis role="bold">Example:</emphasis></para>
 
-      <programlisting> [ecl_type("Decimal10_2")] double RetailPrice;   </programlisting>
+  <programlisting> [ecl_type("Decimal10_2")] double RetailPrice;   </programlisting>
 
-      <para>ESDL does not have a monetary type, so we use
-      <emphasis>ecl_type</emphasis> to define it.</para>
-    </sect2>
+  <para>ESDL does not have a monetary type, so we use
+  <emphasis>ecl_type</emphasis> to define it.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLhelp.xml

@@ -4,9 +4,11 @@
 <sect2 id="ESDL_Attributes_help">
   <title><emphasis role="bold">help</emphasis></title>
 
-  <para>The help attribute (valid only for an ESPMethod) allows you to specify
-  some additional text to display on the form that is automatically generated
-  to execute a method. </para>
+  <para><indexterm>
+      <primary>help</primary>
+    </indexterm>The help attribute (valid only for an ESPMethod) allows you to
+  specify some additional text to display on the form that is automatically
+  generated to execute a method.</para>
 
   <para><emphasis role="bold">Example:</emphasis></para>
 

docs/DynamicESDL/DESDL-Mods/ESDLleading_zero.xml

@@ -1,24 +1,24 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-     <sect2 id="ESDL_Attributes_leading_zero">
-       <title>leading_zero(<emphasis role="bold">n</emphasis><emphasis
-       role="bold">)</emphasis></title>
- 
-       <para>The <emphasis>leading_zero</emphasis> attribute adds zero(s) to
-       the field value to so that the total length is
-       <emphasis>n</emphasis>.</para>
- 
-       <para><emphasis role="bold">Example:</emphasis></para>
- 
-       <programlisting>ESPstruct Date
+<sect2 id="ESDL_Attributes_leading_zero">
+  <title>leading_zero(<indexterm>
+      <primary>leading_zero</primary>
+    </indexterm><emphasis role="bold">n</emphasis><emphasis
+  role="bold">)</emphasis></title>
+
+  <para>The <emphasis>leading_zero</emphasis> attribute adds zero(s) to the
+  field value to so that the total length is <emphasis>n</emphasis>.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>ESPstruct Date
  {
     [leading_zero(4)] Year;
     [leading_zero(2)] Month;
     [leading_zero(2)] Day;
  };  </programlisting>
- 
-       <para>So the Date will always have a 4-digit Year and a 2-digit Month
-       and a 2-digit Day.</para>
-     </sect2>
+
+  <para>So the Date will always have a 4-digit Year and a 2-digit Month and a
+  2-digit Day.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLmax_count_var.xml

@@ -1,17 +1,18 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-     <sect2 id="ESDL_Attributes_max_count_var">
-       <title><emphasis role="bold">max_count_var</emphasis></title>
- 
-       <para>The max_count_var attribute is used to specify the expected max
-       items in a dataset (ESParray).</para>
- 
-       <para><emphasis role="bold">Example:</emphasis></para>
- 
-       <programlisting>[max_count_var("iesp.Constants.JD.MaxRecords")] ESParray &lt;ESPstruct MYRecord, Record&gt; Records;   </programlisting>
- 
-       <para>Roxie will define the constant iesp.Constants.JD.MaxRecords and
-       change it at will without affecting ESP.</para>
-    </sect2>
+<sect2 id="ESDL_Attributes_max_count_var">
+  <title><emphasis role="bold">max_count_var</emphasis></title>
+
+  <para><indexterm>
+      <primary>max_count_var</primary>
+    </indexterm>The max_count_var attribute is used to specify the expected
+  max items in a dataset (ESParray).</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>[max_count_var("iesp.Constants.JD.MaxRecords")] ESParray &lt;ESPstruct MYRecord, Record&gt; Records;   </programlisting>
+
+  <para>Roxie will define the constant iesp.Constants.JD.MaxRecords and change
+  it at will without affecting ESP.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLmax_len.xml

@@ -2,8 +2,10 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect2 id="ESDL_Attributes_max_len">
-  <title><emphasis role="bold">max_len (</emphasis><emphasis
-  role="bold">n</emphasis><emphasis role="bold">)</emphasis></title>
+  <title><emphasis role="bold">max_len <indexterm>
+      <primary>max_len</primary>
+    </indexterm>(</emphasis><emphasis role="bold">n</emphasis><emphasis
+  role="bold">)</emphasis></title>
 
   <para>The <emphasis>max_len</emphasis> attribute specifies the field length
   for ECL string field.</para>

docs/DynamicESDL/DESDL-Mods/ESDLmax_ver.xml

@@ -1,16 +1,16 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-     <sect2 id="ESDL_Attributes_max_ver">
-       <title><emphasis role="bold">max_ver</emphasis></title>
- 
-       <para>The max_ver attribute allows you to define the maximum (latest)
-       version where a field is visible. Requests using a later version will
-       not have access to the field.</para>
- 
-       <para><emphasis role="bold">Example:</emphasis></para>
- 
-       <programlisting> [max_ver("1.04")] bool IsValid;   </programlisting>
-     </sect2>
+<sect2 id="ESDL_Attributes_max_ver">
+  <title><emphasis role="bold">max_ver</emphasis></title>
 
+  <para><indexterm>
+      <primary>max_ver</primary>
+    </indexterm>The max_ver attribute allows you to define the maximum
+  (latest) version where a field is visible. Requests using a later version
+  will not have access to the field.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting> [max_ver("1.04")] bool IsValid;   </programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLmin_ver.xml

@@ -1,15 +1,16 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
- 
-    <sect2 id="ESDL_Attributes_min_ver">
-      <title><emphasis role="bold">min_ver</emphasis></title>
+<sect2 id="ESDL_Attributes_min_ver">
+  <title><emphasis role="bold">min_ver</emphasis></title>
 
-      <para>The min_ver attribute allows you to define the minimum (earliest)
-      version where a field is visible. Requests using a prior version will
-      not have access to the field.</para>
+  <para><indexterm>
+      <primary>min_ver</primary>
+    </indexterm>The min_ver attribute allows you to define the minimum
+  (earliest) version where a field is visible. Requests using a prior version
+  will not have access to the field.</para>
 
-      <para><emphasis role="bold">Example:</emphasis></para>
+  <para><emphasis role="bold">Example:</emphasis></para>
 
-      <programlisting> [min_ver("1.03")] bool IsValid;   </programlisting>
-    </sect2>
+  <programlisting> [min_ver("1.03")] bool IsValid;   </programlisting>
+</sect2>

docs/DynamicESDL/DynamicESDL_Includer.xml

@@ -40,15 +40,18 @@
       <para></para>
     </legalnotice>
 
-    <xi:include href="common/Version.xml" xpointer="xpointer(//*[@id='FooterInfo'])"
+    <xi:include href="common/Version.xml"
+                xpointer="xpointer(//*[@id='FooterInfo'])"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-    <xi:include href="common/Version.xml" xpointer="xpointer(//*[@id='DateVer'])"
+    <xi:include href="common/Version.xml"
+                xpointer="xpointer(//*[@id='DateVer'])"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <corpname>HPCC Systems<superscript>®</superscript></corpname>
 
-    <xi:include href="common/Version.xml" xpointer="xpointer(//*[@id='Copyright'])"
+    <xi:include href="common/Version.xml"
+                xpointer="xpointer(//*[@id='Copyright'])"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <mediaobject role="logo">
@@ -91,10 +94,10 @@
     the service directly using SOAP, XML, or JSON.</para>
   </chapter>
 
-  <chapter>
+  <chapter id="DESDLWorkflowTutorial">
     <title>Workflow Tutorial</title>
 
-    <sect1 role="nobrk">
+    <sect1 id="DESDLWorkflowOverview" role="nobrk">
       <title>Overview</title>
 
       <para>In this section, we will:</para>
@@ -160,7 +163,7 @@
       <para></para>
     </sect1>
 
-    <sect1>
+    <sect1 id="DESDLConfigureAndBind">
       <title>Configure and Bind a Dynamic ESDL service</title>
 
       <para>This step adds an ESP service and a service binding that reserves
@@ -370,7 +373,7 @@ sudo cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.
       </orderedlist>
     </sect1>
 
-    <sect1>
+    <sect1 id="DESDLWriteDef">
       <title>Write the ESDL Service Definition</title>
 
       <para>In this section, we will write a the Service Definitions. The
@@ -406,7 +409,7 @@ ESPresponse AddThisResponse
       </orderedlist>
     </sect1>
 
-    <sect1>
+    <sect1 id="DESDLGenerateDefs">
       <title>Generate ECL definitions from the ESDL Service Definition</title>
 
       <para>In this section, we will generate ECL from this ESDL Service
@@ -477,7 +480,7 @@ OUTPUT(ds_out, NAMED('AddThisResponse')); </programlisting>
         </orderedlist></para>
     </sect1>
 
-    <sect1>
+    <sect1 id="DESDLPublishandBind">
       <title>Publish the ESDL Service Definitions and Bind the ESDL
       Service</title>
 
@@ -552,12 +555,13 @@ OUTPUT(ds_out, NAMED('AddThisResponse')); </programlisting>
   </chapter>
 
   <xi:include href="HPCCClientTools/CT_Mods/CT_ESDL_CLI.xml"
-              xpointer="xpointer(//*[@id='ESDL_CLI'])" xmlns:xi="http://www.w3.org/2001/XInclude" />
+              xpointer="xpointer(//*[@id='ESDL_CLI'])"
+              xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-  <chapter>
+  <chapter id="ESDLLanguageReferenceChapter">
     <title>ESDL Language Reference</title>
 
-    <sect1 role="nobrk">
+    <sect1 id="ESDLStructures" role="nobrk">
       <title>ESDL Structures</title>
 
       <para></para>
@@ -631,7 +635,7 @@ OUTPUT(ds_out, NAMED('AddThisResponse')); </programlisting>
                   xmlns:xi="http://www.w3.org/2001/XInclude" />
     </sect1>
 
-    <sect1>
+    <sect1 id="ESDLAttributes">
       <title>ESDL Attributes</title>
 
       <para>You can use ESDL attributes to extend and override the default
@@ -702,9 +706,10 @@ OUTPUT(ds_out, NAMED('AddThisResponse')); </programlisting>
       <xi:include href="DynamicESDL/DESDL-Mods/ESDLdescription.xml"
                   xpointer="element(/1)"
                   xmlns:xi="http://www.w3.org/2001/XInclude" />
+
       <xi:include href="DynamicESDL/DESDL-Mods/ESDLauth_feature.xml"
                   xpointer="element(/1)"
-                  xmlns:xi="http://www.w3.org/2001/XInclude" />                  
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
     </sect1>
   </chapter>
 </book>

docs/ECLStandardLibraryReference/SLR-Mods/Extract.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
-<sect1 id="ExtractMultiple">
+<sect1 id="Extract">
   <title>Extract</title>
 
   <para><emphasis role="bold">STD.Str.Extract<indexterm>

docs/ECLStandardLibraryReference/SLR-Mods/ExtractMultiple.xml

@@ -5,18 +5,20 @@
   <title>ExtractMultiple</title>
 
   <para><emphasis role="bold">STD.Str.ExtractMultiple<indexterm>
-      <primary>STD.Str.Extract</primary>
+      <primary>STD.Str.ExtractMultiple</primary>
     </indexterm><indexterm>
-      <primary>Str.Extract</primary>
+      <primary>Str.ExtractMultiple</primary>
     </indexterm><indexterm>
-      <primary>Extract</primary>
+      <primary>ExtractMultiple</primary>
     </indexterm>(</emphasis> <emphasis>source, instance</emphasis> <emphasis
   role="bold">)</emphasis></para>
 
-  <para><emphasis role="bold">STD.Uni.Extract<indexterm>
-      <primary>STD.Uni.Extract</primary>
+  <para><emphasis role="bold">STD.Uni.ExtractMultiple<indexterm>
+      <primary>STD.Uni.ExtractMultiple</primary>
     </indexterm><indexterm>
-      <primary>Uni.Extract</primary>
+      <primary>Uni.ExtractMultiple</primary>
+    </indexterm><indexterm>
+      <primary>ExtractMultiple</primary>
     </indexterm>(</emphasis> <emphasis>source, instance</emphasis> <emphasis
   role="bold">)</emphasis></para>
 
@@ -34,57 +36,32 @@
         </row>
 
         <row>
-          <entry><emphasis>instance </emphasis></entry>
+          <entry><emphasis>mask </emphasis></entry>
 
-          <entry>An integer specifying the ordinal position of the data item
-          within the <emphasis>source</emphasis> to return.</entry>
+          <entry>A bitmask specifying the ordinal position of the data item
+          within the <emphasis>source</emphasis> to return where bit 0 is item
+          1, bit 1 is item 2, etc.. </entry>
         </row>
 
         <row>
           <entry>Return:<emphasis> </emphasis></entry>
 
-          <entry>Extract returns either a STRING or UNICODE value, as
+          <entry>ExtractMultiple returns either a STRING or UNICODE value, as
           appropriate.</entry>
         </row>
       </tbody>
     </tgroup>
   </informaltable>
 
-  <para>The <emphasis role="bold">Extract </emphasis>function returns the data
-  at the ordinal position specified by the <emphasis>instance
-  </emphasis>within the comma-delimited <emphasis>source</emphasis>
-  string.</para>
+  <para>The <emphasis role="bold">ExtractMultiple </emphasis>function returns
+  the data at the bitmask positions specified by the <emphasis>mask
+  </emphasis>within the comma-delimited <emphasis>source</emphasis> string.,
+  where bit 0 is item 1, bit 1 is item 2, etc.</para>
 
   <para>Example:</para>
 
-  <programlisting format="linespecific">//all these examples result in 'Success'
-   
-A := IF(STD.Str.Extract('AB,CD,,G,E',0) = '',
-   'Success',
-   'Failure -1');
-    
-B := IF(STD.Str.Extract('AB,CD,,G,E',1) = 'AB',
-   'Success',
-   'Failure -2');
-    
-C := IF(STD.Str.Extract('AB,CD,,G,E',2) = 'CD',
-   'Success',
-   'Failure -3');
-
-D := IF(STD.Str.Extract('AB,CD,,G,E',3) = '',
-   'Success',
-   'Failure -4');
-    
-E := IF(STD.Str.Extract('AB,CD,,G,E',4) = 'G',
-   'Success',
-   'Failure -5');
-
-F := IF(STD.Str.Extract('AB,CD,,G,E',5) = 'E',
-   'Success',
-   'Failure -6');
-    
-G := IF(STD.Str.Extract('AB,CD,,G,E',6) = '',
-   'Success',
-   'Failure -7');
+  <programlisting format="linespecific">IMPORT STD;
+MyTestString:= 'You, only, live, twice';
+STD.Str.ExtractMultiple(MyTestString, 0b10011 ); //returns 'You, only'
 </programlisting>
 </sect1>

docs/ECLStandardLibraryReference/SLR-includer.xml

@@ -279,6 +279,9 @@
     <xi:include href="ECLStandardLibraryReference/SLR-Mods/Extract.xml"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
+    <xi:include href="ECLStandardLibraryReference/SLR-Mods/ExtractMultiple.xml"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLStandardLibraryReference/SLR-Mods/Filter.xml"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 

docs/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -1525,6 +1525,12 @@
               workunit tab shows you the same information that you would see
               if you selected it through the workunit link.</para>
             </listitem>
+
+            <listitem>
+              <para>Select the <emphasis role="bold">History</emphasis> tab to
+              display a list of DFU Actions taken such as copy, remote copy,
+              spray, etc. for the file. </para>
+            </listitem>
           </itemizedlist>On the File Details Summary tab, you can perform some
         actions on the selected file.</para>
 

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -3721,6 +3721,10 @@ ecl bundle install mybundle --keepprior</programlisting>
         file that can be retrieved. In either case, it is fetched to a
         temporary local location, processed as a local file/directory and then
         removed.</para>
+
+        <para>To use the "ecl bundle install &lt;git url&gt;" command, you
+        must have git installed and configured on your system. Git must be
+        accessible to the user (in the path).</para>
       </sect2>
 
       <sect2 id="CT_CLI_SyntaxECLBundleUninstall" role="brk">

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -1671,7 +1671,7 @@ dfsSSLPrivateKeyFile=/keyfilepath/keyfile</programlisting>Set the <emphasis
           master on the same node. Try to keep Dali and ESP on separate nodes.
           Even if you don't have the luxury of very many nodes, you want the
           Thor master and the Dali (at minimum) to be on separate nodes. The
-          best practice is to keep as many components as possible their own
+          best practice is to keep as many components as possible on their own
           nodes.</para>
 
           <para>Another consideration for a multiple node system is to avoid

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml

@@ -1,383 +1,550 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
-<sect1 id="ssl4esp">
-  <title>Configuring ESP Server to use HTTPS (SSL)</title>
+  <chapter id="SSLModuleWrapperChap">
+    <title>Chapter Level Wrapper</title>
 
-  <para>The HPCC Enterprise Services Platform server (ESP) supports Secure
-  Sockets Layer (SSL), a protocol used to send and receive private data or
-  documents.</para>
+    <para>This is a Chapter Wrapper for an included
+    &lt;Sect1&gt; module.</para>
 
-  <para>SSL works by using a private key to encrypt and decrypt data
-  transferred over the SSL connection. By convention, URLs using an SSL
-  connection start with HTTPS instead of HTTP.</para>
+    <sect1 id="ssl4esp">
+      <title>Configuring ESP Server to use HTTPS (SSL)</title>
 
-  <para>The SSL option in the ESP Server allows secure and encrypted
-  communication between a browser or SOAP client application and the HPCC
-  platform.</para>
+      <para>The HPCC Enterprise Services Platform server (ESP) supports Secure
+      Sockets Layer (SSL), a protocol used to send and receive private data or
+      documents.</para>
 
-  <para>SSL capabilities are configured in the Configuration Manager, but
-  require a certificate be installed on the ESP server. The OpenSSL libraries
-  provide a means to create the necessary certificate files in one of two
-  ways.</para>
+      <para>SSL works by using a private key to encrypt and decrypt data
+      transferred over the SSL connection. By convention, URLs using an SSL
+      connection start with HTTPS instead of HTTP.</para>
 
-  <itemizedlist mark="bullet">
-    <listitem>
-      <para>You can use the OpenSSL libraries to create a private key and a
-      Certificate Signing Request (CSR) to purchase a certificate from a
-      Certificate Issuing Authority (such as, VeriSign).</para>
-    </listitem>
+      <para>The SSL option in the ESP Server allows secure and encrypted
+      communication between a browser or SOAP client application and the HPCC
+      platform.</para>
 
-    <listitem>
-      <para>You can use that CSR to generate your own self-signed certificate
-      and then install the certificate and private key to your ESP
-      Server.</para>
-    </listitem>
-  </itemizedlist>
+      <para>SSL capabilities are configured in the Configuration Manager, but
+      require a certificate be installed on the ESP server. The OpenSSL
+      libraries provide a means to create the necessary certificate files in
+      one of two ways.</para>
 
-  <para>In either case, once installed and configured, the network traffic is
-  encrypted and secure. The Public and Private Keys use 1024-bit RSA
-  encryption.</para>
+      <itemizedlist mark="bullet">
+        <listitem>
+          <para>You can use the OpenSSL libraries to create a private key and
+          a Certificate Signing Request (CSR) to purchase a certificate from a
+          Certificate Issuing Authority (such as, VeriSign).</para>
+        </listitem>
+
+        <listitem>
+          <para>You can use that CSR to generate your own self-signed
+          certificate and then install the certificate and private key to your
+          ESP Server.</para>
+        </listitem>
+      </itemizedlist>
 
-  <sect2 id="ConfigureESP_GenerateRSAPrivateKey">
-    <title><emphasis role="bold">Generate an RSA Private
-    Key</emphasis></title>
+      <para>In either case, once installed and configured, the network traffic
+      is encrypted and secure. The Public and Private Keys use 1024-bit RSA
+      encryption.</para>
 
-    <para>Use the OpenSSL toolkit to generate an RSA Private Key and a
-    Certificate Signing Request (CSR). This can also be the basis for a
-    self-signed certificate. Self-signed certificates are useful for internal
-    use or testing.</para>
+      <sect2 id="ConfigureESP_GenerateRSAPrivateKey">
+        <title><emphasis role="bold">Generate an RSA Private
+        Key</emphasis></title>
 
-    <para>In our example, we create a 1024-bit RSA Private Key which is
-    encrypted using Triple-DES encryption and stored in Privacy Enhanced Mail
-    (PEM) format.</para>
+        <para>Use the OpenSSL toolkit to generate an RSA Private Key and a
+        Certificate Signing Request (CSR). This can also be the basis for a
+        self-signed certificate. Self-signed certificates are useful for
+        internal use or testing.</para>
 
-    <programlisting>openssl genrsa -des3 -out server.key 1024</programlisting>
+        <para>In our example, we create a 1024-bit RSA Private Key which is
+        encrypted using Triple-DES encryption and stored in Privacy Enhanced
+        Mail (PEM) format.</para>
 
-    <para>When prompted, provide a passphrase. This is used as the basis for
-    the encryption.</para>
+        <programlisting>openssl genrsa -des3 -out server.key 1024</programlisting>
 
-    <para><emphasis role="bold">Remember this passphrase as you will need to
-    enter it into the Configuration Manager later.</emphasis></para>
-  </sect2>
+        <para>When prompted, provide a passphrase. This is used as the basis
+        for the encryption.</para>
 
-  <sect2 id="ConfigureESP_GenerateCSReq" role="brk">
-    <title><emphasis role="bold">Generate a CSR (Certificate Signing
-    Request)</emphasis></title>
+        <para><emphasis role="bold">Remember this passphrase as you will need
+        to enter it into the Configuration Manager later.</emphasis></para>
+      </sect2>
 
-    <para>After you have a private key, you can use it to create a Certificate
-    Signing Request (CSR). You can use your CSR to request a signed
-    certificate from a Certificate Authority (such as Verisign or Network
-    Solutions). You can also use the CSR to create a self-signed
-    certificate.</para>
+      <sect2 id="ConfigureESP_GenerateCSReq" role="brk">
+        <title><emphasis role="bold">Generate a CSR (Certificate Signing
+        Request)</emphasis></title>
 
-    <programlisting>openssl req -new -key server.key -out server.csr </programlisting>
+        <para>After you have a private key, you can use it to create a
+        Certificate Signing Request (CSR). You can use your CSR to request a
+        signed certificate from a Certificate Authority (such as Verisign or
+        Network Solutions). You can also use the CSR to create a self-signed
+        certificate.</para>
 
-    <para>Answer the questions when prompted:</para>
+        <programlisting>openssl req -new -key server.key -out server.csr </programlisting>
 
-    <para><informaltable colsep="1" frame="all" rowsep="1">
-        <tgroup cols="2">
-          <colspec colwidth="243.90pt" />
+        <para>Answer the questions when prompted:</para>
 
-          <colspec colwidth="234.90pt" />
+        <para><informaltable colsep="1" frame="all" rowsep="1">
+            <tgroup cols="2">
+              <colspec colwidth="243.90pt" />
 
-          <tbody>
-            <row>
-              <entry>Country Name (2 letter code):</entry>
+              <colspec colwidth="234.90pt" />
 
-              <entry></entry>
-            </row>
+              <tbody>
+                <row>
+                  <entry>Country Name (2 letter code):</entry>
 
-            <row>
-              <entry>State or Province Name (full name):</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
+                <row>
+                  <entry>State or Province Name (full name):</entry>
 
-            <row>
-              <entry>Locality Name (eg, city) :</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
+                <row>
+                  <entry>Locality Name (eg, city) :</entry>
 
-            <row>
-              <entry>Organization Name (eg, company) :</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
+                <row>
+                  <entry>Organization Name (eg, company) :</entry>
 
-            <row>
-              <entry>Organizational Unit Name (eg, section) :</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
+                <row>
+                  <entry>Organizational Unit Name (eg, section) :</entry>
 
-            <row>
-              <entry>Common Name (e.g., server’s hostname):</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
+                <row>
+                  <entry>Common Name (e.g., server’s hostname):</entry>
 
-            <row>
-              <entry>Email Address :</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
+                <row>
+                  <entry>Email Address :</entry>
 
-            <row>
-              <entry>A challenge password (optional):</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
+                <row>
+                  <entry>A challenge password (optional):</entry>
 
-            <row>
-              <entry>An optional company name (optional):</entry>
+                  <entry></entry>
+                </row>
 
-              <entry></entry>
-            </row>
-          </tbody>
-        </tgroup>
-      </informaltable></para>
-  </sect2>
+                <row>
+                  <entry>An optional company name (optional):</entry>
 
-  <sect2 id="ConfigureESP_GenerateSelf-Signed_Certificate">
-    <title><emphasis role="bold">Generate a Self-Signed
-    Certificate</emphasis></title>
+                  <entry></entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
 
-    <para>To generate a temporary certificate, which is good for up to 365
-    days, issue the following command:</para>
+      <sect2 id="ConfigureESP_GenerateSelf-Signed_Certificate">
+        <title><emphasis role="bold">Generate a Self-Signed
+        Certificate</emphasis></title>
 
-    <programlisting>openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt </programlisting>
+        <para>To generate a temporary certificate, which is good for up to 365
+        days, issue the following command:</para>
 
-    <para>When prompted, enter the passphrase you used earlier when creating
-    your CSR.</para>
-  </sect2>
+        <programlisting>openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt </programlisting>
 
-  <sect2 id="ConfigureESP_InstallPrivateKeyandCertificate">
-    <title><emphasis role="bold">Installing the Private Key and Certificate to
-    your ESP Server</emphasis></title>
+        <para>When prompted, enter the passphrase you used earlier when
+        creating your CSR.</para>
+      </sect2>
 
-    <para>You must install the certificate and private key on all ESP server
-    node(s) that will host a service binding using SSL.</para>
+      <sect2 id="ConfigureESP_InstallPrivateKeyandCertificate">
+        <title><emphasis role="bold">Installing the Private Key and
+        Certificate to your ESP Server</emphasis></title>
 
-    <para>Your PrivateKey and certificate must be copied to
-    /var/lib/HPCCSystems/myesp/.</para>
+        <para>You must install the certificate and private key on all ESP
+        server node(s) that will host a service binding using SSL.</para>
 
-    <programlisting># For example:
+        <para>Your PrivateKey and certificate must be copied to
+        /var/lib/HPCCSystems/myesp/.</para>
+
+        <programlisting># For example:
 sudo cp server.crt /var/lib/HPCCSystems/myesp/certificate.cer
 sudo cp server.key /var/lib/HPCCSystems/myesp/privatekey.cer
 </programlisting>
-  </sect2>
+      </sect2>
 
-  <sect2 id="ConfigureHTTPSonESPServer" role="brk">
-    <title><emphasis role="bold">Configure HTTPS on your ESP
-    Server</emphasis></title>
+      <sect2 id="ConfigureHTTPSonESPServer" role="brk">
+        <title><emphasis role="bold">Configure HTTPS on your ESP
+        Server</emphasis></title>
 
-    <sect3 id="Start_ConfigMgrAdvancedMode">
-      <title><emphasis role="bold">Start Configuration Manager in Advanced
-      Mode</emphasis></title>
+        <sect3 id="Start_ConfigMgrAdvancedMode">
+          <title><emphasis role="bold">Start Configuration Manager in Advanced
+          Mode</emphasis></title>
 
-      <orderedlist numeration="arabic">
-        <listitem>
-          <para>Start the Configuration Manager Service on one node (usually
-          the first node is considered the head node and is used for this
-          task, but this is up to you).</para>
+          <orderedlist numeration="arabic">
+            <listitem>
+              <para>Start the Configuration Manager Service on one node
+              (usually the first node is considered the head node and is used
+              for this task, but this is up to you).</para>
 
-          <para><programlisting>sudo /opt/HPCCSystems/sbin/configmgr
+              <para><programlisting>sudo /opt/HPCCSystems/sbin/configmgr
 </programlisting></para>
-        </listitem>
-
-        <listitem>
-          <para>Using a Web browser, go to the Configuration Manager's
-          interface.</para>
-
-          <para>Use the url of http://nnn.nnn.nnn.nnn:pppp, where
-          nnn.nnn.nnn.nnn is the IP address of the node running Configuration
-          Manager and pppp is the port (default is 8015).</para>
-
-          <para>The Configuration Manager startup wizard displays.</para>
-        </listitem>
-
-        <listitem>
-          <para>Select <emphasis role="bold">Advanced View</emphasis>.</para>
-        </listitem>
-
-        <listitem>
-          <para>Select an XML file from the drop list.</para>
-
-          <para>This list is populated from versions of an environment XML
-          file in your server's <emphasis
-          role="bold">/etc/HPCCSystems/source/</emphasis> directory.</para>
-
-          <para><emphasis role="bold">Tip</emphasis>: The XML file that
-          matches the active environment.xml is highlighted.</para>
-        </listitem>
-
-        <listitem>
-          <para>Press the <emphasis role="bold">Next</emphasis> button.</para>
-
-          <para>The Configuration Manager Advanced View interface
-          displays.</para>
-        </listitem>
-
-        <listitem>
-          <para>Check the <emphasis role="bold">Write Access </emphasis>box at
-          the top of the page.</para>
-        </listitem>
-      </orderedlist>
-    </sect3>
-
-    <sect3 id="Configure_ESP_forHTTPS" role="brk">
-      <title><emphasis role="bold">Configure ESP</emphasis></title>
-
-      <orderedlist numeration="arabic">
-        <listitem>
-          <para>Select ESP - MyEsp in the Navigator panel on the left
-          side.</para>
-        </listitem>
-
-        <listitem>
-          <para>Select the <emphasis role="bold">HTTPS</emphasis> tab. <figure>
-              <title>Select HTTPS Tab</title>
-
-              <mediaobject>
-                <imageobject>
-                  <imagedata fileref="../../images/ssl-01.jpg" />
-                </imageobject>
-              </mediaobject>
-            </figure></para>
-        </listitem>
-
-        <listitem>
-          <para>In the <emphasis role="bold">passphrase</emphasis> entry
-          control, enter the passphrase you used earlier when you created the
-          private key.</para>
-        </listitem>
-
-        <listitem>
-          <para>When prompted, provide the passphrase again.</para>
-        </listitem>
-
-        <listitem>
-          <para>Click the disk icon to save.</para>
-        </listitem>
-      </orderedlist>
-    </sect3>
-
-    <sect3 id="ConfigureOneorMoreSSLEnabledServiceBindings">
-      <title><emphasis role="bold">Configure one or more SSL-Enabled Service
-      Bindings</emphasis></title>
-
-      <orderedlist numeration="arabic">
-        <listitem>
-          <para>Select the ESP Service Bindings tab.</para>
-        </listitem>
-
-        <listitem>
-          <para>Right-click on the list of services, then select <emphasis
-          role="bold">Add</emphasis>.</para>
-        </listitem>
-
-        <listitem>
-          <para>Provide a name for the binding (e.g., myws_ecl_ssl)</para>
-        </listitem>
-
-        <listitem>
-          <para>Select myws_ecl from the service drop-list.</para>
-
-          <figure>
-            <title>myws_ecl</title>
-
-            <mediaobject>
-              <imageobject>
-                <imagedata fileref="../../images/ssl-02.jpg" />
-              </imageobject>
-            </mediaobject>
-          </figure>
-        </listitem>
-
-        <listitem>
-          <para>Select https from the protocol drop-list.</para>
-
-          <figure>
-            <title>Select HTTPS</title>
-
-            <mediaobject>
-              <imageobject>
-                <imagedata fileref="../../images/ssl-04.jpg" />
-              </imageobject>
-            </mediaobject>
-          </figure>
-
-          <para><emphasis role="bold">Note:</emphasis> If you have not
-          previously edited the port, the change from http to https triggers
-          Configuration Manager to automatically change the port to the
-          default port for https (18002). It only updates automatically if the
-          port has not been edited.</para>
-        </listitem>
-
-        <listitem>
-          <para>Click the disk icon to save</para>
-        </listitem>
-      </orderedlist>
-    </sect3>
-  </sect2>
-
-  <sect2 id="DistributeEnvironmentConfigFileToAllNodes" role="brk">
-    <title><emphasis role="bold">Distribute the environment configuration file
-    to all nodes, Restart, and Certify</emphasis></title>
-
-    <para>Once your environment is set up as desired, you must copy the
-    configuration file out to the other nodes.</para>
-
-    <orderedlist numeration="arabic">
-      <listitem>
-        <para>If it is running, stop the system.</para>
-
-        <para>Make sure system is stopped before attempting to move the
-        environment.xml file.</para>
-      </listitem>
-
-      <listitem>
-        <para>Back up the original environment.xml file</para>
-
-        <programlisting># for example
+            </listitem>
+
+            <listitem>
+              <para>Using a Web browser, go to the Configuration Manager's
+              interface.</para>
+
+              <para>Use the url of http://nnn.nnn.nnn.nnn:pppp, where
+              nnn.nnn.nnn.nnn is the IP address of the node running
+              Configuration Manager and pppp is the port (default is
+              8015).</para>
+
+              <para>The Configuration Manager startup wizard displays.</para>
+            </listitem>
+
+            <listitem>
+              <para>Select <emphasis role="bold">Advanced
+              View</emphasis>.</para>
+            </listitem>
+
+            <listitem>
+              <para>Select an XML file from the drop list.</para>
+
+              <para>This list is populated from versions of an environment XML
+              file in your server's <emphasis
+              role="bold">/etc/HPCCSystems/source/</emphasis>
+              directory.</para>
+
+              <para><emphasis role="bold">Tip</emphasis>: The XML file that
+              matches the active environment.xml is highlighted.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the <emphasis role="bold">Next</emphasis>
+              button.</para>
+
+              <para>The Configuration Manager Advanced View interface
+              displays.</para>
+            </listitem>
+
+            <listitem>
+              <para>Check the <emphasis role="bold">Write Access
+              </emphasis>box at the top of the page.</para>
+            </listitem>
+          </orderedlist>
+        </sect3>
+
+        <sect3 id="Configure_ESP_forHTTPS" role="brk">
+          <title><emphasis role="bold">Configure ESP</emphasis></title>
+
+          <orderedlist numeration="arabic">
+            <listitem>
+              <para>Select ESP - MyEsp in the Navigator panel on the left
+              side.</para>
+            </listitem>
+
+            <listitem>
+              <para>Select the <emphasis role="bold">HTTPS</emphasis> tab.
+              <figure>
+                  <title>Select HTTPS Tab</title>
+
+                  <mediaobject>
+                    <imageobject>
+                      <imagedata fileref="../../images/ssl-01.jpg" />
+                    </imageobject>
+                  </mediaobject>
+                </figure></para>
+            </listitem>
+
+            <listitem>
+              <para>In the <emphasis role="bold">passphrase</emphasis> entry
+              control, enter the passphrase you used earlier when you created
+              the private key.</para>
+            </listitem>
+
+            <listitem>
+              <para>When prompted, provide the passphrase again.</para>
+            </listitem>
+
+            <listitem>
+              <para>Click the disk icon to save.</para>
+            </listitem>
+          </orderedlist>
+        </sect3>
+
+        <sect3 id="ConfigureOneorMoreSSLEnabledServiceBindings">
+          <title><emphasis role="bold">Configure one or more SSL-Enabled
+          Service Bindings</emphasis></title>
+
+          <orderedlist numeration="arabic">
+            <listitem>
+              <para>Select the ESP Service Bindings tab.</para>
+            </listitem>
+
+            <listitem>
+              <para>Right-click on the list of services, then select <emphasis
+              role="bold">Add</emphasis>.</para>
+            </listitem>
+
+            <listitem>
+              <para>Provide a name for the binding (e.g., myws_ecl_ssl)</para>
+            </listitem>
+
+            <listitem>
+              <para>Select myws_ecl from the service drop-list.</para>
+
+              <figure>
+                <title>myws_ecl</title>
+
+                <mediaobject>
+                  <imageobject>
+                    <imagedata fileref="../../images/ssl-02.jpg" />
+                  </imageobject>
+                </mediaobject>
+              </figure>
+            </listitem>
+
+            <listitem>
+              <para>Select https from the protocol drop-list.</para>
+
+              <figure>
+                <title>Select HTTPS</title>
+
+                <mediaobject>
+                  <imageobject>
+                    <imagedata fileref="../../images/ssl-04.jpg" />
+                  </imageobject>
+                </mediaobject>
+              </figure>
+
+              <para><emphasis role="bold">Note:</emphasis> If you have not
+              previously edited the port, the change from http to https
+              triggers Configuration Manager to automatically change the port
+              to the default port for https (18002). It only updates
+              automatically if the port has not been edited.</para>
+            </listitem>
+
+            <listitem>
+              <para>Click the disk icon to save</para>
+            </listitem>
+          </orderedlist>
+        </sect3>
+      </sect2>
+
+      <sect2 id="DistributeEnvironmentConfigFileToAllNodes" role="brk">
+        <title><emphasis role="bold">Distribute the environment configuration
+        file to all nodes, Restart, and Certify</emphasis></title>
+
+        <para>Once your environment is set up as desired, you must copy the
+        configuration file out to the other nodes.</para>
+
+        <orderedlist numeration="arabic">
+          <listitem>
+            <para>If it is running, stop the system.</para>
+
+            <para>Make sure system is stopped before attempting to move the
+            environment.xml file.</para>
+          </listitem>
+
+          <listitem>
+            <para>Back up the original environment.xml file</para>
+
+            <programlisting># for example
 sudo cp /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.bak
 </programlisting>
 
-        <para>Note: the "live" environment.xml file is located in your
-        <emphasis role="bold">/etc/HPCCSystems/ </emphasis>directory.
-        ConfigManager works on files in <emphasis
-        role="bold">/etc/HPCCSystems/source </emphasis>directory. You must
-        copy the XML file from this location to make an environment.xml file
-        active.</para>
-      </listitem>
+            <para>Note: the "live" environment.xml file is located in your
+            <emphasis role="bold">/etc/HPCCSystems/ </emphasis>directory.
+            ConfigManager works on files in <emphasis
+            role="bold">/etc/HPCCSystems/source </emphasis>directory. You must
+            copy the XML file from this location to make an environment.xml
+            file active.</para>
+          </listitem>
 
-      <listitem>
-        <para>Copy the NewEnvironment.xml file from the source directory to
-        the /etc/HPCCSystems and rename the file to environment.xml</para>
+          <listitem>
+            <para>Copy the NewEnvironment.xml file from the source directory
+            to the /etc/HPCCSystems and rename the file to
+            environment.xml</para>
 
-        <programlisting># for example
+            <programlisting># for example
 sudo cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.xml
 </programlisting>
-      </listitem>
-
-      <listitem>
-        <para>Copy the <emphasis
-        role="bold">/etc/HPCCSystems/environment.xml</emphasis> to
-        the<emphasis role="bold"> /etc/HPCCSystems/ </emphasis>on every
-        node.</para>
-
-        <para>You might prefer to use a script to automate this step,
-        especially if you have many nodes. See the Example Scripts section in
-        the Appendix of the Installing and Running the HPCCPlatform
-        manual.</para>
-      </listitem>
-
-      <listitem>
-        <para>Restart the HPCC system and certify the components as
-        usual.</para>
-      </listitem>
-    </orderedlist>
-  </sect2>
-</sect1>
+          </listitem>
+
+          <listitem>
+            <para>Copy the <emphasis
+            role="bold">/etc/HPCCSystems/environment.xml</emphasis> to
+            the<emphasis role="bold"> /etc/HPCCSystems/ </emphasis>on every
+            node.</para>
+
+            <para>You might prefer to use a script to automate this step,
+            especially if you have many nodes. See the Example Scripts section
+            in the Appendix of the Installing and Running the HPCCPlatform
+            manual.</para>
+          </listitem>
+
+          <listitem>
+            <para>Restart the HPCC system and certify the components as
+            usual.</para>
+          </listitem>
+        </orderedlist>
+      </sect2>
+    </sect1>
+
+    <sect1 id="ssl4Roxie">
+      <title>Configuring SSL for Roxie</title>
+
+      <para>Roxie can also be configured to use the Secure Sockets Layer (SSL)
+      protocol. You may have already completed some of these steps if you
+      configured ESP Server to use SSL as described in the previous section.
+      Please refer to the <link linkend="ssl4esp">SSL For ESP</link> section
+      for more information on creating keys and certificates.</para>
+
+      <sect2 id="ConfigureHTTPS_RoxieCluster">
+        <title><emphasis role="bold">Configure HTTPS on your Roxie
+        Cluster</emphasis></title>
+
+        <sect3 id="Start_TheConfigMgrAdvancedMode">
+          <title><emphasis role="bold">Start Configuration Manager in Advanced
+          Mode</emphasis></title>
+
+          <orderedlist numeration="arabic">
+            <listitem>
+              <para>Start the Configuration Manager Service on one node
+              (usually the first node is considered the head node and is used
+              for this task, but this is up to you).</para>
+
+              <para><programlisting>sudo /opt/HPCCSystems/sbin/configmgr
+</programlisting></para>
+            </listitem>
+
+            <listitem>
+              <para>Using a Web browser, go to the Configuration Manager's
+              interface.</para>
+
+              <para>Use the url of http://nnn.nnn.nnn.nnn:pppp, where
+              nnn.nnn.nnn.nnn is the IP address of the node running
+              Configuration Manager and pppp is the port (default is
+              8015).</para>
+
+              <para>The Configuration Manager startup wizard displays.</para>
+            </listitem>
+
+            <listitem>
+              <para>Select <emphasis role="bold">Advanced
+              View</emphasis>.</para>
+            </listitem>
+
+            <listitem>
+              <para>Select an XML file from the drop list.</para>
+
+              <para>This list is populated from versions of an environment XML
+              file in your server's <emphasis
+              role="bold">/etc/HPCCSystems/source/</emphasis>
+              directory.</para>
+
+              <para><emphasis role="bold">Tip</emphasis>: The XML file that
+              matches the active environment.xml is highlighted.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the <emphasis role="bold">Next</emphasis>
+              button.</para>
+
+              <para>The Configuration Manager Advanced View interface
+              displays.</para>
+            </listitem>
+
+            <listitem>
+              <para>Check the <emphasis role="bold">Write Access
+              </emphasis>box at the top of the page.</para>
+            </listitem>
+          </orderedlist>
+        </sect3>
+
+        <sect3 id="Configure_Roxie_forHTTPS" role="brk">
+          <title><emphasis role="bold">Configure Roxie SSL</emphasis></title>
+
+          <orderedlist numeration="arabic">
+            <listitem>
+              <para>Select your Roxie Cluster in the Navigator panel on the
+              left side.</para>
+            </listitem>
+
+            <listitem>
+              <para>Select the <emphasis role="bold">Ports</emphasis>
+              tab.</para>
+            </listitem>
+
+            <listitem>
+              <para>Right-click on the list of ports, then select <emphasis
+              role="bold">Add</emphasis>.<figure>
+                  <title>Select Port Tab</title>
+
+                  <mediaobject>
+                    <imageobject>
+                      <imagedata fileref="../../images/CM-RoxSSL01.jpg" />
+                    </imageobject>
+                  </mediaobject>
+                </figure></para>
+            </listitem>
+
+            <listitem>
+              <para>The default port number is 9876. Change the default port
+              number, for example, to 19876.</para>
+            </listitem>
+
+            <listitem>
+              <para>Change the protocol from <emphasis>Native</emphasis> to
+              <emphasis>SSL</emphasis> from the drop menu (image).</para>
+            </listitem>
+
+            <listitem>
+              <para>In the <emphasis role="bold">passphrase</emphasis> entry
+              control, enter the passphrase you used earlier when you created
+              the private key. Leave this field empty if you did not use a
+              passphrase.</para>
+            </listitem>
+
+            <listitem>
+              <para>When prompted, provide the passphrase again.</para>
+            </listitem>
+
+            <listitem>
+              <para>Enter the certificate filename.</para>
+            </listitem>
+
+            <listitem>
+              <para>Enter the key filename.</para>
+            </listitem>
+
+            <listitem>
+              <para>Click the disk icon to save.</para>
+            </listitem>
+          </orderedlist>
+
+          <para>The default lookup location for the certificate and key files
+          is in <emphasis>/var/lib/HPCCSystems/myroxie</emphasis>. You can
+          specify a full path if you want these files in a different location.
+          The certificate and key files must be available for each Roxie
+          node.</para>
+        </sect3>
+
+        <sect3 id="Distribute_Environment_ConfigFileToAllNodes" role="brk">
+          <title><emphasis role="bold">Distribute the environment
+          configuration file to all nodes, Restart, and
+          Certify</emphasis></title>
+
+          <para>Once your environment is set up as desired, you must copy the
+          configuration file out to the other nodes. For more information
+          about how to distribute your environment, please see the section
+          <link linkend="DistributeEnvironmentConfigFileToAllNodes">Distribute
+          the environment configuration file</link> above.</para>
+        </sect3>
+      </sect2>
+    </sect1>
+  </chapter>
+

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -1349,9 +1349,14 @@ sudo cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-    <xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml"
+    <!-- <xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml"
                 xpointer="element(/1)"
-                xmlns:xi="http://www.w3.org/2001/XInclude" />
+                xmlns:xi="http://www.w3.org/2001/XInclude" />   -->
+                
+  <xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml" xpointer="xpointer(//*[@id='ssl4esp'])"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />  
+  <xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml" xpointer="xpointer(//*[@id='ssl4Roxie'])"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />  
   </chapter>
 
   <chapter id="Installing_MoreECLExamples">

ecl/eclagent/eclagent.cpp

@@ -3138,7 +3138,7 @@ void EclAgent::fatalAbort(bool userabort,const char *excepttext)
         if (userabort) 
             w->setState(WUStateAborted);
         if (excepttext&&*excepttext)
-            addException(SeverityError, "ECLAGENT", 1000, excepttext, NULL, 0, 0, true, false);
+            addException(SeverityError, "eclagent", 1000, excepttext, NULL, 0, 0, true, false);
         w->deleteTempFiles(NULL, false, true);
         wuRead.clear(); 
         w->commit();        // needed because we can't unlock the workunit in this thread

ecl/eclagent/eclgraph.cpp

@@ -591,7 +591,8 @@ bool EclGraphElement::prepare(IAgentContext & agent, const byte * parentExtract,
             break;
         case TAKif:
             {
-                Owned<IHThorArg> helper = createHelper(agent, NULL);
+                assertex(!subgraph->owner);
+                Owned<IHThorArg> helper = createHelper(agent, subgraph->owner);
                 try
                 {
                     helper->onStart(NULL, NULL);
@@ -641,7 +642,8 @@ bool EclGraphElement::prepare(IAgentContext & agent, const byte * parentExtract,
         //However that is far from trivial, so for the moment conditional statements won't be supported by hthor.
         case TAKifaction:
             {
-                Owned<IHThorArg> helper = createHelper(agent, NULL);
+                assertex(!subgraph->owner);
+                Owned<IHThorArg> helper = createHelper(agent, subgraph->owner);
                 //Problem if this is done in a child query...
                 try
                 {
@@ -658,7 +660,7 @@ bool EclGraphElement::prepare(IAgentContext & agent, const byte * parentExtract,
         case TAKsequential:
         case TAKparallel:
             {
-                Owned<IHThorArg> helper = createHelper(agent, NULL);
+                Owned<IHThorArg> helper = createHelper(agent, subgraph->owner);
                 unsigned numBranches = (kind == TAKsequential) ? 
                                         ((IHThorSequentialArg *)helper.get())->numBranches() : 
                                         ((IHThorParallelArg *)helper.get())->numBranches();
@@ -669,7 +671,8 @@ bool EclGraphElement::prepare(IAgentContext & agent, const byte * parentExtract,
 #endif
         case TAKcase:
             {
-                Owned<IHThorArg> helper = createHelper(agent, NULL);
+                assertex(!subgraph->owner);
+                Owned<IHThorArg> helper = createHelper(agent, subgraph->owner);
                 try
                 {
                     helper->onStart(NULL, NULL);

ecl/hql/hqlmeta.cpp

@@ -951,6 +951,27 @@ static bool groupByWithinSortOrder(IHqlExpression * groupBy, IHqlExpression * or
     return false;
 }
 
+static bool groupingWithinSortElement(IHqlExpression * grouping, IHqlExpression * order)
+{
+    ForEachChild(i, grouping)
+    {
+        if (matchesGroupBy(grouping->queryChild(i), order))
+            return true;
+    }
+    return false;
+}
+
+static bool groupingWithinSortOrder(IHqlExpression * grouping, HqlExprArray & sortOrder, unsigned first)
+{
+    unsigned max = sortOrder.ordinality();
+    for (unsigned i = first; i < max; i++)
+    {
+        if (groupingWithinSortElement(grouping, &sortOrder.item(i)))
+            return true;
+    }
+    return false;
+}
+
 //NB: This does not handle ALL groups that is handled in createDataset()
 void CHqlMetaInfo::applyGroupBy(IHqlExpression * groupBy, bool isLocal)
 {
@@ -1563,7 +1584,14 @@ static IHqlExpression * createSubSorted(IHqlExpression * dataset, IHqlExpression
     if (!isLocal && !alwaysLocal)
         subsort.setown(convertSubSortToGroupedSort(subsort));
 
-    assertex(isAlreadySorted(subsort, order, isLocal||alwaysLocal, ignoreGrouping, false));
+    if (!isAlreadySorted(subsort, order, isLocal||alwaysLocal, ignoreGrouping, false))
+    {
+        //If the remaining sort conditions overlap with the already sorted fields then creating a sub sort will cause problems.
+        //This is potentially O(N^2) operation, so avoid checking until the (very unusual) exception is hit.
+        if (groupingWithinSortOrder(alreadySorted, components, sortedElements))
+            return nullptr;
+        throwUnexpectedX("createSubSorted created inconsistent sort order");
+    }
     return subsort.getClear();
 }
 

ecl/hqlcpp/hqlcpp.cpp

@@ -2162,7 +2162,7 @@ void HqlCppTranslator::reportWarning(WarnErrorCategory category, unsigned id, co
 void HqlCppTranslator::addWorkunitException(ErrorSeverity severity, unsigned code, const char * text, IHqlExpression * location)
 {
     Owned<IWUException> msg = wu()->createException();
-    msg->setExceptionSource("Code Generator");
+    msg->setExceptionSource("eclcc");
     if (code)
         msg->setExceptionCode(code);
     msg->setExceptionMessage(text);

ecl/hqlcpp/hqlcppds.cpp

@@ -4465,7 +4465,8 @@ void HqlCppTranslator::doBuildRowAssignProject(BuildCtx & ctx, IReferenceSelecto
     OwnedHqlExpr activeDataset = ensureActiveRow(dataset->queryNormalizedSelector());
 
     // queryChild(1) should be changed to queryTransformExpr()...
-    OwnedHqlExpr transform = queryNewReplaceSelector(queryNewColumnProvider(expr), leftSelect, activeDataset);
+    IHqlExpression * originalTransform = queryNewColumnProvider(expr);
+    OwnedHqlExpr transform = queryNewReplaceSelector(originalTransform, leftSelect, activeDataset);
     Owned<BoundRow> selfCursor;
     if (!transform)
     {
@@ -4473,7 +4474,7 @@ void HqlCppTranslator::doBuildRowAssignProject(BuildCtx & ctx, IReferenceSelecto
         selfCursor.set(bindSelectorAsSelf(subctx, target, expr));
         //Mapping may potentially be ambiguous, so do things correctly (see hqlsource for details)
         BoundRow * prevCursor = resolveSelectorDataset(subctx, dataset->queryNormalizedSelector());
-        transform.set(expr->queryChild(1));
+        transform.set(originalTransform);
 
         bindTableCursor(subctx, dataset, prevCursor->queryBound(), no_left, selSeq);
     }

ecl/hqlcpp/hqlresource.cpp

@@ -1054,7 +1054,7 @@ void ActivityInvariantHoister::gatherChildSplitPoints(IHqlExpression * expr, Act
         alwaysOnce = true;
         break;
     case no_loop:
-        if ((options.targetClusterType == ThorLCRCluster) && !options.isChildQuery)
+        if ((options.targetClusterType != RoxieCluster) && !options.isChildQuery)
         {
             //This is ugly!  The body is executed in parallel, so don't force that as a work unit result
             //It means some child query expressions within loops don't get forced into work unit writes

ecl/hqlcpp/hqlttcpp.cpp

@@ -1871,6 +1871,11 @@ IHqlExpression * SortListSimplifier::simplify(IHqlExpression * sortlist)
                 }
             }
         }
+        else if (isCast(cur) && castPreservesValueAndOrder(cur))
+        {
+            expand = true;
+            appendComponent(invert, cur->queryChild(0));
+        }
         else
         {
 #if 0

ecl/hthor/hthorkey.cpp

@@ -440,6 +440,7 @@ bool CHThorIndexReadActivityBase::doPreopenLimit(unsigned __int64 limit)
     if(superIterator)
     {
         superIterator->first();
+        superIndex = 0;
         do
         {
             df.set(&superIterator->query());

ecl/regress/issue17685.ecl

@@ -0,0 +1,35 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+#option ('targetClusterType', 'roxie');
+
+{unsigned2 a,unsigned2 b,unsigned2 c,unsigned2 d,unsigned2 e} t(unsigned cnt) := TRANSFORM
+    SELF.a := cnt;
+    SELF.b := cnt % 10;
+    SELF.c := HASH64(cnt);
+    SELF.d := HASH32(cnt) % 10;
+    SELF.e := HASH(cnt) % 10;
+END;
+
+ds := dataset(200, t(COUNTER));
+
+s1 := SORT(ds, (unsigned8)d,(unsigned8)e,RECORD);
+f1 := NOCOMBINE(s1);
+
+s2 := SORT(f1, (unsigned8)d,(unsigned8)e,b,RECORD);
+f2 := NOCOMBINE(s2);
+s3 := SORTED(f2, (unsigned8)d,(unsigned8)e,b,RECORD, ASSERT,LOCAL);
+output(s3);

ecllibrary/std/Date.ecl

@@ -621,7 +621,7 @@ Common date formats
     American    '%m/%d/%Y'  mm/dd/yyyy
     Euro        '%d/%m/%Y'  dd/mm/yyyy
     Iso format  '%Y-%m-%d'  yyyy-mm-dd
-    Iso basic   'Y%m%d'     yyyymmdd
+    Iso basic   '%Y%m%d'    yyyymmdd
                 '%d-%b-%Y'  dd-mon-yyyy    e.g., '21-Mar-1954'
  */
 

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -605,9 +605,8 @@ bool CWsWorkunitsEx::onWUCreateAndUpdate(IEspContext &context, IEspWUUpdateReque
         if (!wuid || !*wuid)
         {
             NewWsWorkunit wu(context);
-            wuid = wu->queryWuid();
+            req.setWuid(wu->queryWuid());
         }
-        req.setWuid(wuid);
     }
     catch(IException* e)
     {

esp/src/eclwatch/DynamicESDLMethodWidget.js

@@ -20,12 +20,12 @@ define([
     "dojo/i18n!./nls/hpcc",
     "dojo/_base/array",
     "dojo/dom-construct",
+    "dojo/dom-class",
+    "dojo/topic",
 
     "dijit/registry",
     "dijit/form/Button",
     "dijit/ToolbarSeparator",
-    "dijit/Dialog",
-    "dijit/form/TextBox",
 
     "hpcc/GridDetailsWidget",
     "hpcc/ESPQuery",
@@ -36,33 +36,28 @@ define([
     "dgrid/editor",
     "dgrid/tree"
 
-], function (declare, lang, i18n, nlsHPCC, arrayUtil, domConstruct,
-                registry, Button, ToolbarSeparator, Dialog, TextBox,
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, domConstruct, domClass, topic,
+                registry, Button, ToolbarSeparator,
                 GridDetailsWidget, ESPQuery, ESPUtil, WsESDLConfig,
-                selector, editor, tree) {
+                selector, editor, tree
+            ) {
     return declare("DynamicESDLMethodWidget", [GridDetailsWidget], {
         i18n: nlsHPCC,
 
-        gridTitle: "Methods",
+        gridTitle: nlsHPCC.title_Methods,
         idProperty: "__hpcc_id",
 
         init: function (params) {
             if (this.inherited(arguments))
                 return;
 
-            this.refresh(params);
+            this.refresh();
             this._refreshActionState();
         },
 
-        refresh: function (params) {
-            this.params = params;
-
-            if (params.Methods && !params.Configuration) {
-                this._populatedByUser(params)
-            } else {
-                this.refreshGrid(params);
-            }
-       },
+        refresh: function () {
+            this.refreshGrid();
+        },
 
         startup: function (args) {
             this.inherited(arguments);
@@ -70,53 +65,11 @@ define([
 
         createGrid: function (domID) {
             var context = this;
-            this.dialog = new Dialog({
-                title: context.i18n.AddAttributes,
-                style: "width: 300px;"
-            });
             this.openButton = registry.byId(this.id + "Open");
-            this.saveAttributeButton = new Button({
-                onClick: function (val) {
-                    var att = context.attributeTextBox.get("value");
-                    var val = context.valueTextBox.get("value");
-                    context._addAttribute(att,val);
-                    context.attributeTextBox.reset();
-                    context.valueTextBox.reset();
-                    context.dialog.hide();
-                },
-                label: context.i18n.Save,
-                style: "float: right;"
-            }).placeAt(this.dialog);
-            this.attributeTextBox = new TextBox({
-                label: this.i18n.Attribute,
-                placeholder: this.i18n.Attribute,
-                style: "margin-bottom: 10px;"
-            }).placeAt(this.dialog);
-            this.valueTextBox = new TextBox({
-                label: this.i18n.Value,
-                placeholder: this.i18n.Value
-            }).placeAt(this.dialog);
-            this.addAddAttritbuteButton = new Button({
-                disabled: true,
-                onClick: function (val) {
-                    context.dialog.show();
-                },
-                label: this.i18n.AddAttributes2
-            }).placeAt(this.openButton.domNode, "after");
-            this.addRemoveAttritbuteButton = new Button({
-                disabled: true,
-                onClick: function (val) {
-                    if (confirm(context.i18n.RemoveAttributeQ)) {
-                        var selection = context.grid.getSelected()
-                        context._removeAttribute(selection);
-                    }
-                },
-                label: context.i18n.RemoveAtttributes
-            }).placeAt(this.addAddAttritbuteButton.domNode, "after");
             dojo.destroy(this.id + "Open");
 
             this.store.mayHaveChildren = function (item) {
-                if (item.Attributes && item.Attributes.Attribute.length) {
+                if (!item.__hpcc_parentName) {
                     return true;
                 }
                 return false;
@@ -130,195 +83,81 @@ define([
                 this.__hpcc_parentName.push(child);
             };
 
+            this.saveButton = new Button({
+                onClick: function (evt) {
+                    context.saveMethod();
+                },
+                label: context.i18n.Save
+            }).placeAt(this.openButton, "after");
+
             var retVal = new declare([ESPUtil.Grid(false, true)])({
                 store: this.store,
                 sort: [{ attribute: "Name", descending: false }],
                 columns: {
-                    col1: selector({
-                        width: 27,
-                        selectorType: 'checkbox',
-                        sortable: false
-                    }),
-                    Name: tree({ label: context.i18n.Methods }),
-                    Attribute: editor({
-                        label: context.i18n.Attribute,
+                    Name: tree({ label: context.i18n.Methods}),
+                    Value: editor({
+                        label: this.i18n.MethodConfiguration,
                         autoSave: true,
                         canEdit: function (object, value) {
                             if (object.Attributes || !object.__hpcc_parentName) {
                                 return false;
                             }
-                            return true
+                            return true;
                         },
-                        editor: dijit.form.ValidationTextBox,
+                        editor: 'textarea',
                         editorArgs: {
-                            required: true,
-                            placeholder: context.i18n.AttributesAreRequired,
-                            promptMessage: context.i18n.AttributesAreRequired
-                        }
-                    }),
-                    Value: editor({ label: context.i18n.Value, editor: "text",
-                        autoSave: true,
-                        canEdit: function (object, value) {
-                            if (object.Attributes || !object.__hpcc_parentName) {
-                                return false;
-                            }
-                            return true
+                            rows: 10
                         }
                     })
                 }
             }, domID);
-
-            retVal.on("dgrid-select, dgrid-deselect", function (event) {
-                context.refreshActionState(event);
-            });
-
-            retVal.on("dgrid-datachange", function (event) {
-                context.refreshActionState(event);
-
-            });
             return retVal;
         },
 
-        _removeAttribute: function (selection) {
+        saveMethod: function () {
             var context = this;
-            //building out xml to pass into the service until the service fully complies to a add/remove/update type of mechanism
-            var removedQuery;
-            var passInXML;
-            var dynamicAttributes = "";
-            arrayUtil.forEach(selection, function (row, idx) {
-                if (row.__hpcc_parentName) {
-                    context.store.remove(row.__hpcc_id);
-                    removedQuery = context.store.query({__hpcc_parentName:row.__hpcc_parentName});
-                    if (removedQuery.length === 0) {
-                        passInXML = "<Methods><Method name='" + row.__hpcc_parentName + "'"+ "/></Methods>"
-                    }
-                }
-            });
-            arrayUtil.forEach(removedQuery, function (row, idx) {
-                dynamicAttributes += " " + row.Attribute + "=\'" + row.Value + "\'";
-                passInXML = "<Methods><Method name='" + row.__hpcc_parentName + "\'" + dynamicAttributes + "/></Methods>"
-            });
+            var userXML = "";
+            var results = this.store.query();
 
-            WsESDLConfig.ConfigureESDLBindingMethod({
-                request: {
-                    EspProcName: context.params.Configuration.__hpcc_parentName,
-                    EspBindingName: context.params.Configuration.Name,
-                    EsdlDefinitionID: context.params.Configuration.DefinitionID ? context.params.Configuration.DefinitionID : context.params.Methods.Id,
-                    EsdlServiceName: context.params.Configuration.Service,
-                    Overwrite: true,
-                    Config: passInXML
-                }
-            }).then(function (response) {
-                if (lang.exists("ConfigureESDLBindingMethodResponse.status", response)) {
-                    if (response.ConfigureESDLBindingMethodResponse.status.Code === 0) {
-                        //  MV Is this needed (idx is invalid)?:  context.store.remove(selection[idx].__hpcc_id);
-                    }
-                    context.refresh(context.params);
+            arrayUtil.forEach(results, function(row, idx){
+                if (row.__hpcc_parentName !== null && row.Value !="") {
+                    userXML += row.Value;
                 }
             });
-        },
-
-        _addAttribute: function (attr, val) {
-            //building out xml to pass into the service until the service fully complies to a add/remove/update type of mechanism
-            var context = this;
-            var selection = this.grid.getSelected();
-            var AddedQuery;
-            var passInXML;
-            var dynamicAttributes = "";
 
-            if (selection[0].Attributes) {
-                arrayUtil.forEach(selection, function (row, idx) {
-                    AddedQuery = context.store.query({__hpcc_parentName:row.Name});
-                    arrayUtil.forEach(AddedQuery, function (row, idx) {
-                        dynamicAttributes += " " + row.Attribute + "=\'" + row.Value +"\'";
-                        dynamicAttributes += " " + attr + "=\'" + val + "\'";
-                    });
-                    passInXML = "<Methods><Method name='" + row.Name + "\'" + dynamicAttributes + "/></Methods>"
-                });
-            }  else {
-                var passInXML = "<Methods><Method name='" + selection[0].__hpcc_id + "' " + attr + "='" + val +  "\'" + "/></Methods>";
-            }
-            WsESDLConfig.ConfigureESDLBindingMethod({
+            var xmlBuilder = "<Methods>" + userXML +  "</Methods>";
+            WsESDLConfig.PublishESDLBinding({
                 request: {
                     EspProcName: context.params.Configuration.__hpcc_parentName,
                     EspBindingName: context.params.Configuration.Name,
-                    EsdlDefinitionID: context.params.Configuration.DefinitionID ? context.params.Configuration.DefinitionID : context.params.Methods.Id ,
-                    EsdlServiceName: context.params.Configuration.Service,
+                    EspServiceName: context.params.Configuration.Service,
+                    EsdlDefinitionID: context.params.Configuration.DefinitionID,
                     Overwrite: true,
-                    Config: passInXML
+                    Config: xmlBuilder
                 }
             }).then(function (response) {
-                if (lang.exists("ConfigureESDLBindingMethodResponse.status", response)) {
-                    context.refresh(context.params);
-                }
-            });
-        },
-
-        refreshActionState: function (event) {
-            var context = this;
-            var selection = this.grid.getSelected();
-            var isParent = false;
-            var hasParent = false;
-
-            for (var i = 0; i < selection.length; ++i) {
-                if (selection[i] && selection[i].__hpcc_parentName !== null) {
-                        hasParent = true;
-                    } else {
-                        hasParent = false;
-                    }
-                if (selection[i].__hpcc_parentName === null && selection.length < 2) {
-                    isParent = true;
-                } else {
-                    isParent = false;
-                }
-            }
-
-            this.addAddAttritbuteButton.set("disabled", !isParent);
-            this.addRemoveAttritbuteButton.set("disabled", !hasParent);
-        },
-
-        _populatedByUser: function (params) {
-            var context = this;
-            var results = [];
-            var newRows = [];
-
-            results = params.Methods.Methods.Method;
-
-            arrayUtil.forEach(results, function (row, idx) {
-                lang.mixin(row, {
-                    __hpcc_parentName: null,
-                    __hpcc_id: row.Name
-                });
-
-                if (row.Attributes) {
-                    arrayUtil.forEach(row.Attributes.Attribute, function (attr, idx) {
-                        newRows.push({
-                            __hpcc_parentName: row.Name,
-                            __hpcc_id: row.Name + idx,
-                            Attribute: attr.Name,
-                            Value: attr.Value
+                if (lang.exists("PublishESDLBindingResponse.status", response)) {
+                    if (response.PublishESDLBindingResponse.status.Code === 0) {
+                        dojo.publish("hpcc/brToaster", {
+                            Severity: "Message",
+                            Source: "WsESDLConfig.PublishESDLBinding",
+                            Exceptions: [{ Source: context.i18n.SuccessfullySaved, Message: response.PublishESDLBindingResponse.status.Description }]
                         });
-                    });
+                    }
+                    context.refreshGrid();
                 }
             });
-
-            arrayUtil.forEach(newRows, function (newRow) {
-                results.push(newRow);
-            });
-
-            context.store.setData(results);
-            context.grid.set("query", {__hpcc_parentName: null });
         },
 
-        refreshGrid: function (params) {
+        refreshGrid: function () {
             var context = this;
             var results = [];
             var newRows = [];
 
             WsESDLConfig.GetESDLBinding({
                 request: {
-                    EspProcName: context.params.Configuration.__hpcc_parentName ? context.params.Configuration.__hpcc_parentName : params.Configuration.__hpcc_parentName ,
-                    EspBindingName: context.params.Configuration.Name ? context.params.Configuration.Name : params.Configuration.__hpcc_parentName ,
+                    EspProcName: context.params.Configuration.__hpcc_parentName,
+                    EspBindingName: context.params.Configuration.Name,
                     ReportMethodsAvailable: true
                 }
             }).then(function (response) {
@@ -333,16 +172,18 @@ define([
                     });
 
                     if (row.Attributes) {
-                        arrayUtil.forEach(row.Attributes.Attribute, function (attr, idx) {
-                            newRows.push({
-                                __hpcc_parentName: row.Name,
-                                __hpcc_id: row.Name + idx,
-                                Attribute: attr.Name,
-                                Value: attr.Value
-                            });
+                        newRows.push({
+                            __hpcc_parentName: row.Name,
+                            __hpcc_id: row.Name + idx,
+                            Value: row.XML
+                        });
+                    } else {
+                        newRows.push({
+                            __hpcc_parentName: row.Name,
+                            __hpcc_id: row.Name + idx,
+                            Value: "<Method name=\"" + row.Name + "\"/>"
                         });
                     }
-
                 });
 
                 arrayUtil.forEach(newRows, function (newRow) {
@@ -351,7 +192,7 @@ define([
 
                 context.store.setData(results);
                 context.grid.set("query", {__hpcc_parentName: null });
-            })
+            });
         }
     });
 });

esp/src/eclwatch/ESPGraph.js

@@ -626,8 +626,12 @@ define([
                                 break;
                             case "edge":
                                 var edge = this.walkDocument(childNode, childNode.getAttribute("id"));
-                                if (edge.NumRowsProcessed) {
+                                if (edge.NumRowsProcessed !== undefined) {
                                     edge._eclwatchCount = edge.NumRowsProcessed.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+                                } else if (edge.Count !== undefined) {
+                                    edge._eclwatchCount = edge.Count.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+                                } else if (edge.count !== undefined) {
+                                    edge._eclwatchCount = edge.count.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
                                 }
                                 if (edge.inputProgress) {
                                     edge._eclwatchInputProgress = "[" + edge.inputProgress.replace(/\B(?=(\d{3})+(?!\d))/g, ",") + "]";

esp/src/eclwatch/PackageMapPartsWidget.js

@@ -91,11 +91,15 @@ define([
              this.addPartsDropDown.on("apply", function (evt) {
                 if (context.addPartsDropDown.filterForm.validate()){
                     var addPartInput = context.getFilter();
+                    var packageMapSearch = context.params.packageMap.search("::");
+                    var packageMapClean;
+
+                    packageMapSearch > -1 ? packageMapClean = context.params.packageMap.split('::')[1] : packageMapClean = context.params.packageMap;
+
                     WsPackageMaps.AddPartToPackageMap({
                         request: {
-                            Target: addPartInput.AddPartsTarget,
-                            Process: addPartInput.Process,
-                            PackageMap: addPartInput.PackageMap,
+                            Target: context.params.target,
+                            PackageMap: packageMapClean,
                             PartName: addPartInput.PartName,
                             Content: addPartInput.Content,
                             DaliIp: addPartInput.DaliIp,
@@ -119,9 +123,6 @@ define([
             this.addPartsDropDown.placeAt(this.openButton.domNode, "after");
             this.addPartsDropDown.filterForm.set("style", "width:600px;");
             this.addPartsDropDown.filterDropDown.set("label", context.i18n.Add);
-            this.addPartsTargetSelect = this.createLabelAndElement("AddPartsTarget", this.i18n.Target, "TargetSelect", this.i18n.Target, "", "GetPackageMapTargets");
-            this.addPartsProcessesSelect = this.createLabelAndElement("Process", this.i18n.Processes, "TargetSelect", this.i18n.Processes, "", "GetPackageMapProcesses");
-            this.addPartsPackagemap = this.createLabelAndElement("PackageMap", this.i18n.PackageMap, "ValidationTextBox", this.i18n.PackageMap);
             this.addPartsPartName = this.createLabelAndElement("PartName", this.i18n.PartName, "ValidationTextBox", this.i18n.PartName);
             this.addPartsContent = this.createLabelAndElement("Content", this.i18n.Content, "Textarea", this.i18n.Content);
             this.addPartsDaliIp = this.createLabelAndElement("DaliIp", this.i18n.DaliIP, "TextBox", this.i18n.DaliIP);
@@ -166,7 +167,6 @@ define([
             }, domID);
 
             retVal.on(".dgrid-row:dblclick", function (evt) {
-                event.preventDefault();
                 if (context._onRowDblClick) {
                     var item = retVal.row(evt).data;
                     WsPackageMaps.GetPartFromPackageMap({
@@ -230,7 +230,7 @@ define([
             return this.addPartsDropDown.toObject();
         },
 
-        createLabelAndElement: function (id, label, element, placeholder, value, targetType) {
+        createLabelAndElement: function (id, label, element, placeholder, value) {
             var context = this;
             var control = null;
             switch (element) {
@@ -269,22 +269,6 @@ define([
                         value: value
                     });
                 break;
-                case "TargetSelect":
-                    control = new TargetSelectWidget ({
-                        id: id,
-                        name: id,
-                        style: "width: 40%"
-                    });
-                    if (targetType === "GetPackageMapTargets" ) {
-                        control.init({
-                            GetPackageMapTargets: true
-                        });
-                    } else {
-                        control.init({
-                            GetPackageMapProcesses: true
-                        });
-                    }
-                break;
             }
 
             if (control) {

esp/src/eclwatch/nls/es/hpcc.js

@@ -341,6 +341,7 @@ define(
     MaxSize: "Tamaňo máximo",
     MemberOf: "Miembro de",
     Members: "Miembros",
+    MethodConfiguration: "Configuracion de Método",
     Message: "Mensaje",
     Methods: "Métodos",
     Min: "Minimo",
@@ -600,6 +601,7 @@ define(
     Subgraphs: "Sub-gráficos",
     Submit: "Enviar",
     Subtype: "Subtipo",
+    SuccessfullySaved: "Guardado",
     Summary: "Resumen",
     SummaryMessage: "Mensaje resumido",
     SuperFile: "Super File",
@@ -677,6 +679,7 @@ define(
     title_LibrariesUsed: "Librerias Usadas",
     title_Log: "Archivo de Registro",
     title_LostFilesFor: "Archivos perdidos para",
+    title_Methods: "Métodos",
     title_LZBrowse: "Zonas de descarga",
     title_MemberOf: "Miembros de",
     title_Members: "Miembros",

esp/src/eclwatch/nls/hpcc.js

@@ -348,6 +348,7 @@ define({root:
     MaxRecordLength: "Max Record Length",
     MemberOf: "Member Of",
     Members: "Members",
+    MethodConfiguration: "Method Configuration",
     Message: "Message",
     Methods: "Methods",
     Min: "Min",
@@ -610,6 +611,7 @@ define({root:
     Subgraphs: "Subgraphs",
     Submit: "Submit",
     Subtype: "Subtype",
+    SuccessfullySaved: "Successfully Saved",
     Summary: "Summary",
     SummaryMessage: "Summary Message",
     Superfile: "Superfile",
@@ -685,6 +687,7 @@ define({root:
     title_Log: "Log File",
     title_LFDetails: "Logical File Details",
     title_LZBrowse: "Landing Zones",
+    title_Methods: "Methods",
     title_MemberOf: "Member Of",
     title_Members: "Members",
     title_LostFilesFor: "Lost files for",

esp/tools/soapplus/http.cpp

@@ -430,7 +430,7 @@ void Http::SplitURL(const char* url, StringBuffer& protocol,StringBuffer& UserNa
 
 HttpClient::HttpClient(IProperties* globals, const char* url, const char* inname, 
                        const char* outdir, const char* outfilename, bool writeToFiles,
-                       int doValidation, const char* xsdpath, bool isEspLogFile)
+                       int doValidation, const char* xsdpath, bool isEspLogFile) : m_stopstress(false)
 {
     m_globals = globals;
     if(url && *url)
@@ -489,12 +489,8 @@ HttpClient::HttpClient(IProperties* globals, const char* url, const char* inname
 
     if(globals && globals->hasProp("stressduration"))
     {
-        const char* numstr = globals->queryProp("stressthreads");
-        if(numstr && *numstr)
-            m_stressthreads = atoi(numstr);
-        numstr = globals->queryProp("stressduration");
-        if(numstr && *numstr)
-            m_stressduration = atoi(numstr);
+        m_stressthreads = globals->getPropInt("stressthreads", 0);
+        m_stressduration = globals->getPropInt("stressduration", 0);
     }
     else
     {

initfiles/etc/DIR_NAME/environment.conf.in

@@ -28,8 +28,10 @@ mpStart=7101
 mpEnd=7500
 mpSoMaxConn=128
 mpTraceLevel=0
-#enable SSL for dafilesrv remote file access
-#dfsUseSSL=false
+# enable SSL for dafilesrv remote file access (SSLNone/false | SSLOnly/true | SSLFirst | UnsecureFirst)
+#dfsUseSSL=SSLNone
+# note: if passphrase specified it must be encrypted
+#dfsSSLPassPhrase=
 #dfsSSLCertFile=/certfilepath/certfile
 #dfsSSLPrivateKeyFile=/keyfilepath/keyfile
 

initfiles/etc/DIR_NAME/environment.xml.in

@@ -777,6 +777,7 @@
   <logfields>TIM+DAT+MLT+MID+PID+TID+COD+QUO+PFX</logfields>
   <use_epoll>true</use_epoll>
   <udp_stats>true</udp_stats>
+  <dfsUseSSL>SSLNone</dfsUseSSL>
   <runtime>${RUNTIME_PATH}</runtime>
   <lock>${LOCK_PATH}</lock>
   <configs>${CONFIG_DIR}</configs>

plugins/couchbase/couchbaseembed.cpp

@@ -772,7 +772,7 @@ namespace couchbaseembed
     {
         const char * value = nextField(field);
 
-        if (!value && !*value)
+        if (!value || !*value)
         {
             NullFieldProcessor p(field);
             return p.boolResult;

roxie/ccd/ccdfile.cpp

@@ -1168,13 +1168,11 @@ public:
                 if ((dfsSize != (offset_t) -1 && dfsSize != f->getSize()) ||
                     (!dfsDate.isNull() && !dfsDate.equals(*f->queryDateTime(), false)))
                 {
-                    if (fileType == ROXIE_KEY)
-                    {
-                        // jhtree cache can keep files active and thus prevent us from loading a new version
+                    releaseSlaveDynamicFileCache();  // Slave dynamic file cache or...
+                    if (fileType == ROXIE_KEY)       // ...jhtree cache can keep files active and thus prevent us from loading a new version
                         clearKeyStoreCacheEntry(f);  // Will release iff that is the only link
-                        f.clear(); // Note - needs to be done before calling getValue() again, hence the need to make it separate from the f.set below
-                        f.set(files.getValue(localLocation));
-                    }
+                    f.clear(); // Note - needs to be done before calling getValue() again, hence the need to make it separate from the f.set below
+                    f.set(files.getValue(localLocation));
                     if (f)  // May have been cleared above...
                     {
                         StringBuffer modifiedDt;
@@ -1766,6 +1764,7 @@ protected:
 
 public:
     IMPLEMENT_IINTERFACE;
+
     CResolvedFile(const char *_lfn, const char *_physicalName, IDistributedFile *_dFile, RoxieFileType _fileType, IRoxieDaliHelper* _daliHelper, bool isDynamic, bool cacheIt, bool writeAccess, bool _isSuperFile)
     : daliHelper(_daliHelper), lfn(_lfn), physicalName(_physicalName), dFile(_dFile), fileType(_fileType), isSuper(_isSuperFile)
     {
@@ -2022,9 +2021,9 @@ public:
                     maxParts = numParts;
             }
 
-            IDefRecordMeta *thisDiskMeta = diskMeta.item(subFile);
             if (translators)
             {
+                IDefRecordMeta *thisDiskMeta = diskMeta.isItem(subFile) ? diskMeta.item(subFile) : nullptr;
                 if (fdesc && thisDiskMeta && activityMeta && !thisDiskMeta->equals(activityMeta))
                     if (allowFieldTranslation != IRecordLayoutTranslator::NoTranslation)
                         translators->append(createRecordLayoutTranslator(lfn, thisDiskMeta, activityMeta, allowFieldTranslation));
@@ -2423,7 +2422,7 @@ public:
     IMPLEMENT_IINTERFACE;
     CSlaveDynamicFileCache(unsigned _limit) : tableSize(_limit) {}
 
-    virtual IResolvedFile *lookupDynamicFile(const IRoxieContextLogger &logctx, const char *lfn, CDateTime &cacheDate, unsigned checksum, RoxiePacketHeader *header, bool isOpt, bool isLocal)
+    virtual IResolvedFile *lookupDynamicFile(const IRoxieContextLogger &logctx, const char *lfn, CDateTime &cacheDate, unsigned checksum, RoxiePacketHeader *header, bool isOpt, bool isLocal) override
     {
         if (logctx.queryTraceLevel() > 5)
         {
@@ -2467,6 +2466,11 @@ public:
         files.add(*ret.getLink(), 0);
         return ret.getClear();
     }
+
+    virtual void releaseAll() override
+    {
+        files.kill();
+    }
 };
 
 static CriticalSection slaveDynamicFileCacheCrit;
@@ -2486,7 +2490,8 @@ extern ISlaveDynamicFileCache *querySlaveDynamicFileCache()
 extern void releaseSlaveDynamicFileCache()
 {
     CriticalBlock b(slaveDynamicFileCacheCrit);
-    slaveDynamicFileCache.clear();
+    if (slaveDynamicFileCache)
+        slaveDynamicFileCache->releaseAll();
 }
 
 

roxie/ccd/ccdprotocol.cpp

@@ -291,7 +291,10 @@ public:
                         if (!secureContext)
                             secureContext.setown(createSecureSocketContextEx(certFile.get(), keyFile.get(), passPhrase.get(), ServerSocket));
                         ssock.setown(secureContext->createSecureSocket(client.getClear()));
-                        int status = ssock->secure_accept();
+                        int loglevel = 0;
+                        if (traceLevel > 1)
+                            loglevel = traceLevel;
+                        int status = ssock->secure_accept(loglevel);
                         if (status < 0)
                         {
                             // secure_accept may also DBGLOG() errors ...

roxie/ccd/ccdstate.hpp

@@ -79,6 +79,7 @@ extern IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *pac
 interface ISlaveDynamicFileCache : extends IInterface
 {
     virtual IResolvedFile *lookupDynamicFile(const IRoxieContextLogger &logctx, const char *lfn, CDateTime &cacheDate, unsigned checksum, RoxiePacketHeader *header, bool isOpt, bool isLocal) = 0;
+    virtual void releaseAll() = 0;
 };
 extern ISlaveDynamicFileCache *querySlaveDynamicFileCache();
 extern void releaseSlaveDynamicFileCache();

system/jlib/jlib.hpp

@@ -204,6 +204,8 @@ public:
     inline bool zap(TYPE * x)                   { return PointerArray::zap(x); }
 };
 
+enum SSLCfg { SSLNone = 0, SSLOnly, SSLFirst, UnsecureFirst };
+
 #include "jstring.hpp"
 #include "jarray.hpp"
 #include "jhash.hpp"

system/jlib/jutil.cpp

@@ -28,6 +28,7 @@
 #include "jfile.hpp"
 #include "jprop.hpp"
 #include "jerror.hpp"
+#include "jencrypt.hpp"
 #ifdef _WIN32
 #include <mmsystem.h> // for timeGetTime 
 #include <float.h> //for _isnan and _fpclass
@@ -2396,14 +2397,16 @@ jlib_decl const IProperties &queryEnvironmentConf()
 }
 
 static CriticalSection securitySettingsCrit;
-static bool useSSL = false;
+static SSLCfg useSSL = SSLNone;
 static StringAttr certificate;
 static StringAttr privateKey;
+static StringAttr passPhrase;
 static bool retrieved = false;
-jlib_decl bool querySecuritySettings(bool *          _useSSL,
+jlib_decl bool querySecuritySettings(SSLCfg *        _useSSL,
                                      unsigned short *_port,
                                      const char * *  _certificate,
-                                     const char * *  _privateKey)
+                                     const char * *  _privateKey,
+                                     const char * *  _passPhrase)
 {
     if (!retrieved)
     {
@@ -2417,11 +2420,31 @@ jlib_decl bool querySecuritySettings(bool *          _useSSL,
                 configFileSpec.set(CONFIG_DIR).append(PATHSEPSTR).append("environment.conf");
 #endif
                 Owned<IProperties> conf = createProperties(configFileSpec.str(), true);
-                useSSL = conf->getPropBool("dfsUseSSL", false);
+                StringAttr sslMethod;
+                sslMethod.set(conf->queryProp("dfsUseSSL"));
+                if (sslMethod)
+                {
+                    // checking for true | false for backward compatibility
+                    if ( strieq(sslMethod.str(), "SSLOnly") || strieq(sslMethod.str(), "true") )
+                        useSSL = SSLOnly;
+                    else if ( strieq(sslMethod.str(), "SSLFirst") )
+                        useSSL = SSLFirst;
+                    else if ( strieq(sslMethod.str(), "UnsecureFirst") )
+                        useSSL = UnsecureFirst;
+                    else // SSLNone or false or ...
+                        useSSL = SSLNone;
+                }
                 if (useSSL)
                 {
                     certificate.set(conf->queryProp("dfsSSLCertFile"));
                     privateKey.set(conf->queryProp("dfsSSLPrivateKeyFile"));
+                    const char *passPhrasePtr = conf->queryProp("dfsSSLPassPhrase");
+                    if (!isEmptyString(passPhrasePtr))
+                    {
+                        StringBuffer passPhraseStr;
+                        decrypt(passPhraseStr, passPhrasePtr);
+                        passPhrase.set(passPhraseStr.str());
+                    }
                 }
                 retrieved = true;
             }
@@ -2437,22 +2460,46 @@ jlib_decl bool querySecuritySettings(bool *          _useSSL,
         if (_useSSL)
             *_useSSL = useSSL;
         if (_port)
-            *_port = useSSL ? SECURE_DAFILESRV_PORT : DAFILESRV_PORT;
+        {
+            // port to try first (or only) ...
+            if ( (useSSL == SSLNone) || (useSSL == UnsecureFirst) )
+                *_port = DAFILESRV_PORT;
+            else
+                *_port = SECURE_DAFILESRV_PORT;
+        }
         if (_certificate)
             *_certificate = certificate.get();
         if (_privateKey)
             *_privateKey = privateKey.get();
+        if (_passPhrase)
+            *_passPhrase = passPhrase.get();
     }
     else
     {
         if (_useSSL)
-            *_useSSL = false;
+            *_useSSL = SSLNone;
         if (_port)
             *_port = DAFILESRV_PORT;
     }
     return retrieved;
 }
 
+jlib_decl bool queryDafsSecSettings(SSLCfg *        _useSSL,
+                                    unsigned short *_port,
+                                    unsigned short *_sslport,
+                                    const char * *  _certificate,
+                                    const char * *  _privateKey,
+                                    const char * *  _passPhrase)
+{
+    bool ret = querySecuritySettings(_useSSL, nullptr, _certificate, _privateKey, _passPhrase);
+    // these should really be in env, but currently they are not ...
+    if (_port)
+        *_port = DAFILESRV_PORT;
+    if (_sslport)
+        *_sslport = SECURE_DAFILESRV_PORT;
+    return ret;
+}
+
 static IPropertyTree *getOSSdirTree()
 {
     Owned<IPropertyTree> envtree = getHPCCEnvironment();

system/jlib/jutil.hpp

@@ -360,10 +360,18 @@ extern jlib_decl bool getConfigurationDirectory(const IPropertyTree *dirtree, //
                                                 const char *instance, 
                                                 StringBuffer &dirout);
 
-extern jlib_decl bool querySecuritySettings(bool *          _useSSL,
+extern jlib_decl bool querySecuritySettings(SSLCfg *        _useSSL,
                                             unsigned short *_port,
                                             const char * *  _certificate,
-                                            const char * *  _privateKey);
+                                            const char * *  _privateKey,
+                                            const char * *  _passPhrase);
+
+extern jlib_decl bool queryDafsSecSettings(SSLCfg *        _useSSL,
+                                           unsigned short *_port,
+                                           unsigned short *_sslport,
+                                           const char * *  _certificate,
+                                           const char * *  _privateKey,
+                                           const char * *  _passPhrase);
 
 extern jlib_decl const char * matchConfigurationDirectoryEntry(const char *path,const char *mask,StringBuffer &name, StringBuffer &component, StringBuffer &instance);
 extern jlib_decl bool replaceConfigurationDirectoryEntry(const char *path,const char *frommask,const char *tomask,StringBuffer &out);

system/security/securesocket/securesocket.cpp

@@ -147,8 +147,8 @@ public:
     CSecureSocket(int sockfd, SSL_CTX* ctx, bool verify = false, bool addres_match = false, CStringSet* m_peers = NULL, int loglevel=SSLogNormal);
     ~CSecureSocket();
 
-    virtual int secure_accept();
-    virtual int secure_connect();
+    virtual int secure_accept(int logLevel);
+    virtual int secure_connect(int logLevel);
 
     virtual void logPollError(unsigned revents, const char *rwstr);
     virtual int wait_read(unsigned timeoutms);
@@ -562,7 +562,7 @@ bool CSecureSocket::verify_cert(X509* cert)
     }
 }
 
-int CSecureSocket::secure_accept()
+int CSecureSocket::secure_accept(int logLevel)
 {
     int err;
     err = SSL_accept(m_ssl);
@@ -588,7 +588,8 @@ int CSecureSocket::secure_accept()
         return err;
     }
 
-    DBGLOG("SSL connection using %s", SSL_get_cipher(m_ssl));
+    if (logLevel)
+        DBGLOG("SSL connection using %s", SSL_get_cipher(m_ssl));
 
     if(m_verify)
     {
@@ -612,7 +613,7 @@ int CSecureSocket::secure_accept()
     return 0;
 }
 
-int CSecureSocket::secure_connect()
+int CSecureSocket::secure_connect(int logLevel)
 {
     int err = SSL_connect (m_ssl);                     
     if(err <= 0)
@@ -634,7 +635,8 @@ int CSecureSocket::secure_connect()
         // data exchange to be successful.
         
         // Get the cipher - opt
-        DBGLOG("SSL connection using %s\n", SSL_get_cipher (m_ssl));
+        if (logLevel)
+            DBGLOG("SSL connection using %s\n", SSL_get_cipher (m_ssl));
 
         // Get server's certificate (note: beware of dynamic allocation) - opt
         X509* server_cert = SSL_get_peer_certificate (m_ssl);
@@ -1060,12 +1062,12 @@ public:
 
     ISecureSocket* createSecureSocket(ISocket* sock, int loglevel)
     {
-        return new CSecureSocket(sock, m_ctx, m_verify, m_address_match, m_peers);
+        return new CSecureSocket(sock, m_ctx, m_verify, m_address_match, m_peers, loglevel);
     }
 
     ISecureSocket* createSecureSocket(int sockfd, int loglevel)
     {
-        return new CSecureSocket(sockfd, m_ctx, m_verify, m_address_match, m_peers);
+        return new CSecureSocket(sockfd, m_ctx, m_verify, m_address_match, m_peers, loglevel);
     }
 };
 

system/security/securesocket/securesocket.hpp

@@ -48,8 +48,8 @@ enum SecureSocketType
 // One instance per connection
 interface ISecureSocket : implements ISocket
 {
-    virtual int secure_accept() = 0;
-    virtual int secure_connect() = 0;
+    virtual int secure_accept(int logLevel=1) = 0;
+    virtual int secure_connect(int logLevel=1) = 0;
 };
 
 // One instance per program running

testing/regress/ecl/dynamicoptflag.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+#option ('warnOnImplicitReadLimit', true);
+#onwarning (4522, ignore);
 d := dataset(DYNAMIC('regress::no::such::file'), {string10 f}, FLAT, OPT);
 output(d);
 count(d);

testing/regress/ecl/indexsubstring.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+#option ('warnOnImplicitReadLimit', true);
+
 //version multiPart=false
 
 import ^ as root;

testing/regress/ecl/issue13588.ecl

@@ -15,6 +15,7 @@
     limitations under the License.
 ############################################################################## */
 
+#option ('warnOnImplicitReadLimit', true);
 
 xRec := RECORD
     integer x;

testing/regress/ecl/jsonfetch.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+#option ('warnOnImplicitReadLimit', true);
+
 phoneRecord :=
             RECORD
 string5         areaCode{xpath('@areaCode')};

testing/regress/ecl/key/dfs.xml

@@ -1,3 +1,4 @@
+<Warning><Code>3147</Code><Filename>dfs.ecl</Filename><Line>53</Line><Source>eclcc</Source><Message> Field dg_lastname type mismatch: DFS reports string10 but ECL declared string3</Message></Warning>
 <Dataset name='Result 1'>
  <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange></Row>
  <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange></Row>

testing/regress/ecl/key/dfsfj.xml

@@ -1,3 +1,4 @@
+<Warning><Code>3147</Code><Filename>dfsfj.ecl</Filename><Line>54</Line><Source>eclcc</Source><Message> Field dg_lastname type mismatch: DFS reports string10 but ECL declared string3</Message></Warning>
 <Dataset name='Result 1'>
  <Row><Result_1>Full-keyed joins, RHS translated, LHS not</Result_1></Row>
 </Dataset>

testing/regress/ecl/key/dfsfj2.xml

@@ -1,3 +1,5 @@
+<Warning><Code>3147</Code><Filename>dfsfj2.ecl</Filename><Line>54</Line><Source>eclcc</Source><Message> Field dg_lastname type mismatch: DFS reports string10 but ECL declared string3</Message></Warning>
+<Warning><Code>3147</Code><Filename>dfsfj2.ecl</Filename><Line>55</Line><Source>eclcc</Source><Message> Field dg_lastname type mismatch: DFS reports string10 but ECL declared string3</Message></Warning>
 <Dataset name='Result 1'>
  <Row><Result_1>Full-keyed joins, RHS and LHS translated</Result_1></Row>
 </Dataset>

testing/regress/ecl/key/dynamicoptflag.xml

@@ -1,3 +1,6 @@
+<Warning><Code>4523</Code><Filename>dynamicoptflag.ecl</Filename><Line>38</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'regress::nor::this'</Message></Warning>
+<Warning><Code>4523</Code><Filename>dynamicoptflag.ecl</Filename><Line>41</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'regress::nor::this'</Message></Warning>
+<Warning><Code>4523</Code><Filename>dynamicoptflag.ecl</Filename><Line>51</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'regress::nor::this'</Message></Warning>
 <Dataset name='Result 1'>
 </Dataset>
 <Dataset name='Result 2'>

testing/regress/ecl/key/indexsubstring.xml

@@ -1,3 +1,13 @@
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>55</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>56</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>57</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>58</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>59</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>60</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>64</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>65</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>66</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
+<Warning><Code>4523</Code><Filename>indexsubstring.ecl</Filename><Line>67</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'TST::postcode.key'</Message></Warning>
 <Dataset name='Result 1'>
 </Dataset>
 <Dataset name='Result 2'>

testing/regress/ecl/key/issue13588.xml

@@ -1,3 +1,4 @@
+<Warning><Code>4523</Code><Filename>issue13588.ecl</Filename><Line>39</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'REGRESS::TEMP::ISSUE13588'</Message></Warning>
 <Dataset name='Result 1'>
 </Dataset>
 <Dataset name='Result 2'>

testing/regress/ecl/key/jsonfetch.xml

@@ -1,3 +1,4 @@
+<Warning><Code>4523</Code><Filename>jsonfetch.ecl</Filename><Line>74</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'REGRESS::TEMP::jsonfetch.json.index'</Message></Warning>
 <Dataset name='Result 1'>
 </Dataset>
 <Dataset name='Result 2'>

testing/regress/ecl/key/loop10.xml

@@ -0,0 +1,42 @@
+<Dataset name='Result 1'>
+ <Row><i>4</i></Row>
+ <Row><i>5</i></Row>
+ <Row><i>9</i></Row>
+ <Row><i>10</i></Row>
+ <Row><i>14</i></Row>
+ <Row><i>15</i></Row>
+ <Row><i>19</i></Row>
+ <Row><i>20</i></Row>
+ <Row><i>24</i></Row>
+ <Row><i>25</i></Row>
+ <Row><i>29</i></Row>
+ <Row><i>30</i></Row>
+ <Row><i>34</i></Row>
+ <Row><i>35</i></Row>
+ <Row><i>39</i></Row>
+ <Row><i>40</i></Row>
+ <Row><i>44</i></Row>
+ <Row><i>45</i></Row>
+ <Row><i>49</i></Row>
+ <Row><i>50</i></Row>
+ <Row><i>54</i></Row>
+ <Row><i>55</i></Row>
+ <Row><i>59</i></Row>
+ <Row><i>60</i></Row>
+ <Row><i>64</i></Row>
+ <Row><i>65</i></Row>
+ <Row><i>69</i></Row>
+ <Row><i>70</i></Row>
+ <Row><i>74</i></Row>
+ <Row><i>75</i></Row>
+ <Row><i>79</i></Row>
+ <Row><i>80</i></Row>
+ <Row><i>84</i></Row>
+ <Row><i>85</i></Row>
+ <Row><i>89</i></Row>
+ <Row><i>90</i></Row>
+ <Row><i>94</i></Row>
+ <Row><i>95</i></Row>
+ <Row><i>99</i></Row>
+ <Row><i>100</i></Row>
+</Dataset>

testing/regress/ecl/key/loop3.xml

@@ -0,0 +1,16 @@
+<Dataset name='o1'>
+ <Row><id>3</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>3</id></Row>
+</Dataset>
+<Dataset name='o2'>
+ <Row><id>1</id></Row>
+ <Row><id>1</id></Row>
+ <Row><id>1</id></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+</Dataset>

testing/regress/ecl/key/loopif.xml

@@ -1,3 +1,4 @@
+<Warning><Code>2168</Code><Filename>loopif.ecl</Filename><Line>27</Line><Source>eclcc</Source><Message> Field 'i' in TABLE does not appear to be properly defined by grouping conditions</Message></Warning>
 <Dataset name='Result 1'>
  <Row><i>61</i><j>1</j><c>4</c></Row>
  <Row><i>62</i><j>2</j><c>4</c></Row>

testing/regress/ecl/key/prefetch2.xml

@@ -0,0 +1,24 @@
+<Dataset name='Result 1'>
+ <Row><word>boy  </word><doc>281474976710888</doc></Row>
+ <Row><word>sheep</word><doc>281474976710972</doc></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><word>boy  </word><doc>281474976710888</doc></Row>
+ <Row><word>sheep</word><doc>281474976710972</doc></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><word>boy  </word><doc>0</doc></Row>
+ <Row><word>sheep</word><doc>0</doc></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><word>boy  </word><doc>0</doc></Row>
+ <Row><word>sheep</word><doc>0</doc></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><word>boy  </word><doc>281474976710658</doc></Row>
+ <Row><word>sheep</word><doc>281474976710659</doc></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><word>boy  </word><doc>281474976710658</doc></Row>
+ <Row><word>sheep</word><doc>281474976710659</doc></Row>
+</Dataset>

testing/regress/ecl/key/subsort.xml

@@ -10,3 +10,205 @@
  <Row><a>B</a><b>5</b></Row>
  <Row><a>A</a><b>5</b></Row>
 </Dataset>
+<Dataset name='Result 2'>
+ <Row><a>101</a><b>1</b><c>49182</c><d>0</d><e>0</e></Row>
+ <Row><a>181</a><b>1</b><c>56238</c><d>0</d><e>1</e></Row>
+ <Row><a>193</a><b>3</b><c>17066</c><d>0</d><e>1</e></Row>
+ <Row><a>141</a><b>1</b><c>5926</c><d>0</d><e>2</e></Row>
+ <Row><a>137</a><b>7</b><c>25234</c><d>0</d><e>2</e></Row>
+ <Row><a>5</a><b>5</b><c>7038</c><d>0</d><e>3</e></Row>
+ <Row><a>47</a><b>7</b><c>14928</c><d>0</d><e>3</e></Row>
+ <Row><a>179</a><b>9</b><c>33124</c><d>0</d><e>3</e></Row>
+ <Row><a>83</a><b>3</b><c>56516</c><d>0</d><e>4</e></Row>
+ <Row><a>177</a><b>7</b><c>10010</c><d>0</d><e>5</e></Row>
+ <Row><a>43</a><b>3</b><c>34236</c><d>0</d><e>6</e></Row>
+ <Row><a>45</a><b>5</b><c>57350</c><d>0</d><e>6</e></Row>
+ <Row><a>195</a><b>5</b><c>40180</c><d>0</d><e>6</e></Row>
+ <Row><a>3</a><b>3</b><c>49460</c><d>0</d><e>7</e></Row>
+ <Row><a>31</a><b>1</b><c>45376</c><d>0</d><e>8</e></Row>
+ <Row><a>85</a><b>5</b><c>14094</c><d>0</d><e>8</e></Row>
+ <Row><a>125</a><b>5</b><c>64406</c><d>0</d><e>8</e></Row>
+ <Row><a>29</a><b>9</b><c>22262</c><d>0</d><e>8</e></Row>
+ <Row><a>123</a><b>3</b><c>41292</c><d>0</d><e>9</e></Row>
+ <Row><a>7</a><b>7</b><c>30152</c><d>0</d><e>9</e></Row>
+ <Row><a>99</a><b>9</b><c>26068</c><d>0</d><e>9</e></Row>
+ <Row><a>139</a><b>9</b><c>48348</c><d>0</d><e>9</e></Row>
+ <Row><a>12</a><b>2</b><c>22401</c><d>1</d><e>0</e></Row>
+ <Row><a>8</a><b>8</b><c>41709</c><d>1</d><e>0</e></Row>
+ <Row><a>160</a><b>0</b><c>10149</c><d>1</d><e>1</e></Row>
+ <Row><a>106</a><b>6</b><c>41431</c><d>1</d><e>1</e></Row>
+ <Row><a>198</a><b>8</b><c>9315</c><d>1</d><e>1</e></Row>
+ <Row><a>200</a><b>0</b><c>32429</c><d>1</d><e>2</e></Row>
+ <Row><a>184</a><b>4</b><c>25373</c><d>1</d><e>2</e></Row>
+ <Row><a>48</a><b>8</b><c>26485</c><d>1</d><e>2</e></Row>
+ <Row><a>88</a><b>8</b><c>48765</c><d>1</d><e>2</e></Row>
+ <Row><a>52</a><b>2</b><c>7177</c><d>1</d><e>4</e></Row>
+ <Row><a>64</a><b>4</b><c>33541</c><d>1</d><e>4</e></Row>
+ <Row><a>146</a><b>6</b><c>63711</c><d>1</d><e>5</e></Row>
+ <Row><a>10</a><b>0</b><c>64823</c><d>1</d><e>6</e></Row>
+ <Row><a>182</a><b>2</b><c>2259</c><d>1</d><e>6</e></Row>
+ <Row><a>104</a><b>4</b><c>18317</c><d>1</d><e>6</e></Row>
+ <Row><a>144</a><b>4</b><c>40597</c><d>1</d><e>6</e></Row>
+ <Row><a>186</a><b>6</b><c>48487</c><d>1</d><e>6</e></Row>
+ <Row><a>142</a><b>2</b><c>17483</c><d>1</d><e>7</e></Row>
+ <Row><a>50</a><b>0</b><c>49599</c><d>1</d><e>8</e></Row>
+ <Row><a>90</a><b>0</b><c>6343</c><d>1</d><e>8</e></Row>
+ <Row><a>66</a><b>6</b><c>56655</c><d>1</d><e>9</e></Row>
+ <Row><a>111</a><b>1</b><c>33680</c><d>2</d><e>1</e></Row>
+ <Row><a>93</a><b>3</b><c>41014</c><d>2</d><e>1</e></Row>
+ <Row><a>149</a><b>9</b><c>32846</c><d>2</d><e>3</e></Row>
+ <Row><a>189</a><b>9</b><c>17622</c><d>2</d><e>3</e></Row>
+ <Row><a>71</a><b>1</b><c>48904</c><d>2</d><e>4</e></Row>
+ <Row><a>53</a><b>3</b><c>18734</c><d>2</d><e>4</e></Row>
+ <Row><a>165</a><b>5</b><c>2398</c><d>2</d><e>5</e></Row>
+ <Row><a>187</a><b>7</b><c>60044</c><d>2</d><e>5</e></Row>
+ <Row><a>15</a><b>5</b><c>57072</c><d>2</d><e>6</e></Row>
+ <Row><a>95</a><b>5</b><c>64128</c><d>2</d><e>6</e></Row>
+ <Row><a>57</a><b>7</b><c>64962</c><d>2</d><e>6</e></Row>
+ <Row><a>109</a><b>9</b><c>10566</c><d>2</d><e>6</e></Row>
+ <Row><a>151</a><b>1</b><c>55960</c><d>2</d><e>7</e></Row>
+ <Row><a>13</a><b>3</b><c>33958</c><d>2</d><e>7</e></Row>
+ <Row><a>147</a><b>7</b><c>9732</c><d>2</d><e>7</e></Row>
+ <Row><a>69</a><b>9</b><c>25790</c><d>2</d><e>7</e></Row>
+ <Row><a>163</a><b>3</b><c>44820</c><d>2</d><e>8</e></Row>
+ <Row><a>107</a><b>7</b><c>52988</c><d>2</d><e>8</e></Row>
+ <Row><a>55</a><b>5</b><c>41848</c><d>2</d><e>9</e></Row>
+ <Row><a>17</a><b>7</b><c>14650</c><d>2</d><e>9</e></Row>
+ <Row><a>170</a><b>0</b><c>60183</c><d>3</d><e>1</e></Row>
+ <Row><a>114</a><b>4</b><c>2815</c><d>3</d><e>1</e></Row>
+ <Row><a>76</a><b>6</b><c>41153</c><d>3</d><e>1</e></Row>
+ <Row><a>18</a><b>8</b><c>26207</c><d>3</d><e>1</e></Row>
+ <Row><a>20</a><b>0</b><c>49321</c><d>3</d><e>2</e></Row>
+ <Row><a>60</a><b>0</b><c>34097</c><d>3</d><e>2</e></Row>
+ <Row><a>130</a><b>0</b><c>9871</c><d>3</d><e>2</e></Row>
+ <Row><a>112</a><b>2</b><c>45237</c><d>3</d><e>2</e></Row>
+ <Row><a>152</a><b>2</b><c>1981</c><d>3</d><e>2</e></Row>
+ <Row><a>34</a><b>4</b><c>61295</c><d>3</d><e>2</e></Row>
+ <Row><a>22</a><b>2</b><c>6899</c><d>3</d><e>3</e></Row>
+ <Row><a>156</a><b>6</b><c>48209</c><d>3</d><e>3</e></Row>
+ <Row><a>58</a><b>8</b><c>10983</c><d>3</d><e>3</e></Row>
+ <Row><a>168</a><b>8</b><c>37069</c><d>3</d><e>3</e></Row>
+ <Row><a>116</a><b>6</b><c>25929</c><d>3</d><e>4</e></Row>
+ <Row><a>36</a><b>6</b><c>18873</c><d>3</d><e>5</e></Row>
+ <Row><a>74</a><b>4</b><c>18039</c><d>3</d><e>6</e></Row>
+ <Row><a>128</a><b>8</b><c>52293</c><d>3</d><e>7</e></Row>
+ <Row><a>62</a><b>2</b><c>57211</c><d>3</d><e>8</e></Row>
+ <Row><a>154</a><b>4</b><c>25095</c><d>3</d><e>9</e></Row>
+ <Row><a>81</a><b>1</b><c>33402</c><d>4</d><e>0</e></Row>
+ <Row><a>121</a><b>1</b><c>18178</c><d>4</d><e>1</e></Row>
+ <Row><a>27</a><b>7</b><c>64684</c><d>4</d><e>1</e></Row>
+ <Row><a>63</a><b>3</b><c>3232</c><d>4</d><e>2</e></Row>
+ <Row><a>173</a><b>3</b><c>29318</c><d>4</d><e>2</e></Row>
+ <Row><a>133</a><b>3</b><c>44542</c><d>4</d><e>3</e></Row>
+ <Row><a>97</a><b>7</b><c>2954</c><d>4</d><e>3</e></Row>
+ <Row><a>117</a><b>7</b><c>37486</c><d>4</d><e>3</e></Row>
+ <Row><a>39</a><b>9</b><c>53544</c><d>4</d><e>4</e></Row>
+ <Row><a>159</a><b>9</b><c>17344</c><d>4</d><e>4</e></Row>
+ <Row><a>1</a><b>1</b><c>26346</c><d>4</d><e>5</e></Row>
+ <Row><a>25</a><b>5</b><c>41570</c><d>4</d><e>5</e></Row>
+ <Row><a>119</a><b>9</b><c>60600</c><d>4</d><e>5</e></Row>
+ <Row><a>135</a><b>5</b><c>2120</c><d>4</d><e>6</e></Row>
+ <Row><a>157</a><b>7</b><c>59766</c><d>4</d><e>6</e></Row>
+ <Row><a>41</a><b>1</b><c>11122</c><d>4</d><e>7</e></Row>
+ <Row><a>175</a><b>5</b><c>52432</c><d>4</d><e>7</e></Row>
+ <Row><a>23</a><b>3</b><c>18456</c><d>4</d><e>8</e></Row>
+ <Row><a>79</a><b>9</b><c>10288</c><d>4</d><e>9</e></Row>
+ <Row><a>192</a><b>2</b><c>5509</c><d>5</d><e>0</e></Row>
+ <Row><a>46</a><b>6</b><c>3371</c><d>5</d><e>0</e></Row>
+ <Row><a>98</a><b>8</b><c>14511</c><d>5</d><e>0</e></Row>
+ <Row><a>138</a><b>8</b><c>36791</c><d>5</d><e>0</e></Row>
+ <Row><a>140</a><b>0</b><c>59905</c><d>5</d><e>1</e></Row>
+ <Row><a>180</a><b>0</b><c>44681</c><d>5</d><e>2</e></Row>
+ <Row><a>44</a><b>4</b><c>45793</c><d>5</d><e>2</e></Row>
+ <Row><a>178</a><b>8</b><c>21567</c><d>5</d><e>2</e></Row>
+ <Row><a>4</a><b>4</b><c>61017</c><d>5</d><e>3</e></Row>
+ <Row><a>30</a><b>0</b><c>33819</c><d>5</d><e>4</e></Row>
+ <Row><a>124</a><b>4</b><c>52849</c><d>5</d><e>4</e></Row>
+ <Row><a>126</a><b>6</b><c>10427</c><d>5</d><e>4</e></Row>
+ <Row><a>100</a><b>0</b><c>37625</c><d>5</d><e>5</e></Row>
+ <Row><a>122</a><b>2</b><c>29735</c><d>5</d><e>5</e></Row>
+ <Row><a>194</a><b>4</b><c>28623</c><d>5</d><e>5</e></Row>
+ <Row><a>28</a><b>8</b><c>10705</c><d>5</d><e>5</e></Row>
+ <Row><a>84</a><b>4</b><c>2537</c><d>5</d><e>6</e></Row>
+ <Row><a>6</a><b>6</b><c>18595</c><d>5</d><e>7</e></Row>
+ <Row><a>196</a><b>6</b><c>51737</c><d>5</d><e>7</e></Row>
+ <Row><a>102</a><b>2</b><c>60739</c><d>5</d><e>9</e></Row>
+ <Row><a>86</a><b>6</b><c>25651</c><d>5</d><e>9</e></Row>
+ <Row><a>127</a><b>7</b><c>21984</c><d>6</d><e>0</e></Row>
+ <Row><a>197</a><b>7</b><c>63294</c><d>6</d><e>0</e></Row>
+ <Row><a>199</a><b>9</b><c>20872</c><d>6</d><e>0</e></Row>
+ <Row><a>91</a><b>1</b><c>17900</c><d>6</d><e>2</e></Row>
+ <Row><a>143</a><b>3</b><c>29040</c><d>6</d><e>2</e></Row>
+ <Row><a>67</a><b>7</b><c>2676</c><d>6</d><e>3</e></Row>
+ <Row><a>103</a><b>3</b><c>6760</c><d>6</d><e>4</e></Row>
+ <Row><a>89</a><b>9</b><c>60322</c><d>6</d><e>4</e></Row>
+ <Row><a>49</a><b>9</b><c>38042</c><d>6</d><e>5</e></Row>
+ <Row><a>11</a><b>1</b><c>10844</c><d>6</d><e>6</e></Row>
+ <Row><a>51</a><b>1</b><c>61156</c><d>6</d><e>7</e></Row>
+ <Row><a>161</a><b>1</b><c>21706</c><d>6</d><e>7</e></Row>
+ <Row><a>183</a><b>3</b><c>13816</c><d>6</d><e>7</e></Row>
+ <Row><a>145</a><b>5</b><c>52154</c><d>6</d><e>7</e></Row>
+ <Row><a>185</a><b>5</b><c>36930</c><d>6</d><e>7</e></Row>
+ <Row><a>65</a><b>5</b><c>45098</c><d>6</d><e>8</e></Row>
+ <Row><a>105</a><b>5</b><c>29874</c><d>6</d><e>8</e></Row>
+ <Row><a>87</a><b>7</b><c>37208</c><d>6</d><e>9</e></Row>
+ <Row><a>9</a><b>9</b><c>53266</c><d>6</d><e>9</e></Row>
+ <Row><a>110</a><b>0</b><c>22123</c><d>7</d><e>0</e></Row>
+ <Row><a>14</a><b>4</b><c>45515</c><d>7</d><e>1</e></Row>
+ <Row><a>54</a><b>4</b><c>30291</c><d>7</d><e>2</e></Row>
+ <Row><a>150</a><b>0</b><c>44403</c><d>7</d><e>3</e></Row>
+ <Row><a>92</a><b>2</b><c>29457</c><d>7</d><e>3</e></Row>
+ <Row><a>164</a><b>4</b><c>56377</c><d>7</d><e>3</e></Row>
+ <Row><a>16</a><b>6</b><c>3093</c><d>7</d><e>3</e></Row>
+ <Row><a>68</a><b>8</b><c>14233</c><d>7</d><e>3</e></Row>
+ <Row><a>70</a><b>0</b><c>37347</c><d>7</d><e>5</e></Row>
+ <Row><a>32</a><b>2</b><c>38181</c><d>7</d><e>5</e></Row>
+ <Row><a>56</a><b>6</b><c>53405</c><d>7</d><e>5</e></Row>
+ <Row><a>166</a><b>6</b><c>13955</c><d>7</d><e>5</e></Row>
+ <Row><a>108</a><b>8</b><c>64545</c><d>7</d><e>5</e></Row>
+ <Row><a>94</a><b>4</b><c>52571</c><d>7</d><e>6</e></Row>
+ <Row><a>148</a><b>8</b><c>21289</c><d>7</d><e>6</e></Row>
+ <Row><a>188</a><b>8</b><c>6065</c><d>7</d><e>6</e></Row>
+ <Row><a>72</a><b>2</b><c>60461</c><d>7</d><e>7</e></Row>
+ <Row><a>162</a><b>2</b><c>33263</c><d>7</d><e>7</e></Row>
+ <Row><a>190</a><b>0</b><c>29179</c><d>7</d><e>9</e></Row>
+ <Row><a>115</a><b>5</b><c>14372</c><d>8</d><e>0</e></Row>
+ <Row><a>59</a><b>9</b><c>22540</c><d>8</d><e>0</e></Row>
+ <Row><a>35</a><b>5</b><c>7316</c><d>8</d><e>1</e></Row>
+ <Row><a>75</a><b>5</b><c>29596</c><d>8</d><e>1</e></Row>
+ <Row><a>37</a><b>7</b><c>30430</c><d>8</d><e>2</e></Row>
+ <Row><a>61</a><b>1</b><c>45654</c><d>8</d><e>4</e></Row>
+ <Row><a>33</a><b>3</b><c>49738</c><d>8</d><e>4</e></Row>
+ <Row><a>167</a><b>7</b><c>25512</c><d>8</d><e>4</e></Row>
+ <Row><a>19</a><b>9</b><c>37764</c><d>8</d><e>4</e></Row>
+ <Row><a>171</a><b>1</b><c>6204</c><d>8</d><e>5</e></Row>
+ <Row><a>113</a><b>3</b><c>56794</c><d>8</d><e>5</e></Row>
+ <Row><a>129</a><b>9</b><c>63850</c><d>8</d><e>5</e></Row>
+ <Row><a>191</a><b>1</b><c>40736</c><d>8</d><e>6</e></Row>
+ <Row><a>73</a><b>3</b><c>6482</c><d>8</d><e>6</e></Row>
+ <Row><a>169</a><b>9</b><c>48626</c><d>8</d><e>6</e></Row>
+ <Row><a>131</a><b>1</b><c>21428</c><d>8</d><e>7</e></Row>
+ <Row><a>77</a><b>7</b><c>52710</c><d>8</d><e>7</e></Row>
+ <Row><a>21</a><b>1</b><c>60878</c><d>8</d><e>8</e></Row>
+ <Row><a>153</a><b>3</b><c>13538</c><d>8</d><e>9</e></Row>
+ <Row><a>155</a><b>5</b><c>36652</c><d>8</d><e>9</e></Row>
+ <Row><a>82</a><b>2</b><c>44959</c><d>9</d><e>0</e></Row>
+ <Row><a>96</a><b>6</b><c>56933</c><d>9</d><e>0</e></Row>
+ <Row><a>136</a><b>6</b><c>13677</c><d>9</d><e>0</e></Row>
+ <Row><a>24</a><b>4</b><c>30013</c><d>9</d><e>1</e></Row>
+ <Row><a>176</a><b>6</b><c>63989</c><d>9</d><e>1</e></Row>
+ <Row><a>2</a><b>2</b><c>37903</c><d>9</d><e>3</e></Row>
+ <Row><a>42</a><b>2</b><c>22679</c><d>9</d><e>3</e></Row>
+ <Row><a>172</a><b>2</b><c>17761</c><d>9</d><e>3</e></Row>
+ <Row><a>38</a><b>8</b><c>41987</c><d>9</d><e>3</e></Row>
+ <Row><a>158</a><b>8</b><c>5787</c><d>9</d><e>3</e></Row>
+ <Row><a>78</a><b>8</b><c>64267</c><d>9</d><e>4</e></Row>
+ <Row><a>80</a><b>0</b><c>21845</c><d>9</d><e>5</e></Row>
+ <Row><a>174</a><b>4</b><c>40875</c><d>9</d><e>5</e></Row>
+ <Row><a>134</a><b>4</b><c>56099</c><d>9</d><e>6</e></Row>
+ <Row><a>26</a><b>6</b><c>53127</c><d>9</d><e>6</e></Row>
+ <Row><a>40</a><b>0</b><c>65101</c><d>9</d><e>8</e></Row>
+ <Row><a>120</a><b>0</b><c>6621</c><d>9</d><e>8</e></Row>
+ <Row><a>118</a><b>8</b><c>49043</c><d>9</d><e>8</e></Row>
+ <Row><a>132</a><b>2</b><c>32985</c><d>9</d><e>9</e></Row>
+</Dataset>

testing/regress/ecl/key/xmlfetch2.xml

@@ -1,3 +1,4 @@
+<Warning><Code>4523</Code><Filename>xmlfetch2.ecl</Filename><Line>73</Line><Source>eclcc</Source><Message> Neither LIMIT() nor CHOOSEN() supplied for index read on  'REGRESS::TEMP::xmlfetch2.xml.index'</Message></Warning>
 <Dataset name='Result 1'>
 </Dataset>
 <Dataset name='Result 2'>

testing/regress/ecl/loop10.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//noroxie   - roxie doesn't support spilling to disk with the results being read from a child graph, the spill file is too large for memory
+
+//generate an error if a workunit spill is used
+#option ('outputLimitMb', 1);
+
+r := record
+  integer i;
+end;
+
+r t(integer c) := transform
+  SELF.i := c;
+end;
+
+ds1 := NOCOMBINE(dataset(1000000, t(COUNTER)));
+ds2 := sort(ds1, i)(i != 5);
+
+ds3 := DATASET(100, t(COUNTER));
+
+f(dataset(r) inDs) := join(inDs, ds2, LEFT.i % 10 = RIGHT.i, t(LEFT.i+1));
+
+l := loop(ds3, 3, f(ROWS(LEFT)));
+
+sort(l, i);

testing/regress/ecl/loop3.ecl

@@ -0,0 +1,31 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+r := { unsigned id };
+ds := dataset([1,2,3,4], r);
+
+
+f(dataset(r) inDs) := FUNCTION
+
+    o1 := output(inDs((id % 4) = 3),named('o1'),extend);
+    o2 := output(inDs((id % 4) = 1),named('o2'),extend);
+    return WHEN(inDs, parallel(o1, o2));
+END;
+
+l := LOOP(ds, 3, f(ROWS(LEFT)));
+
+output(l);

testing/regress/ecl/prefetch2.ecl

@@ -0,0 +1,62 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=file
+//version multiPart=true
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, true);
+useLocal := #IFDEFINED(root.useLocal, false);
+useTranslation := #IFDEFINED(root.useTranslation, false);
+
+//--- end of version configuration ---
+
+#option ('layoutTranslationEnabled', useTranslation);
+
+import $.setup;
+import setup.TS;
+Files := setup.Files(multiPart, useLocal, useTranslation);
+
+in := dataset([{'boy'}, {'sheep'}], {string5 word});
+
+outrec := record
+  string5 word;
+  unsigned doc;
+END;
+
+outrec trans1(in L) := TRANSFORM
+  SELF.word := L.word;
+  SELF.doc := SORTED(Files.getSearchSuperIndex()(KEYED(kind = TS.kindType.TextEntry),keyed(word = l.word)), doc)[1].doc;
+END;
+
+outrec trans2(in L) := TRANSFORM
+  SELF.word := L.word;
+  SELF.doc := SORTED(Files.getSearchSuperIndex()(KEYED(kind = TS.kindType.TextEntry),keyed(word = 'gobbledegook'+l.word)), doc)[1].doc;
+END;
+
+outrec trans3(in L) := TRANSFORM
+  SELF.word := L.word;
+  SELF.doc := STEPPED(Files.getSearchSuperIndex()(KEYED(kind = TS.kindType.TextEntry),keyed(word = l.word)), doc)[1].doc;
+END;
+
+
+output(project(in, trans1(LEFT))) : independent;
+output(project(in, trans1(LEFT), PREFETCH(20))) : independent;
+output(project(in, trans2(LEFT))) : independent;
+output(project(in, trans2(LEFT), PREFETCH(20, PARALLEL))) : independent;
+output(project(in, trans3(LEFT))) : independent;
+output(project(in, trans3(LEFT), PREFETCH(20, PARALLEL))) : independent;

testing/regress/ecl/setup/files.ecl

@@ -184,5 +184,6 @@ EXPORT NameWordIndex() := '~REGRESS::' + indexPrefix + '::wordIndex' + IF(useLoc
 EXPORT NameSearchIndex      := '~REGRESS::' + indexPrefix + '::searchIndex';
 EXPORT getWordIndex() := INDEX(TS.textSearchIndex, NameWordIndex());
 EXPORT getSearchIndex() := INDEX(TS.textSearchIndex, NameSearchIndex);
+EXPORT getSearchSuperIndex() := INDEX(TS.textSearchIndex, '{' + NameSearchIndex + ',' + NameWordIndex() + '}');
 
 END;

testing/regress/ecl/subsort.ecl

@@ -12,3 +12,24 @@ d := dataset([
 ], { string a, integer b });
 
 SUBSORT(d, {a}, {b});
+
+
+//Test implicit subsort where some of the sort fields are duplicated with cast versions of the same fields.
+
+{unsigned2 a,unsigned2 b,unsigned2 c,unsigned2 d,unsigned2 e} t(unsigned cnt) := TRANSFORM
+    SELF.a := cnt;
+    SELF.b := cnt % 10;
+    SELF.c := HASH64(cnt);
+    SELF.d := HASH32(cnt) % 10;
+    SELF.e := HASH(cnt) % 10;
+END;
+
+ds := dataset(200, t(COUNTER));
+
+s1 := SORT(ds, (unsigned8)d,(unsigned8)e,RECORD);
+f1 := NOCOMBINE(s1);
+
+s2 := SORT(f1, (unsigned8)d,(unsigned8)e,b,RECORD);
+f2 := NOCOMBINE(s2);
+s3 := SORTED(f2, (unsigned8)d,(unsigned8)e,b,RECORD, ASSERT,LOCAL);
+output(s3);

testing/regress/ecl/thor/update.xml

@@ -1,12 +1,12 @@
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph3[28], diskwrite[30]: output file = &apos;thor::regress::base.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[38]: output file = &apos;thor::regress::out5.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[44]: output file = &apos;thor::regress::out2.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[42]: output file = &apos;thor::regress::out3.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[40]: output file = &apos;thor::regress::out4.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[46]: output file = &apos;thor::regress::out1.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph5[47], diskwrite[56]: output file = &apos;thor::regress::out6.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph6[57], diskwrite[66]: output file = &apos;thor::regress::out6.d00&apos; - is up to date - it will not be rebuilt</Message></Warning>
-<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph7[67], indexwrite[70]: output file = &apos;thor::regress::outi1.idx&apos; - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph3[28], diskwrite[30]: output file = 'thor::regress::base.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[38]: output file = 'thor::regress::out5.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[44]: output file = 'thor::regress::out2.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[42]: output file = 'thor::regress::out3.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[40]: output file = 'thor::regress::out4.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph4[31], diskwrite[46]: output file = 'thor::regress::out1.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph5[47], diskwrite[56]: output file = 'thor::regress::out6.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph6[57], diskwrite[66]: output file = 'thor::regress::out6.d00' - is up to date - it will not be rebuilt</Message></Warning>
+<Warning><Code>10125</Code><Source>master</Source><Message>Graph graph7[67], indexwrite[70]: output file = 'thor::regress::outi1.idx' - is up to date - it will not be rebuilt</Message></Warning>
 <Dataset name='Result 1'>
 </Dataset>
 <Dataset name='Result 2'>

testing/regress/ecl/xmlfetch2.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+#option ('warnOnImplicitReadLimit', true);
+
 phoneRecord :=
             RECORD
 string5         areaCode{xpath('@areaCode')};