Merge remote-tracking branch 'upstream/closedown-4.0.x' into jakes-merged-master

Conflicts:
	dali/base/dadfs.cpp

Signed-off-by: Jake Smith <jake.smith@lexisnexis.com>

baseaddr.txt

@@ -103,7 +103,6 @@ icudata         0x01300000 0x00810000
 snmputils       0x014A0000 0x00040000
 schedulectrl    0x014E0000 0x00040000
 
-roxiemanager    0x01560000 0x00080000
 roxiecommlib    0x01600000 0x00080000
 
 ; plugins.

cmake_modules/dependencies/raring.cmake

@@ -0,0 +1 @@
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.49.0, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync")

common/CMakeLists.txt

@@ -20,7 +20,6 @@ HPCC_ADD_SUBDIRECTORY (fileview2 "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (monitoring "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (remote)
 HPCC_ADD_SUBDIRECTORY (roxiecommlib)
-HPCC_ADD_SUBDIRECTORY (roxiemanager "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (thorhelper)
 HPCC_ADD_SUBDIRECTORY (workunit)
 HPCC_ADD_SUBDIRECTORY (wuwebview "PLATFORM")

common/roxiemanager/CMakeLists.txt

@@ -1,90 +0,0 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-
-# Component: roxiemanager 
-#####################################################
-# Description:
-# ------------
-#    Cmake Input File for roxiemanager
-#####################################################
-
-project( roxiemanager ) 
-
-include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
-
-set (    SRCS 
-         ../../dali/dfu/dfuutil.cpp 
-         ${ESPSCM_GENERATED_DIR}/roxiecommlibscm_esp.cpp 
-         ${ESPSCM_GENERATED_DIR}/roxiemanagerscm_esp.cpp 
-         roxiequerycompiler.cpp 
-         roxiequerymanager.cpp 
-         roxiewuprocessor.cpp 
-                 
-                 roxiemanager.hpp
-                 roxiequerycompiler.hpp
-                 roxiewuprocessor.hpp
-                 
-                ${HPCC_SOURCE_DIR}/esp/scm/roxiemanagerscm.ecm
-    )
-
-include_directories ( 
-         ./../../system/security/shared
-         ./../../common/remote 
-         ./../../system/mp 
-         ./../../common/workunit 
-         ./../../dali/dfu 
-         ./../../common/environment 
-         ./../../roxie/roxie
-         ./../../roxie/ccd
-         ./../../common/fileview2 
-         ./../../system/include 
-         ./../../dali/base 
-         ./../../rtl/include 
-         ./../../common/dllserver 
-         ./../../system/xmllib
-         ./../../esp/clients 
-         ./../../esp/platform 
-         ./../../esp/bindings 
-         ./../../esp/bindings/SOAP/xpp 
-         ./../../system/jlib 
-         ./../../rtl/eclrtl 
-    )
-
-ADD_DEFINITIONS( -D_USRDLL -DROXIEMANAGER_EXPORTS )
-
-HPCC_ADD_LIBRARY( roxiemanager SHARED ${SRCS} )
-add_dependencies( roxiemanager espscm )
-install ( TARGETS roxiemanager RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
-target_link_libraries ( roxiemanager
-         jlib 
-         roxiecommlib 
-         mp 
-         hrpc 
-         remote 
-         dalibase 
-         environment 
-         dllserver 
-         nbcd 
-         eclrtl 
-         deftype 
-         workunit 
-         jhtree 
-         hql 
-         fileview2 
-    )
-
-

common/roxiemanager/roxiemanager.hpp

@@ -1,50 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef ROXIEQUERYMANAGER_INCL
-#define ROXIEQUERYMANAGER_INCL
-
-#ifdef _WIN32
-    #ifdef ROXIEMANAGER_EXPORTS
-        #define ROXIEMANAGER_API __declspec(dllexport)
-    #else
-        #define ROXIEMANAGER_API __declspec(dllimport)
-    #endif
-#else
-    #define ROXIEMANAGER_API
-#endif
-
-#include "roxiemanagerscm.hpp"
-
-#include "errorlist.h"
-
-#define ROXIEMANAGER_IGNORE_EXCEPTION   0   // not a real exception
-
-#define ROXIEMANAGER_UNRESOLVED_FILE    ROXIE_MGR_START
-#define ROXIEMANAGER_MISSING_FILE_PARTS ROXIE_MGR_START+1
-#define ROXIEMANAGER_MISSING_ID         ROXIE_MGR_START+2
-#define ROXIEMANAGER_SOCKET_ERROR       ROXIE_MGR_START+3
-#define ROXIEMANAGER_UNEXPECTION_WU_ERROR ROXIE_MGR_START+4
-#define ROXIEMANAGER_FILE_MISMATCH      ROXIE_MGR_START+5
-#define ROXIEMANAGER_DEPLOY_FAILED      ROXIE_MGR_START+6
-#define ROXIEMANAGER_DALI_LOOKUP_ERROR  ROXIE_MGR_START+7
-#define ROXIEMANAGER_ROD_NOT_SUPPORTED  ROXIE_MGR_START+8
-#define ROXIEMANAGER_FILE_PERMISSION_ERR ROXIE_MGR_START+9
-#define ROXIEMANAGER_FILE_SIZE_ERROR    ROXIE_MGR_START+10
-
-#endif
-

common/roxiemanager/roxiequerycompiler.cpp

@@ -1,323 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#pragma warning (disable : 4786)
-
-#include "jlib.hpp"
-#include "roxiequerycompiler.hpp"
-
-class CRoxieQueryCompileInfo : public CInterface, implements IRoxieQueryCompileInfo
-{
-private:
-    unsigned memoryLimit;
-    unsigned wuTimeOut;
-    unsigned timeLimit;
-    unsigned warnTimeLimit;
-
-    bool poolGraphs;
-    bool highPriority;
-
-    StringBuffer repositoryLabel;
-    StringBuffer moduleName;
-    StringBuffer defaultStyleName;
-    StringBuffer wuDebugOptions;
-
-    StringBuffer userName;
-    StringBuffer password;
-    StringBuffer jobName;
-    StringBuffer ecl;
-    StringBuffer appName;
-    StringBuffer clusterName;
-    int queryPriority;
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CRoxieQueryCompileInfo(const char *_ecl, const char *_jobName, const char *_clusterName, const char *_appName)
-        :ecl(_ecl), jobName(_jobName), clusterName(_clusterName)
-    {
-        memoryLimit = 0;
-        wuTimeOut = 0;
-        timeLimit = 0;
-        warnTimeLimit = 0;
-        poolGraphs = false;
-        highPriority = false;
-        queryPriority = UNKNOWN_PRIORITY;
-        
-        if (_appName && *_appName)
-            appName.append(_appName);
-        else
-            appName.append("RoxieManager");
-    }
-
-    virtual void setMemoryLimit(unsigned val) { memoryLimit = val; }
-    virtual unsigned getMemoryLimit() { return memoryLimit; }
-    
-    virtual void setWuTimeOut(unsigned val) { wuTimeOut = val; }
-    virtual unsigned getWuTimeOut() { return wuTimeOut; }
-
-    virtual void setTimeLimit(unsigned val) { timeLimit = val; }
-    virtual unsigned getTimeLimit() { return timeLimit; }
-
-    virtual void setWarnTimeLimit(unsigned val) { warnTimeLimit = val; }
-    virtual unsigned getWarnTimeLimit() { return warnTimeLimit; }
-
-    virtual void setPoolGraphs(bool val) { poolGraphs = val; }
-    virtual bool getPoolGraphs() { return poolGraphs; }
-
-    virtual void setHighPriority(bool val) { highPriority = val; }
-    virtual bool getHighPriority() { return highPriority; }
-
-    virtual void setRepositoryLabel(const char *val) { repositoryLabel.append(val); }
-    virtual const char * queryRepositoryLabel() { return repositoryLabel.str(); }
-
-    virtual void enableWebServiceInfoRetrieval(const char *_moduleName, const char *_defaultStyleName) 
-        { moduleName.append(_moduleName); defaultStyleName.append(_defaultStyleName);}
-
-    virtual void setQueryPriority(int val) { queryPriority = val; }
-    virtual int getQueryPriority() { return queryPriority; }
-
-    virtual void setWuDebugOptions(const char *val) { wuDebugOptions.append(val); }
-    virtual const char * queryWuDebugOptions() { return wuDebugOptions.str(); }
-
-    virtual const char * queryUserId() { return userName.str(); }
-    virtual const char * queryPassword() { return password.str(); }
-    virtual const char * queryJobName() { return jobName.str(); }
-    virtual const char * queryEcl() { return ecl.str(); }
-    virtual const char * queryAppName() { return appName.str(); }
-    virtual const char * queryClusterName() { return clusterName.str(); }
-    virtual const char * queryDefaultStyleName() { return defaultStyleName.str(); }
-    virtual const char * queryModuleName() { return moduleName.str(); }
-};
-
-
-
-
-class CRoxieQueryCompiler : public CInterface, implements IRoxieQueryCompiler
-{
-private:
-    SocketEndpoint ep;
-
-    void processCompilationErrors(IConstWorkUnit *workunit, SCMStringBuffer &errors)
-    {
-        if (workunit->getExceptionCount())
-        {
-            Owned<IConstWUExceptionIterator> exceptions = &workunit->getExceptions();
-            StringArray errorStr;
-            StringArray warnStr;
-            ForEach (*exceptions)
-            {
-                IConstWUException &exception = exceptions->query();
-
-                unsigned exceptCode = exception.getExceptionCode();
-                SCMStringBuffer name;
-                exception.getExceptionFileName(name);
-
-                unsigned lineNo = exception.getExceptionLineNo();
-                unsigned column = exception.getExceptionColumn();
-
-                SCMStringBuffer source, message;
-                exception.getExceptionSource(source);
-                exception.getExceptionMessage(message);
-
-                if (exception.getSeverity() == ExceptionSeverityWarning)
-                {
-                    StringBuffer err;
-                    if ( (lineNo != 0) || (column != 0))
-                        err.appendf("WARNING: %s (%d, %d): %d %s", name.str(), lineNo, column, exceptCode, message.str());
-                    else
-                        err.appendf("WARNING: %s: %d %s", name.str(), exceptCode, message.str());
-                    warnStr.append(err.str());
-                }
-                else if (exception.getSeverity() == ExceptionSeverityError)
-                {
-                    StringBuffer err;
-                    if ( (lineNo != 0) || (column != 0))
-                        err.appendf("ERROR: %s (%d, %d): %d %s", name.str(), lineNo, column, exceptCode, message.str());
-                    else
-                        err.appendf("ERROR: %s: %d %s", name.str(), exceptCode, message.str());
-                    errorStr.append(err.str());
-                }
-            }
-
-            ForEachItemIn(err_idx, errorStr)
-            {
-                const char *err = errorStr.item(err_idx);
-                errors.s.appendf("%s\n", err);
-            }
-
-            ForEachItemIn(warn_idx, warnStr)
-            {
-                const char *err = warnStr.item(warn_idx);
-                errors.s.appendf("%s\n", err);
-            }
-
-            errors.s.appendf("%d error(s),  %d warning(s)\n", errorStr.ordinality(), warnStr.ordinality());
-        }
-    }
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CRoxieQueryCompiler()
-    {
-    }
-
-    IConstWorkUnit *createWorkunit(SCMStringBuffer &wuid, const char *userName, const char *queryAppName)
-    {
-        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
-        Owned<IWorkUnit> workunit = factory->createWorkUnit(NULL, queryAppName, userName);
-        workunit->getWuid(wuid);
-        workunit->setState(WUStateUnknown);
-        workunit->setUser(userName);
-        return workunit.getClear();
-    }
-
-    IConstWorkUnit *compileEcl(SCMStringBuffer &wuid, const char *userName, const char *password, IRoxieQueryCompileInfo &compileInfo, IRoxieQueryProcessingInfo &processingInfo, SCMStringBuffer &status)
-    {
-        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
-        {
-            Owned<IWorkUnit> workunit;
-            
-            if (wuid.length())
-                workunit.setown(factory->updateWorkUnit(wuid.str()));
-            else
-                workunit.setown(factory->createWorkUnit(NULL, compileInfo.queryAppName(), userName));
-
-            workunit->setDebugValue("RoxieConfigStatus", "Compiling...", true);
-
-            Owned<IWUQuery> query = workunit->updateQuery();
-            query->setQueryText(compileInfo.queryEcl());
-            query->setQueryType(QueryTypeEcl);
-
-            const char *queryJobName = compileInfo.queryJobName();
-            if (queryJobName && *queryJobName)
-                workunit->setJobName(queryJobName);
-
-            workunit->setAction(WUActionCompile);
-            workunit->setUser(userName);
-            workunit->getWuid(wuid);
-
-            // set generic debug options
-            StringBuffer options(compileInfo.queryWuDebugOptions());
-            if (options.length())
-            {
-                Owned<IPropertyTree> dbgOptions = createPTreeFromXMLString(options, ipt_caseInsensitive);
-                Owned<IPropertyTreeIterator> iter = dbgOptions->getElements("Option");
-                ForEach(*iter)
-                {
-                    IPropertyTree &item = iter->query();
-                    const char *name = item.queryProp("@name");
-                    const char *value = item.queryProp("@value");
-                    workunit->setDebugValue(name, value, true);
-                }
-            }
-
-            // set specific debug options
-            workunit->setDebugValueInt("forceRoxie", 1, true);
-            workunit->setDebugValueInt("traceRowXml", 1, true);
-            workunit->setDebugValue("targetClusterType", "roxie", true);
-
-            workunit->setClusterName(compileInfo.queryClusterName()); // MORE - eclserver should not require cluster set for roxy compiles (probably)
-
-            if (compileInfo.getHighPriority())
-                workunit->setPriority(PriorityClassHigh);
-
-            int queryPriority = compileInfo.getQueryPriority();
-            if (queryPriority != UNKNOWN_PRIORITY)  // MORE - if set use queryPriority to set wu priority
-            {
-                workunit->setDebugValueInt("queryPriority", queryPriority, true);
-                if (queryPriority == LOW_PRIORITY)
-                    workunit->setPriority(PriorityClassLow);
-                else
-                    workunit->setPriority(PriorityClassHigh);
-            }
-            else
-            {
-                if (!compileInfo.getHighPriority())
-                    workunit->setDebugValue("queryPriority", "High", true);
-            }
-
-            workunit->setDebugValueInt("memoryLimit", compileInfo.getMemoryLimit(), true);
-            workunit->setDebugValueInt("timeLimit", compileInfo.getTimeLimit(), true);
-            workunit->setDebugValueInt("warnTimeLimit", compileInfo.getWarnTimeLimit(), true);
-            workunit->setDebugValueInt("poolGraphs", compileInfo.getPoolGraphs(), true);
-
-            workunit->setDebugValue("comment", processingInfo.queryComment(), true);
-            workunit->setDebugValue("package", processingInfo.queryPackageName(), true);
-
-            const char *label = compileInfo.queryRepositoryLabel();
-            if (label && *label)
-                workunit->setSnapshot(label);
-        }
-        submitWorkUnit(wuid.str(), userName, password);
-        if (waitForWorkUnitToCompile(wuid.str(), compileInfo.getWuTimeOut()))
-        {
-            IConstWorkUnit *workunit = factory->openWorkUnit(wuid.str(), false);
-            Owned<IWorkUnit> wu = &workunit->lock();
-            SCMStringBuffer jn;
-            wu->getJobName(jn);
-            if (jn.length() == 0)
-            {
-                SCMStringBuffer name;
-                wu->getDebugValue("name", name);
-                if (name.length())
-                    wu->setJobName(name.str());
-                else
-                    wu->setJobName(wuid.str());
-            }
-            processCompilationErrors(workunit, status);
-            return workunit;
-        }
-        else
-        {
-            Owned<IConstWorkUnit> workunit = factory->openWorkUnit(wuid.str(), false);
-            WUState wu_state = workunit->getState();
-
-            // there was a problem, just report it - either compilation problems, eclserver timeout, etc
-            processCompilationErrors(workunit, status);
-            if (!status.length())  // no compilations errors so it must be something else
-            {
-                //StringBuffer errStr;
-                if (wu_state == WUStateCompiling)
-                    status.s.append("Time out trying to compile query - need to increase the wuTimeOut");
-                else
-                {
-                    SCMStringBuffer state;
-                    workunit->getStateDesc(state);
-                    status.s.appendf("Unknown error: Could not compile query - make sure eclserver is running - workunit status = %s", state.str());
-                }
-            }
-            return NULL;
-        }
-    }
-
-};
-
-
-
-IRoxieQueryCompileInfo *createRoxieQueryCompileInfo(const char *_ecl, const char *_jobName, const char *_clusterName, const char *_appName)
-{
-    return new CRoxieQueryCompileInfo(_ecl, _jobName, _clusterName, _appName);
-}
-
-
-IRoxieQueryCompiler *createRoxieQueryCompiler()
-{
-    return new CRoxieQueryCompiler();
-}
-

common/roxiemanager/roxiequerycompiler.hpp

@@ -1,42 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef _ROXIEQUERYCOMPILER_HPP__
-#define _ROXIEQUERYCOMPILER_HPP__
-
-
-interface IConstWorkUnit;
-interface IWorkUnit;
-interface IEmbeddedXslTransformer;
-interface IAttributeMetaDataResolver;
-
-#include "jptree.hpp"
-#include "esp.hpp"
-
-#include "roxiemanager.hpp"
-
-
-interface IRoxieQueryCompiler : extends IInterface
-{
-    virtual IConstWorkUnit *createWorkunit(SCMStringBuffer &wuid, const char *userName, const char *queryAppName) = 0;
-    virtual IConstWorkUnit *compileEcl(SCMStringBuffer &wuid, const char *userName, const char *password, IRoxieQueryCompileInfo &compileInfo, IRoxieQueryProcessingInfo &processingInfo, SCMStringBuffer &status) = 0;
-};
-
-extern IRoxieQueryCompiler *createRoxieQueryCompiler();
-
-#endif
-

common/roxiemanager/roxiequerymanager.cpp

@@ -1,637 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#pragma warning (disable : 4786)
-
-#include "jlib.hpp"
-#include "roxiemanager.hpp"
-#include "roxiequerycompiler.hpp"
-#include "roxiewuprocessor.hpp"
-
-#include "roxiecommlibscm.hpp"
-
-#include "daclient.hpp"
-#include "dadfs.hpp"
-
-
-/////////////////////////////////////////////////////////////////////////////////// 
-class CRoxieQueryManager : public CInterface, implements IRoxieQueryManager
-{
-private:
-    SocketEndpoint ep;
-    unsigned roxieTimeout;
-    StringBuffer roxieName;
-    StringBuffer workunitDali;
-    Owned<IPropertyTree> xml;
-    Owned<IRoxieCommunicationClient> queryCommClient;
-    StringBuffer user;
-    StringBuffer password;
-    int logLevel;
-
-private:
-
-    const char *resolveQuerySetName(const char *querySetName)
-    {
-        return (querySetName && *querySetName) ? querySetName : roxieName.str();
-    }
-
-    IConstWorkUnit *getWorkUnit(const char *wuid)
-    {
-        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
-        IConstWorkUnit *workunit = factory->openWorkUnit(wuid, false);
-        if (!workunit)
-            throw MakeStringException(ROXIEMANAGER_MISSING_ID, "Unknown workunit id %s", wuid);
-            
-        return workunit;
-    }
-
-    void setWUException(IException* e, IWorkUnit *wu, SCMStringBuffer &status, bool &warningsOnly)
-    {
-        StringBuffer msg;
-        e->errorMessage(msg);
-        Owned<IWUException> we = wu->createException();
-
-        if (e->errorCode() == ROXIEMANAGER_UNRESOLVED_FILE)
-        {
-            DBGLOG("WARNING %s", msg.str());
-            we->setSeverity(ExceptionSeverityWarning);
-        }
-        else
-        {
-            DBGLOG("ERROR %s", msg.str());
-            we->setSeverity(ExceptionSeverityError);
-            warningsOnly = false;
-        }
-
-        status.s.appendf("%s\n", msg.str());
-
-        we->setExceptionMessage(msg);
-        StringBuffer s;
-        s.append("roxiequerymanager.cpp");
-        we->setExceptionSource(s.str());
-    }
-
-
-    IConstWorkUnit *processQuerySetWorkunit(SCMStringBuffer &wuid, 
-                                            IRoxieQueryCompileInfo &compileInfo, 
-                                            IRoxieQueryProcessingInfo &processingInfo,
-                                            SCMStringBuffer &status
-                                            )
-    {
-        Owned<IConstWorkUnit> workunit;
-        try
-        {
-            Owned<IRoxieQueryCompiler> compiler = createRoxieQueryCompiler();
-            workunit.setown(compiler->compileEcl(wuid, user, password, compileInfo, processingInfo, status));
-            if (!workunit)
-            {
-                DBGLOG("ERROR compiling query %s", status.str());
-                return NULL;
-            }
-        }
-        catch(IException *e)
-        {
-            // don't try and update a workunit - eclserver already did it
-            StringBuffer msg;
-            e->errorMessage(msg);
-            status.set(msg.str());
-
-            DBGLOG("ERROR compiling query %s", msg.str());
-            e->Release();
-            
-            return NULL;
-        }
-
-        Owned<IWorkUnit> wu = &workunit->lock();
-        wu->setState(WUStateCompiled);
-        return workunit.getClear();
-    }
-
-    void processWorkunit(IConstWorkUnit *workunit, SCMStringBuffer &queryName, IRoxieQueryProcessingInfo &processingInfo, WUQueryActivationOptions activateOption, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus)
-    {
-        Owned<IWorkUnit> wu = &workunit->lock();
-
-        SCMStringBuffer jobName;
-        wu->getJobName(jobName);
-
-        if (stricmp(jobName.str(),queryName.str())!=0)
-            wu->setJobName(queryName.str());
-
-        try
-        {
-            // look up data file info
-            Owned<IRoxieWuProcessor> wuProcessor = createRoxieWuProcessor(roxieName, queryCommClient, logLevel);
-            wuProcessor->lookupFileNames(wu, processingInfo, status);
-            IPropertyTree *pkgInfo = wuProcessor->queryPackageInfo();
-            StringBuffer newQueryId;
-            const char *qsName = resolveQuerySetName(querySetName);
-            addQueryToQuerySet(wu, qsName, queryName.str(), pkgInfo, activateOption, newQueryId, NULL);
-
-            const char *queryComment = processingInfo.queryComment();
-            if (queryComment)
-                setQueryCommentForNamedQuery(qsName, newQueryId.str(), queryComment);
-        }
-        catch(IException *e)
-        {
-            int errCode = e->errorCode();
-            StringBuffer err;
-            e->errorMessage(err);
-            status.s.appendf("%d %s", errCode, err.str());
-            DBGLOG("ERROR updating query list %s", status.str());
-            e->Release();
-        }
-
-        if (notifyRoxie)
-        {
-            queryCommClient->sendRoxieReloadControlRequest();
-        }
-    }
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CRoxieQueryManager(SocketEndpoint& _ep, const char *_roxieName, const char *_workunitDali, unsigned _roxieTimeout, const char *_user, const char *_password, int _logLevel)
-        :ep(_ep), roxieTimeout(_roxieTimeout), workunitDali(_workunitDali), roxieName(_roxieName), user(_user), password(_password), logLevel(_logLevel)
-    {
-        queryCommClient.setown(createRoxieCommunicationClient(ep, roxieTimeout));
-    }
-
-    void getNewQueryWorkunitId(SCMStringBuffer &wuid, SCMStringBuffer &roxieQueryName, const char *queryAppName)
-    {
-        Owned<IRoxieQueryCompiler> compiler = createRoxieQueryCompiler();
-        Owned<IConstWorkUnit> workunit = compiler->createWorkunit(wuid, user, queryAppName);
-        if (!workunit)
-        {
-            DBGLOG("ERROR creating workunit");
-            return;
-        }
-    }
-
-
-    bool compileQuery(SCMStringBuffer &wuid, SCMStringBuffer &roxieQueryName, IRoxieQueryCompileInfo &compileInfo, IRoxieQueryProcessingInfo &processingInfo, const char *targetClusterName, SCMStringBuffer &status)
-    {
-        Owned<IConstWorkUnit> workunit = processQuerySetWorkunit(wuid, compileInfo, processingInfo, status);
-
-        if (workunit)
-            return true;
-        else
-            return false;
-    }
-
-    bool deployQuery(SCMStringBuffer &wuid, SCMStringBuffer &roxieQueryName, IRoxieQueryCompileInfo &compileInfo, IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, bool allowNewRoxieOnDemandQuery, const char *targetClusterName, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus)
-    {
-        Owned<IConstWorkUnit> workunit = processQuerySetWorkunit(wuid, compileInfo, processingInfo, status);
-        if (!workunit)
-            return false;
-
-        processWorkunit(workunit, roxieQueryName, processingInfo, activateOption, querySetName, notifyRoxie, status, roxieDeployStatus);
-        return true;
-    }
-
-    bool deployWorkunit(SCMStringBuffer &wuid,  SCMStringBuffer &roxieQueryName, IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus)
-    {
-        if (!wuid.length())
-            throw MakeStringException(ROXIEMANAGER_MISSING_ID, "Missing workunit id");
-
-        Owned<IConstWorkUnit> workunit = getWorkUnit(wuid.str());
-        return publishWorkunit(workunit, roxieQueryName, processingInfo, userId, activateOption, querySetName, notifyRoxie, status, roxieDeployStatus);
-    }
-
-    bool publishWorkunit(IConstWorkUnit *workunit,  SCMStringBuffer &roxieQueryName, IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus)
-    {
-        if (!workunit)
-            throw MakeStringException(ROXIEMANAGER_MISSING_ID, "Missing workunit id");
-
-        if (!roxieQueryName.length()) {
-            SCMStringBuffer jobName;
-            roxieQueryName.set(workunit->getJobName(jobName).str());
-        }
-
-        processWorkunit(workunit, roxieQueryName, processingInfo, activateOption, querySetName, notifyRoxie, status, roxieDeployStatus);
-        return true;
-    }
-
-    const char *getQuerySetWuId(const char *name, const char *sourceQuerySetName, SCMStringBuffer &querySetWuId)
-    {
-        Owned<IPropertyTree> tree = getQueryRegistry(sourceQuerySetName, true);
-        if (!tree)
-        {
-            DBGLOG("Could not find query %s in queryset %s", name, sourceQuerySetName);
-            return NULL;
-        }
-        StringBuffer xpath;
-        xpath.appendf("Query[@wuid='%s']", name);
-        IPropertyTree *query = tree->queryPropTree(xpath.str());
-        if (!query)
-        {
-            StringBuffer id;
-            StringBuffer aliasxpath;
-            aliasxpath.appendf("Alias[@name='%s']", name);
-            IPropertyTree *alias = tree->queryPropTree(aliasxpath.str());
-            if (alias)
-                id.append(alias->queryProp("@id"));
-            else
-                id.append(name);
-            xpath.clear().appendf("Query[@id='%s']", id.str());
-            query = tree->queryPropTree(xpath.str());
-
-            if (!query)
-                return NULL;
-
-            querySetWuId.set(query->queryProp("@wuid"));
-        }
-        else
-            querySetWuId.set(name);
-
-        return querySetWuId.str();
-    }
-
-    bool publishFromQuerySet(SCMStringBuffer &name, SCMStringBuffer &roxieQueryName, IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, const char *sourceQuerySetName, const char *targetQuerySetName, const char *sourceDaliIP, const char *queryComment, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus)
-    {
-        if (!name.length())
-            throw MakeStringException(ROXIEMANAGER_MISSING_ID, "Missing id");
-
-        SCMStringBuffer wuid;
-        getQuerySetWuId(name.str(), sourceQuerySetName, wuid);
-
-        SCMStringBuffer jobName;
-        Owned<IConstWorkUnit> workunit;
-        try
-        {
-            workunit.setown(getWorkUnit(wuid.str()));
-            roxieQueryName.set(workunit->getJobName(jobName).str());
-        }
-        catch(IException *e)
-        {
-            e->Release();
-        }
-
-        processWorkunit(workunit, name, processingInfo, activateOption, targetQuerySetName, notifyRoxie, status, roxieDeployStatus);
-    
-        return true;
-    }
-
-    IPropertyTree *retrieveQueryList(const char *filter, bool excludeQueryNames, bool excludeAliasNames, bool excludeLibraryNames, bool excludeDataOnlyNames, unsigned version)
-    {
-        Owned<IPropertyTree> queryRegistry = getQueryRegistry(roxieName.str(), false);
-
-        StringBuffer xml;
-        xml.append("<QueryNames>");
-
-        Owned<IPropertyTreeIterator> queries = queryRegistry->getElements("Query");
-        ForEach(*queries)
-        {
-            IPropertyTree &query = queries->query();
-            const char *id = query.queryProp("@id");
-
-            if (!filter || WildMatch(id, filter, true))
-            {
-                const char *wuid = query.queryProp("@wuid");
-                if (!wuid)
-                    continue;
-
-                try
-                {
-                    Owned<IConstWorkUnit> wu = getWorkUnit(wuid);
-
-                    unsigned libraryInterfaceHash = wu->getApplicationValueInt("LibraryModule", "interfaceHash", 0);
-                    if (libraryInterfaceHash && excludeLibraryNames)
-                        continue;
-
-                    bool isDataOnlyQuery = query.getPropBool("@loadDataOnly");
-                    if (isDataOnlyQuery && excludeDataOnlyNames)
-                        continue;
-
-                    if (!libraryInterfaceHash && !isDataOnlyQuery && excludeQueryNames)
-                        continue;
-
-                    StringBuffer err;
-                    xml.appendf(" <Query id='%s'", id);
-
-                    xml.appendf(" wuid='%s'", wuid);
-                    Owned<IConstWUQuery> q = wu->getQuery();
-            
-                    Owned<IConstWUAssociatedFile> associatedFile = q->getAssociatedFile(FileTypeDll, 0);
-                    if (associatedFile)
-                    {
-                        SCMStringBuffer associateFileName;
-                        associatedFile->getName(associateFileName);
-                        xml.appendf(" associatedName='%s'", associateFileName.str());
-                    }
-                                    
-                    SCMStringBuffer label;
-                    wu->getSnapshot(label);
-                    if (label.length())
-                        xml.appendf(" label ='%s'", label.str());
-
-                    SCMStringBuffer user;
-                    wu->getDebugValue("created_for", user);
-                    if (user.length())
-                        xml.appendf(" deployedBy='%s'", user.str());
-
-                    SCMStringBuffer comment;
-                    wu->getDebugValue("comment", comment);
-                    if (comment.length())
-                        xml.appendf(" comment='%s'", comment.str());
-
-                    xml.appendf(" priority='%d'", wu->getDebugValueInt("querypriority", 0));
-
-                    if (libraryInterfaceHash != 0)
-                    {
-                        xml.appendf(" isLibrary='1'");
-                        xml.appendf(" libraryInterfaceHash='%u'", libraryInterfaceHash);
-                    }
-
-                    // Must be from the queryset
-                    if (query.getPropBool("@suspended", false))
-                    {
-                        const char *user = query.queryProp("@updatedByUserId");
-                        xml.appendf(" suspendedBy='%s'", (user) ? user : "Roxie");
-                        xml.append(" suspended='1'");
-                    }
-                    err.append(query.queryProp("@error"));
-                    if (err.length())
-                    {
-                        StringBuffer temp;
-                        encodeXML(err.str(), temp);
-                        xml.appendf(" error='%s'", temp.str());
-                    }
-
-                    if (isDataOnlyQuery)
-                        xml.appendf(" isLoadDataOnly='1'");
-
-                    Owned<IPropertyTreeIterator> libraries = query.getElements("Library");
-                    xml.append(">\n");
-                    xml.append(" <librariesUsed>");
-                    ForEach(*libraries)
-                    {
-                        IPropertyTree &library = libraries->query();
-                        const char *libName = library.queryProp("@name");
-                        xml.appendf("<library>%s</library>", libName);
-                    }
-                    xml.append("</librariesUsed>\n");
-                    xml.append("</Query>\n");
-                }
-                catch(IException *e)
-                {
-                    StringBuffer err;
-                    e->errorMessage(err);
-                    xml.appendf(" <Query id='%s'", id);
-                    xml.appendf(" wuid='%s'", wuid);
-                    xml.appendf(" error='%s'", err.str());
-                    xml.appendf(" comment='%s'", err.str());
-                    xml.append(">\n");
-                    xml.append("</Query>\n");
-                    e->Release();
-                }
-            }
-        }
-        if (!excludeAliasNames)
-        {
-            Owned<IPropertyTreeIterator> aliases = queryRegistry->getElements("Alias");
-            ForEach(*aliases)
-            {
-                IPropertyTree &alias = aliases->query();
-                const char *id = alias.queryProp("@id");
-                const char *name = alias.queryProp("@name");
-                if (!filter || WildMatch(name, filter, true) || WildMatch(id, filter, true))
-                    xml.appendf(" <Alias id='%s' name='%s'/>\n", id, name);
-            }
-        }
-        xml.append("</QueryNames>");
-
-        Owned<IPropertyTree> tree = createPTreeFromXMLString(xml);
-        return tree.getClear();
-    }
-
-    void addAlias(const char *alias, const char *queryId, const char *querySetName, bool notifyRoxie, SCMStringBuffer &oldActive)
-    {
-        try
-        {
-            addQuerySetAlias(resolveQuerySetName(querySetName), alias, queryId);
-        }
-        catch(IException *e)
-        {
-            int errCode = e->errorCode();
-            StringBuffer err;
-            e->errorMessage(err);
-            DBGLOG("ERROR addingAlias %d %s", errCode, err.str());
-            e->Release();
-        }
-
-        if (notifyRoxie)
-            queryCommClient->sendRoxieReloadControlRequest();
-    }
-
-    void suspendQuery(const char *id, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status)
-    {
-        try
-        {
-            setSuspendQuerySetQuery(resolveQuerySetName(querySetName), id, true, NULL);
-            status.s.appendf("successfully suspended query %s", id);
-        }
-        catch(IException *e)
-        {
-            int errCode = e->errorCode();
-            StringBuffer err;
-            e->errorMessage(err);
-            status.s.appendf("%d %s", errCode, err.str());
-            DBGLOG("ERROR suspending query %s", status.str());
-            e->Release();
-        }
-        if (notifyRoxie)
-            queryCommClient->sendRoxieReloadControlRequest();
-    }
-
-    void unsuspendQuery(const char *id, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status)
-    {
-        try
-        {
-            setSuspendQuerySetQuery(resolveQuerySetName(querySetName), id, false, NULL);
-            status.s.appendf("successfully unsuspended query %s", id);
-        }
-        catch(IException *e)
-        {
-            int errCode = e->errorCode();
-            StringBuffer err;
-            e->errorMessage(err);
-            status.s.appendf("%d %s", errCode, err.str());
-            DBGLOG("ERROR unsuspending query %s", status.str());
-            e->Release();
-        }
-        if (notifyRoxie)
-            queryCommClient->sendRoxieReloadControlRequest();
-    }
-
-    void deleteQuery(const char *id, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status)
-    {
-        try
-        {
-            deleteQuerySetQuery(resolveQuerySetName(querySetName), id);
-            status.s.appendf("successfully deleted:\n query %s", id);
-        }
-        catch(IException *e)
-        {
-            int errCode = e->errorCode();
-            StringBuffer err;
-            e->errorMessage(err);
-            status.s.appendf("%d %s", errCode, err.str());
-            DBGLOG("ERROR deleting query %s", status.str());
-            e->Release();
-        }
-        if (notifyRoxie)
-            queryCommClient->sendRoxieReloadControlRequest();
-
-    }
-
-    void removeAllAliasForQuery(const char *queryId, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status)
-    {
-        try
-        {
-            removeQuerySetAliasesFromNamedQuery(resolveQuerySetName(querySetName), queryId);
-            status.s.appendf("successfully removed all aliases for query %s", queryId);
-        }
-        catch(IException *e)
-        {
-            int errCode = e->errorCode();
-            StringBuffer err;
-            e->errorMessage(err);
-            status.s.appendf("%d %s", errCode, err.str());
-            DBGLOG("ERROR deleting query %s", status.str());
-            e->Release();
-        }
-
-        if (notifyRoxie)
-                queryCommClient->sendRoxieReloadControlRequest();
-    }
-
-    void removeAlias(const char *alias, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status)
-    {
-        try
-        {
-            removeQuerySetAlias(resolveQuerySetName(querySetName), alias);
-            status.s.appendf("successfully removed alias %s", alias);
-        }
-        catch(IException *e)
-        {
-            int errCode = e->errorCode();
-            StringBuffer err;
-            e->errorMessage(err);
-            status.s.appendf("%d %s", errCode, err.str());
-            DBGLOG("ERROR removing alias %s", status.str());
-            e->Release();
-        }
-
-        if (notifyRoxie)
-            queryCommClient->sendRoxieReloadControlRequest();
-    }
-
-    const char * runQuery(IConstWorkUnit *workunit, const char *roxieQueryName, bool outputToSocket, bool allowNewRoxieOnDemandQuery, SCMStringBuffer &response)
-    {
-        StringBuffer query;
-        SCMStringBuffer wuid;
-        query.appendf("<%s wuid='%s' roxieOnDemand='%d' outputToSocket='%d'/>", roxieQueryName, workunit->getWuid(wuid).str(), (int) allowNewRoxieOnDemandQuery, (int) outputToSocket);
-        try
-        {
-            queryCommClient->sendRoxieOnDemandRequest(query.str(), response, false);
-        }
-        catch (IException *E)
-        {
-            Owned<IWorkUnit> wu = &workunit->lock();
-            Owned<IWUException> we = wu->createException();
-            we->setExceptionCode(E->errorCode());
-            StringBuffer msg;
-            we->setExceptionMessage(E->errorMessage(msg).str());
-            we->setExceptionSource("Roxie");
-            we->setSeverity(ExceptionSeverityError);
-            wu->setState(WUStateFailed);
-            throw;
-        }
-        return response.str();
-    }
-
-
-    void setQueryWarningTime(const char *id, unsigned warnTime, SCMStringBuffer &status)
-    {
-    }
-
-    unsigned getQueryWarningTime(const char *id)
-    {
-        return 0;
-    }
-
-    bool updateACLInfo(bool allow, const char *restrict_ip, const char *mask, const char *query, const char *errorMsg, int errorCode, int port, SCMStringBuffer &status)
-    {
-        if (port == 0)   // use the default
-            queryCommClient->updateACLInfo(allow, restrict_ip, mask, query, errorMsg, errorCode, NULL, status);
-        else if (port == -1)  // use all ports
-        {
-            IntArray portNumbers;  // store unique ports
-            Owned<IPropertyTree> topology = queryCommClient->retrieveTopology();
-
-            Owned<IPropertyTreeIterator> servers = topology->getElements("Endpoint/RoxieTopology/RoxieServerProcess");
-            ForEach (*servers)
-            {
-                IPropertyTree &server = servers->query();
-                int serverPort = server.getPropInt("@port");
-                const char *netAddress = server.queryProp("@netAddress");
-
-                portNumbers.append(port);
-                StringBuffer roxieAddress;
-                roxieAddress.appendf("%s:%d", netAddress, serverPort);
-                queryCommClient->updateACLInfo(allow, restrict_ip, mask, query, errorMsg, errorCode, roxieAddress.str(), status);
-            }       
-        }
-        else  // use only the specific port passed in
-        {
-            Owned<IPropertyTree> topology = queryCommClient->retrieveTopology();
-            StringBuffer xpath;
-            xpath.appendf("Endpoint/RoxieTopology/RoxieServerProcess[@port='%d']", port);
-            bool foundIt = false;
-
-            Owned<IPropertyTreeIterator> servers = topology->getElements(xpath.str());
-            ForEach (*servers)
-            {
-                IPropertyTree &server = servers->query();
-                int serverPort = server.getPropInt("@port");
-                const char *netAddress = server.queryProp("@netAddress");
-
-                StringBuffer roxieAddress;
-                roxieAddress.appendf("%s:%d", netAddress, serverPort);
-                queryCommClient->updateACLInfo(allow, restrict_ip, mask, query, errorMsg, errorCode, roxieAddress.str(), status);
-                foundIt = true;
-                break;  // only need to send 1
-            }
-
-            if (!foundIt)
-                status.s.appendf("Could not find port %d in the roxie configuration", port);
-        }
-
-
-        return true;
-    }
-
-    IRoxieCommunicationClient * queryRoxieCommunicationClient() { return queryCommClient; }
-
-};
-
-
-IRoxieQueryManager* createRoxieQueryManager(SocketEndpoint &roxieEP, const char *roxieName, const char *workunitDali, unsigned roxieTimeout, const char *_user, const char *_password, int logLevel)
-{
-    return new CRoxieQueryManager(roxieEP, roxieName, workunitDali, roxieTimeout, _user, _password, logLevel);
-}
-

common/roxiemanager/roxiewuprocessor.hpp

@@ -1,56 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef _ROXIEWUPROCESSOR_HPP__
-#define _ROXIEWUPROCESSOR_HPP__
-
-
-interface IConstWorkUnit;
-interface IWorkUnit;
-interface IEmbeddedXslTransformer;
-interface IAttributeMetaDataResolver;
-
-#include "jptree.hpp"
-#include "esp.hpp"
-#include "roxie.hpp"
-
-#include "roxiemanager.hpp"
-
-interface IRoxieCommunicationClient;
-
-#define GET_LOCK_FAILURE            1100
-//#define DALI_FILE_LOOKUP_TIMEOUT (1000*60*1)  // 1 minute
-#define DALI_FILE_LOOKUP_TIMEOUT (1000*15*1)  // 15 seconds
-
-enum QueryAction { DEPLOY_ATTR, DEPLOY_WU, COMPILE, ROXIE_ON_DEMAND };
-
-interface IRoxieWuProcessor : extends IInterface
-{
-    virtual void retrieveFileNames(IPropertyTree *remoteQuery, IPropertyTree *remoteTopology, bool copyFileLocationInfo, const char *lookupDaliIp, const char *user, const char *password) = 0;
-    virtual void retrieveFileNames(IPropertyTree *xml, IPropertyTree *remoteQuery, IPropertyTree *remoteState, IPropertyTree *remoteTopology, bool copyFileLocationInfo, const char *lookupDaliIp, const char *user, const char *password) = 0;
-    virtual bool lookupFileNames(IWorkUnit *wu, IRoxieQueryProcessingInfo &processingInfo, SCMStringBuffer &status) = 0;
-
-    virtual bool processAddRoxieFileInfoToDali(const char *src_filename, const char *dest_filename, const char *lookupDaliIp, IUserDescriptor *userdesc, StringBuffer &status) = 0;
-    virtual bool processAddRoxieFileRelationshipsToDali(const char *src_filename, const char *remoteRoxieClusterName, const char *lookupDaliIp, IUserDescriptor *userdesc, StringBuffer &msg) = 0;
-    virtual IPropertyTree *queryPackageInfo() = 0;
-
-};
-
-extern IRoxieWuProcessor *createRoxieWuProcessor(const char *roxieClusterName, IRoxieCommunicationClient *_roxieCommClient, int logLevel);
-
-#endif
-

common/roxiemanager/sourcedoc.xml

@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
--->
-<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
-<section>
-    <title>common/roxiemanager</title>
-
-    <para>
-        The common/roxiemanager directory contains the sources for the common/roxiemanager library.
-    </para>
-</section>

common/sourcedoc.xml

@@ -32,7 +32,6 @@
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="remote/sourcedoc.xml"/>
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxiecommlib/sourcedoc.xml"/>
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxiehelper/sourcedoc.xml"/>
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxiemanager/sourcedoc.xml"/>
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="thorhelper/sourcedoc.xml"/>
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="workunit/sourcedoc.xml"/>
 </section>

common/workunit/pkgimpl.hpp

@@ -426,7 +426,10 @@ public:
                 Owned<IReferencedFileIterator> refFiles = filelist->getFiles();
                 ForEach(*refFiles)
                 {
-                    VStringBuffer fullname("%s/%s", queryid, refFiles->query().getLogicalName());
+                    IReferencedFile &rf = refFiles->query();
+                    if (rf.getFlags() & RefFileInPackage)
+                        continue;
+                    VStringBuffer fullname("%s/%s", queryid, rf.getLogicalName());
                     unmatchedFiles.append(fullname);
                 }
 

dali/base/dadfs.cpp

@@ -2365,6 +2365,34 @@ inline void dfCheckRoot(const char *trc,Owned<IPropertyTree> &root,IRemoteConnec
     }
 }
 
+class CFileChangeWriteLock
+{
+    Owned<IRemoteConnection> &conn;
+    Owned<IPropertyTree> &root;
+    unsigned timeoutMs, prevMode;
+public:
+    CFileChangeWriteLock(Owned<IRemoteConnection> &_conn, unsigned _timeoutMs, Owned<IPropertyTree> &_root)
+        : conn(_conn), timeoutMs(_timeoutMs), root(_root)
+    {
+        prevMode = conn->queryMode();
+        unsigned newMode = (prevMode & ~RTM_LOCKBASIC_MASK) | RTM_LOCK_WRITE;
+        conn->changeMode(RTM_LOCK_WRITE, timeoutMs);
+    }
+    ~CFileChangeWriteLock()
+    {
+        if (conn.get())
+            conn->changeMode(prevMode, timeoutMs);
+    }
+    IPropertyTree *detach(bool close)
+    {
+        Owned<IPropertyTree> detachedRoot = createPTreeFromIPT(root);
+        root.clear();
+        conn->close(close);
+        conn.clear();
+        return detachedRoot.getClear();
+    }
+};
+
 static bool setFileProtectTree(IPropertyTree &p,const char *owner, bool protect)
 {
     bool ret = false;
@@ -2512,81 +2540,10 @@ protected:
         root->removeProp("Attr");
         return NULL;
     }
-
-    // Call this function *ONLY* from detach()
-    // MORE: When the refactoring tips below get implemented, this method can go
-    // And the appropriate sub methods can be created depending whether this is
-    // a super-file or simply a file
-    void doRemoveEntry(CDfsLogicalFileName &lfn, IUserDescriptor *user, unsigned timeoutms=INFINITE)
+    void updateFS(const CDfsLogicalFileName &lfn, unsigned timeoutMs)
     {
-        StringBuffer cname;
-        lfn.getCluster(cname);
-        DfsXmlBranchKind bkind;
-        CFileConnectLock fconnlock;
-        {
-            IPropertyTree *froot=NULL;
-            if (fconnlock.initany("doRemoveEntry", lfn, bkind, true, false, false, timeoutms))
-                froot = fconnlock.queryRoot();
-            if (!froot)
-                ThrowStringException(-1, "Can't find SDS node for %s", lfn.get());
-
-            // Remove Cluster from Logical file
-            // MORE: Move this to a doRemoveCluster method
-            if (cname.length()) {
-                if (bkind==DXB_SuperFile)
-                    ThrowStringException(-1, "Trying to remove cluster %s from superfile %s",cname.str(),lfn.get());
-
-                const char *group = froot->queryProp("@group");
-                if (group&&(strcmp(group,cname.str())!=0)) {    // see if only cluster (if it is remove entire)
-                    StringBuffer query;
-                    query.appendf("Cluster[@name=\"%s\"]",cname.str());
-                    IPropertyTree *t = froot->queryPropTree(query.str());
-                    if (t) {
-                        if (froot->removeTree(t))
-                            return;
-                        else
-                            ThrowStringException(-1, "Can't remove cluster %s from %s",cname.str(),lfn.get());
-                    }
-                    else
-                        ThrowStringException(-1, "Cluster %s not present in file %s",cname.str(),lfn.get());
-                }
-            }
-
-            // Remove SuperOwners from all sub files
-            if (bkind==DXB_SuperFile) {
-                Owned<IPropertyTreeIterator> iter = froot->getElements("SubFile");
-                StringBuffer oquery;
-                oquery.append("SuperOwner[@name=\"").append(lfn.get()).append("\"]");
-                Owned<IMultiException> exceptions = MakeMultiException("CDelayedDelete::doRemoveEntry::SuperOwners");
-                ForEach(*iter) {
-                    const char *name = iter->query().queryProp("@name");
-                    if (name&&*name) {
-                        CDfsLogicalFileName subfn;
-                        subfn.set(name);
-                        CFileConnectLock fconnlockSub;
-                        DfsXmlBranchKind subbkind;
-                        // MORE: Use CDistributedSuperFile::linkSuperOwner(false) - ie. unlink
-                        if (fconnlockSub.initany("CDelayedDelete::doRemoveEntry", subfn, subbkind, false, false, false, timeoutms))
-                        {
-                            IPropertyTree *subfroot = fconnlockSub.queryRoot();
-                            if (subfroot) {
-                                if (!subfroot->removeProp(oquery.str()))
-                                    exceptions->append(*MakeStringException(-1, "CDelayedDelete::removeEntry: SubFile %s of %s not found for removal",name?name:"(NULL)",lfn.get()));
-                            }
-                        }
-                    }
-                }
-                if (exceptions->ordinality())
-                    throw exceptions.getClear();
-            }
-        }
-
-        // Remove from SDS and clean the lock
-        fconnlock.remove();
-        fconnlock.kill();
-
         // Update the file system
-        removeFileEmptyScope(lfn,timeoutms);
+        removeFileEmptyScope(lfn, timeoutMs);
         // MORE: We shouldn't have to update all relationships if we had done a better job making sure
         // that all correct relationships were properly cleaned up
         queryDistributedFileDirectory().removeAllFileRelationships(lfn.get());
@@ -2944,7 +2901,6 @@ public:
 class CDistributedFile: public CDistributedFileBase<IDistributedFile>
 {
 protected:
-    Owned<IFileDescriptor> fdesc;
     CDistributedFilePartArray parts;            // use queryParts to access
     CriticalSection sect;
     StringAttr directory;
@@ -3001,6 +2957,142 @@ protected:
                 parts.item(i1++).clearDirty();
         }
     }
+    void detach(unsigned timeoutMs=INFINITE, bool removePhysicals=true)
+    {
+        // Removes either a cluster in case of multi cluster file or the whole File entry from DFS
+
+        assert(proplockcount == 0 && "CDistributedFile detach: Some properties are still locked");
+        assertex(!isAnon()); // not attached!
+
+        if (removePhysicals)
+        {
+            // Avoid removing physically when there is no physical representation
+            if (logicalName.isMulti() || logicalName.isExternal())
+                removePhysicals = false;
+        }
+
+        StringBuffer clusterName;
+        Owned<IFileDescriptor> fileDescCopy;
+#ifdef EXTRA_LOGGING
+        PROGLOG("CDistributedFile::detach(%s)",logicalName.get());
+        LOGPTREE("CDistributedFile::detach root.1",root);
+#endif
+        {
+            CriticalBlock block(sect); // JCSMORE - not convinced this is still necessary
+            CFileChangeWriteLock writeLock(conn, timeoutMs, root);
+
+            logicalName.getCluster(clusterName);
+
+            // copy file descriptor before altered, used by physical file removal routines
+            if (removePhysicals)
+            {
+                MemoryBuffer mb;
+                Owned<IFileDescriptor> fdesc = getFileDescriptor(clusterName);
+                fdesc->serialize(mb);
+                fileDescCopy.setown(deserializeFileDescriptor(mb));
+            }
+
+            bool removeFile=true;
+            if (clusterName.length())
+            {
+                // Remove just cluster specified, unless it's the last, in which case detach below will remove File entry.
+                if (clusters.ordinality()>1)
+                {
+                    if (removeCluster(clusterName.str()))
+                        removeFile=false;
+                    else
+                        ThrowStringException(-1, "Cluster %s not present in file %s", clusterName.str(), logicalName.get());
+                }
+            }
+
+            // detach this IDistributeFile
+            root.setown(writeLock.detach(removeFile));
+            // NB: The file is now unlocked
+            if (removeFile)
+                updateFS(logicalName, timeoutMs);
+
+            logicalName.clear();
+        }
+        // NB: beyond unlock
+        if (removePhysicals)
+        {
+            CriticalBlock block(physicalChange);
+            Owned<IMultiException> exceptions = MakeMultiException("CDistributedFile::detach");
+            removePhysicalPartFiles(fileDescCopy, exceptions);
+            if (exceptions->ordinality())
+                throw exceptions.getClear();
+        }
+    }
+    bool removePhysicalPartFiles(IFileDescriptor *fileDesc, IMultiException *mexcept)
+    {
+        if (logicalName.isExternal())
+        {
+            if (logicalName.isQuery())
+                return false;
+            throw MakeStringException(-1,"cannot remove an external file (%s)",logicalName.get());
+        }
+        if (logicalName.isForeign())
+            throw MakeStringException(-1,"cannot remove a foreign file (%s)",logicalName.get());
+
+        class casyncfor: public CAsyncFor
+        {
+            IFileDescriptor *fileDesc;
+            CriticalSection errcrit;
+            IMultiException *mexcept;
+        public:
+            bool ok;
+            bool islazy;
+            casyncfor(IFileDescriptor *_fileDesc, IMultiException *_mexcept)
+            {
+                fileDesc = _fileDesc;
+                ok = true;
+                islazy = false;
+                mexcept = _mexcept;
+            }
+            void Do(unsigned i)
+            {
+                IPartDescriptor *part = fileDesc->queryPart(i);
+                unsigned nc = part->numCopies();
+                for (unsigned copy = 0; copy < nc; copy++)
+                {
+                    RemoteFilename rfn;
+                    part->getFilename(copy, rfn);
+                    Owned<IFile> partfile = createIFile(rfn);
+                    StringBuffer eps;
+                    try
+                    {
+                        unsigned start = msTick();
+                        if (!partfile->remove()&&(copy==0)&&!islazy) // only warn about missing primary files
+                            LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
+                        else
+                        {
+                            unsigned t = msTick()-start;
+                            if (t>5*1000)
+                                LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getUrlStr(eps).str(), t/1000);
+                        }
+                    }
+                    catch (IException *e)
+                    {
+                        CriticalBlock block(errcrit);
+                        if (mexcept)
+                            mexcept->append(*e);
+                        else
+                        {
+                            StringBuffer s("Failed to remove file part ");
+                            s.append(partfile->queryFilename()).append(" from ");
+                            rfn.queryEndpoint().getUrlStr(s);
+                            EXCLOG(e, s.str());
+                            e->Release();
+                        }
+                        ok = false;
+                    }
+                }
+            }
+        } afor(fileDesc, mexcept);
+        afor.islazy = fileDesc->queryProperties().getPropBool("@lazy");
+        afor.For(fileDesc->numParts(),10,false,true);
+        return afor.ok;
+    }
 
 protected: friend class CDistributedFilePart;
     CDistributedFilePartArray &queryParts()
@@ -3283,7 +3375,7 @@ public:
         saveClusters();
     }
 
-    void removeCluster(const char *clustername)
+    bool removeCluster(const char *clustername)
     {
         CClustersLockedSection cls(CDistributedFileBase<IDistributedFile>::logicalName);
         reloadClusters();
@@ -3293,7 +3385,9 @@ public:
                 throw MakeStringException(-1,"CFileClusterOwner::removeCluster cannot remove sole cluster %s",clustername);
             clusters.remove(i);
             saveClusters();
+            return true;
         }
+        return false;
     }
 
     void setPreferredClusters(const char *clusterlist)
@@ -3562,7 +3656,6 @@ public:
 #endif
     }
 
-private:
     /*
      * Internal method (not in IDistributedFile interface) that is used
      * when renaming files (so don't delete the physical representation).
@@ -3572,155 +3665,16 @@ private:
      *
      * See removeLogical()
      */
-#ifdef _USE_CPPUNIT
 public:
-#endif
     void detachLogical(unsigned timeoutms=INFINITE)
     {
         detach(timeoutms, false);
     }
 
 public:
-
-    void detach(unsigned timeoutms=INFINITE, bool removePhysicalParts=true)
+    virtual void detach(unsigned timeoutMs=INFINITE)
     {
-        assert(proplockcount == 0 && "CDistributedFile detach: Some properties are still locked");
-        assertex(!isAnon()); // not attached!
-
-        // If cluster name was passed via query (filename@cluster), try to find it
-        StringBuffer clustername;
-        logicalName.getCluster(clustername);
-
-        // Avoid removing physically when there is no physical representation
-        if (logicalName.isMulti() || logicalName.isExternal())
-            removePhysicalParts = false;
-
-        // Clean up file and remove from SDS only if this is the last
-        // cluster it belongs to, since we should be able to still remove
-        // the other clusters' files. Or if there are no clusters.
-        if ((clustername.length()==0) /* no cluster passed - ie. detach all */
-           ||((findCluster(clustername.str())==0)&&(numClusters()==1))) /* cluster is first, and no other clusters */
-        {
-            CriticalBlock block (sect);
-            MemoryBuffer mb;
-#ifdef EXTRA_LOGGING
-            PROGLOG("CDistributedFile::detach(%s)",logicalName.get());
-            LOGPTREE("CDistributedFile::detach root.1",root);
-#endif
-            root->serialize(mb);
-            conn.clear();
-            root.setown(createPTree(mb));
-            CDfsLogicalFileName lname;
-            lname.set(logicalName);
-            logicalName.clear();
-#ifdef EXTRA_LOGGING
-            LOGPTREE("CDistributedFile::detach root.2",root);
-#endif
-            // Remove from SDS
-            doRemoveEntry(lname,udesc,timeoutms);
-            // Make sure we remove *all* physical instances
-            clustername.clear();
-        }
-
-        // Remove parts, physically
-        if (removePhysicalParts)
-        {
-            CriticalBlock block(physicalChange);
-            Owned<IMultiException> exceptions = MakeMultiException("CDistributedFile::detach");
-            removePhysicalPartFiles(clustername.str(),exceptions);
-            if (exceptions->ordinality())
-                throw exceptions.getClear();
-        }
-    }
-
-    bool removePhysicalPartFiles(const char *cluster,IMultiException *mexcept)
-    {
-        Owned<IGroup> grpfilter;
-        if (cluster&&*cluster) {
-            unsigned cn = findCluster(cluster);
-            if (cn==NotFound)
-                return false;
-            if (clusters.ordinality()==0)
-                cluster = NULL; // cannot delete last cluster
-            else
-                grpfilter.setown(clusters.getGroup(cn));
-        }
-        if (logicalName.isExternal()) {
-            if (logicalName.isQuery())
-                return false;
-            throw MakeStringException(-1,"cannot remove an external file (%s)",logicalName.get());
-        }
-        if (logicalName.isForeign())
-            throw MakeStringException(-1,"cannot remove a foreign file (%s)",logicalName.get());
-
-        unsigned width = numParts();
-        CriticalSection errcrit;
-        class casyncfor: public CAsyncFor
-        {
-            IDistributedFile *file;
-            CriticalSection &errcrit;
-            IMultiException *mexcept;
-            unsigned width;
-            IGroup *grpfilter;
-        public:
-            bool ok;
-            bool islazy;
-            casyncfor(IDistributedFile *_file,unsigned _width,IGroup *_grpfilter,IMultiException *_mexcept,CriticalSection &_errcrit)
-                : errcrit(_errcrit)
-            {
-                file = _file;
-                ok = true;
-                islazy = false;
-                mexcept = _mexcept;
-                width = _width;
-                grpfilter = _grpfilter;
-            }
-            void Do(unsigned i)
-            {
-                Owned<IDistributedFilePart> part = file->getPart(i);
-                unsigned nc = part->numCopies();
-                for (unsigned copy = 0; copy < nc; copy++)
-                {
-                    RemoteFilename rfn;
-                    part->getFilename(rfn,copy);
-                    if (grpfilter&&(grpfilter->rank(rfn.queryEndpoint())==RANK_NULL))
-                        continue;
-                    Owned<IFile> partfile = createIFile(rfn);
-                    StringBuffer eps;
-                    try
-                    {
-                        unsigned start = msTick();
-                        if (!partfile->remove()&&(copy==0)&&!islazy) // only warn about missing primary files
-                            LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
-                        else {
-                            unsigned t = msTick()-start;
-                            if (t>5*1000) 
-                                LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getUrlStr(eps).str(), t/1000);
-                        }
-
-                    }
-                    catch (IException *e)
-                    {
-                        CriticalBlock block(errcrit);
-                        if (mexcept) 
-                            mexcept->append(*e);
-                        else {
-                            StringBuffer s("Failed to remove file part ");
-                            s.append(partfile->queryFilename()).append(" from ");
-                            rfn.queryEndpoint().getUrlStr(s);
-                            EXCLOG(e, s.str());
-                            e->Release();
-                        }
-                        ok = false;
-                    }
-                }
-            }
-        } afor(this,width,grpfilter,mexcept,errcrit);
-        afor.islazy = queryAttributes().getPropInt("@lazy")!=0;
-        afor.For(width,10,false,true);
-        if (cluster&&*cluster) 
-            removeCluster(cluster);
-        return afor.ok;
+        detach(timeoutMs, true);
     }
 
     bool existsPhysicalPartFiles(unsigned short port)
@@ -3833,7 +3787,7 @@ public:
         if (newbasedir)
             diroverride = newbasedir;
 
-        const char *myBase = queryBaseDirectory(false,os);
+        const char *myBase = queryBaseDirectory(0, os);
         StringBuffer baseDir, newPath;
         makePhysicalPartName(logicalName.get(), 0, 0, newPath, false, os, diroverride);
         if (!getBase(directory, newPath, baseDir))
@@ -4607,6 +4561,37 @@ protected:
     int interleaved; // 0 not interleaved, 1 interleaved old, 2 interleaved new
     IArrayOf<IDistributedFile> subfiles;
 
+    void clearSuperOwners(unsigned timeoutMs)
+    {
+        Owned<IPropertyTreeIterator> iter = root->getElements("SubFile");
+        StringBuffer oquery;
+        oquery.append("SuperOwner[@name=\"").append(logicalName.get()).append("\"]");
+        Owned<IMultiException> exceptions = MakeMultiException("CDelayedDelete::doRemoveEntry::SuperOwners");
+        ForEach(*iter)
+        {
+            const char *name = iter->query().queryProp("@name");
+            if (name&&*name)
+            {
+                CDfsLogicalFileName subfn;
+                subfn.set(name);
+                CFileConnectLock fconnlockSub;
+                DfsXmlBranchKind subbkind;
+                // MORE: Use CDistributedSuperFile::linkSuperOwner(false) - ie. unlink
+                if (fconnlockSub.initany("CDelayedDelete::doRemoveEntry", subfn, subbkind, false, false, false, timeoutMs))
+                {
+                    IPropertyTree *subfroot = fconnlockSub.queryRoot();
+                    if (subfroot)
+                    {
+                        if (!subfroot->removeProp(oquery.str()))
+                            exceptions->append(*MakeStringException(-1, "CDelayedDelete::removeEntry: SubFile %s of %s not found for removal",name?name:"(NULL)", logicalName.get()));
+                    }
+                }
+            }
+        }
+        if (exceptions->ordinality())
+            throw exceptions.getClear();
+    }
+
     static StringBuffer &getSubPath(StringBuffer &path,unsigned idx)
     {
         return path.append("SubFile[@num=\"").append(idx+1).append("\"]");
@@ -5124,8 +5109,6 @@ public:
         return ret; 
     }
 
-
-
     void attach(const char *_logicalname,IUserDescriptor *user)
     {
         // will need more thought but this gives limited support for anon
@@ -5145,32 +5128,22 @@ public:
         root.setown(conn->getRoot());
     }
 
-    void detach(unsigned timeoutms=INFINITE, bool removePhysicalParts=true)
+    void detach(unsigned timeoutMs=INFINITE)
     {   
         // will need more thought but this gives limited support for anon
         if (isAnon())
             return;
         assertex(conn.get()); // must be attached
-        CriticalBlock block (sect);
+        CriticalBlock block(sect);
         checkModify("CDistributedSuperFile::detach");
         subfiles.kill();    
-        MemoryBuffer mb;
-        root->serialize(mb);
-        root.clear();
-        conn.clear();
-        root.setown(createPTree(mb));
-        CDfsLogicalFileName lname;
-        lname.set(logicalName);
-        logicalName.clear();
 
         // Remove from SDS
-        doRemoveEntry(lname,udesc,timeoutms);
-    }
-
-    bool removePhysicalPartFiles(const char *clustername,IMultiException *mexcept)
-    {
-        throw MakeStringException(-1,"removePhysicalPartFiles not supported for SuperFiles");
-        return false; 
+        CFileChangeWriteLock writeLock(conn, timeoutMs, root);
+        clearSuperOwners(timeoutMs);
+        root.setown(writeLock.detach(true));
+        updateFS(logicalName, timeoutMs);
+        logicalName.clear();
     }
 
     bool existsPhysicalPartFiles(unsigned short port)
@@ -5842,14 +5815,16 @@ public:
         subfiles.item(0).addCluster(clustername,mspec);
     }
 
-    virtual void removeCluster(const char *clustername)
+    virtual bool removeCluster(const char *clustername)
     {
+        bool clusterRemoved=false;
         CriticalBlock block (sect);
         clusterscache.clear();
         ForEachItemIn(i,subfiles) {
             IDistributedFile &f=subfiles.item(i);
-            f.removeCluster(clustername);
+            clusterRemoved |= f.removeCluster(clustername);
         }       
+        return clusterRemoved;
     }
 
     void setPreferredClusters(const char *clusters)
@@ -6316,16 +6291,50 @@ public:
 
 #define GROUP_CACHE_INTERVAL (1000*60)
 
+static const char *translateGroupType(GroupType groupType)
+{
+    switch (groupType)
+    {
+        case grp_thor:
+            return "Thor";
+        case grp_roxie:
+            return "Roxie";
+        case grp_roxiefarm:
+            return "RoxieFarm";
+        case grp_hthor:
+            return "hthor";
+        default:
+            return NULL;
+    }
+}
+
+static GroupType translateGroupType(const char *groupType)
+{
+    if (!groupType)
+        return grp_unknown;
+    if (strieq(groupType, "Thor"))
+        return grp_thor;
+    else if (strieq(groupType, "Roxie"))
+        return grp_roxie;
+    else if (strieq(groupType, "RoxieFarm"))
+        return grp_roxiefarm;
+    else if (strieq(groupType, "hthor"))
+        return grp_hthor;
+    else
+        return grp_unknown;
+}
+
 class CNamedGroupCacheEntry: public CInterface
 {
 public:
     Linked<IGroup> group;
     StringAttr name;
-    StringAttr groupdir;
+    StringAttr groupDir;
     unsigned cachedtime;
+    GroupType groupType;
 
-    CNamedGroupCacheEntry(IGroup *_group, const char *_name, const char *_dir)
-    : group(_group), name(_name), groupdir(_dir)
+    CNamedGroupCacheEntry(IGroup *_group, const char *_name, const char *_dir, GroupType _groupType)
+    : group(_group), name(_name), groupDir(_dir), groupType(_groupType)
     {
         cachedtime = msTick();
     }
@@ -6345,7 +6354,7 @@ public:
         defaultTimeout = INFINITE;
     }
 
-    IGroup *dolookup(const char *logicalgroupname,IRemoteConnection *conn, StringBuffer *dirret)
+    IGroup *dolookup(const char *logicalgroupname,IRemoteConnection *conn, StringBuffer *dirret, GroupType &groupType)
     {
         SocketEndpointArray epa;
         StringBuffer gname(logicalgroupname);
@@ -6396,6 +6405,7 @@ public:
             logicalgroupname = gname.str();
         }
         StringAttr groupdir;
+        GroupType type;
         bool cached = false;
         unsigned timeNow = msTick();
         {
@@ -6413,12 +6423,14 @@ public:
                     if (range.length()==0)
                     {
                         if (dirret)
-                            dirret->append(entry.groupdir);
+                            dirret->append(entry.groupDir);
+                        groupType = entry.groupType;
                         return entry.group.getLink();
                     }
                     // there is a range so copy to epa
                     entry.group->getSocketEndpoints(epa);
-                    groupdir.set(entry.groupdir);
+                    groupdir.set(entry.groupDir);
+                    type = entry.groupType;
                     break;
                 }
             }
@@ -6432,7 +6444,7 @@ public:
                 s+=2;
                 if (*s) {
                     Owned<INode> dali = createINode(eps.str());
-                    if (!dali || !getRemoteGroup(dali, s, FOREIGN_DALI_TIMEOUT, groupdir, epa))
+                    if (!dali || !getRemoteGroup(dali, s, FOREIGN_DALI_TIMEOUT, groupdir, type, epa))
                         return NULL;
                 }
             }
@@ -6454,6 +6466,7 @@ public:
             if (!pt)
                 return NULL;
             groupdir.set(pt->queryProp("@dir"));
+            type = translateGroupType(pt->queryProp("@kind"));
             Owned<IPropertyTreeIterator> pe2 = pt->getElements("Node");
             ForEach(*pe2) {
                 SocketEndpoint ep(pe2->query().queryProp("@ip"));
@@ -6464,7 +6477,7 @@ public:
         if (!cached)
         {
             CriticalBlock block(cachesect);
-            cache.append(*new CNamedGroupCacheEntry(ret, gname, groupdir));
+            cache.append(*new CNamedGroupCacheEntry(ret, gname, groupdir, type));
         }
         if (range.length())
         {
@@ -6516,17 +6529,19 @@ public:
         }
         if (dirret)
             dirret->append(groupdir);
+        groupType = type;
         return ret.getClear();
     }
 
     IGroup *lookup(const char *logicalgroupname)
     {
-        return dolookup(logicalgroupname,NULL,NULL);
+        GroupType dummy;
+        return dolookup(logicalgroupname, NULL, NULL, dummy);
     }
 
-    IGroup *lookup(const char *logicalgroupname, StringBuffer &dir)
+    IGroup *lookup(const char *logicalgroupname, StringBuffer &dir, GroupType &groupType)
     {
-        return dolookup(logicalgroupname,NULL,&dir);
+        return dolookup(logicalgroupname, NULL, &dir, groupType);
     }
 
     INamedGroupIterator *getIterator()
@@ -6593,7 +6608,7 @@ public:
         lname.append(name);
     }
     
-    void add(const char *logicalgroupname,IGroup *group,bool cluster,const char *dir)
+    void add(const char *logicalgroupname, IGroup *group, bool cluster, const char *dir, GroupType groupType)
     {
         StringBuffer name(logicalgroupname);
         name.toLowerCase();
@@ -6607,13 +6622,13 @@ public:
             CriticalBlock block(cachesect);
             cache.kill();
             if (group)
-                cache.append(*new CNamedGroupCacheEntry(group, name.str(), dir));
+                cache.append(*new CNamedGroupCacheEntry(group, name.str(), dir, groupType));
         }
     }
 
     void remove(const char *logicalgroupname)
     {
-        add(logicalgroupname,NULL,false,NULL);
+        add(logicalgroupname, NULL, false, NULL, grp_unknown);
     }
 
     bool find(IGroup *grp, StringBuffer &gname, bool add)
@@ -6685,7 +6700,8 @@ public:
     }
 
 private:
-    bool getRemoteGroup(const INode *foreigndali, const char *gname, unsigned foreigndalitimeout, StringAttr &groupdir, SocketEndpointArray &epa)
+    bool getRemoteGroup(const INode *foreigndali, const char *gname, unsigned foreigndalitimeout,
+                           StringAttr &groupdir, GroupType &type, SocketEndpointArray &epa)
     {
         StringBuffer lcname(gname);
         gname = lcname.trim().toLowerCase().str();
@@ -6712,6 +6728,7 @@ private:
         Owned<IPropertyTree> pt = createPTree(mb);
         Owned<IPropertyTreeIterator> pe = pt->getElements("Node");
         groupdir.set(pt->queryProp("@dir"));
+        type = translateGroupType(pt->queryProp("@kind"));
         ForEach(*pe) {
             SocketEndpoint ep(pe->query().queryProp("@ip"));
             epa.append(ep);
@@ -6733,7 +6750,8 @@ bool CNamedGroupIterator::match()
             const char *name = pe->query().queryProp("@name");
             if (!name||!*name)
                 return false;
-            Owned<IGroup> lgrp = groupStore->dolookup(name,conn,NULL);
+            GroupType dummy;
+            Owned<IGroup> lgrp = groupStore->dolookup(name, conn, NULL, dummy);
             if (lgrp) {
                 if (exactmatch)
                     return lgrp->equals(matchgroup);
@@ -7428,7 +7446,8 @@ private:
                 ClusterPartDiskMapSpec mspec = file->queryPartDiskMapping(0);
                 // Unlock the old file
                 unlock();
-                file->detach(INFINITE, false); // don't delete physicals, now used by newFile
+                CDistributedFile *_file = dynamic_cast<CDistributedFile *>(file.get());
+                _file->detachLogical(INFINITE); // don't delete physicals, now used by newFile
                 transaction->clearFile(file); // no long used in transaction
                 newFile->addCluster(newcluster.str(),mspec);
                 parent->fixDates(newFile);
@@ -8159,7 +8178,6 @@ class CInitGroups
         grp->setProp("@name", name);
     }
 
-    enum GroupType { grp_thor, grp_thorspares, grp_roxie, grp_roxiefarm, grp_hthor };
     IGroup *getGroupFromCluster(GroupType groupType, IPropertyTree &cluster)
     {
         SocketEndpointArray eps;
@@ -8206,24 +8224,20 @@ class CInitGroups
                 {
                     Owned<IPropertyTreeIterator> channels;
                     channels.setown(node.getElements("RoxieChannel"));
-                    unsigned j = 0;
-                    unsigned mindrive = (unsigned)-1;
+                    unsigned thisNodePrimaryChannel = 0;
                     ForEach(*channels) {
-                        unsigned k = channels->query().getPropInt("@number");
-                        const char * dir = channels->query().queryProp("@dataDirectory");
-                        unsigned d = dir?getPathDrive(dir):0;
-                        if (d<mindrive) {
-                            j = k;
-                            mindrive = d;
-                        }
+                        unsigned channel = channels->query().getPropInt("@number");
+                        unsigned level = channels->query().getPropInt("@level", 0);
+                        if (level == 0)  // level 0 means primary copy
+                            thisNodePrimaryChannel = channel;
                     }
-                    if (j==0) {
+                    if (thisNodePrimaryChannel==0) {
                         ERRLOG("Cannot construct roxie cluster %s, no channel for node",cluster.queryProp("@name"));
                         return NULL;
                     }
-                    while (eps.ordinality()<j)
+                    while (eps.ordinality()<thisNodePrimaryChannel)
                         eps.append(nullep);
-                    eps.item(j-1) = ep;
+                    eps.item(thisNodePrimaryChannel-1) = ep;
                     break;
                 }
                 case grp_thor:
@@ -8333,13 +8347,9 @@ class CInitGroups
                 realCluster = false;
                 break;
             case grp_roxie:
-                defDir = cluster.queryProp("@slaveDataDir");
-                if (!defDir||!*defDir)
-                    defDir = cluster.queryProp("@baseDataDir");
                 gname.append(cluster.queryProp("@name"));
                 break;
             case grp_roxiefarm:
-                defDir = cluster.queryProp("@dataDirectory");
                 break;
             default:
                 throwUnexpected();

dali/base/dadfs.hpp

@@ -214,7 +214,7 @@ interface IDistributedFile: extends IInterface
     virtual const char *queryPartMask() = 0;                                    // default part name mask
 
     virtual void attach(const char *logicalname,IUserDescriptor *user) = 0;     // attach to name in DFS
-    virtual void detach(unsigned timeoutms=INFINITE, bool removePhysicalParts=true) = 0;                       // no longer attached to name in DFS
+    virtual void detach(unsigned timeoutms=INFINITE) = 0;                       // no longer attached to name in DFS
 
     virtual IPropertyTree &queryAttributes() = 0;                               // DFile attributes
 
@@ -231,9 +231,6 @@ interface IDistributedFile: extends IInterface
 
     virtual unsigned numCopies(unsigned partno) = 0;                            // number of copies
 
-    virtual bool removePhysicalPartFiles(const char *cluster=NULL,IMultiException *exceptions=NULL) = 0;          // removes all physical part files
-                                                                                // returns true if no major errors
-
     virtual bool existsPhysicalPartFiles(unsigned short port) = 0;              // returns true if physical patrs all exist (on primary OR secondary)
 
     virtual bool renamePhysicalPartFiles(const char *newlfn,const char *cluster=NULL,IMultiException *exceptions=NULL,const char *newbasedir=NULL) = 0;           // renames all physical part files
@@ -262,7 +259,7 @@ interface IDistributedFile: extends IInterface
     virtual void setECL(const char *ecl) = 0;
 
     virtual void addCluster(const char *clustername,const ClusterPartDiskMapSpec &mspec) = 0;
-    virtual void removeCluster(const char *clustername) = 0;    // doesn't delete parts
+    virtual bool removeCluster(const char *clustername) = 0;    // doesn't delete parts
     virtual bool checkClusterCompatible(IFileDescriptor &fdesc, StringBuffer &err) = 0;
     virtual void updatePartDiskMapping(const char *clustername,const ClusterPartDiskMapSpec &spec)=0;
 
@@ -563,6 +560,8 @@ extern da_decl IDistributedFileDirectory &queryDistributedFileDirectory();
 
 // ==GROUP STORE=================================================================================================
 
+enum GroupType { grp_thor, grp_thorspares, grp_roxie, grp_roxiefarm, grp_hthor, grp_unknown };
+
 interface INamedGroupIterator: extends IInterface
 {
     virtual bool first() = 0;
@@ -575,16 +574,15 @@ interface INamedGroupIterator: extends IInterface
 
 interface INamedGroupStore: implements IGroupResolver
 {
-    
     virtual IGroup *lookup(const char *logicalgroupname) = 0;
     virtual INamedGroupIterator *getIterator() = 0;
-    virtual INamedGroupIterator *getIterator(IGroup *match,bool exact=false) = 0;
-    virtual void add(const char *logicalgroupname,IGroup *group,bool cluster=false, const char *dir=NULL) = 0;
+    virtual INamedGroupIterator *getIterator(IGroup *match, bool exact=false) = 0;
+    virtual void add(const char *logicalgroupname,IGroup *group, bool cluster=false, const char *dir=NULL, GroupType groupType = grp_unknown) = 0;
     virtual void remove(const char *logicalgroupname) = 0;
     virtual bool find(IGroup *grp, StringBuffer &lname, bool add=false) = 0;
     virtual void addUnique(IGroup *group,StringBuffer &lname,const char *dir=NULL) = 0;
     virtual void swapNode(const IpAddress &from, const IpAddress &to) = 0;
-    virtual IGroup *lookup(const char *logicalgroupname, StringBuffer &dir) = 0;
+    virtual IGroup *lookup(const char *logicalgroupname, StringBuffer &dir, GroupType &groupType) = 0;
     virtual unsigned setDefaultTimeout(unsigned timems) = 0;                                    // sets default timeout for SDS connections and locking                                                                                         // returns previous value
 
 };

dali/base/dafdesc.cpp

@@ -212,7 +212,7 @@ void ClusterPartDiskMapSpec::fromProp(IPropertyTree *tree)
     // if directory is specified then must match default base to be default replicated
     StringBuffer dir;
     if (tree&&tree->getProp("@directory",dir)) {
-        const char * base = queryBaseDirectory(false,SepCharBaseOs(getPathSepChar(dir.str())));
+        const char * base = queryBaseDirectory(0, SepCharBaseOs(getPathSepChar(dir.str())));
         size32_t l = strlen(base);
         if ((memcmp(base,dir.str(),l)!=0)||((l!=dir.length())&&!isPathSepChar(dir.charAt(l))))
             defrep = 0;
@@ -394,7 +394,7 @@ public:
             group.setown(createIGroup(grptext));
         mspec.deserialize(mb);
         mb.read(name);
-        if ((name.length()==1)&&(*name.get()=='!')) { // flag for roxie lable
+        if ((name.length()==1)&&(*name.get()=='!')) { // flag for roxie label
             mb.read(roxielabel);
             name.clear();
         }
@@ -525,7 +525,7 @@ public:
     void getBaseDir(StringBuffer &basedir,DFD_OS os)
     {
         if (mspec.defaultBaseDir.isEmpty())  // assume thor default
-            basedir.append(queryBaseDirectory(false,os));
+            basedir.append(queryBaseDirectory(0, os));
         else
             basedir.append(mspec.defaultBaseDir);
     }
@@ -533,7 +533,7 @@ public:
     void getReplicateDir(StringBuffer &basedir,DFD_OS os)
     {
         if (mspec.defaultReplicateDir.isEmpty())  // assume thor default
-            basedir.append(queryBaseDirectory(true,os));
+            basedir.append(queryBaseDirectory(1, os));
         else
             basedir.append(mspec.defaultReplicateDir);
     }
@@ -1206,6 +1206,19 @@ class CFileDescriptor:  public CFileDescriptorBase, implements ISuperFileDescrip
         return NULL;
     }
 
+    IClusterInfo *queryCluster(const char *_clusterName)
+    {
+        StringAttr clusterName = _clusterName;
+        clusterName.toLowerCase();
+        StringBuffer name;
+        ForEachItemIn(c, clusters)
+        {
+            if (0 == strcmp(clusters.item(c).getClusterLabel(name.clear()).str(), clusterName))
+                return &clusters.item(c);
+        }
+        return NULL;
+    }
+
     void replaceClusterDir(unsigned partno,unsigned copy, StringBuffer &path)
     {
         // assumes default dir matches one of clusters
@@ -1447,6 +1460,8 @@ public:
         unsigned n = clusters.ordinality();
         pt.setPropInt("@numclusters",n);
         unsigned cn = 0;
+
+// JCSMORE - afaics, IFileDescriptor @group is no longer used
         StringBuffer grplist;
         ForEachItemIn(i1,clusters) {
             Owned<IPropertyTree> ct = createPTree("Cluster");
@@ -1467,6 +1482,8 @@ public:
             pt.setProp("@group",grplist.str());
         else
             pt.removeProp("@group");
+/// ^^
+
         n = numParts();
         pt.setPropInt("@numparts",n);
         if ((flags&IFDSF_EXCLUDE_PARTS)==0) {
@@ -1523,7 +1540,7 @@ public:
             if (!sc)
                 sc = getPathSepChar(dirname);
             StringBuffer tmp;
-            tmp.append(queryBaseDirectory(false,SepCharBaseOs(sc))).append(sc).append(s);
+            tmp.append(queryBaseDirectory(0, SepCharBaseOs(sc))).append(sc).append(s);
             directory.set(tmp.str());
         }
         else
@@ -1882,12 +1899,15 @@ public:
         closePending();
         unsigned done = 0;
         StringBuffer cname;
-        ForEachItemIn(i,names) {
+        ForEachItemIn(i,names)
+        {
             StringAttr name = names.item(i);
             name.toLowerCase();
-            for (unsigned j=done;j<clusters.ordinality();j++) {
+            for (unsigned j=done;j<clusters.ordinality();j++)
+            {
                 clusters.item(j).getClusterLabel(cname.clear());
-                if (strcmp(cname.str(),name)==0) {
+                if (strcmp(cname.str(),name)==0)
+                {
                     if (done!=j)
                         clusters.swap(done,j);
                     done++;
@@ -1895,11 +1915,31 @@ public:
                 }
             }
         }
-        if (exclusive) {
+        if (exclusive)
+        {
             if (!done)
                 done = 1;
+            StringAttr oldDefaultDir;
+            StringBuffer baseDir1;
             while (clusters.ordinality()>done)
+            {
+                clusters.item(clusters.ordinality()-1).getBaseDir(baseDir1.clear(),SepCharBaseOs(getPathSepChar(directory)));
+
+                // if baseDir is leading component this file's default directory..
+                if (!oldDefaultDir.length() && directory.length()>=baseDir1.length() && 0==strncmp(directory, baseDir1, baseDir1.length()) &&
+                    (directory.length()==baseDir1.length() || isPathSepChar(directory[baseDir1.length()])))
+                    oldDefaultDir.set(baseDir1.str());
                 clusters.remove(clusters.ordinality()-1);
+            }
+            if (oldDefaultDir.length() && clusters.ordinality())
+            {
+                StringBuffer baseDir2;
+                clusters.item(0).getBaseDir(baseDir2.clear(), SepCharBaseOs(getPathSepChar(directory)));
+                StringBuffer newDir(baseDir2.str());
+                if (directory.length()>oldDefaultDir.length())
+                    newDir.append(directory.get()+oldDefaultDir.length());
+                directory.set(newDir.str());
+            }
         }
     }
 
@@ -2179,11 +2219,8 @@ inline bool validFNameChar(char c)
     return (c>=32 && c<127 && !strchr(invalids, c));
 }
 
-static StringAttr winbasedir;
-static StringAttr winreplicatedir;
-static StringAttr unixbasedir;
-static StringAttr unixreplicatedir;
-
+static StringAttr winbasedirs[MAX_REPLICATION_LEVELS];
+static StringAttr unixbasedirs[MAX_REPLICATION_LEVELS];
 
 static StringAttr defaultpartmask("$L$._$P$_of_$N$");
 
@@ -2197,38 +2234,37 @@ void loadDefaultBases()
     ldbDone = true;
     // assumed default first thor
     // first set 
-    if (winreplicatedir.isEmpty())
-        winreplicatedir.set("d:\\thordata");
-    if (winbasedir.isEmpty())
-        winbasedir.set("c:\\thordata");
+    if (winbasedirs[1].isEmpty())
+        winbasedirs[1].set("d:\\thordata");
+    if (winbasedirs[0].isEmpty())
+        winbasedirs[0].set("c:\\thordata");
     SessionId mysessid = myProcessSession();
-    if (mysessid&&(unixreplicatedir.isEmpty()||unixbasedir.isEmpty())) {
+    if (mysessid && (unixbasedirs[1].isEmpty() || unixbasedirs[0].isEmpty())) {
         Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software/Directories", mysessid, RTM_LOCK_READ, SDS_CONNECT_TIMEOUT);
         if (conn) {
             IPropertyTree* dirs = conn->queryRoot();
             StringBuffer dirout;
-            if (unixreplicatedir.isEmpty())
+            if (unixbasedirs[1].isEmpty())
                 if (getConfigurationDirectory(dirs,"mirror", "thor",
                     "mythor",   // NB this is dummy value should really get 1st thor (but actually hopefully not used anyway)
                     dirout))
-                    unixreplicatedir.set(dirout.str());
-            if (unixbasedir.isEmpty())
+                    unixbasedirs[1].set(dirout.str());
+            if (unixbasedirs[0].isEmpty())
                 if (getConfigurationDirectory(dirs,"data", "thor",
                     "mythor",   // NB this is dummy value should really get 1st thor (but actually hopefully not used anyway)
                     dirout.clear()))
-                    unixbasedir.set(dirout.str());
+                    unixbasedirs[0].set(dirout.str());
 
         }
     }
-    if (unixreplicatedir.isEmpty())
-        unixreplicatedir.set("/d$/thordata");
-    if (unixbasedir.isEmpty())
-        unixbasedir.set("/c$/thordata");
-
+    if (unixbasedirs[0].isEmpty())
+        unixbasedirs[0].set("/var/lib/HPCCSystems/hpcc-data/thor");
+    if (unixbasedirs[1].isEmpty())
+        unixbasedirs[1].set("/var/lib/HPCCSystems/hpcc-mirror/thor");
 }
 
 
-const char *queryBaseDirectory(bool replicate,DFD_OS os)
+const char *queryBaseDirectory(unsigned replicateLevel, DFD_OS os)
 {
     if (os==DFD_OSdefault)
 #ifdef _WIN32
@@ -2236,17 +2272,19 @@ const char *queryBaseDirectory(bool replicate,DFD_OS os)
 #else
         os = DFD_OSunix;
 #endif
+    assertex(replicateLevel < MAX_REPLICATION_LEVELS);
     loadDefaultBases();
-    switch (os) {
+    switch (os)
+    {
     case DFD_OSwindows:
-        return replicate?winreplicatedir.get():winbasedir.get();
+        return winbasedirs[replicateLevel];
     case DFD_OSunix:
-        return replicate?unixreplicatedir.get():unixbasedir.get();
+        return unixbasedirs[replicateLevel];
     }
     return NULL;
 }
 
-void setBaseDirectory(const char * dir,bool replicate,DFD_OS os)
+void setBaseDirectory(const char * dir, unsigned replicateLevel, DFD_OS os)
 {
     // 2 possibilities
     // either its an absolute path
@@ -2257,35 +2295,19 @@ void setBaseDirectory(const char * dir,bool replicate,DFD_OS os)
 #else
         os = DFD_OSunix;
 #endif
+    assertex(replicateLevel < MAX_REPLICATION_LEVELS);
     StringBuffer out;
-    if (!dir||!*dir||!isAbsolutePath(dir)) {
-        if (os==DFD_OSwindows) {
-            if (replicate)
-                out.append("d:\\thordata");
-            else
-                out.append("c:\\thordata");
-        }
-        else if (replicate)
-            out.append("/d$/thordata");
-        else
-            out.append("/c$/thordata");
-        dir = out.str();
-    }
+    if (!dir||!*dir||!isAbsolutePath(dir))
+        throw MakeStringException(-1,"setBaseDirectory(%s) requires an absolute path",dir ? dir : "null");
     size32_t l = strlen(dir);
     if ((l>3)&&(isPathSepChar(dir[l-1])))
         l--;
     switch (os) {
     case DFD_OSwindows:
-        if (replicate)
-            winreplicatedir.set(dir,l);
-        else
-            winbasedir.set(dir,l);
+        winbasedirs[replicateLevel].set(dir,l);
         break;
     case DFD_OSunix:
-        if (replicate)
-            unixreplicatedir.set(dir,l);
-        else
-            unixbasedir.set(dir,l);
+        unixbasedirs[replicateLevel].set(dir,l);
         break;
     }
 }
@@ -2404,7 +2426,7 @@ inline const char *skipRoot(const char *lname)
 
 
 
-StringBuffer &makePhysicalPartName(const char *lname, unsigned partno, unsigned partmax, StringBuffer &result, bool replicate, DFD_OS os,const char *diroverride)
+StringBuffer &makePhysicalPartName(const char *lname, unsigned partno, unsigned partmax, StringBuffer &result, unsigned replicateLevel, DFD_OS os,const char *diroverride)
 {
     assertex(lname);
     if (strstr(lname,"::>")) { // probably query
@@ -2439,7 +2461,7 @@ StringBuffer &makePhysicalPartName(const char *lname, unsigned partno, unsigned
         result.append(diroverride);
     }
     else
-        result.append(queryBaseDirectory(replicate,os));
+        result.append(queryBaseDirectory(replicateLevel, os));
 
     size32_t l = result.length();
     if ((l>3)&&(result.charAt(l-1)!=OsSepChar(os))) {
@@ -2517,7 +2539,7 @@ bool setReplicateDir(const char *dir,StringBuffer &out,bool isrep,const char *ba
     if (!sep)
         return false;
     DFD_OS os = SepCharBaseOs(*sep);
-    const char *d = baseDir?baseDir:queryBaseDirectory(!isrep,os);
+    const char *d = baseDir?baseDir:queryBaseDirectory(isrep ? 0 : 1,os);
     if (!d)
         return false;
     unsigned match = 0;
@@ -2528,7 +2550,7 @@ bool setReplicateDir(const char *dir,StringBuffer &out,bool isrep,const char *ba
             match = i;
             count++;
         }
-    const char *r = repDir?repDir:queryBaseDirectory(isrep,os);
+    const char *r = repDir?repDir:queryBaseDirectory(isrep ? 1 : 0,os);
     if (d[i]==0) {
         if ((dir[i]==0)||isPathSepChar(dir[i])) {
             out.append(r).append(dir+i);

dali/base/dafdesc.hpp

@@ -34,6 +34,8 @@ interface IReplicatedFile;
 
 #define SUPPORTS_MULTI_CLUSTERS  // always now set
 
+#define MAX_REPLICATION_LEVELS 4
+
 enum DFD_OS
 {
     DFD_OSdefault,
@@ -211,8 +213,9 @@ if endCluster is not called it will assume only one cluster and not replicated
 
     virtual IGroup *getGroup() = 0;                                             // will be for first cluster
 
-    virtual unsigned numClusters()=0;
-    virtual ClusterPartDiskMapSpec &queryPartDiskMapping(unsigned clusternum)=0;
+    virtual unsigned numClusters() = 0;
+    virtual IClusterInfo *queryCluster(const char *clusterName) = 0;
+    virtual ClusterPartDiskMapSpec &queryPartDiskMapping(unsigned clusternum) = 0;
     virtual IGroup *queryClusterGroup(unsigned clusternum) = 0;                     // returns group for cluster if known
     virtual void setClusterGroup(unsigned clusternum,IGroup *grp) = 0;              // sets group for cluster
     virtual StringBuffer &getClusterGroupName(unsigned clusternum,StringBuffer &ret,IGroupResolver *resolver=NULL) = 0;                 // returns group name of cluster (if set)
@@ -290,13 +293,13 @@ void getClusterInfo(IPropertyTree &pt, IGroupResolver *resolver, unsigned flags,
 
 
 
-// default logical to physcal filename routined
+// default logical to physical filename routines
 extern da_decl StringBuffer &makePhysicalPartName(
                                 const char *lname,                  // logical name
                                 unsigned partno,                    // part number (1..)
                                 unsigned partmax,                   // number of parts (1..)
                                 StringBuffer &result,               // result filename (or directory name if part 0)
-                                bool replicate = false,             // uses replication directory
+                                unsigned replicateLevel = 0,       // uses replication directory
                                 DFD_OS os=DFD_OSdefault,            // os must be specified if no dir specified
                                 const char *diroverride=NULL);      // override default directory
 extern da_decl StringBuffer &makeSinglePhysicalPartName(const char *lname, // single part file
@@ -307,12 +310,12 @@ extern da_decl StringBuffer &makeSinglePhysicalPartName(const char *lname, // si
                                                         );    
 
 // set/get defaults
-extern da_decl const char *queryBaseDirectory(bool replicatedir=false,DFD_OS os=DFD_OSdefault);
-extern da_decl void setBaseDirectory(const char * dir,bool replicatedir=false,DFD_OS os=DFD_OSdefault);
+extern da_decl const char *queryBaseDirectory(unsigned replicateLevel=0, DFD_OS os=DFD_OSdefault);
+extern da_decl void setBaseDirectory(const char * dir, unsigned replicateLevel=0, DFD_OS os=DFD_OSdefault);
 extern da_decl const char *queryPartMask();
 extern da_decl StringBuffer &getPartMask(StringBuffer &ret,const char *lname=NULL,unsigned partmax=0);
 extern da_decl void setPartMask(const char * mask);
-extern da_decl bool setReplicateDir(const char *name,StringBuffer &out, bool isrep=true,const char *baseDir=NULL,const char *repDir=NULL); // changes direcctory of name passed to backup directory
+extern da_decl bool setReplicateDir(const char *name,StringBuffer &out, bool isrep=true,const char *baseDir=NULL,const char *repDir=NULL); // changes directory of name passed to backup directory
 
 extern da_decl IFileDescriptor *createFileDescriptor();
 extern da_decl IFileDescriptor *createFileDescriptor(IPropertyTree *attr);      // ownership of attr tree is taken

dali/base/dautils.cpp

@@ -722,9 +722,12 @@ const char *CDfsLogicalFileName::get(bool removeforeign) const
     return ret;
 }
 
-StringBuffer &CDfsLogicalFileName::get(StringBuffer &str,bool removeforeign) const
+StringBuffer &CDfsLogicalFileName::get(StringBuffer &str, bool removeforeign, bool withCluster) const
 {
-    return str.append(get(removeforeign));
+    str.append(get(removeforeign));
+    if (withCluster && cluster.length())
+        str.append("@").append(cluster);
+    return str;
 }
 
 
@@ -976,7 +979,7 @@ bool CDfsLogicalFileName::setFromMask(const char *fname,const char *rootdir)
         return false;
     // first remove base dir from fname if present
     DFD_OS os = SepCharBaseOs(getPathSepChar(fname));
-    const char *dir = (rootdir&&*rootdir)?rootdir:queryBaseDirectory(false,os);
+    const char *dir = (rootdir&&*rootdir)?rootdir:queryBaseDirectory(0, os);
     // ignore drive if present
     if (os==DFD_OSwindows) {
         if (dir[1]==':')

dali/base/dautils.hpp

@@ -105,7 +105,7 @@ public:
     StringBuffer &getCluster(StringBuffer &cname) const { return cname.append(cluster); }
 
     const char *get(bool removeforeign=false) const;
-    StringBuffer &get(StringBuffer &str,bool removeforeign=false) const;
+    StringBuffer &get(StringBuffer &str,bool removeforeign=false, bool withCluster=false) const;
     const char *queryTail() const;
     StringBuffer &getTail(StringBuffer &buf) const;
 

dali/daunittest/dautdfs.cpp

@@ -305,7 +305,8 @@ protected:
         mapping.setRepeatedCopies(7,false);
         fdesc->addCluster(grp,mapping);
         StringBuffer dir2;
-        grp.setown(dfsgroup->lookup(DFSUTGROUP "7b", dir2));
+        GroupType groupType;
+        grp.setown(dfsgroup->lookup(DFSUTGROUP "7b", dir2, groupType));
         ClusterPartDiskMapSpec mapping2;
         mapping2.setDefaultBaseDir(dir2);
         mapping2.setRepeatedCopies(7,true);

dali/dfu/dfurun.cpp

@@ -345,7 +345,8 @@ class CDFUengine: public CInterface, implements IDFUengine
         if (!cluster||!*cluster)
             return;
         StringBuffer dir;
-        Owned<IGroup> grp = queryNamedGroupStore().lookup(cluster,dir);
+        GroupType groupType;
+        Owned<IGroup> grp = queryNamedGroupStore().lookup(cluster, dir, groupType);
         if (!grp) {
             throw MakeStringException(-1,"setFileRepeatOptions cluster %s not found",cluster);
             return;
@@ -793,7 +794,6 @@ public:
 //                      destination->setClusterPartDefaultBaseDir(tmp.str(),basedir);
                 }
             }
-            options->setNoDelete(ctx.superoptions->getNoDelete());
             options->setNoSplit(ctx.superoptions->getNoSplit());
             options->setOverwrite(ctx.superoptions->getOverwrite());
             options->setReplicate(ctx.superoptions->getReplicate());
@@ -868,9 +868,8 @@ public:
         if (dfile) {
             if (!ctx.superoptions->getOverwrite()) 
                 throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
-            if (dfile->querySuperFile()) 
-                dfile->detach();
-            else {
+            if (!dfile->querySuperFile())
+            {
                 if (ctx.superoptions->getIfModified()&&
                     (ftree->hasProp("Attr/@fileCrc")&&ftree->getPropInt64("Attr/@size")&&
                     ((unsigned)ftree->getPropInt64("Attr/@fileCrc")==(unsigned)dfile->queryAttributes().getPropInt64("@fileCrc"))&&
@@ -878,9 +877,8 @@ public:
                     PROGLOG("File copy of %s not done as file unchanged",srclfn);
                     return;
                 }
-                dfile->detach();
-                dfile->removePhysicalPartFiles(NULL,NULL);
             }
+            dfile->detach();
             dfile.clear();
         }
         if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_File))==0) {
@@ -1402,10 +1400,7 @@ public:
                     source->getLogicalName(tmp.clear());
                     if (tmp.length()) {
                         runningconn.setown(setRunning(runningpath.str()));;
-                        if (options->getNoDelete())
-                            fdir.removeEntry(tmp.str(),userdesc);
-                        else
-                            fdir.removeEntry(tmp.str(),userdesc);
+                        fdir.removeEntry(tmp.str(),userdesc);
                         Audit("REMOVE",userdesc,tmp.clear(),NULL);
                         runningconn.clear();
                     }

dali/dfu/dfuutil.cpp

@@ -454,6 +454,7 @@ class CFileCloner
         {
             StringBuffer s;
             dstfdesc->queryProperties().setProp("@cloneFrom", srcdali->endpoint().getUrlStr(s).str());
+            dstfdesc->queryProperties().setProp("@cloneFromDir", srcfdesc->queryDefaultDir());
             unsigned numClusters = srcfdesc->numClusters();
             for (unsigned clusterNum = 0; clusterNum < numClusters; clusterNum++)
             {
@@ -538,7 +539,8 @@ public:
             break;
         }
         StringBuffer defdir1;
-        grp1.setown(queryNamedGroupStore().lookup(_cluster1,defdir1));
+        GroupType groupType;
+        grp1.setown(queryNamedGroupStore().lookup(_cluster1, defdir1, groupType));
         if (!grp1)
             throw MakeStringException(-1,"Cannot find cluster %s",_cluster1);
         if (defdir1.length())
@@ -547,7 +549,7 @@ public:
             spec2 = spec1;
             cluster2.set(_cluster2);
             StringBuffer defdir2;
-            grp2.setown(queryNamedGroupStore().lookup(_cluster2,defdir2));
+            grp2.setown(queryNamedGroupStore().lookup(_cluster2, defdir2, groupType));
             if (!grp2)
                 throw MakeStringException(-1,"Cannot find cluster %s",_cluster2);
             spec2.setRepeatedCopies(CPDMSRP_lastRepeated,true); // only TLK on cluster2
@@ -614,12 +616,7 @@ public:
         if (dfile) {
             if (!overwrite)
                 throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
-            if (dfile->querySuperFile())
-                dfile->detach();
-            else {
-                dfile->detach();
-                dfile->removePhysicalPartFiles(NULL,NULL);
-            }
+            dfile->detach();
             dfile.clear();
         }
         if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_File))==0) {
@@ -690,12 +687,7 @@ public:
         if (dfile) {
             if (!overwrite)
                 throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
-            if (dfile->querySuperFile())
-                dfile->detach();
-            else {
-                dfile->detach();
-                dfile->removePhysicalPartFiles(NULL,NULL);
-            }
+            dfile->detach();
             dfile.clear();
         }
         cloneSubFile(ftree,dlfn.get(),srcdali);
@@ -867,56 +859,63 @@ public:
         SocketEndpoint daliep = srcdali;
         CDfsLogicalFileName slfn;
         slfn.set(srclfn);
-        if (slfn.isForeign()) { // trying to confuse me
+        if (slfn.isForeign()) // trying to confuse me
+        {
             slfn.getEp(daliep);
             slfn.clearForeign();
         }
         if (daliep.port==0)
             daliep.port= DALI_SERVER_PORT;
         Owned<INode> srcnode = createINode(daliep);
-        if (queryCoven().inCoven(srcnode)) {
+        if (queryCoven().inCoven(srcnode))
+        {
             // if dali is local and filenames same
             CDfsLogicalFileName dlfn;
             dlfn.set(lfn);
-            if (strcmp(slfn.get(),dlfn.get())==0) {
+            if (strcmp(slfn.get(),dlfn.get())==0)
+            {
                 PROGLOG("File copy of %s not done as file local",srclfn);
                 return;
             }
         }
         Owned<IPropertyTree> ftree = queryDistributedFileDirectory().getFileTree(srclfn,srcuser,srcnode, FOREIGN_DALI_TIMEOUT, false);
-        if (!ftree.get()) {
+        if (!ftree.get())
+        {
             StringBuffer s;
             throw MakeStringException(-1,"Source file %s could not be found in Dali %s",srclfn,daliep.getUrlStr(s).str());
         }
         // first see if target exists (and remove if does and overwrite specified)
         Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(lfn,user,true);
-        if (dfile) {
+        if (dfile)
+        {
             if (!overwrite)
                 throw MakeStringException(-1,"Destination file %s already exists",lfn);
-            if (dfile->querySuperFile())
-                dfile->detach();
-            else {
+            if (!dfile->querySuperFile())
+            {
                 if (ftree->hasProp("Attr/@fileCrc")&&ftree->getPropInt64("Attr/@size")&&
                     ((unsigned)ftree->getPropInt64("Attr/@fileCrc")==(unsigned)dfile->queryAttributes().getPropInt64("@fileCrc"))&&
-                    (ftree->getPropInt64("Attr/@size")==dfile->getFileSize(false,false))) {
+                    (ftree->getPropInt64("Attr/@size")==dfile->getFileSize(false,false)))
+                {
                     PROGLOG("File copy of %s not done as file unchanged",srclfn);
                     return;
                 }
-                dfile->detach();
-                dfile->removePhysicalPartFiles(NULL,NULL);
             }
+            dfile->detach();
             dfile.clear();
         }
-        if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_File))==0) {
+        if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_File))==0)
+        {
             assertex(copier);
             if (!copier->copyFile(lfn,daliep,srclfn,srcuser,user))
                 throw MakeStringException(-1,"File %s could not be copied",lfn);
 
         }
-        else if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_SuperFile))==0) {
+        else if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_SuperFile))==0)
+        {
             StringArray subfiles;
             Owned<IPropertyTreeIterator> piter = ftree->getElements("SubFile");
-            ForEach(*piter) {
+            ForEach(*piter)
+            {
                 const char *name = piter->query().queryProp("@name");
                 CDfsLogicalFileName dlfn;
                 dlfn.set(name);
@@ -930,12 +929,11 @@ public:
             Owned<IDistributedSuperFile> sfile = queryDistributedFileDirectory().createSuperFile(lfn,user,true,false);
             if (!sfile)
                 throw MakeStringException(-1,"SuperFile %s could not be created",lfn);
-            ForEachItemIn(i,subfiles) {
+            ForEachItemIn(i,subfiles)
                 sfile->addSubFile(subfiles.item(i));
-            }
-
         }
-        else {
+        else
+        {
             StringBuffer s;
             throw MakeStringException(-1,"Source file %s in Dali %s is not a file or superfile",srclfn,daliep.getUrlStr(s).str());
         }

dali/dfu/dfuwu.cpp

@@ -1678,11 +1678,6 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions
 public:
     IMPLEMENT_DFUWUCHILD;
 
-    bool getNoDelete() const
-    {
-        return queryRoot()->getPropInt("@nodelete")!=0;
-    }
-
     bool getNoSplit() const
     {
         return queryRoot()->getPropInt("@nosplit")!=0;

dali/dfu/dfuwu.hpp

@@ -136,7 +136,6 @@ enum DFUclusterPartDiskMapping // legacy - should use ClusterPartDiskMapSpec ins
 
 interface IConstDFUoptions : extends IInterface
 {
-    virtual bool getNoDelete() const = 0;
     virtual bool getNoSplit() const = 0;
     virtual bool getReplicate() const = 0;
     virtual bool getRecover() const = 0;

dali/dfuXRefLib/dfuxreflib.cpp

@@ -2677,8 +2677,8 @@ IPropertyTree *  runXRef(unsigned nclusters,const char **clusters,IXRefProgressC
     Owned<IGroup> group = queryNamedGroupStore().lookup(clusters[0]);
     if (group)
         islinux = queryOS(group->queryNode(0).endpoint())==MachineOsLinux;
-    dirs[0] = queryBaseDirectory(false,islinux?DFD_OSunix:DFD_OSwindows);
-    dirs[1] = queryBaseDirectory(true,islinux?DFD_OSunix:DFD_OSwindows);
+    dirs[0] = queryBaseDirectory(0,islinux?DFD_OSunix:DFD_OSwindows);
+    dirs[1] = queryBaseDirectory(1,islinux?DFD_OSunix:DFD_OSwindows);
     numdirs = 2;
     IPropertyTree *ret=NULL;
     try {

dali/dfuplus/dfuplus.cpp

@@ -886,14 +886,6 @@ int CDfuPlusHelper::rundafs()
 
 int CDfuPlusHelper::remove()
 {
-    bool nodelete = false;
-    const char* ndstr = NULL;
-    if((ndstr = globals->queryProp("nodelete")) != NULL)
-    {
-        if(strcmp(ndstr, "1") == 0)
-        nodelete = true;
-    }
-
     StringArray files;
     const char* name = globals->queryProp("name");
     const char* names = globals->queryProp("names");
@@ -960,7 +952,6 @@ int CDfuPlusHelper::remove()
     
     Owned<IClientDFUArrayActionRequest> req = dfuclient->createDFUArrayActionRequest();
     req->setType("Delete");
-    req->setNoDelete(nodelete);
     req->setLogicalFiles(files);
 
     Owned<IClientDFUArrayActionResponse> resp = dfuclient->DFUArrayAction(req);

dali/dfuplus/main.cpp

@@ -106,7 +106,6 @@ void handleSyntax()
     out.append("        name=<logical-name>\n");
     out.append("        names=<multiple-logical-names-separated-by-comma>\n");
     out.append("        namelist=<logical-name-list-in-file>\n");
-    out.append("        nodelete=0|1    -- optional\n");
     out.append("    rename options:\n");
     out.append("        srcname=<source-logical-name>\n");
     out.append("        dstname=<destination-logical-name>\n");

dali/ft/daft.cpp

@@ -106,7 +106,7 @@ void CDistributedFileSystem::move(IDistributedFile * from, IDistributedFile * to
     sprayer->setTarget(to);
     sprayer->spray();
     //sprayer->removeSource();
-    remove(from);
+    from->detach();
 }
 
 void CDistributedFileSystem::replicate(IDistributedFile * from, IGroup *destgroup, IPropertyTree * recovery, IRemoteConnection * recoveryConnection, IDFPartFilter *filter, IPropertyTree * options, IDaftProgress * progress, IAbortRequestCallback * abort, const char *wuid)
@@ -212,17 +212,6 @@ offset_t CDistributedFileSystem::getSize(IDistributedFile * file, bool forceget,
     return totalSize;
 }
 
-// FIXME: This should NOT call detach / removePhysical directly!
-bool CDistributedFileSystem::remove(IDistributedFile * file,const char *cluster,IMultiException *mexcept, unsigned timeoutms)
-{
-    // this is now equivalent to removePhysicalPartFiles as linux uses dafilesrv by default
-    if (!cluster||((file->findCluster(cluster)==0)&&(file->numClusters()==1))) {
-        cluster = NULL; // deleting the last cluster removes the file
-        file->detach(timeoutms);
-    }
-    return file->removePhysicalPartFiles(cluster,mexcept); // this is bit cavalier with errors - but better orphans than inconsistant DFS
-}
-
 bool CDistributedFileSystem::compress(IDistributedFile * file)
 {
     bool ok = true;

dali/ft/daft.hpp

@@ -72,7 +72,6 @@ interface IDistributedFileSystem : public IInterface
     virtual offset_t getSize(IDistributedFile * file,
                              bool forceget=false,                               // if true gets physical size (ignores cached attribute)
                              bool dontsetattr=true) = 0;                        // if true doesn't set attribute when physical size got
-    virtual bool remove(IDistributedFile * file,const char *clustername=NULL,IMultiException *mexcept=NULL, unsigned timeoutms=INFINITE) = 0;
     virtual bool compress(IDistributedFile * file) = 0;                                                  
     virtual offset_t getCompressedSize(IDistributedFile * part) = 0;
 

dali/ft/daft.ipp

@@ -42,7 +42,6 @@ public:
 
 //operations on a single file.
     virtual offset_t getSize(IDistributedFile * file,bool forceget,bool dontsetattr);
-    virtual bool remove(IDistributedFile * file,const char *cluster=NULL,IMultiException *mexcept=NULL, unsigned timeoutms=INFINITE);
     virtual bool compress(IDistributedFile * file);                                                  
     virtual offset_t getCompressedSize(IDistributedFile * part);
 

dali/sasha/saxref.cpp

@@ -747,7 +747,8 @@ public:
         grpstr.toLowerCase();
         StringAttr grpname(grpstr.str());
         StringBuffer basedir;
-        grp.setown(queryNamedGroupStore().lookup(grpstr.str(),basedir));
+        GroupType groupType;
+        grp.setown(queryNamedGroupStore().lookup(grpstr.str(), basedir, groupType));
         if (!grp) {
             ERRLOG(LOGPFX "Cluster %s node group %s not found",clustname.get(),grpstr.str());
             return false;
@@ -804,9 +805,9 @@ public:
             if (getConfigurationDirectory(serverConfig->queryPropTree("Directories"),"mirror","thor",_clustname,repdir))
                 rdir = repdir.str();
             iswin = grp->ordinality()?(getDaliServixOs(grp->queryNode(0).endpoint())==DAFS_OSwindows):false;
-            setBaseDirectory(ddir,false,iswin?DFD_OSwindows:DFD_OSunix);
-            setBaseDirectory(rdir,true,iswin?DFD_OSwindows:DFD_OSunix);
-            rootdir.set(queryBaseDirectory(false,iswin?DFD_OSwindows:DFD_OSunix));
+            setBaseDirectory(ddir,0,iswin?DFD_OSwindows:DFD_OSunix);
+            setBaseDirectory(rdir,1,iswin?DFD_OSwindows:DFD_OSunix);
+            rootdir.set(queryBaseDirectory(0,iswin?DFD_OSwindows:DFD_OSunix));
         }
         else {
             rootdir.set(basedir);

deployment/deploy/XMLTags.h

@@ -97,6 +97,7 @@
 #define XML_ATTR_BREAKOUTLIMIT         "@breakoutLimit"
 #define XML_ATTR_BUILD                 "@build"
 #define XML_ATTR_BUILDSET              "@buildSet"
+#define XML_ATTR_CHANNEL               "@channel"
 #define XML_ATTR_CLUSTER               "@cluster"
 #define XML_ATTR_CLUSTERNAME           "@clusterName"
 #define XML_ATTR_CHECKCANCELLED        "@checkCancelled"
@@ -115,7 +116,7 @@
 #define XML_ATTR_CUSTOMERNAME          "@customerName"
 #define XML_ATTR_DALISERVER            "@daliServer"
 #define XML_ATTR_DATABUILD             "@dataBuild"
-#define XML_ATTR_DATADIRECTORY         "@dataDirectory"
+#define XML_ATTR_LEVEL                 "@level"
 #define XML_ATTR_DATAMODEL             "@dataModel"
 #define XML_ATTR_DATASEGMENT           "@dataSegment"
 #define XML_ATTR_DEBUGXML              "@debugXml"
@@ -253,7 +254,7 @@
 #define TAG_NETADDRESS                  "netAddress"
 #define TAG_DOMAIN                      "domain"
 #define TAG_PROCESS                     "process"
-#define TAG_DATADIRECTORY               "dataDirectory"
+#define TAG_LEVEL                       "level"
 #define TAG_COMPUTER                    "computer"
 #define TAG_LISTENQUEUE                 "listenQueue"
 #define TAG_NUMTHREADS                  "numThreads"

deployment/deployutils/configenvhelper.cpp

@@ -121,6 +121,23 @@ bool CConfigEnvHelper::handleThorTopologyOp(const char* cmd, const char* xmlArg,
     return retVal;
 }
 
+void CConfigEnvHelper::addSlaveProcessConfig(IPropertyTree* pRoxie, IPropertyTree *pSlaveNode, int channel, int level, const char* netAddress)
+{
+  if (pRoxie == NULL || pSlaveNode == NULL || netAddress == NULL)
+    return;
+
+
+  IPropertyTree* pSlaveProcess = pRoxie->addPropTree(XML_TAG_ROXIE_SLAVE, createPTree());
+
+  if (pSlaveProcess == NULL)
+    return;
+
+  pSlaveProcess->addProp(XML_ATTR_COMPUTER, pSlaveNode->queryProp(XML_ATTR_COMPUTER));
+  pSlaveProcess->addPropInt(XML_ATTR_CHANNEL, channel);
+  pSlaveProcess->addPropInt(XML_ATTR_LEVEL, level);
+  pSlaveProcess->addProp(XML_ATTR_NETADDRESS, netAddress);
+}
+
 IPropertyTree* CConfigEnvHelper::getSoftwareNode(const char* compType, const char* compName)
 {
   StringBuffer xpath;
@@ -145,6 +162,10 @@ bool CConfigEnvHelper::addRoxieServers(const char* xmlArg)
   {
     xpath.clear().appendf("RoxieCluster[@name='%s']/"XML_TAG_ROXIE_FARM, pszRoxieCluster);
     pFarm = getSoftwareNode(xpath.str(), pszFarm);
+
+    if (pFarm == NULL)
+      return false;
+
     sFarmName = pFarm->queryProp(XML_ATTR_NAME);
     bNewFarm = false;
 
@@ -163,25 +184,7 @@ bool CConfigEnvHelper::addRoxieServers(const char* xmlArg)
     if (!pFarm->hasProp("@aclName"))
       pFarm->addProp("@aclName", "");
 
-    StringBuffer dataDir = pFarm->queryProp(XML_ATTR_DATADIRECTORY);
-    if (dataDir.length()==0)
-      dataDir.append(RUNTIME_DIR"/roxiedata");
-
-    if (!pFarm->queryPropTree(XML_TAG_ROXIE_SERVER"[1]")) //no servers in farm
-    {
-      //if (nComputers > 0)
-      //g_pDocument->makePlatformSpecificAbsolutePath(computers[0]->queryProp(XML_ATTR_NAME), dataDir);
-      Owned<IPropertyTreeIterator> iter = pSrcTree->getElements(XML_TAG_COMPONENT);
-      
-      ForEach (*iter)
-      {
-        IPropertyTree* pFolder = &iter->query();
-        makePlatformSpecificAbsolutePath(pFolder->queryProp(XML_ATTR_NAME), dataDir);
-        break;
-      }
-
-      pFarm->setProp(XML_ATTR_DATADIRECTORY, dataDir.str());
-    }
+    StringBuffer dataDir = pFarm->queryProp(XML_ATTR_LEVEL);
   }
   else
   {
@@ -189,31 +192,12 @@ bool CConfigEnvHelper::addRoxieServers(const char* xmlArg)
     createUniqueName("farm", xpath.str(), sFarmName);
     bNewFarm = true;
 
-    StringBuffer sDataDir;
-    sDataDir.append(RUNTIME_DIR"/roxiedata");
-
-    //get datadir from existing farm if any
-    xpath.clear().appendf("Software/RoxieCluster[@name='%s']/"XML_TAG_ROXIE_FARM"[1]", pszRoxieCluster);
-    IPropertyTree* pFarm1 =  m_pRoot->queryPropTree(xpath.str());
-    if (pFarm1)
-      sDataDir.clear().append(pFarm1->queryProp(XML_ATTR_DATADIRECTORY));
-
-    //if (nComputers > 0)
-    //g_pDocument->makePlatformSpecificAbsolutePath(computers[0]->queryProp(XML_ATTR_NAME), sDataDir);
     Owned<IPropertyTreeIterator> iter = pSrcTree->getElements(XML_TAG_COMPONENT);
       
-    ForEach (*iter)
-    {
-      IPropertyTree* pFolder = &iter->query();
-      makePlatformSpecificAbsolutePath(pFolder->queryProp(XML_ATTR_NAME), sDataDir);
-      break;
-    }
-
     xpath.clear().appendf("RoxieCluster[@name='%s']/"XML_TAG_ROXIE_FARM, pszRoxieCluster);
     pFarm = pParent->addPropTree(xpath.str(), createPTree());
     pFarm->addProp    (XML_ATTR_NAME,       sFarmName.str());
     pFarm->addPropInt("@port", pSrcTree->getPropInt("@port", 9876));
-    pFarm->addProp    (XML_ATTR_DATADIRECTORY,  sDataDir.str());
     pFarm->addPropInt("@listenQueue", 200);
     pFarm->addPropInt("@numThreads",    30);
     pFarm->addPropInt("@requestArrayThreads", 5);
@@ -961,25 +945,8 @@ bool CConfigEnvHelper::handleRoxieSlaveConfig(const char* xmlArg)
         pRoxie->setProp("@baseDataDir", sDataDir.str());
         pRoxie->setProp("@localSlave", computers.size() > 1 ? "false" : "true");
 
-        //update Roxie data directories for all farms and all legacy servers
-        //change all farms
-        Owned<IPropertyTreeIterator> iterFarms = pRoxie->getElements(XML_TAG_ROXIE_FARM);
-        ForEach (*iterFarms)
-        {
-            IPropertyTree* pTmpComp = &iterFarms->query();
-            if (strcmp(pTmpComp->queryProp(XML_ATTR_DATADIRECTORY), sDataDir.str()))
-                pTmpComp->setProp(XML_ATTR_DATADIRECTORY, sDataDir.str());
-        }
-
         //change all legacy servers
         Owned<IPropertyTreeIterator> iterServers = pRoxie->getElements(XML_TAG_ROXIE_SERVER);
-
-        ForEach (*iterServers)
-        {
-            IPropertyTree* pTmpComp = &iterServers->query();
-            if (strcmp(pTmpComp->queryProp(XML_ATTR_DATADIRECTORY), sDataDir.str()))
-                pTmpComp->setProp(XML_ATTR_DATADIRECTORY, sDataDir.str());
-        }
     }
     catch (IException *e)
     {
@@ -994,26 +961,6 @@ bool CConfigEnvHelper::handleRoxieSlaveConfig(const char* xmlArg)
     return true;
 }
 
-
-void CConfigEnvHelper::addReplicateConfig(IPropertyTree* pSlaveNode, int channel, const char* dir, 
-                                                                const char* netAddress, IPropertyTree* pRoxie)
-{
-    StringBuffer directory;
-    directory.appendf("%s", dir);
-    makePlatformSpecificAbsolutePath( pSlaveNode->queryProp(XML_ATTR_COMPUTER), directory);
-
-    IPropertyTree* pInstance = pSlaveNode->addPropTree(XML_TAG_ROXIE_CHANNEL, createPTree());
-    pInstance->setPropInt("@number", channel);
-    pInstance->addProp(XML_ATTR_DATADIRECTORY, directory.str());
-    
-    //maintain a copy as an old style slave procss
-    IPropertyTree* pSlaveProcess = pRoxie->addPropTree(XML_TAG_ROXIE_SLAVE, createPTree());
-    pSlaveProcess->addProp(XML_ATTR_COMPUTER, pSlaveNode->queryProp(XML_ATTR_COMPUTER));
-    pSlaveProcess->addPropInt("@channel", channel);
-    pSlaveProcess->addProp(XML_ATTR_DATADIRECTORY, directory.str());
-    pSlaveProcess->addProp(XML_ATTR_NETADDRESS, netAddress);
-}
-
 bool CConfigEnvHelper::GenerateCyclicRedConfig(IPropertyTree* pRoxie, IPropertyTreePtrArray& computers, 
                                                                                              const char* copies, const char* pszOffset,
                                                                                              const char* dir1, const char* dir2, const char* dir3)
@@ -1060,7 +1007,15 @@ bool CConfigEnvHelper::GenerateCyclicRedConfig(IPropertyTree* pRoxie, IPropertyT
             const char drive = 'c' + c;
             channel = 1 + ((baseChannel + c*(nComputers-offset)) % nComputers);
 
-            addReplicateConfig(pSlave, channel, c==0? dir1: (c==1?dir2:dir3), netAddress, pRoxie);
+            IPropertyTree* pInstance = pSlave->addPropTree(XML_TAG_ROXIE_CHANNEL, createPTree());
+            pInstance->setPropInt("@number", channel);
+
+            StringBuffer strTemp;
+            strTemp.append(c);
+
+            pInstance->addProp(XML_ATTR_LEVEL, strTemp.str());
+
+            addSlaveProcessConfig(pRoxie, pSlave, channel, c, netAddress);
         }
     }
     m_numChannels = nComputers;
@@ -1098,8 +1053,15 @@ bool CConfigEnvHelper::GenerateOverloadedConfig(IPropertyTree* pRoxie, IProperty
 
         for (int c=0; c<m_numDataCopies; c++)
         {
-            const char drive = 'c' + c;
-            addReplicateConfig(pSlave, channel + c*nComputers, c==0?dir1:(c==1?dir2:dir3), netAddress, pRoxie);
+            IPropertyTree* pInstance = pSlave->addPropTree(XML_TAG_ROXIE_CHANNEL, createPTree());
+            pInstance->setPropInt("@number", channel + c*nComputers);
+
+            StringBuffer strTemp;
+            strTemp.append(c);
+
+            pInstance->addProp(XML_ATTR_LEVEL, strTemp.str());
+
+            addSlaveProcessConfig(pRoxie, pSlave, channel + c*nComputers, c, netAddress);
         }
         channel++;
     }
@@ -1139,7 +1101,14 @@ bool CConfigEnvHelper::GenerateFullRedConfig(IPropertyTree* pRoxie, int copies,
         pSlave->addProp(XML_ATTR_NAME, name.str());
         pSlave->addProp(XML_ATTR_COMPUTER, szComputer);
 
-        addReplicateConfig(pSlave, 1 + (channel++ % maxChannel), dir1, netAddress, pRoxie);
+        IPropertyTree* pInstance = pSlave->addPropTree(XML_TAG_ROXIE_CHANNEL, createPTree());
+        pInstance->setPropInt("@number", 1 + (channel % maxChannel));
+        pInstance->addProp(XML_ATTR_LEVEL, "0");
+
+        addSlaveProcessConfig(pRoxie, pSlave, 1 + (channel % maxChannel), 0, netAddress);
+
+        channel++;
+
     }
     m_numChannels = maxChannel;
     return true;

deployment/deployutils/configenvhelper.hpp

@@ -64,7 +64,7 @@ private:
     bool EnsureInRange(const char* psz, UINT low, UINT high, const char* caption);
     bool handleRoxieSlaveConfig(const char* params);
     bool handleReplaceRoxieServer(const char* xmlArg);
-    void addReplicateConfig(IPropertyTree* pSlaveNode, int channel, const char* drive, const char* netAddress, IPropertyTree* pRoxie);
+    void addSlaveProcessConfig(IPropertyTree *pRoxie, IPropertyTree *pSlaveNode, int channel, int level, const char* netAddress);
     bool GenerateCyclicRedConfig(IPropertyTree* pRoxie, IPropertyTreePtrArray& computers, const char* copies, const char* pszOffset,
                                                              const char* dir1, const char* dir2, const char* dir3);
     bool GenerateOverloadedConfig(IPropertyTree* pRoxie, IPropertyTreePtrArray& computers, const char* copies,

deployment/deployutils/deployutils.cpp

@@ -506,7 +506,7 @@ public:
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_SERVER, TAG_PROCESS, "", 0, 1, "", 0);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_NAME, "", 0, 1, "", 0);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_PROCESS, "", 0, 1, "", 0);
-        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_DATADIRECTORY, "", 0, 1, "", 0);
+        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_LEVEL, "", 0, 1, "", 0);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_LISTENQUEUE, "", 0, 1, "", 1);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_NUMTHREADS, "", 0, 1, "", 1);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_PORT, "", 0, 1, "", 1);
@@ -520,7 +520,7 @@ public:
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_CHANNEL, TAG_NAME, "", 0, 1, "", 0);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_CHANNEL, TAG_ITEMTYPE, "", 0, 1, "", 0);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_CHANNEL, TAG_COMPUTER, "", 0, 1, "", 0);
-        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_CHANNEL, TAG_DATADIRECTORY, "", 0, 1, "", 0);
+        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_CHANNEL, TAG_LEVEL, "", 0, 1, "", 0);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_CHANNEL, TAG_NUMBER, "", 0, 1, "", 0);
       }
 
@@ -536,7 +536,7 @@ public:
           m_colIndex.appendf("colIndex['process%s']=%d;", serverStr, index++);
           m_colIndex.appendf("colIndex['netAddress%s']=%d;", serverStr, index++);
           m_colIndex.appendf("colIndex['port%s']=%d;", serverStr, index++);
-          m_colIndex.appendf("colIndex['dataDirectory%s']=%d;", serverStr, index++);
+          m_colIndex.appendf("colIndex['level%s']=%d;", serverStr, index++);
           m_colIndex.appendf("colIndex['listenQueue%s']=%d;", serverStr, index++);
           m_colIndex.appendf("colIndex['numThreads%s']=%d;", serverStr, index++);
           m_colIndex.appendf("colIndex['requestArrayThreads%s']=%d;", serverStr, index++);
@@ -547,7 +547,7 @@ public:
           m_colIndex.appendf("colIndex['name%s']=%d;", agentStr, index++); 
           m_colIndex.appendf("colIndex['itemType%s']=%d;", agentStr, index++);
           m_colIndex.appendf("colIndex['netAddress%s']=%d;", agentStr, index++);
-          m_colIndex.appendf("colIndex['dataDirectory%s']=%d;", agentStr, index++);
+          m_colIndex.appendf("colIndex['level%s']=%d;", agentStr, index++);
           m_colIndex.appendf("colIndex['number%s']=%d;", agentStr, index++);
         }
         else if (!strcmp(m_compName.str(), XML_TAG_THORCLUSTER))

docs/ECLLanguageReference/ECLR_mods/Basics-Constants.xml

@@ -25,7 +25,7 @@
 STRING20 MyString3 := 'Fred\\Ginger\'s Place';
                     //evaluated as: "Fred\Ginger's Place"</programlisting>
 
-    <para>Other available escape characters are: </para>
+    <para>Other available escape characters are:</para>
 
     <para><informaltable colsep="1" frame="all" rowsep="1">
         <tgroup cols="2">
@@ -35,32 +35,43 @@ STRING20 MyString3 := 'Fred\\Ginger\'s Place';
 
           <tbody>
             <row>
-              <entry><emphasis>\t</emphasis></entry>
+              <entry><emphasis role="code">\t</emphasis></entry>
 
               <entry>tab</entry>
             </row>
 
             <row>
-              <entry><emphasis>\n</emphasis></entry>
+              <entry><emphasis role="code">\n</emphasis></entry>
 
               <entry>new line</entry>
             </row>
 
             <row>
-              <entry><emphasis>\r</emphasis></entry>
+              <entry><emphasis role="code">\r</emphasis></entry>
 
               <entry>carriage return</entry>
             </row>
 
             <row>
-              <entry><emphasis>\nnn</emphasis></entry>
+              <entry><emphasis role="code">\nnn</emphasis></entry>
 
               <entry>3 octal digits (for any other character)</entry>
             </row>
+
+            <row>
+              <entry><emphasis role="code">\uhhhh</emphasis></entry>
+
+              <entry>lowercase "u" followed by 4 hexadecimal digits (for any
+              other UNICODE-only character)</entry>
+            </row>
           </tbody>
         </tgroup>
       </informaltable></para>
 
+    <programlisting>MyString1 := 'abcd'; 
+MyString2 := U'abcd\353';    // becomes 'abcdë'
+</programlisting>
+
     <para><emphasis role="bold">Hexadecimal<indexterm>
         <primary>Hexadecimal</primary>
       </indexterm> string constants<indexterm>
@@ -86,8 +97,9 @@ STRING20 MyString3 := 'Fred\\Ginger\'s Place';
     character. Characters between the quotes are utf8-encoded and the type of
     the constant is UNICODE.</para>
 
-    <programlisting>MyUnicodeString := U'abcd'; // same as: (UNICODE)'abcd'
-MyUnicodeString := U'abcd\353'; // becomes 'abcdë'</programlisting>
+    <programlisting>MyUnicodeString1 := U'abcd';        // same as: (UNICODE)'abcd'
+MyUnicodeString2 := U'abcd\353';    // becomes 'abcdë'
+MyUnicodeString3 := U'abcd\u00EB'; // becomes 'abcdë'</programlisting>
 
     <para><emphasis role="bold">VARSTRING string constants<indexterm>
         <primary>VARSTRING string constants</primary>

docs/ECLWatch/ECLWTechPrev.xml

@@ -99,5 +99,10 @@
     <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
                 xpointer="LogicalFiles"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+                
+    <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
+		xpointer="Landing_Zones"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+                
   </chapter>
 </book>

docs/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -216,7 +216,7 @@
 
           <listitem>
             <para><emphasis role="bold">Cluster</emphasis> - filter workunits
-            by cluster. Select the cluster from the drop list. </para>
+            by cluster. Select the cluster from the drop list.</para>
           </listitem>
 
           <listitem>
@@ -329,7 +329,7 @@
 
             <mediaobject>
               <imageobject>
-                <imagedata fileref="../../images/ECLW402.jpg"
+                <imagedata fileref="../../images/ECLWA402.jpg"
                            vendor="eclwatchSS" />
               </imageobject>
             </mediaobject>
@@ -513,12 +513,12 @@
 
           <listitem>
             <para><emphasis role="bold">Cluster</emphasis> - filter workunits
-            by cluster. Select the cluster from the drop list. </para>
+            by cluster. Select the cluster from the drop list.</para>
           </listitem>
 
           <listitem>
             <para><emphasis role="bold">State</emphasis> - filter workunits by
-            state. Select the state from the drop list. </para>
+            state. Select the state from the drop list.</para>
           </listitem>
 
           <listitem>
@@ -768,7 +768,7 @@
 
           <listitem>
             <para><emphasis role="bold">Cluster</emphasis> - filter files by
-            cluster. Select the cluster from the drop list. </para>
+            cluster. Select the cluster from the drop list.</para>
           </listitem>
 
           <listitem>
@@ -926,5 +926,197 @@
         </itemizedlist>
       </sect2>
     </sect1>
+
+    <sect1 id="Landing_Zones" vendor="eclwatchSS">
+      <title>Landing Zones</title>
+
+      <para>The Landing Zone link displays the Landing Zones page. The Landing
+      Zone Page shows you each landing zone you have configured for your
+      cluster and its contents. <figure>
+          <title>Landing Zone Page</title>
+
+          <mediaobject>
+            <imageobject>
+              <imagedata fileref="../../images/ECLWA440.jpg"
+                         vendor="eclwatchSS" />
+            </imageobject>
+          </mediaobject>
+        </figure></para>
+
+      <para>Click on the arrow next to a drop zone container to expand. The
+      files on the drop zone display. You can choose to upload, download, or
+      delete any files on the drop zone using the landing zone action buttons.
+      You can also spray files to a cluster from this page.</para>
+
+      <sect2 id="Upload_Files">
+        <title>Upload files</title>
+
+        <para>You can upload files to your landing zone from the Landing Zone
+        page.</para>
+
+        <orderedlist>
+          <listitem>
+            <para>Press the <emphasis role="bold">Upload</emphasis> action
+            button.<informaltable colsep="1" frame="all" rowsep="1">
+                <?dbfo keep-together="always"?>
+
+                <tgroup cols="2">
+                  <colspec colwidth="49.50pt" />
+
+                  <colspec />
+
+                  <tbody>
+                    <row>
+                      <entry><inlinegraphic
+                      fileref="../../images/tip.jpg" /></entry>
+
+                      <entry>The upload utility in ECL Watch is limited by the
+                      browser's file size limitation. This is typically 4 GB.
+                      For production systems, we recommend a secure copy
+                      protocol (scp) utility.</entry>
+                    </row>
+                  </tbody>
+                </tgroup>
+              </informaltable></para>
+          </listitem>
+
+          <listitem>
+            <para>Choose the file from the window that displays.</para>
+          </listitem>
+
+          <listitem>
+            <para>Verify the file and dropzone are correct in the <emphasis
+            role="bold">Info Dialog</emphasis> that displays. <figure>
+                <title>Info Dialog</title>
+
+                <mediaobject>
+                  <imageobject>
+                    <imagedata fileref="../../images/ECLWA440b.jpg"
+                               vendor="eclwatchSS" />
+                  </imageobject>
+                </mediaobject>
+              </figure></para>
+          </listitem>
+
+          <listitem>
+            <para>Press the <emphasis role="bold">Start</emphasis> button to
+            begin the upload.</para>
+
+            <para>The file progress indicator shows as the file uploads. When
+            upload is complete, the window closes.</para>
+          </listitem>
+        </orderedlist>
+
+        <para></para>
+      </sect2>
+
+      <sect2 id="Download_Files">
+        <title>Download files</title>
+
+        <para>You can download files from your landing zone to your computer.
+        <orderedlist>
+            <listitem>
+              <para>From the Landing Zone page, select a file (or files) to
+              download by checking the box next to it.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the <emphasis role="bold">Download</emphasis> button
+              to download the file.</para>
+            </listitem>
+          </orderedlist>The file will download to your browser's download
+        directory as specified in your browser settings.</para>
+      </sect2>
+
+      <sect2 id="Delete_Files">
+        <title>Delete files</title>
+
+        <para>You can delete files from your landing zone. <orderedlist>
+            <listitem>
+              <para>From the Landing Zone page, select a file (or files) to
+              delete by checking the box next to it.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the <emphasis role="bold">Delete</emphasis> action
+              button to delete the file from your landing zone.</para>
+            </listitem>
+          </orderedlist></para>
+      </sect2>
+
+      <sect2>
+        <title>Hex Preview</title>
+
+        <para>You can view a Hexadecimal representation of files on the
+        landing zone. <orderedlist>
+            <listitem>
+              <para>Select a file by checking the box next to it.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the<emphasis role="bold">Hex Preview</emphasis>
+              action button to display the selected file(s) in a hex
+              format.</para>
+            </listitem>
+          </orderedlist></para>
+
+        <para><figure>
+            <title>Hex Preview</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="../../images/ECLWA440h.jpg" />
+              </imageobject>
+            </mediaobject>
+          </figure></para>
+
+        <para>You can adjust the width of the view on the hex preview page
+        using the spinbox controls on the <emphasis
+        role="bold">Width</emphasis> box.</para>
+
+        <para>If you have an EBCDIC file check the box next to <emphasis
+        role="bold">EBCDIC:</emphasis> for it to display properly.</para>
+      </sect2>
+
+      <sect2 id="LZ_Spray">
+        <title>Spray Files</title>
+
+        <para>You can spray files to your clusters from the Landing Zone page.
+        <orderedlist>
+            <listitem>
+              <para>Select the file from your drop zone by checking the box
+              next to it.</para>
+            </listitem>
+
+            <listitem>
+              <para>Select the appropriate drop menu option for the type of
+              spray you want.</para>
+
+              <para>For example, to spray a delimited file, select the
+              <emphasis role="bold">Delimited</emphasis> drop menu
+              option.</para>
+
+              <para><figure>
+                  <title>Landing Zone Spray</title>
+
+                  <mediaobject>
+                    <imageobject>
+                      <imagedata fileref="../../images/ECLWA441.jpg" />
+                    </imageobject>
+                  </mediaobject>
+                </figure></para>
+            </listitem>
+
+            <listitem>
+              <para>Fill in the values as appropriate for the spray.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the <emphasis role="bold">Spray</emphasis> button to
+              spray the file.</para>
+            </listitem>
+          </orderedlist></para>
+      </sect2>
+    </sect1>
   </chapter>
 </book>

docs/HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml

@@ -750,16 +750,6 @@ dfuplus action=dspray srcname=le::imagedb
 
                     <entry>The logical name of the file to remove.</entry>
                   </row>
-
-                  <row>
-                    <entry><emphasis><emphasis><emphasis>nodelete</emphasis>
-                    </emphasis></emphasis></entry>
-
-                    <entry>A boolean flag (0 | 1) indicating whether to
-                    physically delete the file in addition to removing its
-                    listing from the DFU. If omitted, the default is
-                    0—physically delete the file.</entry>
-                  </row>
                 </tbody>
               </tgroup>
             </informaltable> Example:</para>

ecl/hqlcpp/hqlsource.cpp

@@ -5439,6 +5439,8 @@ bool MonitorExtractor::matchSubstringFilter(KeyConditionInfo & matches, node_ope
     IHqlExpression * selector = left->queryChild(0);
     if (!isKeySelect(selector) || !okToKey(selector, keyedKind))
         return false;
+    if (!isIndexInvariant(right))
+        return false;
     ITypeInfo * fieldType = selector->queryType();
     unsigned fieldLength = fieldType->getStringLen();
     if (fieldLength == UNKNOWN_LENGTH)

ecl/hthor/hthor.cpp

@@ -6918,13 +6918,12 @@ const void * CHThorEnthActivity::nextInGroup()
 CHThorTopNActivity::CHThorTopNActivity(IAgentContext & _agent, unsigned _activityId, unsigned _subgraphId, IHThorTopNArg & _arg, ThorActivityKind _kind)
     : CHThorSimpleActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg), compare(*helper.queryCompare())
 {
-    limit = helper.getLimit();
-    assertex(limit == (size_t)limit);
-    sorted = (const void * *)checked_calloc((size_t)(limit+1), sizeof(void *), "topn");
     hasBest = helper.hasBest();
     grouped = outputMeta.isGrouped();
     curIndex = 0;
     sortedCount = 0;
+    limit = 0;
+    sorted = NULL;
 }
 
 CHThorTopNActivity::~CHThorTopNActivity()
@@ -6937,6 +6936,9 @@ CHThorTopNActivity::~CHThorTopNActivity()
 void CHThorTopNActivity::ready()
 {
     CHThorSimpleActivityBase::ready();
+    limit = helper.getLimit();
+    assertex(limit == (size_t)limit);
+    sorted = (const void * *)checked_calloc((size_t)(limit+1), sizeof(void *), "topn");
     sortedCount = 0;
     curIndex = 0;
     eof = false;
@@ -6948,6 +6950,8 @@ void CHThorTopNActivity::done()
     CHThorSimpleActivityBase::done();
     while(curIndex < sortedCount)
         ReleaseRoxieRow(sorted[curIndex++]);
+    free(sorted);
+    sorted = NULL;
 }
 
 const void * CHThorTopNActivity::nextInGroup()

esp/services/ws_workunits/roxiequeryhandler.hpp

@@ -15,4 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-// no longer needed - code moved to roxiemanager
+d := dataset([], {string10 a, unsigned b});
+
+i := INDEX(d, {a, b}, 'fred::smith');
+
+i(a[1..length(a)] = a);

esp/files/scripts/configmgr/configmgr.js

@@ -708,9 +708,9 @@ function initItemForRoxieSlaves(item) {
   item.netAddress = "";
   item.netAddress_extra = "";
   item.netAddress_ctrlType = 0;
-  item.dataDirectory = "";
-  item.dataDirectory_extra = "";
-  item.dataDirectory_ctrlType = 0;
+  item.level = "";
+  item.level_extra = "";
+  item.level_ctrlType = 0;
   item.itemType = "";
   item.itemType_extra = "";
   item.itemType_ctrlType = 0;
@@ -745,9 +745,9 @@ function initItemForRoxieServers(item) {
   item.port = "";
   item.port_extra = "";
   item.port_ctrlType = 0;
-  item.dataDirectory = "";
-  item.dataDirectory_extra = "";
-  item.dataDirectory_ctrlType = 0;
+  item.level = "";
+  item.level_extra = "";
+  item.level_ctrlType = 0;
   item.listenQueue = "";
   item.listenQueue_extra = "";
   item.listenQueue_ctrlType = 0;
@@ -877,13 +877,13 @@ function handleConfigCellClickEvent(oArgs, caller, isComplex) {
       }
     }
 
-    if (bldSet === "roxie" && attrName === "dataDirectory") {
+    if (bldSet === "roxie" && attrName === "level") {
       if (newValue === '') {
         alert('Roxie data directory cannot be empty');
         return;
       }
 
-      if (!confirm("The primary data directory for other farms and agents will be changed to the same value. Do you want to continue?")) {
+      if (!confirm("The level for other farms and agents will be changed to the same value. Do you want to continue?")) {
         for (i = 1; i < 7; i++)
           if (caller.editors[i].isActive)
           caller.editors[i].cancel();

esp/scm/espscm.cmake

@@ -34,7 +34,6 @@ set ( ESPSCM_SRCS
       ws_access.ecm
       esploggingservice.ecm
       roxiecommlibscm.ecm
-      roxiemanagerscm.ecm
       soapesp.ecm
       ws_ecl_client.ecm
       ws_fs.ecm

esp/scm/roxiemanagerscm.ecm

@@ -1,171 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef ROXIEMANAGER_API
-#define ROXIEMANAGER_API __declspec(dllimport)
-#endif
-
-#include "esp.hpp"
-#include "jptree.hpp"
-#include "jsocket.hpp"
-#include "workunit.hpp"
-
-interface IUserDescriptor;
-interface IRoxieCommunicationClient;
-
-SCMenum ActivationOptions1
-{
-    NO_ACTIVATE = 0,
-    ACTIVATE = 1,
-    ACTIVATE_SUSPEND = 2,
-    ACTIVATE_DELETE = 3,
-    LOAD_DATA_ONLY = 4,
-    ACTIVATE_LOAD_DATA_ONLY = 5
-};
-
-enum RoxieQueryPriorities
-{
-    UNKNOWN_PRIORITY = -1,
-    LOW_PRIORITY = 0,
-    HIGH_PRIORITY = 1,
-    SLA_PRIORITY = 2
-};
-
-SCMenum LayoutTranslationEnabledType
-{
-    RLT_OFF,
-    RLT_ON,
-    RLT_UNKNOWN
-};
-
-
-SCMinterface IConstRoxieQueryCompileInfo(IInterface)
-{
-    unsigned getMemoryLimit();
-    unsigned getWuTimeOut();
-    unsigned getTimeLimit();
-    unsigned getWarnTimeLimit();
-    bool getPoolGraphs();
-    bool getHighPriority();
-    int getQueryPriority();
-    const char *queryRepositoryLabel();
-    const char *queryJobName();
-    const char *queryEcl();
-    const char *queryAppName();
-    const char *queryClusterName();
-    const char *queryDefaultStyleName();
-    const char *queryModuleName();
-    const char *queryWuDebugOptions();
-};
-
-
-SCMinterface IRoxieQueryCompileInfo(IConstRoxieQueryCompileInfo)
-{
-    void setMemoryLimit(unsigned val);
-    void setWuTimeOut(unsigned val);
-    void setTimeLimit(unsigned val);
-    void setWarnTimeLimit(unsigned val);
-    void setPoolGraphs(bool val);
-    void setHighPriority(bool val);
-    void setQueryPriority(int val);
-    void setRepositoryLabel(const char *val);
-    void enableWebServiceInfoRetrieval(const char *_moduleName, const char *_defaultStyleName);
-    void setWuDebugOptions(const char *val);
-};
-
-
-SCMinterface IConstRoxieQueryProcessingInfo(IInterface)
-{
-    bool getLoadDataOnly();
-    bool getResolveFileInfo();
-    bool getNoForms();
-    bool getUseRenamedFileInfo();
-    const char *queryComment();
-    const char *queryPackageName();
-    const char *queryDfsDaliIp();
-    const char *querySourceRoxieClusterName();
-    bool getGeneratePackageFileInfo();
-    const char *queryScope();
-    IUserDescriptor *queryUserDescriptor();
-    int getLayoutTranslationEnabled();
-    bool getResolveKeyDiffInfo();
-    bool getCopyKeyDiffLocationInfo();
-};
-
-
-SCMinterface IRoxieQueryProcessingInfo(IConstRoxieQueryProcessingInfo)
-{
-    void setLoadDataOnly(bool val);
-    void setResolveFileInfo(bool val);
-    void setUseRenamedFileInfo(bool val);
-    void setComment(const char *val);
-    void setPackageName(const char *val);
-    void setNoForms(bool val);
-    void setDfsDaliIp(const char *val);
-    void setSourceRoxieClusterName(const char *val);
-    void setGeneratePackageFileInfo(bool val);
-    void setScope(const char *val);
-    void setUserDescriptor(IUserDescriptor *val);
-    void setLayoutTranslationEnabled(int val);
-    void setResolveKeyDiffInfo(bool val);
-    void setCopyKeyDiffLocationInfo(bool val);
-};
-
-
-SCMinterface IRoxieQueryManager(IInterface)
-{
-    bool compileQuery(SCMStringBuffer &wuid, SCMStringBuffer &roxieQueryName, IRoxieQueryCompileInfo &compileInfo,
-                      IRoxieQueryProcessingInfo&processingInfo, const char *targetClusterName, SCMStringBuffer &status);
-
-    bool deployQuery(SCMStringBuffer &wuid, SCMStringBuffer &roxieQueryName, IRoxieQueryCompileInfo &compileInfo,
-                     IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, bool allowNewRoxieOnDemandQuery, const char *targetClusterName, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus);
-
-    bool deployWorkunit(SCMStringBuffer &wuid,  SCMStringBuffer &roxieQueryName, IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus);
-    bool publishWorkunit(IConstWorkUnit *workunit,  SCMStringBuffer &roxieQueryName, IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus);
-    
-    bool publishFromQuerySet(SCMStringBuffer &name, SCMStringBuffer &roxieQueryName, IRoxieQueryProcessingInfo &processingInfo, const char *userId, WUQueryActivationOptions activateOption, const char *sourceQuerySetName, const char *targetQuerySetName, const char *sourceDaliIP, const char *queryComment, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus);
-    
-    void getNewQueryWorkunitId(SCMStringBuffer &wuid, SCMStringBuffer &roxieQueryName, const char *queryAppName);
-
-    const char *runQuery(IConstWorkUnit *workunit, const char *roxieQueryName, bool resultsToSocket, bool allowNewRoxieOnDemandQuery, SCMStringBuffer &response);
-
-    void addAlias(const char *alias, const char *queryId, const char *querySetName, bool notifyRoxie, SCMStringBuffer &oldActive);
-    void suspendQuery(const char *id, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status);
-    void unsuspendQuery(const char *id, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status);
-    void deleteQuery(const char *id, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status);
-    void removeAlias(const char *alias, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status);
-    void removeAllAliasForQuery(const char *id, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status);
-    
-    IPropertyTree *retrieveQueryList(const char *filter, bool excludeQueryNames, bool excludeAliasNames, bool excludeLibraryNames, bool excludeDataOnlyNames, unsigned version);
-
-    void setQueryWarningTime(const char *id, unsigned warnTime, SCMStringBuffer &status);
-    unsigned getQueryWarningTime(const char *id);
-
-    bool updateACLInfo(bool allow, const char *restrict_ip, const char *mask, const char *query, const char *errorMsg, int errorCode, int port, SCMStringBuffer &status);
-};
-
-
-
-
-extern "C" ROXIEMANAGER_API IRoxieQueryManager *createRoxieQueryManager(SocketEndpoint &roxieEP, const char *roxieName, const char *workunitDali, unsigned roxieTimeout, const char *_userName, const char *_password, int logLevel);
-
-
-extern "C" ROXIEMANAGER_API IRoxieQueryCompileInfo *createRoxieQueryCompileInfo(const char *_ecl, const char *_jobName, const char *_clusterName, const char *_appName);
-extern "C" ROXIEMANAGER_API IRoxieQueryProcessingInfo *createRoxieQueryProcessingInfo();
-
-
-

esp/scm/smcscm.cmake

@@ -32,7 +32,6 @@ set ( ESPSCM_SRCS
       ws_dfuXref.ecm
       ws_fs.ecm
       ws_roxie.ecm
-      ws_roxiequery.ecm
       ws_smc.ecm
       ws_topology.ecm
       ws_workunits.ecm

esp/scm/ws_roxiequery.ecm

@@ -1,198 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#include "xslprocessor.hpp" 
-//  ===========================================================================
-
-ESPStruct RoxieQueryAlias
-{
-    string ID;
-    string Original;
-};
-
-ESPStruct RoxieQuery
-{
-    string ID;
-    [min_ver("1.02")] string WUID;
-    //string Cluster;
-    //string Label;
-    //string AssociatedName;
-    [min_ver("1.01")] string DeployedBy;
-    [min_ver("1.01")] string UpdatedBy;
-    string Error;
-    string Comment;
-    string Extra;
-    string HighPriority;
-    string HasAliases;
-    string Suspended;
-};
-
-ESPrequest 
-[
-]
-RoxieQuerySearchRequest
-{
-};
-
-ESPresponse 
-[
-   exceptions_inline
-]
-RoxieQuerySearchResponse
-{
-    ESParray<string, ClusterName> ClusterNames;
-    ESParray<string, SuspendedSelection> SuspendedSelections;
-};
-
-
-ESPrequest 
-[
-]
-RoxieQueryListRequest
-{
-    ESParray<string> IDs;
-
-    string LogicalName;
-    string Cluster;
-    string Suspended;
-    int PageSize;
-    int PageStartFrom;
-    string Sortby;
-    bool Descending(false);
-};
-
-ESPresponse 
-[
-    exceptions_inline,
-    encode(0)
-]
-RoxieQueryListResponse
-{
-    ESParray<ESPstruct RoxieQuery> RoxieQueries;
-
-    int PageSize(100);
-    int64 PageStartFrom(1);
-    int64 LastPageFrom(-1);
-    int64 PageEndAt;
-    int64 PrevPageFrom(-1);
-    int64 NextPageFrom(-1);
-    int64 NumFiles;
-    string Sortby;
-    bool Descending(false);
-    string LogicalName;
-    string Cluster;
-    string ParametersForPaging;
-    string ParametersForSorting;
-};
-
-ESPrequest 
-[
-]
-RoxieQueryDetailsRequest
-{
-    string QueryID;
-    string Cluster;
-};
-
-ESPresponse 
-[
-   exceptions_inline
-]
-RoxieQueryDetailsResponse
-{
-    string QueryID;
-    string Cluster;
-    string WUID;
-    string AssociatedName;
-    string HighPriority;
-    string Suspended;
-    string Label;
-    string Error;
-    string Comment;
-    string Extra;
-    [min_ver("1.01")] string DeployedBy;
-    [min_ver("1.01")] string UpdatedBy;
-    
-    ESParray<ESPStruct RoxieQueryAlias> Aliases;
-};
-
-ESPrequest GVCAjaxGraphRequest
-{
-    string Cluster;
-    string Name;
-    string GraphName;
-};
-
-ESPresponse [exceptions_inline, http_encode(0)] GVCAjaxGraphResponse
-{
-    string Cluster;
-    string Name;
-    string GraphName;
-    string GraphType;
-};
-
-ESPrequest ShowGVCGraphRequest
-{
-    string Cluster;
-    string QueryId;
-    string GraphName;
-};
-
-ESPresponse [exceptions_inline, http_encode(0)] ShowGVCGraphResponse
-{
-    string Cluster;
-    string QueryId;
-    string GraphName;
-    ESParray<string> GraphNames;
-    string TheGraph;
-};
-
-ESPrequest RoxieQueryProcessGraphRequest
-{
-    string  Cluster;
-    string  QueryId;
-    string  GraphName;
-    bool    XmlGraph;
-    bool    Stats;
-};
-
-ESPresponse  [encode(0), exceptions_inline] RoxieQueryProcessGraphResponse
-{
-    string theGraph;
-};
-
-
-//  ===========================================================================
-ESPservice [
-    version("1.10"), default_client_version("1.10"),
-    noforms, 
-    exceptions_inline("./smc_xslt/exceptions.xslt")] WsRoxieQuery
-{
-    ESPuses ESPStruct RoxieQueryAlias;
-    ESPuses ESPStruct RoxieQuery;
-
-    ESPmethod [resp_xsl_default("/esp/xslt/roxiequery_search.xslt")] RoxieQuerySearch(RoxieQuerySearchRequest, RoxieQuerySearchResponse);
-    ESPmethod [resp_xsl_default("/esp/xslt/roxiequery.xslt")] RoxieQueryList(RoxieQueryListRequest, RoxieQueryListResponse);
-    ESPmethod [resp_xsl_default("/esp/xslt/roxiequerydetails.xslt")] QueryDetails(RoxieQueryDetailsRequest, RoxieQueryDetailsResponse);
-    ESPmethod [description("Stub for Ajax GVC Graph."), help(""), resp_xsl_default("/esp/xslt/GvcGraph.xslt")] GVCAjaxGraph(GVCAjaxGraphRequest, GVCAjaxGraphResponse);
-    ESPmethod [description("Show GVC graph for a Roxie query"), help(""), resp_xsl_default("/esp/xslt/RoxieGVCGraph.xslt")] ShowGVCGraph(ShowGVCGraphRequest, ShowGVCGraphResponse);
-    ESPmethod [resp_xsl_default("/esp/xslt/graphStats.xslt")] RoxieQueryProcessGraph(RoxieQueryProcessGraphRequest, RoxieQueryProcessGraphResponse);
-};
-
-SCMexportdef(WsRoxieQuery);
-
-SCMapi(WsRoxieQuery) IClientWsRoxieQuery *createWsRoxieQueryClient();

esp/services/CMakeLists.txt

@@ -24,7 +24,6 @@ HPCC_ADD_SUBDIRECTORY (ws_ecl "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_fileio "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_fs "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_machine "PLATFORM")
-HPCC_ADD_SUBDIRECTORY (ws_roxiequery "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_smc "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_topology "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_workunits "PLATFORM")

esp/services/WsDeploy/WsDeployService.cpp

@@ -1467,93 +1467,6 @@ bool CWsDeployFileInfo::saveSetting(IEspContext &context, IEspSaveSettingRequest
           }
         }
 
-        //if dataDirectory for a roxie farm is being changed, also change baseDataDir for roxie
-        if(!strcmp(pszCompType, "Directories") && !strcmp(pszSubType, "Category"))
-        {
-          StringBuffer sbNewValue;
-          bool bdata = strstr(pszSubTypeKey, "[@name='data']") || strstr(pszSubTypeKey, "[@name=\"data\"]");\
-          bool bdata2 = strstr(pszSubTypeKey, "[@name='data2']") || strstr(pszSubTypeKey, "[@name=\"data2\"]");
-          bool bdata3 = strstr(pszSubTypeKey, "[@name='data3']") || strstr(pszSubTypeKey, "[@name=\"data3\"]");
-
-          if (bdata || bdata2 || bdata3)
-          {
-            Owned<IPropertyTreeIterator> iterRoxies = pEnvSoftware->getElements("RoxieCluster");
-            ForEach (*iterRoxies)
-            {
-              IPropertyTree* pRoxie = &iterRoxies->query();
-
-              if (bdata)
-              {
-                getCommonDir(pEnvRoot, "data", "roxie", pRoxie->queryProp(XML_ATTR_NAME), sbNewValue.clear());
-                pRoxie->setProp("@baseDataDir", sbNewValue.str());
-
-                //change all farms
-                Owned<IPropertyTreeIterator> iterFarms = pRoxie->getElements(XML_TAG_ROXIE_FARM);
-                ForEach (*iterFarms)
-                {
-                  IPropertyTree* pTmpComp = &iterFarms->query();
-                  if (strcmp(pTmpComp->queryProp(XML_ATTR_DATADIRECTORY), sbNewValue.str()))
-                    pTmpComp->setProp(XML_ATTR_DATADIRECTORY, sbNewValue.str());
-                }
-
-                //change all legacy Roxie servers
-                Owned<IPropertyTreeIterator> iterRoxieServers = pRoxie->getElements(XML_TAG_ROXIE_SERVER);
-
-                ForEach (*iterRoxieServers)
-                {
-                  IPropertyTree* pTmpComp = &iterRoxieServers->query();
-                  if (strcmp(pTmpComp->queryProp(XML_ATTR_DATADIRECTORY), sbNewValue.str()))
-                    pTmpComp->setProp(XML_ATTR_DATADIRECTORY, sbNewValue.str());
-                }
-
-                //also change roxie slave primary data directory for all RoxieSlave and RoxieSlaveProcess
-                Owned<IPropertyTreeIterator> iterSlvs = pRoxie->getElements(XML_TAG_ROXIE_ONLY_SLAVE);
-
-                ForEach (*iterSlvs)
-                {
-                  IPropertyTree* pTmpComp = &iterSlvs->query();
-                  const char* pRoxieComputer = pTmpComp->queryProp(XML_ATTR_COMPUTER);
-                  IPropertyTree* pChannel = pTmpComp->queryPropTree(XML_TAG_ROXIE_CHANNEL"[1]");
-                  if (pChannel)
-                  {
-                    pChannel->setProp(XML_ATTR_DATADIRECTORY, sbNewValue.str());
-                    const char* number = pChannel->queryProp("@number");
-                    xpath.clear().appendf(XML_TAG_ROXIE_SLAVE"[@channel='%s'][@computer='%s']", number, pRoxieComputer);
-                    
-                    IPropertyTree* pSlvProc = pRoxie->queryPropTree(xpath.str());
-                    if (pSlvProc)
-                      pSlvProc->setProp(XML_ATTR_DATADIRECTORY, sbNewValue.str());
-                  }
-                }
-              }
-              else if (bdata2 || bdata3)
-              {
-                getCommonDir(pEnvRoot, bdata2 ? "data2" : "data3" , "roxie", pRoxie->queryProp(XML_ATTR_NAME), sbNewValue.clear());
-                Owned<IPropertyTreeIterator> iterSlvs = pRoxie->getElements(XML_TAG_ROXIE_ONLY_SLAVE);
-                StringBuffer sb(XML_TAG_ROXIE_CHANNEL);
-                sb.appendf("%s", bdata2?"[2]":"[3]");
-
-                ForEach (*iterSlvs)
-                {
-                  IPropertyTree* pTmpComp = &iterSlvs->query();
-                  const char* pRoxieComputer = pTmpComp->queryProp(XML_ATTR_COMPUTER);
-                  IPropertyTree* pChannel = pTmpComp->queryPropTree(sb.str());
-                  if (pChannel)
-                  {
-                    pChannel->setProp(XML_ATTR_DATADIRECTORY, sbNewValue.str());
-                    const char* number = pChannel->queryProp("@number");
-                    xpath.clear().appendf(XML_TAG_ROXIE_SLAVE"[@channel='%s'][@computer='%s']", number, pRoxieComputer);
-                    
-                    IPropertyTree* pSlvProc = pRoxie->queryPropTree(xpath.str());
-                    if (pSlvProc)
-                      pSlvProc->setProp(XML_ATTR_DATADIRECTORY, sbNewValue.str());
-                  }
-                }
-              }
-            }
-          }
-        }
-
         //if we are changing the eclServer field of wsattributes, set the following 
         //extra params from that eclserver. dbPassword, dbUser, mySQL, repository
         if (!strcmp(pszCompType, "WsAttributes") && !strcmp(pszAttrName, "eclServer"))
@@ -3735,7 +3648,7 @@ bool CWsDeployFileInfo::getDeployableComps(IEspContext &context, IEspGetDeployab
                 if (*nodeName != 'R')// || //neither RoxieServerProcess nor RoxieSlaveProcess
                 {
                   IPropertyTree* pInstanceNode = pCompNode->addPropTree(XML_TAG_INSTANCES, createPTree());
-                  const char* directory = pNode->queryProp(*nodeName == 'R' ? XML_ATTR_DATADIRECTORY : XML_ATTR_DIRECTORY);
+                  const char* directory = pNode->queryProp(*nodeName == 'R' ? XML_ATTR_LEVEL : XML_ATTR_DIRECTORY);
                   if (directory && *directory)
                     pInstanceNode->addProp(XML_ATTR_BUILD, directory);
 

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1150,144 +1150,73 @@ bool CWsDfuEx::DFUDeleteFiles(IEspContext &context, IEspDFUArrayActionRequest &r
     StringArray superFileNames, filesCannotBeDeleted;
     for(int j = 0; j < 2; j++) //j=0: delete superfiles first
     {
-        for(unsigned i = 0; i < req.getLogicalFiles().length();i++)
+        for(unsigned i = 0; i < req.getLogicalFiles().length(); i++)
         {
-            const char* file = req.getLogicalFiles().item(i);
-            if(!file || !*file)
+            const char* filename = req.getLogicalFiles().item(i);
+            if(!filename || !*filename)
                 continue;
 
-            unsigned len = strlen(file);
-            const char* cluster = NULL;
-            const char *pCh = strchr(file, '@');
-            if (pCh)
-            {
-                len = pCh - file;
-                if (len+1 < strlen(file))
-                    cluster = pCh + 1;
-            }
-
-            StringBuffer logicalFileName;
-            char* curfile = new char[len+1];
-            strncpy(curfile, file, len);
-            curfile[len] = 0;
-            logicalFileName.append(curfile);
-            delete [] curfile;
-
             if (j>0)
-            { //now, we want to skip superfiles and the files which cannot do the lookup.
-                bool superFile = false;
-                ForEachItemIn(ii, superFileNames)
-                {
-                    const char* file = superFileNames.item(ii);
-                    if (file && streq(file, logicalFileName.str()))
-                    {
-                        superFile = true;
-                        break;
-                    }
-                }
+            { // 2nd pass, now we want to skip superfiles and the files which cannot do the lookup.
 
-                if (superFile)
+                if (superFileNames.contains(filename))
                     continue;
 
-                bool fileCannotBeDeleted = false;
-                ForEachItemIn(i, filesCannotBeDeleted)
-                {
-                    const char* file = filesCannotBeDeleted.item(i);
-                    if (file && streq(file, logicalFileName.str()))
-                    {
-                        fileCannotBeDeleted = true;
-                        break;
-                    }
-                }
-
-                if (fileCannotBeDeleted)
+                if (filesCannotBeDeleted.contains(filename))
                     continue;
             }
 
+            Owned<IDistributedFile> df;
             try
             {
-                Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalFileName.str(), userdesc, true) ;
+                df.setown(queryDistributedFileDirectory().lookup(filename, userdesc, true));
                 if(!df)
                 {
-                    returnStr.appendf("<Message><Value>Cannot delete %s: file not found</Value></Message>", logicalFileName.str());
-                    filesCannotBeDeleted.append(logicalFileName);
+                    returnStr.appendf("<Message><Value>Cannot delete %s: file not found</Value></Message>", filename);
+                    filesCannotBeDeleted.append(filename);
                     continue;
                 }
-
-                if (j<1) //j=0: delete superfiles first
-                {
-                    if(!df->querySuperFile())
-                        continue;
-
-                    superFileNames.append(logicalFileName);
-                }
-
-                DBGLOG("CWsDfuEx::DFUDeleteFiles User=%s Action=Delete File=%s",username.str(), logicalFileName.str());
-
-                CDfsLogicalFileName lfn;
-                StringBuffer cname(cluster);
-                lfn.set(logicalFileName.str());
-                if (!cname.length())
-                    lfn.getCluster(cname);  // see if cluster part of LogicalFileName
-
-                bool deleted;
-                if(!req.getNoDelete() && !df->querySuperFile())
-                {
-                    Owned<IMultiException> pExceptionHandler = MakeMultiException();
-                    deleted = queryDistributedFileSystem().remove(df,cname.length()?cname.str():NULL,pExceptionHandler, REMOVE_FILE_SDS_CONNECT_TIMEOUT);
-                    StringBuffer errorStr;
-                    pExceptionHandler->errorMessage(errorStr);
-                    if (errorStr.length() > 0)
-                    {
-                        returnStr.appendf("<Message><Value>%s</Value></Message>",errorStr.str());
-                        DBGLOG("%s", errorStr.str());
-                    }
-                    else if (!deleted)
-                    {
-                        returnStr.appendf("<Message><Value>Logical File %s not deleted. No error message.</Value></Message>",logicalFileName.str());
-                        DBGLOG("Logical File %s not deleted. No error message.\n",logicalFileName.str());
-                    }
-                    else
-                    {
-                        PrintLog("Deleted Logical File: %s\n",logicalFileName.str());
-                        returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>",logicalFileName.str());
-                    }
-                }
-                else
-                {
-                    df.clear(); 
-                    deleted = queryDistributedFileDirectory().removeEntry(logicalFileName.str(), userdesc, NULL, REMOVE_FILE_SDS_CONNECT_TIMEOUT); // this can remove clusters also
-                    if (deleted)
-                    {
-                        PrintLog("Detached File: %s\n",logicalFileName.str());
-                        returnStr.appendf("<Message><Value>Detached File %s</Value></Message>", logicalFileName.str());
-                    }
-                    else
-                    {
-                        returnStr.appendf("<Message><Value>File %s not detached.</Value></Message>",logicalFileName.str());
-                        DBGLOG("File %s not detached.\n",logicalFileName.str());
-                    }
-                }
-
-                if (!deleted)
-                    return false;
             }
             catch(IException* e)
             {
-                filesCannotBeDeleted.append(logicalFileName);
+                filesCannotBeDeleted.append(filename);
 
                 StringBuffer emsg;
                 e->errorMessage(emsg);
                 if((e->errorCode() == DFSERR_CreateAccessDenied) && (req.getType() != NULL))
-                {
-                    emsg.replaceString("Create ", "Delete ");               
-                }
+                    emsg.replaceString("Create ", "Delete ");
 
-                returnStr.appendf("<Message><Value>Cannot delete %s: %s</Value></Message>", logicalFileName.str(), emsg.str());
+                returnStr.appendf("<Message><Value>Cannot delete %s: %s</Value></Message>", filename, emsg.str());
             }
             catch(...)
             {
-                returnStr.appendf("<Message><Value>Cannot delete %s: unknown exception.</Value></Message>", logicalFileName.str());
+                returnStr.appendf("<Message><Value>Cannot delete %s: unknown exception.</Value></Message>", filename);
+            }
+            if (df)
+            {
+                if (0==j) // skip non-super files on 1st pass
+                {
+                    if(!df->querySuperFile())
+                        continue;
+
+                    superFileNames.append(filename);
+                }
+
+                DBGLOG("CWsDfuEx::DFUDeleteFiles User=%s Action=Delete File=%s",username.str(), filename);
+                try
+                {
+                    df->detach(REMOVE_FILE_SDS_CONNECT_TIMEOUT);
+                    PROGLOG("Deleted Logical File: %s\n",filename);
+                    returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>", filename);
+                }
+                catch (IException *e)
+                {
+                    StringBuffer errorMsg;
+                    e->errorMessage(errorMsg);
+                    PROGLOG("%s", errorMsg.str());
+                    e->Release();
+                    returnStr.appendf("<Message><Value>%s</Value></Message>", errorMsg.str());
+                }
             }
         }
     }
@@ -1376,7 +1305,7 @@ bool CWsDfuEx::onDFUArrayAction(IEspContext &context, IEspDFUArrayActionRequest
             DBGLOG("CWsDfuEx::onDFUArrayAction User=%s Action=%s File=%s",username.str(),req.getType(), file);
             try
             {
-                onDFUAction(userdesc.get(), curfile, cluster, req.getType(), req.getNoDelete(), returnStr);
+                onDFUAction(userdesc.get(), curfile, cluster, req.getType(), returnStr);
             }
             catch(IException* e)
             {
@@ -1421,13 +1350,13 @@ bool CWsDfuEx::onDFUArrayAction(IEspContext &context, IEspDFUArrayActionRequest
     return true;
 }
 
-bool CWsDfuEx::onDFUAction(IUserDescriptor* udesc, const char* LogicalFileName, const char* ClusterName,const char* ActionType,bool nodelete, StringBuffer& returnStr)
+bool CWsDfuEx::onDFUAction(IUserDescriptor* udesc, const char* LogicalFileName, const char* ClusterName, const char* ActionType, StringBuffer& returnStr)
 {
     //No 'try/catch' is needed for this method since it will be called internally.
     if (strcmp(Action_Delete ,ActionType) == 0)
     {
         LogicFileWrapper Logicfile;
-        if (!Logicfile.doDeleteFile(LogicalFileName,ClusterName,nodelete, returnStr, udesc))
+        if (!Logicfile.doDeleteFile(LogicalFileName,ClusterName, returnStr, udesc))
             return false;
     }
     else if (strcmp(Action_AddtoSuperfile ,ActionType) == 0)

esp/services/ws_dfu/ws_dfuService.hpp

@@ -111,7 +111,7 @@ private:
     bool checkDescription(const char *description, const char *descriptionFilter);
     void getDefFile(IUserDescriptor* udesc, const char* FileName,StringBuffer& returnStr);
     void xsltTransformer(const char* xsltPath,StringBuffer& source,StringBuffer& returnStr);
-    bool onDFUAction(IUserDescriptor* udesc, const char* LogicalFileName,const char* ClusterName,const char* ActionType, bool nodelete, StringBuffer& returnStr);
+    bool onDFUAction(IUserDescriptor* udesc, const char* LogicalFileName, const char* ClusterName, const char* ActionType, StringBuffer& returnStr);
     bool checkFileContent(IEspContext &context, IUserDescriptor* udesc, const char * logicalName, const char * cluster);
     void getRoxieClusterConfig(char const * clusterType, char const * clusterName, char const * processName, StringBuffer& netAddress, int& port);
     //bool getRoxieQueriesForFile(const char* logicalName, const char* cluster, StringArray& roxieQueries);

esp/services/ws_fs/ws_fsService.cpp

@@ -440,7 +440,8 @@ static void DeepAssign(IEspContext &context, IConstDFUWorkUnit *src, IEspDFUWork
     }
 }
 
-bool CFileSprayEx::ParseLogicalPath(const char * pLogicalPath, const char* cluster, StringBuffer &folder, StringBuffer &title, StringBuffer &defaultFolder, StringBuffer &defaultReplicateFolder)
+bool CFileSprayEx::ParseLogicalPath(const char * pLogicalPath, const char* cluster,
+                                    StringBuffer &folder, StringBuffer &title, StringBuffer &defaultFolder, StringBuffer &defaultReplicateFolder)
 {
     if(!pLogicalPath || !*pLogicalPath)
         return false;
@@ -454,7 +455,9 @@ bool CFileSprayEx::ParseLogicalPath(const char * pLogicalPath, const char* clust
 
     if(cluster != NULL && *cluster != '\0')
     {
-        Owned<IGroup> group = queryNamedGroupStore().lookup(cluster);
+        StringBuffer basedir;
+        GroupType groupType;
+        Owned<IGroup> group = queryNamedGroupStore().lookup(cluster, basedir, groupType);
         if (group) {
             switch (queryOS(group->queryNode(0).endpoint())) {
             case MachineOsW2K:
@@ -463,9 +466,23 @@ bool CFileSprayEx::ParseLogicalPath(const char * pLogicalPath, const char* clust
             case MachineOsLinux:
                 os = DFD_OSunix; break;
             }
-            if (directories.get()) {
-                getConfigurationDirectory(directories, "data", "thor", cluster, defaultFolder);
-                getConfigurationDirectory(directories, "mirror", "thor", cluster, defaultReplicateFolder);
+            if (directories.get())
+            {
+                switch (groupType)
+                {
+                case grp_roxie:
+                    getConfigurationDirectory(directories, "data", "roxie", cluster, defaultFolder);
+                    getConfigurationDirectory(directories, "data2", "roxie", cluster, defaultReplicateFolder);
+                    // MORE - should extend to systems with higher redundancy
+                    break;
+                case grp_hthor:
+                    getConfigurationDirectory(directories, "data", "hthor", cluster, defaultFolder);
+                    break;
+                case grp_thor:
+                default:
+                    getConfigurationDirectory(directories, "data", "thor", cluster, defaultFolder);
+                    getConfigurationDirectory(directories, "mirror", "thor", cluster, defaultReplicateFolder);
+                }
             }
         }
         else 

esp/services/ws_packageprocess/CMakeLists.txt

@@ -40,7 +40,6 @@ include_directories (
          ${HPCC_SOURCE_DIR}/common/environment
          ${HPCC_SOURCE_DIR}/dali/dfu
          ${HPCC_SOURCE_DIR}/common/remote
-         ${HPCC_SOURCE_DIR}/common/roxiemanager
          ${HPCC_SOURCE_DIR}/common/workunit
          ${HPCC_SOURCE_DIR}/rtl/include
     )
@@ -57,6 +56,5 @@ target_link_libraries ( ws_packageprocess
          securesocket
          dalibase
          ws_fs
-         roxiemanager
     )
 

esp/services/ws_roxiequery/CMakeLists.txt

@@ -1,93 +0,0 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-
-# Component: ws_roxiequery 
-#####################################################
-# Description:
-# ------------
-#    Cmake Input File for ws_roxiequery
-#####################################################
-
-project( ws_roxiequery ) 
-
-include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
-
-set (    SRCS 
-         ${ESPSCM_GENERATED_DIR}/ws_roxiequery_esp.cpp 
-         ws_roxiequeryplugin.cpp 
-         ws_roxiequeryservice.cpp 
-         
-         ws_roxiequeryservice.cpp
-         
-         ${HPCC_SOURCE_DIR}/esp/scm/ws_roxiequery.ecm
-    )
-
-include_directories ( 
-         ./../../../dali/dfu 
-         ./../../../system/mp 
-         ./../../platform 
-         ./../../../system/jlib 
-         ./../../../common/environment 
-         ./../../services 
-         ./../../../dali/ft 
-         ./../common 
-         ./../../../system/xmllib 
-         ./../../../esp/bindings/http/platform 
-         ./../../../system/security/securesocket 
-         ./../../../system/security/shared 
-         ./../../../system/include 
-         ./../../../common/workunit 
-         ./../../../common/remote 
-         ./../../clients 
-         ./../../../esp/esplib 
-         ./../../../dali/base 
-         ./../../bindings/SOAP/scrubbed 
-         ./../ws_workunits 
-         ./../../../rtl/include 
-         ./../../../common/dllserver 
-         ./../../bindings 
-         ./../../smc/SMCLib 
-         ./../../bindings/SOAP/xpp 
-         ./../../../common/fileview2 
-         ./../../../rtl/eclrtl 
-    )
-
-ADD_DEFINITIONS( -D_USRDLL )
-
-HPCC_ADD_LIBRARY( ws_roxiequery SHARED ${SRCS} )
-add_dependencies ( ws_roxiequery espscm )
-install ( TARGETS ws_roxiequery RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
-target_link_libraries ( ws_roxiequery 
-         jlib
-         fileview2
-         remote 
-         ${XALAN_LIBRARIES} ${XERCES_LIBRARIES}
-         xmllib 
-         esphttp 
-         dalibase 
-         environment 
-         dalift 
-         dllserver 
-         nbcd 
-         eclrtl 
-         deftype 
-         workunit 
-         SMCLib 
-         roxiecommlib 
-         roxiemanager
-    )
-

esp/services/ws_roxiequery/sourcedoc.xml

@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
--->
-<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
-<section>
-    <title>esp/services/ws_roxiequery</title>
-
-    <para>
-        The esp/services/ws_roxiequery directory contains the sources for the esp/services/ws_roxiequery library.
-    </para>
-</section>

esp/services/ws_roxiequery/ws_roxiequeryplugin.cpp

@@ -1,91 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#pragma warning (disable : 4786)
-
-#ifndef WsRoxieQuery_API
-#ifdef _WIN32
-#define WsRoxieQuery_API __declspec(dllexport)
-#else
-#define WsRoxieQuery_API
-#endif //_WIN32
-#endif //WsRoxieQuery_API
-
-#include "ws_roxiequery_esp.ipp"
-
-//ESP Bindings
-#include "http/platform/httpprot.hpp"
-
-//ESP Service
-#include "ws_roxiequeryservice.hpp"
-
-#include "espplugin.hpp"
-
-extern "C"
-{
-//when we aren't loading dynamically
-// Change the function names when we stick with dynamic loading.
-ESP_FACTORY IEspService * esp_service_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
-{
-    if (strcmp(type, "WsRoxieQuery")==0)
-    {
-        CWsRoxieQueryEx* service = new CWsRoxieQueryEx;
-        service->init(cfg, process, name);
-        return service;
-    }
-    return NULL;
-}
-
-
-
-ESP_FACTORY IEspRpcBinding * esp_binding_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
-{
-    if (strcmp(type, "WsRoxieQuery")==0)
-    {
-#ifdef _DEBUG
-        http_soap_log_level log_level_ = hsl_all;
-#else
-        http_soap_log_level log_level_ = hsl_none;
-#endif
-        return new CWsRoxieQuerySoapBindingEx(cfg, name, process, log_level_);
-    }
-
-    return NULL;
-}
-
-
-
-ESP_FACTORY IEspProtocol * esp_protocol_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
-{
-    if (strcmp(type, "http_protocol")==0)
-    {
-        return new CHttpProtocol;
-    }
-    else if(strcmp(type, "secure_http_protocol") == 0)
-    {
-        IPropertyTree *sslSettings;
-        sslSettings = cfg->getPropTree(StringBuffer("Software/EspProcess[@name=\"").append(process).append("\"]").append("/EspProtocol[@name=\"").append(name).append("\"]").str());
-        if(sslSettings != NULL)
-        {
-            return new CSecureHttpProtocol(sslSettings);
-        }
-    }
-    
-    return NULL;
-}
-
-};

esp/services/ws_roxiequery/ws_roxiequeryservice.cpp

@@ -1,812 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#pragma warning (disable : 4786)
-
-#include <math.h>
-#include "jfile.hpp"
-#include "wshelpers.hpp"
-#include "portlist.h"
-#include "jiface.hpp"
-#include "environment.hpp"
-#include "TpWrapper.hpp"
-
-#include "jstring.hpp"
-
-#include "dautils.hpp"
-#include "eclrtl.hpp"
-#include "fverror.hpp"
-#include "exception_util.hpp"
-
-#include "ws_roxiequeryservice.hpp"
-
-const unsigned int  ROXIEQUERYID                    = 1;
-const unsigned int  ROXIEQUERYWUID                  = 2;
-const unsigned int  ROXIEQUERYCLUSTER               = 3;
-const unsigned int  ROXIEQUERYLABEL             = 4;
-const unsigned int  ROXIEQUERYSUSPENDED         = 5;
-const unsigned int  ROXIEQUERYHIGHPRIORITY      = 6;
-const unsigned int  ROXIEQUERYERROR             = 7;
-const unsigned int  ROXIEQUERYCOMMENT               = 8;
-const unsigned int  ROXIEQUERYASSOCIATEDNAME    = 9;
-const unsigned int  ROXIEQUERYHASALIASES            = 10;
-const unsigned int  ROXIEQUERYDEPLOYEDBY            = 11;
-const unsigned int  ROXIEQUERYUPDATEDBY         = 12;
-const unsigned int  ROXIEMANAGER_TIMEOUT        = 60000;
-
-static const char* FEATURE_URL="RoxieQueryAccess";
-static const char* ROXIE_CLUSTER="RoxieCluster";
-static const char* ROXIE_FARMERPROCESS1="RoxieServerProcess[1]";
-
-__int64 findPositionInRoxieQueryList(int type, const char *value, bool descend, IArrayOf<IEspRoxieQuery>& queries)
-{
-    if (!value || (strlen(value) < 1))
-    {
-        if (descend)
-            return -1;
-        else
-            return 0;
-    }
-
-    __int64 addToPos = -1;
-    ForEachItemIn(i, queries)
-    {
-        IEspRoxieQuery& query = queries.item(i);
-        char *Value = NULL;
-        switch (type)
-        {
-        case ROXIEQUERYID:
-            Value = (char *) query.getID();
-            break;
-        case ROXIEQUERYDEPLOYEDBY:
-            Value = (char *) query.getDeployedBy();
-            break;
-        case ROXIEQUERYUPDATEDBY:
-            Value = (char *) query.getUpdatedBy();
-            break;
-        case ROXIEQUERYWUID:
-            Value = (char *) query.getWUID();
-            break;
-        case ROXIEQUERYSUSPENDED:
-            Value = (char *) query.getSuspended();
-            break;
-        case ROXIEQUERYHIGHPRIORITY:
-            Value = (char *) query.getHighPriority();
-            break;
-        case ROXIEQUERYERROR:
-            Value = (char *) query.getError();
-            break;
-        case ROXIEQUERYCOMMENT:
-            Value = (char *) query.getComment();
-            break;
-        case ROXIEQUERYHASALIASES:
-            Value = (char *) query.getHasAliases();
-            break;
-        }
-
-        if (!Value)
-            continue;
-
-        if (type != ROXIEQUERYID)
-        {
-            if (descend && strcmp(value, Value)>0)
-            {
-                addToPos = i;
-                break;
-            }
-            if (!descend && strcmp(value, Value)<0)
-            {
-                addToPos = i;
-                break;
-            }
-        }
-        else
-        {
-            if (descend && stricmp(value, Value)>0)
-            {
-                addToPos = i;
-                break;
-            }
-            if (!descend && stricmp(value, Value)<0)
-            {
-                addToPos = i;
-                break;
-            }
-        }
-    }
-
-    return addToPos;
-}
-
-void getUserInformation(IEspContext &context, StringBuffer &username, StringBuffer &password)
-{
-    context.getUserID(username);
-    context.getPassword(password);
-}
-
-void CWsRoxieQueryEx::init(IPropertyTree *cfg, const char *process, const char *service)
-{
-
-    IPropertyTree* pStyleSheets = cfg->queryPropTree("StyleSheets");
-    const char* xslt = cfg->queryProp("xslt[@name='atts']");
-    m_attsXSLT.append(getCFD()).append( xslt && *xslt ? xslt : "smc_xslt/atts.xslt");
-
-    xslt = cfg->queryProp("xslt[@name='graphStats']");
-    m_graphStatsXSLT.append(getCFD()).append( xslt && *xslt ? xslt : "smc_xslt/graphStats.xslt");
-}
-
-void CWsRoxieQueryEx::addToQueryString(StringBuffer &queryString, const char *name, const char *value)
-{
-    if (queryString.length() > 0)
-    {
-        queryString.append("&amp;");
-    }
-
-    queryString.append(name);
-    queryString.append("=");
-    queryString.append(value);
-}
-
-void CWsRoxieQueryEx::addToQueryStringFromInt(StringBuffer &queryString, const char *name, int value)
-{
-    if (queryString.length() > 0)
-    {
-        queryString.append("&amp;");
-    }
-
-    queryString.append(name);
-    queryString.append("=");
-    queryString.append(value);
-}
-
-void CWsRoxieQueryEx::getClusterConfig(char const * clusterType, char const * clusterName, char const * processName, StringBuffer& netAddress, int& port)
-{
-    Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
-#if 0
-    Owned<IConstEnvironment> environment = factory->openEnvironment();
-#else
-    Owned<IConstEnvironment> environment = factory->openEnvironmentByFile();
-#endif
-
-    Owned<IPropertyTree> pRoot = &environment->getPTree();
-
-    StringBuffer xpath;
-    xpath.appendf("Software/%s[@name='%s']", clusterType, clusterName);
-
-    IPropertyTree* pCluster = pRoot->queryPropTree( xpath.str() );
-    if (!pCluster)
-        throw MakeStringException(ECLWATCH_CLUSTER_NOT_IN_ENV_INFO, "Failed to get environment information for the cluster '%s %s'.", clusterType, clusterName);
-    xpath.clear().append(processName);
-    xpath.append("@computer");
-    const char* computer = pCluster->queryProp(xpath.str());
-    if (!computer || strlen(computer) < 1)
-        throw MakeStringException(ECLWATCH_CLUSTER_NOT_IN_ENV_INFO, "Failed to get the 'computer' information for the process '%s'. Please check environment settings for the cluster '%s %s'.", processName, clusterType, clusterName);
-
-    xpath.clear().append(processName);
-    xpath.append("@port");
-    const char* portStr = pCluster->queryProp(xpath.str());
-    port = ROXIE_SERVER_PORT;
-    if (portStr && *portStr)
-    {
-        port = atoi(portStr);
-    }
-
-    Owned<IConstMachineInfo> pMachine = environment->getMachine(computer);
-    if (pMachine)
-    {
-        SCMStringBuffer scmNetAddress;
-        pMachine->getNetAddress(scmNetAddress);
-        netAddress = scmNetAddress.str();
-#ifdef MACHINE_IP
-        if (!strcmp(netAddress.str(), "."))
-            netAddress = MACHINE_IP;
-#endif
-    }
-    
-    return;
-}
-
-bool CWsRoxieQueryEx::onRoxieQuerySearch(IEspContext &context, IEspRoxieQuerySearchRequest & req, IEspRoxieQuerySearchResponse & resp)
-{
-    try
-    {
-        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
-            throw MakeStringException(ECLWATCH_ROXIE_QUERY_ACCESS_DENIED, "Failed to Search Roxie Queries. Permission denied.");
-
-        StringBuffer username;
-        context.getUserID(username);
-        DBGLOG("CWsRoxieQueryEx::onRoxieQuerySearch User=%s",username.str());
-
-        CTpWrapper dummy;
-        IArrayOf<IEspTpCluster> clusters;
-        dummy.getClusterProcessList(eqRoxieCluster, clusters);
-
-        if (clusters.length() <= 0)
-            throw MakeStringException(ECLWATCH_CLUSTER_NOT_IN_ENV_INFO, "Roxie cluster not found.");
-
-        StringArray roxieclusters;
-        ForEachItemIn(k, clusters)
-        {
-            IEspTpCluster& cluster = clusters.item(k);
-            roxieclusters.append(cluster.getName());
-        }
-
-        StringArray ftarray;
-        ftarray.append("Suspended and Non-Suspended");
-        ftarray.append("Non-Suspended Only");
-        ftarray.append("Suspended Only");
-
-        resp.setClusterNames(roxieclusters);
-        resp.setSuspendedSelections(ftarray);
-    }
-    catch(IException* e)
-    {   
-        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
-    }
-    return true;
-}
-
-bool CWsRoxieQueryEx::getAllRoxieQueries(IEspContext &context, const char* cluster, const char* suspended, const char* sortBy, bool descending, __int64 displayEnd, IArrayOf<IEspRoxieQuery>& RoxieQueryList, __int64& totalFiles)
-{
-    int port;
-    StringBuffer netAddress;
-    SocketEndpoint ep;
-    getClusterConfig(ROXIE_CLUSTER, cluster, ROXIE_FARMERPROCESS1, netAddress, port);
-
-    int suspendedFlag = 0;
-    if (suspended && *suspended)
-    {
-        if (stricmp(suspended, "Suspended Only") == 0)
-        {
-            suspendedFlag = 2;
-        }
-        else if (stricmp(suspended, "Non-Suspended Only") == 0)
-        {
-            suspendedFlag = 1;
-        }
-    }
-
-    ep.set(netAddress.str(), port);
-
-    StringBuffer username, password;
-    getUserInformation(context, username, password);
-
-    StringBuffer currentDaliIp;
-    const SocketEndpoint &ep1 = queryCoven().queryComm().queryGroup().queryNode(0).endpoint();
-    ep1.getUrlStr(currentDaliIp);
-
-    LogLevel loglevel = getEspLogLevel(&context);
-    Owned<IRoxieQueryManager> queryManager = createRoxieQueryManager(ep, cluster, currentDaliIp, ROXIEMANAGER_TIMEOUT, username.str(), password.str(), loglevel);
-    Owned<IPropertyTree> result = queryManager->retrieveQueryList("*", false, false, false, false, 0);
-
-    Owned<IPropertyTreeIterator> queries = result->getElements("Query");
-    ForEach(*queries)
-    {
-        IPropertyTree &query = queries->query();
-        const char* id = query.queryProp("@id");
-        const char* wuid = query.queryProp("@wuid");
-        const char* deployedby = query.queryProp("@deployedBy");
-        const char* updatedby = query.queryProp("@suspendedBy");
-        const char* comment = query.queryProp("@comment");
-        const char* error = query.queryProp("@error");
-        bool bSuspended = query.getPropBool("@suspended", false);
-        bool bHighPriority = query.getPropBool("@priority", false);
-        bool bHasAliases = query.getPropBool("@hasAlias", false);
-
-        if (suspendedFlag > 1)
-        {
-            if (!bSuspended) //Suspended Only
-                continue;
-        }
-        else    if (suspendedFlag > 0)
-        {
-            if (bSuspended) //Non-Suspended Only
-                continue;
-        }
-
-        StringBuffer strSuspended, strHighPriority, strHasAliases;
-        if (bSuspended)
-            strSuspended.append("Yes");
-        else
-            strSuspended.append("No");
-        if (bHighPriority)
-            strHighPriority.append("Yes");
-        else
-            strHighPriority.append("No");
-        if (bHasAliases)
-            strHasAliases.append("Yes");
-        else
-            strHasAliases.append("No");
-
-        Owned<IEspRoxieQuery> roxieQuery = createRoxieQuery("","");
-        if (id && *id)
-            roxieQuery->setID(id);
-        if (comment && *comment)
-            roxieQuery->setComment(comment);
-        if (error && *error)
-            roxieQuery->setError("Yes");
-        else
-            roxieQuery->setError("No");
-
-        roxieQuery->setHasAliases(strHasAliases.str());
-        roxieQuery->setSuspended(strSuspended.str());
-        roxieQuery->setHighPriority(strHighPriority.str());
-
-        double version = context.getClientVersion();
-        if (version > 1.00)
-        {
-            if (deployedby && *deployedby)
-                roxieQuery->setDeployedBy(deployedby);
-            if (updatedby && *updatedby)
-                roxieQuery->setUpdatedBy(updatedby);
-        }
-        if (version > 1.01)
-        {
-            if (wuid && *wuid)
-                roxieQuery->setWUID(wuid);
-        }
-
-        __int64 addToPos = -1; //Add to tail
-        if (!sortBy)
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYID, id, false, RoxieQueryList);
-        }
-        else if (stricmp(sortBy, "ID")==0)
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYID, id, descending, RoxieQueryList);
-        }
-        else if (stricmp(sortBy, "DeployedBy")==0)
-        {
-            if (version > 1.00)
-            {
-                addToPos = findPositionInRoxieQueryList(ROXIEQUERYDEPLOYEDBY, deployedby, descending, RoxieQueryList);
-            }
-        }
-        else if (stricmp(sortBy, "UpdatedBy")==0)
-        {
-            if (version > 1.00)
-            {
-                addToPos = findPositionInRoxieQueryList(ROXIEQUERYUPDATEDBY, updatedby, descending, RoxieQueryList);
-            }
-        }
-        else if (stricmp(sortBy, "WUID")==0)
-        {
-            if (version > 1.01)
-            {
-                addToPos = findPositionInRoxieQueryList(ROXIEQUERYWUID, wuid, descending, RoxieQueryList);
-            }
-        }
-        else if (stricmp(sortBy, "Suspended")==0)
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYSUSPENDED, strSuspended.str(), descending, RoxieQueryList);
-        }
-        else if (stricmp(sortBy, "HighPriority")==0)
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYHIGHPRIORITY, strHighPriority.str(), descending, RoxieQueryList);
-        }
-        else if (stricmp(sortBy, "Error")==0)
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYERROR, error, descending, RoxieQueryList);
-        }
-        else if (stricmp(sortBy, "Comment")==0)
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYCOMMENT, comment, descending, RoxieQueryList);
-        }
-        else if (stricmp(sortBy, "HasAliases")==0)
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYHASALIASES, strHighPriority.str(), descending, RoxieQueryList);
-        }
-        else
-        {
-            addToPos = findPositionInRoxieQueryList(ROXIEQUERYID, id, false, RoxieQueryList);
-        }
-
-        totalFiles++;
-
-        if (addToPos < 0)
-            RoxieQueryList.append(*roxieQuery.getClear());
-        else
-            RoxieQueryList.add(*roxieQuery.getClear(), (int) addToPos);
-    }
-
-    return true;
-}
-
-bool CWsRoxieQueryEx::onRoxieQueryList(IEspContext &context, IEspRoxieQueryListRequest & req, IEspRoxieQueryListResponse & resp)
-{
-    try
-    {
-        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
-            throw MakeStringException(ECLWATCH_ROXIE_QUERY_ACCESS_DENIED, "Failed to List Roxie Queries. Permission denied.");
-
-        StringBuffer username;
-        context.getUserID(username);
-        DBGLOG("CWsRoxieQueryEx::onRoxieQueryList User=%s",username.str());
-
-        const char* sortBy = req.getSortby();
-        bool descending = req.getDescending();
-
-        unsigned pagesize = req.getPageSize();
-        if (pagesize < 1)
-        {
-            pagesize = 100;
-        }
-
-        __int64 displayStartReq = 1;
-        if (req.getPageStartFrom() > 0)
-            displayStartReq = req.getPageStartFrom();
-
-        __int64 displayStart = displayStartReq - 1;
-        __int64 displayEnd = displayStart + pagesize;
-
-        StringBuffer size;
-        __int64 totalFiles = 0;
-        IArrayOf<IEspRoxieQuery> RoxieQueryList;
-
-        const char* name = req.getLogicalName();
-        const char* cluster = req.getCluster();
-
-        if (!cluster || !*cluster)
-            throw MakeStringException(ECLWATCH_INVALID_INPUT,"Cluster is not specified for retrieving Roxie queries.");
-
-        getAllRoxieQueries(context, cluster, req.getSuspended(), sortBy, descending, displayEnd, RoxieQueryList, totalFiles);
-
-        resp.setNumFiles(totalFiles);
-        resp.setPageSize(pagesize);
-        resp.setPageStartFrom(displayStart+1);
-        resp.setPageEndAt(displayEnd);
-
-        if (displayStart - pagesize > 0)
-            resp.setPrevPageFrom(displayStart - pagesize + 1);
-        else if(displayStart > 0)
-            resp.setPrevPageFrom(1);
-
-        if(displayEnd < totalFiles)
-        {
-            resp.setNextPageFrom(displayEnd+1);
-            resp.setLastPageFrom(pagesize * (int) floor((double) totalFiles/pagesize) + 1);
-        }
-
-        
-        StringBuffer ParametersForSorting;
-        addToQueryString(ParametersForSorting, "Cluster", cluster);
-        resp.setCluster(cluster);
-
-        if (name && *name)
-        {
-            addToQueryString(ParametersForSorting, "LogicalName", name);
-            resp.setLogicalName(name);
-        }
-        if (req.getSuspended() && *req.getSuspended())
-            addToQueryString(ParametersForSorting, "Suspended", req.getSuspended());
-
-        StringBuffer ParametersForPaging;
-        addToQueryStringFromInt(ParametersForPaging, "PageSize", pagesize);
-
-        descending = false;
-        if (sortBy && *sortBy)
-        {
-            if (req.getDescending())
-                descending = req.getDescending();
-
-            resp.setSortby(sortBy);
-            resp.setDescending(descending);
-
-            addToQueryString(ParametersForPaging, "Sortby", sortBy);
-            if (descending)
-            {
-                addToQueryString(ParametersForPaging, "Descending", "1");
-            }
-        }
-
-        if (ParametersForSorting.length() > 0)
-            resp.setParametersForSorting(ParametersForSorting.str());
-        if (ParametersForPaging.length() > 0)
-            resp.setParametersForPaging(ParametersForPaging.str());
-
-        resp.setRoxieQueries(RoxieQueryList);
-    }
-    catch(IException* e)
-    {   
-        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
-    }
-    return true;
-}
-
-bool CWsRoxieQueryEx::onQueryDetails(IEspContext &context, IEspRoxieQueryDetailsRequest & req, IEspRoxieQueryDetailsResponse & resp)
-{
-    try
-    {
-        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
-            throw MakeStringException(ECLWATCH_ROXIE_QUERY_ACCESS_DENIED, "Failed to get Query Details. Permission denied.");
-
-        StringBuffer username, password;
-        getUserInformation(context, username, password);
-        DBGLOG("CWsRoxieQueryEx::onQueryDetails User=%s",username.str());
-
-        const char* queryID = req.getQueryID();
-        const char* cluster = req.getCluster();
-        if (!queryID || !*queryID || !cluster || !*cluster)
-        {
-            throw MakeStringException(ECLWATCH_INVALID_INPUT,"Query ID or Cluster is not specified for retrieving query information.");
-        }
-
-        int port;
-        StringBuffer netAddress;
-        SocketEndpoint ep;
-        getClusterConfig(ROXIE_CLUSTER, cluster, ROXIE_FARMERPROCESS1, netAddress, port);
-
-        ep.set(netAddress.str(), port);
-
-        StringBuffer currentDaliIp;
-        const SocketEndpoint &ep1 = queryCoven().queryComm().queryGroup().queryNode(0).endpoint();
-        ep1.getUrlStr(currentDaliIp);
-
-        LogLevel loglevel = getEspLogLevel(&context);
-        Owned<IRoxieQueryManager> queryManager = createRoxieQueryManager(ep, cluster, currentDaliIp, ROXIEMANAGER_TIMEOUT, username.str(), password.str(), loglevel);
-        Owned<IPropertyTree> result = queryManager->retrieveQueryList(queryID, false, false, false, false, 0);
-
-        if (result)
-        {
-            Owned<IPropertyTreeIterator> queries = result->getElements("Query");
-            ForEach(*queries)
-            {
-                IPropertyTree &query = queries->query();
-                const char* wuid = query.queryProp("@wuid");
-                const char* label = query.queryProp("@label");
-                const char* associatedName = query.queryProp("@associatedName");
-                const char* comment = query.queryProp("@comment");
-                const char* error = query.queryProp("@error");
-                const char* deployedby = query.queryProp("@deployedBy");
-                const char* updatedby = query.queryProp("@suspendedBy");
-                bool bSuspended = query.getPropBool("@suspended", false);
-                bool bHighPriority = query.getPropBool("@priority", false);
-                StringBuffer strSuspended, strHighPriority;
-                if (bSuspended)
-                    resp.setSuspended("Yes");
-                else
-                    resp.setSuspended("No");
-                if (bHighPriority)
-                    resp.setHighPriority("Yes");
-                else
-                    resp.setHighPriority("No");
-
-                resp.setQueryID(queryID);
-                resp.setCluster(cluster);
-                if (wuid && *wuid)
-                    resp.setWUID(wuid);
-                if (associatedName && *associatedName)
-                    resp.setAssociatedName(associatedName);
-                if (label && *label)
-                    resp.setLabel(label);
-                if (error && *error)
-                    resp.setError(error);
-                if (comment && *comment)
-                    resp.setComment(comment);
-                    
-                double version = context.getClientVersion();
-                if (version > 1.00)
-                {
-                    if (deployedby && *deployedby)
-                        resp.setDeployedBy(deployedby);
-                    if (updatedby && *updatedby)
-                        resp.setUpdatedBy(updatedby);
-                }
-                break;
-            }
-
-            IArrayOf<IEspRoxieQueryAlias> AliasList;
-            Owned<IPropertyTreeIterator> queries1 = result->getElements("Alias");
-            ForEach(*queries1)
-            {
-                IPropertyTree &query = queries1->query();
-                const char* id = query.queryProp("@id");
-                const char* original = query.queryProp("@original");
-
-                Owned<IEspRoxieQueryAlias> alias = createRoxieQueryAlias("","");
-
-                if (id && *id)
-                    alias->setID(id);
-                if (original && *original)
-                    alias->setOriginal(original);
-
-                AliasList.append(*alias.getClear());
-            }
-
-            resp.setAliases(AliasList);
-        }
-    }
-    catch(IException* e)
-    {   
-        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
-    }
-    return true;
-}
-
-bool CWsRoxieQueryEx::onGVCAjaxGraph(IEspContext &context, IEspGVCAjaxGraphRequest &req, IEspGVCAjaxGraphResponse &resp)
-{
-    resp.setCluster(req.getCluster());
-    resp.setName(req.getName());
-    resp.setGraphName(req.getGraphName());
-    resp.setGraphType("roxiequery");
-    return true;
-}
-bool CWsRoxieQueryEx::onShowGVCGraph(IEspContext &context, IEspShowGVCGraphRequest &req, IEspShowGVCGraphResponse &resp)
-    {
-        const char *queryName = req.getQueryId();
-        if (!queryName || !*queryName)
-            throw MakeStringException(0, "Missing Roxie query name!");
-
-        const char* cluster = req.getCluster();
-
-        int port;
-        StringBuffer netAddress;
-        SocketEndpoint ep;
-        getClusterConfig(ROXIE_CLUSTER, cluster, ROXIE_FARMERPROCESS1, netAddress, port);
-        ep.set(netAddress.str(), port);
-
-        StringArray graphNames;
-        enumerateGraphs(queryName, ep, graphNames);
-
-        char graphName[256];
-        if (req.getGraphName() && *req.getGraphName())
-        {
-            strcpy(graphName, req.getGraphName());
-        }
-        else if (graphNames.length() > 0)
-        {
-            StringBuffer graphName0 = graphNames.item(0);
-            strcpy(graphName, graphName0.str());
-        }
-        else
-        {
-            throw MakeStringException(0, "Graph not found!");
-        }
-
-        Owned<IPropertyTree> pXgmmlGraph = getXgmmlGraph(queryName, ep, graphName);
-
-        StringBuffer xml;
-        toXML(pXgmmlGraph, xml);
-
-        if (xml.length() > 0)
-        {
-            StringBuffer str;
-            encodeUtf8XML(xml.str(), str, 0);
-
-            StringBuffer xml1;
-            xml1.append("<Control>");
-            xml1.append("<Endpoint>");
-            xml1.append("<Query id=\"Gordon.Extractor.0\">");
-            xml1.appendf("<Graph id=\"%s\">", graphName);
-            xml1.append("<xgmml>");
-            xml1.append(str);
-            xml1.append("</xgmml>");
-            xml1.append("</Graph>");
-            xml1.append("</Query>");
-            xml1.append("</Endpoint>");
-            xml1.append("</Control>");
-
-            resp.setTheGraph(xml1);
-        }
-
-        if (graphName && *graphName)
-            resp.setGraphName( graphName );
-        resp.setQueryId( queryName );
-        resp.setGraphNames( graphNames );
-        return true;
-    }
-
-void CWsRoxieQueryEx::enumerateGraphs(const char *queryName, SocketEndpoint &ep, StringArray& graphNames)
-{
-
-    try
-    {
-
-        Owned<IRoxieCommunicationClient> roxieClient = createRoxieCommunicationClient(ep, ROXIEMANAGER_TIMEOUT);
-        Owned<IPropertyTree> result = roxieClient->retrieveQueryStats(queryName, "listGraphNames", NULL, false);
-        IPropertyTree* rootTree = result->queryPropTree("Endpoint/Query");
-
-        if (rootTree)
-        {
-            Owned<IPropertyTreeIterator> graphit = rootTree->getElements("Graph");
-            ForEach(*graphit)
-            {
-                const char* graphName = graphit->query().queryProp("@id");
-                graphNames.append( graphName );
-            }
-        }
-    }
-    catch (IException *e)
-    {
-        StringBuffer errmsg;
-        ERRLOG("Exception when enumerating graphs for %s: %s", queryName, e->errorMessage(errmsg).str());
-        throw e;
-    }
-    catch(...)
-    {
-        ERRLOG("Unknown exception when enumerating graphs for %s", queryName);
-        throw MakeStringException(-1, "Unknown exception while enumerating graphs for %s", queryName);
-    }
-}
-
-IPropertyTree* CWsRoxieQueryEx::getXgmmlGraph(const char *queryName, SocketEndpoint &ep, const char* graphName)
-{
-
-    try
-    {
-        Owned<IRoxieCommunicationClient> roxieClient = createRoxieCommunicationClient(ep, ROXIEMANAGER_TIMEOUT);
-        Owned<IPropertyTree> result = roxieClient->retrieveQueryStats(queryName, "selectGraph", graphName, true);
-
-        StringBuffer xpath;
-        xpath   << "Endpoint/Query/Graph[@id='" 
-                << graphName
-                << "']/xgmml/graph";
-        IPropertyTree* pXgmmlGraph = result->queryPropTree(xpath.str());
-        if (!pXgmmlGraph)
-            throw MakeStringException(-1, "Graph not found!");
-        return LINK( pXgmmlGraph );
-    }
-    catch (IException *e)
-    {
-        StringBuffer errmsg;
-        ERRLOG("Exception when fetching graph %s for %s: %s", graphName, queryName, e->errorMessage(errmsg).str());
-        throw e;
-    }
-    catch(...)
-    {
-        ERRLOG("Unknown exception when fetching graph %s for %s", graphName, queryName);
-        throw MakeStringException(-1, "Unknown exception while generating graph %s for %s", graphName, queryName);
-    }
-}
-
-bool CWsRoxieQueryEx::onRoxieQueryProcessGraph(IEspContext &context, IEspRoxieQueryProcessGraphRequest &req, IEspRoxieQueryProcessGraphResponse &resp)
-{
-    const char* cluster = req.getCluster();
-    const char* graphName = req.getGraphName();
-    const char *queryName = req.getQueryId();
-    if (!queryName || !*queryName)
-        throw MakeStringException(0, "Missing Roxie query name!");
-
-    try
-    {
-        int port;
-        StringBuffer netAddress;
-        SocketEndpoint ep;
-        getClusterConfig(ROXIE_CLUSTER, cluster, ROXIE_FARMERPROCESS1, netAddress, port);
-        ep.set(netAddress.str(), port);
-        Owned<IRoxieCommunicationClient> roxieClient = createRoxieCommunicationClient(ep, ROXIEMANAGER_TIMEOUT);
-
-        Owned<IPropertyTree> xgmml = getXgmmlGraph(queryName, ep, graphName);
-
-        StringBuffer x;
-
-        toXML(xgmml.get(), x);
-        resp.setTheGraph(x.str());
-    }
-    catch (IException *e)
-    {
-        StringBuffer errmsg;
-        ERRLOG("Exception when fetching graph stats %s for %s: %s", graphName, queryName, e->errorMessage(errmsg).str());
-        throw e;
-    }
-    catch(...)
-    {
-        ERRLOG("Unknown exception when fetching graph stats %s for %s", graphName, queryName);
-        throw MakeStringException(-1, "Unknown exception while generating graph %s for %s", graphName, queryName);
-    }
-    return true;
-}

esp/services/ws_roxiequery/ws_roxiequeryservice.hpp

@@ -1,89 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef _ESPWIZ_WsRoxieQuery_HPP__
-#define _ESPWIZ_WsRoxieQuery_HPP__
-
-#ifdef _WIN32
-    #define FILEVIEW_API __declspec(dllimport)
-#else
-    #define FILEVIEW_API
-#endif
-
-#include "fileview.hpp"
-#include "fvrelate.hpp"
-
-#include "roxiecommlibscm.hpp"
-#include "roxiemanagerscm.hpp"
-
-#include <list>
-#include "WUWrapper.hpp"
-
-#include "ws_roxiequery_esp.ipp"
-
-class CWsRoxieQuerySoapBindingEx : public CWsRoxieQuerySoapBinding
-{
-    StringBuffer m_portalURL;
-public:
-    CWsRoxieQuerySoapBindingEx(IPropertyTree *cfg, const char *name, const char *process, http_soap_log_level llevel=hsl_none) : CWsRoxieQuerySoapBinding(cfg, name, process, llevel)
-    {
-        StringBuffer xpath;
-        xpath.appendf("Software/EspProcess[@name='%s']/@portalurl", process);
-        const char* portalURL = cfg->queryProp(xpath.str());
-        if (portalURL && *portalURL)
-            m_portalURL.append(portalURL);
-    }
-
-    virtual void getNavigationData(IEspContext &context, IPropertyTree & data)
-    {
-    }
-};
-
-
-class CWsRoxieQueryEx : public CWsRoxieQuery
-{
-private:
-    Owned<IXslProcessor> m_xsl;
-    Mutex m_superfilemutex;
-
-    StringBuffer m_attsXSLT;
-    StringBuffer m_graphStatsXSLT;
-
-public:
-    IMPLEMENT_IINTERFACE;
-    virtual ~CWsRoxieQueryEx(){};
-    virtual void init(IPropertyTree *cfg, const char *process, const char *service);
-
-    bool onRoxieQuerySearch(IEspContext &context, IEspRoxieQuerySearchRequest & req, IEspRoxieQuerySearchResponse & resp);
-    bool onRoxieQueryList(IEspContext &context, IEspRoxieQueryListRequest &req, IEspRoxieQueryListResponse &resp);
-    bool onQueryDetails(IEspContext &context, IEspRoxieQueryDetailsRequest & req, IEspRoxieQueryDetailsResponse & resp);
-    bool onGVCAjaxGraph(IEspContext &context, IEspGVCAjaxGraphRequest &req, IEspGVCAjaxGraphResponse &resp);
-    bool onShowGVCGraph(IEspContext &context, IEspShowGVCGraphRequest &req, IEspShowGVCGraphResponse &resp);
-    bool onRoxieQueryProcessGraph(IEspContext &context, IEspRoxieQueryProcessGraphRequest &req, IEspRoxieQueryProcessGraphResponse &resp);
-
-private:
-    void addToQueryString(StringBuffer &queryString, const char *name, const char *value);
-    void addToQueryStringFromInt(StringBuffer &queryString, const char *name, int value);
-    void getClusterConfig(char const * clusterType, char const * clusterName, char const * processName, StringBuffer& netAddress, int& port);
-
-    bool getAllRoxieQueries(IEspContext &context, const char* cluster, const char* suspended, const char* sortBy, bool descending, __int64 displayEnd, IArrayOf<IEspRoxieQuery>& RoxieQueryList, __int64& totalFiles);
-    IPropertyTree* getXgmmlGraph(const char *queryName, SocketEndpoint &ep, const char* graphName);
-    void enumerateGraphs(const char *queryName, SocketEndpoint &ep, StringArray& graphNames);
-};
-
-#endif //_ESPWIZ_WsRoxieQuery_HPP__
-

esp/services/ws_smc/CMakeLists.txt

@@ -96,7 +96,6 @@ target_link_libraries ( ws_smc
          ws_fs 
          ws_topology 
          schedulectrl 
-         roxiemanager 
          ws_workunits 
     )
 IF (USE_OPENLDAP)

esp/services/ws_workunits/CMakeLists.txt

@@ -49,7 +49,6 @@ include_directories (
          ./../../platform
          ./../../../dali/dfu
          ./../../../dali/sasha
-         ./../../../common/roxiemanager
          ./../../../common/remote
          ./../../../system/jlib
          ./../../../common/environment
@@ -101,7 +100,6 @@ target_link_libraries ( ws_workunits
          SMCLib
          schedulectrl
          roxiecommlib
-         roxiemanager
          hql
          jhtree
          fileview2

esp/services/ws_workunits/roxiequeryhandler.cpp

@@ -1,18 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-// no longer needed - code moved to roxiemanager

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -34,7 +34,6 @@
 #include "dllserver.hpp"
 #include "schedulectrl.hpp"
 #include "scheduleread.hpp"
-#include "roxiemanager.hpp"
 #include "dadfs.hpp"
 #include "dfuwu.hpp"
 #include "thorplugin.hpp"

esp/smc/SMCLib/LogicFileWrapper.cpp

@@ -55,49 +55,39 @@ void LogicFileWrapper::FindClusterName(const char* logicalName, StringBuffer& re
     }
 }
 
-bool LogicFileWrapper::doDeleteFile(const char* LogicalFileName,const char *cluster, bool nodelete,StringBuffer& returnStr, IUserDescriptor* udesc)
+bool LogicFileWrapper::doDeleteFile(const char* logicalName,const char *cluster, StringBuffer& returnStr, IUserDescriptor* udesc)
 {
     CDfsLogicalFileName lfn;
-    StringBuffer cname(cluster);
-    lfn.set(LogicalFileName);
-    if (!cname.length())
-        lfn.getCluster(cname);  // see if cluster part of LogicalFileName
+    lfn.set(logicalName);
+    StringBuffer cname;
+    lfn.getCluster(cname);
+    if (0 == cname.length()) // if has no cluster, use supplied cluster
+        lfn.setCluster(cluster);
+    lfn.get(cname.clear(), false, true); // get file@cluster form;
 
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(LogicalFileName, udesc, true) ;
-    if(!df)
-    {
-        returnStr.appendf("<Message><Value>File %s not found</Value></Message>", LogicalFileName);
-        return false;
-    }
-
-    bool deleted;
-    if(!nodelete && !df->querySuperFile())
+    try
     {
-        Owned<IMultiException> pExceptionHandler = MakeMultiException();
-        deleted = queryDistributedFileSystem().remove(df,cname.length()?cname.str():NULL,pExceptionHandler);
-        StringBuffer errorStr;
-        pExceptionHandler->errorMessage(errorStr);
-        if (errorStr.length() > 0)
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(cname.str(), udesc, true) ;
+        if(!df)
         {
-            returnStr.appendf("<Message><Value>%s</Value></Message>",errorStr.str());
-            DBGLOG("%s", errorStr.str());
-        }
-        else {
-            PrintLog("Deleted Logical File: %s\n",LogicalFileName);
-            returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>",LogicalFileName);
+            returnStr.appendf("<Message><Value>File %s not found</Value></Message>", cname.str());
+            return false;
         }
 
+        df->detach();
+        returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>", cname.str());
+        DBGLOG("%s", returnStr.str());
+        return true;
     }
-    else
+    catch (IException *e)
     {
-        df.clear(); 
-        deleted = queryDistributedFileDirectory().removeEntry(LogicalFileName,udesc); // this can remove clusters also
-        if (deleted)
-            returnStr.appendf("<Message><Value>Detached File %s</Value></Message>", LogicalFileName);
+        StringBuffer errorMsg;
+        e->errorMessage(returnStr);
+        e->Release();
+        PROGLOG("%s", errorMsg.str());
+        returnStr.appendf("<Message><Value>Failed to delete File %s, error: %s</Value></Message>", cname.str(), errorMsg.str());
     }
-
-
-    return deleted; 
+    return false;
 }
 
 bool LogicFileWrapper::doCompressFile(const char* name,StringBuffer& returnStr, IUserDescriptor* udesc)

esp/smc/SMCLib/LogicFileWrapper.hpp

@@ -47,7 +47,7 @@ public:
     IMPLEMENT_IINTERFACE;
     LogicFileWrapper();
     virtual ~LogicFileWrapper();
-    bool doDeleteFile(const char* name, const char *cluster, bool nodelete, StringBuffer& returnStr, IUserDescriptor* udesc = NULL);
+    bool doDeleteFile(const char* name, const char *cluster, StringBuffer& returnStr, IUserDescriptor* udesc = NULL);
     bool doCompressFile(const char* name,StringBuffer& returnStr, IUserDescriptor* udesc = 0);
     void FindClusterName(const char* logicalName,StringBuffer& returnCluster, IUserDescriptor* udesc = 0);
 

initfiles/componentfiles/configxml/@temp/esp_service_WsSMC.xsl

@@ -438,32 +438,6 @@ This is required by its binding with ESP service '<xsl:value-of select="$espServ
         </EspBinding>
     </xsl:template>
     
-    <!-- WS-ROXIEQUERY -->
-    <xsl:template match="EspService" mode="WsRoxieQuery">
-        <xsl:param name="bindingNode"/>
-        <xsl:param name="authNode"/>
-
-        <xsl:variable name="serviceType" select="'WsRoxieQuery'"/>
-        <xsl:variable name="serviceName" select="concat($serviceType, '_', @name, '_', $process)"/>
-        <xsl:variable name="bindName" select="concat($serviceType, '_', $bindingNode/@name, '_', $process)"/>
-        <xsl:variable name="bindType" select="'WsRoxieQuery'"/>
-        <xsl:variable name="servicePlugin">
-            <xsl:call-template name="defineServicePlugin">
-                <xsl:with-param name="plugin" select="'ws_roxiequery'"/>
-            </xsl:call-template>
-        </xsl:variable>
-
-
-        <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}"/>
-        <EspBinding name="{$bindName}" service="{$serviceName}" protocol="{$bindingNode/@protocol}" type="{$bindType}" plugin="{$servicePlugin}" netAddress="0.0.0.0" port="{$bindingNode/@port}">
-            <xsl:call-template name="bindAuthentication">
-                <xsl:with-param name="bindingNode" select="$bindingNode"/>
-                <xsl:with-param name="authMethod" select="$authNode/@method"/>
-                <xsl:with-param name="service" select="'ws_roxiequery'"/>
-            </xsl:call-template>
-        </EspBinding>
-    </xsl:template>
-
     <!-- ws_machine-->
     <xsl:template match="EspService" mode="ws_machine">
         <xsl:param name="bindingNode"/>

initfiles/componentfiles/configxml/RoxieTopology.xsl

@@ -151,47 +151,20 @@
                 <xsl:element name="RoxieServerProcess">
                     <xsl:attribute name="netAddress"><xsl:value-of select="/Environment/Hardware/Computer[@name=$computer]/@netAddress"/></xsl:attribute>
                     <xsl:copy-of select="@port"/>
-                <xsl:if test="string(@dataDirectory)=''">
-                    <xsl:message terminate="yes">Data directory is not specified for Roxie server '<xsl:value-of select="@computer"/>'.</xsl:message>
-                </xsl:if>
-                <xsl:variable name="dataDir">
-                            <xsl:call-template name="makeAbsolutePath">
-                                <xsl:with-param name="path" select="@dataDirectory"/>
-                            </xsl:call-template>                        
-                </xsl:variable>
-                    <xsl:attribute name="baseDataDirectory">
-                        <xsl:value-of select="$dataDir"/>
-                    </xsl:attribute>
-                    <xsl:attribute name="dataDirectory">
-                        <xsl:value-of select="concat($dataDir, $pathSep, $roxieClusterNode/@name)"/>
-                    </xsl:attribute>
-                    <xsl:copy-of select="@*[name()!='netAddress' and name()!='dataDirectory' and name()!='computer' and name()!='name' and name()!='port']"/>
+                    <xsl:copy-of select="@*[name()!='netAddress' and name()!='computer' and name()!='name' and name()!='port']"/>
                 </xsl:element>
             </xsl:for-each>
             <xsl:for-each select="RoxieSlaveProcess">
-                <xsl:sort select="@dataDirectory"/>
+                <xsl:sort select="@level"/>
                 <xsl:sort select="@channel" data-type="number"/>
                 <xsl:element name="RoxieSlaveProcess">
                     <xsl:variable name="computer" select="@computer"/>
                     <xsl:attribute name="netAddress"><xsl:value-of select="/Environment/Hardware/Computer[@name=$computer]/@netAddress"/></xsl:attribute>
                     <xsl:attribute name="channel"><xsl:value-of select="@channel"/></xsl:attribute>
-                <xsl:if test="string(@dataDirectory)=''">
-                    <xsl:message terminate="yes">
-                        <xsl:text>Data directory is not specified for Roxie slave '</xsl:text>
-                        <xsl:value-of select="@computer"/>
-                        <xsl:text>' for channel </xsl:text>
-                        <xsl:value-of select="@channel"/>.</xsl:message>
-                </xsl:if>                           
-
-                                        <xsl:variable name="dataDir">
-                            <xsl:call-template name="makeAbsolutePath">
-                            <xsl:with-param name="path" select="@dataDirectory"/>
-                        </xsl:call-template>                        
-                                        </xsl:variable>
-                    <xsl:attribute name="dataDirectory">
-                        <xsl:value-of select="concat($dataDir, $pathSep, $roxieClusterNode/@name)"/>
-                    </xsl:attribute>
-                                  
+                    <xsl:if test="string(@level)=''">
+                        <xsl:attribute name="level">0</xsl:attribute>
+                    </xsl:if>
+                    <xsl:copy-of select="@*[name()!='netAddress' and name()!='computer' and name()!='name' and name()!='channel']"/>
                 </xsl:element>
             </xsl:for-each>
             <xsl:for-each select="RoxieMonitorProcess">

initfiles/componentfiles/configxml/buildsetCC.xml.in

@@ -169,10 +169,6 @@
                           path="WsEclAccess"
                           resource="WsEclAccess"
                           service="ws_ecl"/>
-     <AuthenticateFeature description="Access to Roxie queries and files"
-                          path="RoxieQueryAccess"
-                          resource="RoxieQueryAccess"
-                          service="ws_roxiequery"/>
      <AuthenticateFeature description="Access to cluster topology"
                           path="ClusterTopologyAccess"
                           resource="ClusterTopologyAccess"

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -49,7 +49,6 @@
                 </xs:appinfo>
               </xs:annotation>
             </xs:attribute>
-            <xs:attribute name="dataDirectory" type="absolutePath" use="required"/>
             <xs:attribute name="numThreads" type="xs:nonNegativeInteger" use="optional" default="30">
               <xs:annotation>
                 <xs:appinfo>
@@ -105,7 +104,13 @@
                 </xs:appinfo>
               </xs:annotation>
             </xs:attribute>
-            <xs:attribute name="dataDirectory" type="absolutePath" use="required"/>
+            <xs:attribute name="level" type="xs:nonNegativeInteger" use="optional" default="0">
+              <xs:annotation>
+                <xs:appinfo>
+                  <tooltip>Replication level of this slave</tooltip>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
           </xs:complexType>
         </xs:element>
         <xs:element name="ACL" maxOccurs="unbounded">
@@ -449,14 +454,14 @@
     <xs:attribute name="checkFileDate" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Compare file dates of physical files with the information stored in state file when RoxieConfig was executed.</tooltip>
+          <tooltip>Compare file dates of physical files with the information in DFS.</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="copyResources" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Copies any missing data files/keys from the position they were in when RoxieConfig was executed.</tooltip>
+          <tooltip>Copies any missing data files/keys from the position they were in when query was deployed.</tooltip>
           <autogenforwizard>1</autogenforwizard>
           <autogendefaultvalue>false</autogendefaultvalue>
           <autogendefaultformultinode>true</autogendefaultformultinode>
@@ -480,49 +485,49 @@
     <xs:attribute name="defaultHighPriorityTimeLimit" type="xs:nonNegativeInteger" use="optional" default="0">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Maximum run time (in ms) for any single active high-priority query (if not overriden in roxieconfig)</tooltip>
+          <tooltip>Maximum run time (in ms) for any single active high-priority query (if not overriden)</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="defaultHighPriorityTimeWarning" type="xs:nonNegativeInteger" use="optional" default="5000">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Time (in ms) before generating SNMP warning for a high-priority query (if not overriden in roxieconfig)</tooltip>
+          <tooltip>Time (in ms) before generating SNMP warning for a high-priority query (if not overriden)</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="defaultLowPriorityTimeLimit" type="xs:nonNegativeInteger" use="optional" default="0">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Maximum run time (in ms) for any single active low-priority query (if not overriden in roxieconfig)</tooltip>
+          <tooltip>Maximum run time (in ms) for any single active low-priority query (if not overriden)</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="defaultLowPriorityTimeWarning" type="xs:nonNegativeInteger" use="optional" default="0">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Time (in ms) before generating SNMP warning for a low-priority query (if not overriden in roxieconfig)</tooltip>
+          <tooltip>Time (in ms) before generating SNMP warning for a low-priority query (if not overriden)</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="defaultMemoryLimit" type="xs:nonNegativeInteger" use="optional" default="0">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Maximum amount of memory available for row data in any single active query (if not overriden in roxieconfig)</tooltip>
+          <tooltip>Maximum amount of memory available for row data in any single active query (if not overriden)</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="defaultSLAPriorityTimeLimit" type="xs:nonNegativeInteger" use="optional" default="0">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Maximum run time (in ms) for any single active SLA-high-priority query (if not overriden in roxieconfig)</tooltip>
+          <tooltip>Maximum run time (in ms) for any single active SLA-high-priority query (if not overriden)</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="defaultSLAPriorityTimeWarning" type="xs:nonNegativeInteger" use="optional" default="5000">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Time (in ms) before generating SNMP warning for a SLA-high-priority query (if not overriden in roxieconfig)</tooltip>
+          <tooltip>Time (in ms) before generating SNMP warning for a SLA-high-priority query (if not overriden)</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
@@ -757,7 +762,7 @@
     <xs:attribute name="useRemoteResources" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Loads any missing data files/keys from the position they were in when RoxieConfig was executed.</tooltip>
+          <tooltip>Reads any missing data files/keys from the position they were in when deployed.</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -363,6 +363,7 @@
    <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data/[COMPONENT]" name="data"/>
    <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data2/[COMPONENT]" name="data2"/>
    <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data3/[COMPONENT]" name="data3"/>
+   <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data4/[COMPONENT]" name="data4"/>
    <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-mirror/[COMPONENT]" name="mirror"/>
    <Category dir="${EXEC_PREFIX}/lib/[NAME]/queries/[INST]" name="query"/>
    <Category dir="${EXEC_PREFIX}/lock/[NAME]/[INST]" name="lock"/>
@@ -894,7 +895,7 @@
                 useRemoteResources="true"
                 useTreeCopy="false">
    <RoxieFarmProcess 
-                     dataDirectory="${RUNTIME_PATH}/hpcc-data/roxie"
+                     level="0"
                      listenQueue="200"
                      name="farm1"
                      numThreads="30"
@@ -903,7 +904,7 @@
     <RoxieServerProcess computer="localhost" name="farm1_s1"/>
    </RoxieFarmProcess>
    <RoxieFarmProcess
-                      dataDirectory="${RUNTIME_PATH}/hpcc-data/roxie"
+                      level="0"
                       listenQueue="200"
                       name="farm2"
                       numThreads="30"
@@ -913,7 +914,7 @@
     </RoxieFarmProcess>
     <RoxieServerProcess
                        computer="localhost"
-                       dataDirectory="${RUNTIME_PATH}/hpcc-data/roxie"
+                       level="0"
                        listenQueue="200"
                        name="farm1_s1"
                        netAddress="."
@@ -922,7 +923,7 @@
                        requestArrayThreads="5"/>
    <RoxieServerProcess 
                        computer="localhost"
-                       dataDirectory="${RUNTIME_PATH}/hpcc-data/roxie"
+                       level="0"
                        listenQueue="200"
                        name="farm2_s1"
                        netAddress="."
@@ -930,11 +931,11 @@
                        port="0"
                        requestArrayThreads="5"/>
    <RoxieSlave computer="localhost" name="s1">
-    <RoxieChannel dataDirectory="${RUNTIME_PATH}/hpcc-data/roxie" number="1"/>
+    <RoxieChannel level="0" number="1"/>
    </RoxieSlave>
    <RoxieSlaveProcess channel="1"
                       computer="localhost"
-                      dataDirectory="${RUNTIME_PATH}/hpcc-data/roxie"
+                      level="0"
                       name="s1"
                       netAddress="."/>
   </RoxieCluster>

roxie/ccd/ccd.hpp

@@ -64,6 +64,7 @@ extern unsigned channelCount;                   // number of channels this node
 extern unsigned subChannels[MAX_CLUSTER_SIZE];  // maps channel numbers to subChannels for this node
 extern bool suspendedChannels[MAX_CLUSTER_SIZE];// indicates suspended channels for this node
 extern unsigned numSlaves[MAX_CLUSTER_SIZE];    // number of slaves listening on this channel
+extern unsigned replicationLevel[MAX_CLUSTER_SIZE];  // Which copy of the data this channel uses on this slave
 
 extern unsigned myNodeIndex;
 #define OUTOFBAND_SEQUENCE    0x8000        // indicates an out-of-band reply
@@ -427,7 +428,6 @@ extern StringBuffer logDirectory;
 extern StringBuffer pluginDirectory;
 extern StringBuffer pluginsList;
 extern StringBuffer queryDirectory;
-extern StringBuffer baseDataDirectory;
 extern StringBuffer codeDirectory;
 extern StringBuffer tempDirectory;
 

roxie/ccd/ccdactivities.cpp

@@ -5349,15 +5349,17 @@ public:
         try  // operations does not want any missing file errors to be fatal, or throw traps - just log it
         {
             bool isOpt = _graphNode.getPropBool("att[@name='_isOpt']/@value") || pretendAllOpt;
-            if (queryNodeIndexName(_graphNode))
+            const char *fileName = queryNodeFileName(_graphNode);
+            const char *indexName = queryNodeIndexName(_graphNode);
+            if (indexName && (!fileName || !streq(indexName, fileName)))
             {
-                indexfile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeIndexName(_graphNode), isOpt, true, _queryFactory.queryWorkUnit()));
+                indexfile.setown(_queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, _queryFactory.queryWorkUnit()));
                 if (indexfile)
                     keyArray.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, queryFactory.queryChannel(), queryFactory.getEnableFieldTranslation()));
             }
-            if (queryNodeFileName(_graphNode))
+            if (fileName)
             {
-                datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isOpt, true, _queryFactory.queryWorkUnit()));
+                datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, _queryFactory.queryWorkUnit()));
                 if (datafile)
                     fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
             }

roxie/ccd/ccddali.cpp

@@ -288,11 +288,7 @@ private:
         dstfdesc->setPartMask(dstpartmask.str());
         unsigned np = srcfdesc->numParts();
         dstfdesc->setNumParts(srcfdesc->numParts());
-        DFD_OS os = (getPathSepChar(srcfdesc->queryDefaultDir())=='\\')?DFD_OSwindows:DFD_OSunix;
-        StringBuffer dir;
-        StringBuffer dstdir;
-        makePhysicalPartName(dstlfn.get(),0,0,dstdir,false,os,NULL);
-        dstfdesc->setDefaultDir(dstdir.str());
+        dstfdesc->setDefaultDir(srcfdesc->queryProperties().queryProp("@cloneFromDir"));
 
         for (unsigned pn=0;pn<srcfdesc->numParts();pn++) {
             offset_t sz = srcfdesc->queryPart(pn)->queryProperties().getPropInt64("@size",-1);
@@ -309,18 +305,39 @@ private:
         {
             IPropertyTree &elem = groups->query();
             const char *groupName = elem.queryProp("@groupName");
+            StringBuffer dir;
             StringBuffer foreignGroup("foreign::");
             foreignGroup.append(cloneFrom).append("::").append(groupName);
-            Owned<IGroup> group = queryNamedGroupStore().lookup(foreignGroup);  // NOTE - this is cached by the named group store
+            GroupType groupType;
+            Owned<IGroup> group = queryNamedGroupStore().lookup(foreignGroup, dir, groupType);
             ClusterPartDiskMapSpec dmSpec;
             dmSpec.fromProp(&elem);
+            if (!dmSpec.defaultBaseDir.length())
+            {
+                if (dir.length())
+                {
+                    dmSpec.setDefaultBaseDir(dir);
+                }
+                else
+                {
+                    // Due to the really weird code in dadfs, this MUST be set to match the leading portion of cloneFromDir
+                    // in order to properly handle remote systems with different default directory locations
+                    StringBuffer tail;
+                    DFD_OS os = (getPathSepChar(srcfdesc->queryDefaultDir())=='\\')?DFD_OSwindows:DFD_OSunix;
+                    makePhysicalPartName(dstlfn.get(),0,0,tail,0,os,PATHSEPSTR);  // if lfn is a::b::c, tail will be /a/b/
+                    assertex(tail.length() > 1);
+                    tail.setLength(tail.length()-1);   // strip off the trailing /
+                    StringBuffer head(srcfdesc->queryProperties().queryProp("@cloneFromDir")); // Will end with /a/b
+                    assertex(streq(head.str() + head.length() - tail.length(), tail.str()));
+                    head.setLength(head.length() - tail.length()); // So strip off the end...
+                    dmSpec.setDefaultBaseDir(head.str());
+                }
+            }
             dstfdesc->addCluster(groupName, group, dmSpec);
         }
-
         return dstfdesc.getClear();
     }
 
-
 public:
 
     IMPLEMENT_IINTERFACE;

roxie/ccd/ccdfile.cpp

@@ -619,6 +619,8 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                     {
                         const char *remoteName = remoteLocationInfo.item(idx);
                         Owned<IFile> remote = createIFile(remoteName);
+                        if (traceLevel > 5)
+                            DBGLOG("checking remote location %s", remoteName);
                         if (fileUpToDate(remote, size, modified, crc, isCompressed)==FileIsValid)
                         {
                             if (miscDebugTraceLevel > 10)
@@ -1042,11 +1044,12 @@ public:
     }
 
     virtual ILazyFileIO *lookupFile(const char *lfn, RoxieFileType fileType,
-                                     IPartDescriptor *pdesc, unsigned numParts,
+                                     IPartDescriptor *pdesc, unsigned numParts, unsigned replicationLevel,
                                      const StringArray &deployedLocationInfo, bool startFileCopy)
     {
         IPropertyTree &partProps = pdesc->queryProperties();
         offset_t dfsSize = partProps.getPropInt64("@size", -1);
+        bool local = partProps.getPropBool("@local");
         unsigned crc;
         if (!pdesc->getCrc(crc))
             crc = 0;
@@ -1060,15 +1063,20 @@ public:
         unsigned partNo = pdesc->queryPartIndex() + 1;
         StringBuffer localLocation;
 
-        // MORE - not at all sure about this. Foreign files should stay foreign ?
-        CDfsLogicalFileName dlfn;
-        dlfn.set(lfn);
-        if (dlfn.isForeign())
-            dlfn.clearForeign();
-        const char *logicalname = dlfn.get();
-
-        makePhysicalPartName(logicalname, partNo, numParts, localLocation, false, DFD_OSdefault, baseDataDirectory);  // MORE - if we get the dataDirectory we can pass it in and possibly reuse an existing file
-
+        if (local)
+        {
+            assertex(partNo==1 && numParts==1);
+            localLocation.append(lfn);  // any resolution done earlier
+        }
+        else
+        {
+            // MORE - not at all sure about this. Foreign files should stay foreign ?
+            CDfsLogicalFileName dlfn;
+            dlfn.set(lfn);
+            if (dlfn.isForeign())
+                dlfn.clearForeign();
+            makePhysicalPartName(dlfn.get(), partNo, numParts, localLocation, replicationLevel, DFD_OSdefault);
+        }
         Owned<ILazyFileIO> ret;
         try
         {
@@ -1278,13 +1286,13 @@ public:
     }
 };
 
-ILazyFileIO *createDynamicFile(const char *id, IPartDescriptor *pdesc, IPartDescriptor *remotePDesc, RoxieFileType fileType, int numParts, bool startCopy)
+ILazyFileIO *createPhysicalFile(const char *id, IPartDescriptor *pdesc, IPartDescriptor *remotePDesc, RoxieFileType fileType, int numParts, bool startCopy, unsigned channel)
 {
     StringArray remoteLocations;
     if (remotePDesc)
         appendRemoteLocations(remotePDesc, remoteLocations, false);
 
-    return queryFileCache().lookupFile(id, fileType, pdesc, numParts, remoteLocations, startCopy);
+    return queryFileCache().lookupFile(id, fileType, pdesc, numParts, replicationLevel[channel], remoteLocations, startCopy);
 }
 
 //====================================================================================================
@@ -1611,7 +1619,8 @@ protected:
     mutable CriticalSection lock;
     mutable Owned<IFilePartMap> fileMap;
     mutable PerChannelCacheOf<IInMemoryIndexManager> indexMap;
-//  MORE - cache the others, using per-channel cache support. 
+    mutable PerChannelCacheOf<IFileIOArray> ioArrayMap;
+    mutable PerChannelCacheOf<IKeyArray> keyArrayMap;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -1775,7 +1784,18 @@ public:
         else
             mb.append(false);
     }
-    virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const 
+    virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const
+    {
+        CriticalBlock b(lock);
+        IFileIOArray *ret = ioArrayMap.get(channel);
+        if (!ret)
+        {
+            ret = createIFileIOArray(isOpt, channel);
+            ioArrayMap.set(ret, channel);
+        }
+        return LINK(ret);
+    }
+    IFileIOArray *createIFileIOArray(bool isOpt, unsigned channel) const
     {
         Owned<CFileIOArray> f = new CFileIOArray();
         f->addFile(NULL, 0);
@@ -1795,7 +1815,7 @@ public:
                             IPartDescriptor *pdesc = fdesc->queryPart(i-1);
                             assertex(pdesc);
                             IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, i-1);
-                            Owned<ILazyFileIO> file = createDynamicFile(subNames.item(0), pdesc, remotePDesc, ROXIE_FILE, numParts, cached != NULL);
+                            Owned<ILazyFileIO> file = createPhysicalFile(subNames.item(0), pdesc, remotePDesc, ROXIE_FILE, numParts, cached != NULL, channel);
                             IPropertyTree &partProps = pdesc->queryProperties();
                             f->addFile(file.getClear(), partProps.getPropInt64("@offset"));
                         }
@@ -1820,7 +1840,6 @@ public:
     }
     virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, bool allowFieldTranslation) const
     {
-        Owned<IKeyArray> ret = ::createKeyArray();
         unsigned maxParts = 0;
         ForEachItemIn(subFile, subFiles)
         {
@@ -1851,7 +1870,18 @@ public:
             else
                 translators->append(NULL);
         }
-
+        CriticalBlock b(lock);
+        IKeyArray *ret = keyArrayMap.get(channel);
+        if (!ret)
+        {
+            ret = createKeyArray(isOpt, channel, maxParts);
+            keyArrayMap.set(ret, channel);
+        }
+        return LINK(ret);
+    }
+    IKeyArray *createKeyArray(bool isOpt, unsigned channel, unsigned maxParts) const
+    {
+        Owned<IKeyArray> ret = ::createKeyArray();
         if (channel)
         {
             ret->addKey(NULL);
@@ -1873,7 +1903,7 @@ public:
                             if (pdesc)
                             {
                                 IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, partNo-1);
-                                part.setown(createDynamicFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, fdesc->numParts(), cached != NULL));
+                                part.setown(createPhysicalFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, fdesc->numParts(), cached != NULL, channel));
                                 pdesc->getCrc(crc);
                             }
                         }
@@ -1912,7 +1942,7 @@ public:
                     assertex(numParts > 0);
                     IPartDescriptor *pdesc = fdesc->queryPart(numParts - 1);
                     IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, numParts - 1);
-                    Owned<ILazyFileIO> keyFile = createDynamicFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, numParts, cached != NULL);
+                    Owned<ILazyFileIO> keyFile = createPhysicalFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, numParts, cached != NULL, channel);
                     unsigned crc = 0;
                     pdesc->getCrc(crc);
                     StringBuffer pname;
@@ -1941,17 +1971,16 @@ public:
 
     virtual IInMemoryIndexManager *getIndexManager(bool isOpt, unsigned channel, IFileIOArray *files, IRecordSize *recs, bool preload, int numKeys) const 
     {
-        // MORE - if we want to share this then we need to pass in channel too and cache per channel. Should combine the get() and the load().
-        // MORE - I don't know that it makes sense to pass isOpt in to these calls rather than just when creating the IResolvedFile... think about it.
+        // MORE - I don't know that it makes sense to pass isOpt in to these calls
+        // Failures to resolve will not be cached, only successes.
         // MORE - preload and numkeys are all messed up - can't be specified per query have to be per file
-//      return createInMemoryIndexManager(isOpt, lfn);
 
         CriticalBlock b(lock);
         IInMemoryIndexManager *ret = indexMap.get(channel);
         if (!ret)
         {
             ret = createInMemoryIndexManager(isOpt, lfn);
-            ret->load(files, recs, preload, numKeys);
+            ret->load(files, recs, preload, numKeys);   // note - files (passed in) are channel specific
             indexMap.set(ret, channel);
         }
         return LINK(ret);
@@ -1997,6 +2026,7 @@ public:
         Owned<IFileDescriptor> fdesc = createFileDescriptor();
         Owned<IPropertyTree> pp = createPTree("Part");
         pp->setPropInt64("@size",size);
+        pp->setPropBool("@local", true);
         fdesc->setPart(0, queryMyNode(), localFileName, pp);
         addSubFile(fdesc.getClear(), NULL);
     }

roxie/ccd/ccdfile.hpp

@@ -54,11 +54,10 @@ interface ILazyFileIO : extends IFileIO
     virtual void removeCache(const IRoxieFileCache *) = 0;
 };
 
-extern ILazyFileIO *createDynamicFile(const char *id, IPartDescriptor *pdesc, RoxieFileType fileType, int numParts, SocketEndpoint &cloneFrom);
-
 interface IRoxieFileCache : extends IInterface
 {
-    virtual ILazyFileIO *lookupFile(const char *lfn, RoxieFileType fileType, IPartDescriptor *pdesc, unsigned numParts, const StringArray &deployedLocationInfo, bool startFileCopy) = 0;
+    virtual ILazyFileIO *lookupFile(const char *lfn, RoxieFileType fileType, IPartDescriptor *pdesc, unsigned numParts,
+                                      unsigned replicationLevel, const StringArray &deployedLocationInfo, bool startFileCopy) = 0;
     virtual RoxieFileStatus fileUpToDate(IFile *f, offset_t size, const CDateTime &modified, unsigned crc, bool isCompressed) = 0;
     virtual int numFilesToCopy() = 0;
     virtual void closeExpired(bool remote) = 0;

roxie/ccd/ccdmain.cpp

@@ -148,7 +148,6 @@ StringBuffer logDirectory;
 StringBuffer pluginDirectory;
 StringBuffer queryDirectory;
 StringBuffer codeDirectory;
-StringBuffer baseDataDirectory;
 StringBuffer tempDirectory;
 
 ClientCertificate clientCert;
@@ -283,7 +282,7 @@ void getAccessList(const char *aclName, IPropertyTree *topology, IPropertyTree *
     serverInfo->removeProp(xpath);
 }
 
-void addServerChannel(const char *dataDirectory, unsigned port, unsigned threads, const char *access, IPropertyTree *topology)
+void addServerChannel(unsigned port, unsigned threads, const char *access, IPropertyTree *topology)
 {
     if (!ownEP.port)
         ownEP.set(port, queryHostIP());
@@ -291,13 +290,10 @@ void addServerChannel(const char *dataDirectory, unsigned port, unsigned threads
     ForEach(*servers)
     {
         IPropertyTree &f = servers->query();
-        if (strcmp(f.queryProp("@dataDirectory"), dataDirectory) != 0)
-            throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - Roxie server dataDirectory respecified");
         if (f.getPropInt("@port", 0) == port)
             throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - Roxie server port repeated");
     }
     IPropertyTree *ci = createPTree("RoxieServerProcess");
-    ci->setProp("@dataDirectory", dataDirectory);
     ci->setPropInt("@port", port);
     ci->setPropInt("@numThreads", threads);
     if (access && *access)
@@ -312,7 +308,7 @@ bool ipMatch(IpAddress &ip)
     return ip.isLocal();
 }
 
-void addSlaveChannel(unsigned channel, const char *dataDirectory, bool suspended)
+void addSlaveChannel(unsigned channel, unsigned level, bool suspended)
 {
     StringBuffer xpath;
     xpath.appendf("RoxieSlaveProcess[@channel=\"%d\"]", channel);
@@ -321,18 +317,19 @@ void addSlaveChannel(unsigned channel, const char *dataDirectory, bool suspended
     IPropertyTree *ci = createPTree("RoxieSlaveProcess");
     ci->setPropInt("@channel", channel);
     ci->setPropBool("@suspended", suspended);
-    ci->setPropInt("@subChannel", numSlaves[channel]);
+    ci->setPropInt("@subChannel", numSlaves[channel]);   // Alternatively could probably use replication level as subchannel ?
     suspendedChannels[channel] = suspended;
-    ci->setProp("@dataDirectory", dataDirectory);
+    assertex(!replicationLevel[channel]);  // implies channel repeated, caught above
+    replicationLevel[channel] = level;
     ccdChannels->addPropTree("RoxieSlaveProcess", ci);
 }
 
-void addChannel(unsigned channel, const char *dataDirectory, bool isMe, bool suspended, IpAddress& slaveIp)
+void addChannel(unsigned channel, unsigned level, bool isMe, bool suspended, IpAddress& slaveIp)
 {
     numSlaves[channel]++;
     if (isMe && channel > 0 && channel <= numChannels)
     {
-        addSlaveChannel(channel, dataDirectory, suspended);
+        addSlaveChannel(channel, level, suspended);
     }
     if (!localSlave)
     {
@@ -520,8 +517,8 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             }
             topology=createPTreeFromXMLString(
                 "<RoxieTopology numChannels='1' localSlave='1'>"
-                 "<RoxieServerProcess dataDirectory='.' netAddress='.'/>"
-                 "<RoxieSlaveProcess dataDirectory='.' netAddress='.' channel='1'/>"
+                 "<RoxieServerProcess netAddress='.'/>"
+                 "<RoxieSlaveProcess netAddress='.' channel='1' level='0'/>"
                 "</RoxieTopology>"
                 );
             int port = globals->getPropInt("--port", 9876);
@@ -557,9 +554,27 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         soapTraceLevel = topology->getPropInt("@soapTraceLevel", runOnce ? 0 : 1);
         miscDebugTraceLevel = topology->getPropInt("@miscDebugTraceLevel", 0);
 
-        IPropertyTree *directoryTree = topology->queryPropTree("Directories");
+        Linked<IPropertyTree> directoryTree = topology->queryPropTree("Directories");
+        if (!directoryTree)
+        {
+            Owned<IPropertyTree> envFile = getHPCCEnvironment();
+            if (envFile)
+                directoryTree.set(envFile->queryPropTree("Software/Directories"));
+        }
         if (directoryTree)
-            getConfigurationDirectory(directoryTree,"query","roxie", roxieName, queryDirectory);
+        {
+            getConfigurationDirectory(directoryTree, "query", "roxie", roxieName, queryDirectory);
+            for (unsigned replicationLevel = 0; replicationLevel < MAX_REPLICATION_LEVELS; replicationLevel++)
+            {
+                StringBuffer dataDir;
+                StringBuffer dirId("data");
+                if (replicationLevel)
+                    dirId.append(replicationLevel+1);
+                if (getConfigurationDirectory(directoryTree, dirId, "roxie", roxieName, dataDir))
+                    setBaseDirectory(dataDir, replicationLevel, DFD_OSdefault);
+            }
+        }
+        directoryTree.clear();
 
         //Logging stuff
         if (globals->getPropBool("--stdlog", traceLevel != 0) || topology->getPropBool("@forceStdLog", false))
@@ -821,14 +836,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 queryDirectory.append(codeDirectory).append("queries");
         }
         addNonEmptyPathSepChar(queryDirectory);
-
-        // if no Dali, files are local
-        if (fileNameServiceDali.length() == 0)
-            baseDataDirectory.append("./"); // Path separator will be replaced, if necessary
-        else
-            baseDataDirectory.append(topology->queryProp("@baseDataDir"));
         queryFileCache().start();
-
         getTempFilePath(tempDirectory, "roxie", topology);
 
 #ifdef _WIN32
@@ -854,11 +862,9 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 unsigned roxieServerHost = roxieServer.getPropInt("@multihost", 0);
                 if (ipMatch(ep) && ((roxieServerHost == myHostNumber) || (myHostNumber==-1)))
                 {
-                    if (baseDataDirectory.length() == 0)  // if not set by other topology settings default to this ...
-                        baseDataDirectory.append(roxieServer.queryProp("@baseDataDirectory"));
                     unsigned numThreads = roxieServer.getPropInt("@numThreads", numServerThreads);
                     const char *aclName = roxieServer.queryProp("@aclName");
-                    addServerChannel(roxieServers->query().queryProp("@dataDirectory"), port, numThreads, aclName, topology);
+                    addServerChannel(port, numThreads, aclName, topology);
                     if (!myIPadded || (myHostNumber==-1))
                     {
                         myNodeIndex = nodeIndex;
@@ -910,7 +916,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             }
         }
         Owned<IPropertyTreeIterator> slaves = topology->getElements("./RoxieSlaveProcess");
-        unsigned slaveCount = 0;
         IpAddress *primaries = new IpAddress[numChannels+1];    // check each channel has a different primary, if possible. Leaks on fatal errors, but who cares
         ForEach(*slaves)
         {
@@ -925,17 +930,14 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 unsigned channel = slave.getPropInt("@channel", 0);
                 if (!channel)
                     channel = slave.getPropInt("@channels", 0); // legacy support
-                const char *dataDirectory = slave.queryProp("@dataDirectory");
+                unsigned replicationLevel = slave.getPropInt("@level", 0);
                 if (channel && channel <= numChannels)
                 {
                     if (isMe)
                         isCCD = true;
-                    if (!numSlaves[channel])
-                    {
+                    if (!replicationLevel)
                         primaries[channel] = slaveIp;
-                        slaveCount++;
-                    }
-                    addChannel(channel, dataDirectory, isMe, suspended, slaveIp);
+                    addChannel(channel, replicationLevel, isMe, suspended, slaveIp);
                     if (isMe)
                         joinMulticastChannel(channel);
                 }
@@ -950,7 +952,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
 
         for (unsigned n = 1; n < numActiveChannels; n++)
         {
-            if (primaries[n].isNull())
+            if (!numSlaves[n])
                 throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - no slaves for channel %d", n);
             if (checkPrimaries)
             {

roxie/ccd/ccdquery.cpp

@@ -1095,7 +1095,7 @@ public:
                 f->getXrefInfo(*xref, logctx);
             }
         }
-        toXML(xref, reply);
+        toXML(xref, reply, 1);
     }
     virtual void resetQueryTimings()
     {

roxie/ccd/ccdqueue.cpp

@@ -40,6 +40,7 @@ unsigned channels[MAX_CLUSTER_SIZE];
 unsigned channelCount;
 unsigned subChannels[MAX_CLUSTER_SIZE];
 unsigned numSlaves[MAX_CLUSTER_SIZE];
+unsigned replicationLevel[MAX_CLUSTER_SIZE];
 unsigned IBYTIDelays[MAX_CLUSTER_SIZE]; // MORE: this will cover only 2 slaves per channel, change to cover all. 
 
 SpinLock suspendCrit;

roxie/ccd/ccdserver.cpp

@@ -23040,15 +23040,17 @@ public:
                 return;  // ignore 'spills'
             bool isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
             bool isOpt = _graphNode.getPropBool("att[@name='_isOpt']/@value") || pretendAllOpt;
-            if (queryNodeIndexName(_graphNode))
+            const char *fileName = queryNodeFileName(_graphNode);
+            const char *indexName = queryNodeIndexName(_graphNode);
+            if (indexName && (!fileName || !streq(indexName, fileName)))
             {
-                indexfile.setown(queryFactory.queryPackage().lookupFileName(queryNodeIndexName(_graphNode), isOpt, true, queryFactory.queryWorkUnit()));
+                indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, queryFactory.queryWorkUnit()));
                 if (indexfile)
                     keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, false));
             }
-            if (queryNodeFileName(_graphNode))
+            if (fileName)
             {
-                datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isOpt, true, queryFactory.queryWorkUnit()));
+                datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, queryFactory.queryWorkUnit()));
                 if (datafile)
                 {
                     if (isLocal)

roxie/ccd/ccdstate.cpp

@@ -281,8 +281,7 @@ protected:
             if (strstr(fileName,"::"))
             {
                 bool wasDFS;
-                // MORE - really we don't want the 1 of 1 bit of this...
-                makeSinglePhysicalPartName(fileName, useName, true, wasDFS, baseDataDirectory.str());
+                makeSinglePhysicalPartName(fileName, useName, true, wasDFS);
             }
             else
                 useName.append(fileName);
@@ -376,8 +375,8 @@ public:
 
     CRoxiePackageNode(IPropertyTree *p) : CPackageNode(p)
     {
-        daliHelper.setown(connectToDali()); // MORE - should make this conditional
-        if (!daliHelper.get() || !daliHelper->connected())
+        daliHelper.setown(connectToDali()); // MORE - should make this conditional?
+        if (!fileNameServiceDali.length())
             node->setPropBool("@localFiles", true);
     }
 

roxie/roxiemem/DOCUMENTATION.rst

@@ -0,0 +1,137 @@
+========================
+The Roxie Memory Manager
+========================
+
+************
+Introduction
+************
+
+This memory manager started life as the memory manager which was only used for the Roxie engine.  It had several
+original design goals:
+
+* Support link counted rows.
+* Be as fast as possible on allocate and deallocate of small rows.
+* Allow rows serialized from slaves to be used directly without being cloned first.
+* Allow the memory used by a single query, or by all queries combined, to be limited, with graceful recovery.
+* Isolate roxie queries from one another, so that one query can't bring
+  down all the rest by allocating too much memory.
+* Allow all the memory used by a query to be guaranteed to get freed when the query finishes, thus reducing the
+  possibility of memory leaks.
+* Predictable behaviour with no pathogenic cases.
+
+(Note that efficient usage of memory does not appear on that list - the expectation when the memory
+manager was first designed was that Roxie queries would use minimal amounts of memory and speed was
+more important.  Some subsequent changes e.g., Packed heaps help mitigate that.)
+
+The basic design is to reserve (but not commit) a single large block of memory in the virtual address space.  This
+memory is subdivided into "pages".  (These are not the same as the os virtual memory pages.  The memory manager pages
+are currently defined as 1Mb in size.)
+
+Memory is allocated from a set of "heaps.  Each heap owns a set of pages, and sub allocates memory of a
+single size from those pages.  All allocations from a particular page belong to the same heap.  Rounding the requested
+memory size up to the next heap-size means that memory
+is not lost due to fragmentation.
+
+Information about each heap is stored in the base of the page (using a class with virtual functions) and the
+address of an allocated row is masked to determine which heap object it belongs to, and how it should be linked/released
+etc.  Any pointer not in the allocated virtual address (e.g., constant data) can be linked/released with no effect.
+
+Each allocation has a link count and an allocator id associated with it.  The allocator id represents the type of
+the row, and is used to determine what destructor needs to be called when the row is destroyed.  (The row also
+contains a flag to indicate if it is fully constructed so it is valid for the destructor to be called.)
+
+An implementation of IRowManager processes all allocations for a particular roxie query.  This provides the
+mechanism for limiting how much memory a query uses.
+
+For fixed size allocations it is possible to get a more efficient interface for allocating rows.  There are options
+to create unique fixed size heaps (to reduce thread contention) and packed heaps - where all rows share the same
+allocator id.
+
+(Note to self: Is there ever any advantage having a heap that is unique but not packed??)
+
+****************
+Dynamic Spilling
+****************
+
+Thor has different requirements to roxie.  In roxie, if a query exceeds its memory requirements then it is terminated.  Thor
+needs to be able to spill rows and other memory to disk and continue.  This is achieved by allowing any process that
+stores buffered rows to register a callback with the memory manager.  When more memory is required these are called
+to free up memory, and allow the job to continues.
+
+Each callback can specify a priority - lower priority callbacks are called first since they are assumed to have a
+lower cost associated with spilling.  When more memory is required the callbacks are called in priority order until
+one of them succeeds.  The can also be passed a flag to indicate it is critical to force them to free up as much memory
+as possible.
+
+
+Complications
+=============
+
+There are several different complications involved with the memory spilling:
+
+* There will be many different threads allocating rows.
+* Callbacks could be triggered at any time.
+* There is a large scope for deadlock between the callbacks and allocations.
+* It may be better to not resize a large array if rows had to be evicted to resize it.
+* Filtered record streams can cause significant wasted space in the memory blocks.
+* Resizing a multi-page allocation is non trivial.
+
+
+Resizing Large memory blocks
+============================
+Some of the memory allocations cover more than one "page" - e.g., arrays used to store blocks of rows.  (These
+are called huge pages internally, not to be confused with operating system support for huge pages...)  When
+one of these memory blocks needs to be expanded you need to be careful:
+
+* Allocating a new page, copying, updating the pointer (within a cs) and then freeing is safe.  Unfortunately
+  it may involve copying a large chunk of memory.  It may also fail if there isn't memory for the new and old
+  block, even if the existing block could have been expanded into an adjacent block.
+
+* You can't lock, call a resize routine and update the pointer because the resize routine may need to allocate
+  a new memory block- that may trigger a callback, which could in turn deadlock trying to gain the lock.
+  (The callback may be from another thread...)
+
+* Therefore the memory manager contains a call which allows you to resize a block, but with a callback
+  which is used to atomically update the pointer so it always remains thead safe.
+
+
+Compacting heaps
+================
+Occasionally you have processes which read a large number of rows and then filter them so only a few are still
+held in memory.  Rows tend to be allocated in sequence through the heap pages, which can mean those few remaining
+rows are scattered over many pages.  If they could all be moved to a single page it would free up a significant
+amount of memory.
+
+The memory manager contains a function to pack a set of rows into a smaller number of pages: IRowManager->compactRows().
+
+This works by iterating through each of the rows in a list.  If the row belongs to a heap that could be compacted,
+and isn't part of a full heaplet, then the row is moved.  Since subsequent rows tend to be allocated from the same
+heaplet this has the effect of compacting the rows.
+
+Rules
+=====
+Some rules to follow when implementing callbacks:
+
+* A callback cannot allocate any memory from the memory manager.  If it does it is likely to deadlock.
+
+* You cannot allocate memory while holding a lock if that lock is also required by a callback.
+
+  Again this will cause deadlock.  If it proves impossible you can use a try-lock primitive in the callback,
+  but it means you won't be able to spill those rows.
+
+* If the heaps are fragmented it may be more efficient to repack the heaps than spill to disk.
+
+* If you're resizing a potentially big block of memory use the resize function with the callback.
+
+*************
+Shared Memory
+*************
+
+Much of the time Thor doesn't uses full memory available to it.  If you are running multiple Thor processes
+on the same machine you may want to configure the system so that each Thor has a private block of memory,
+but there is also a shared block of memory which can be used by whichever process needs it.
+
+The ILargeMemCallback provides a mechanism to dynamically allocate more memory to a process as it requires it.
+This could potentially be done in stages rather than all or nothing.
+
+(Currently unused as far as I know...)

roxie/roxiemem/README.rst

@@ -0,0 +1,5 @@
+This directory contains the memory manager which is used by all of the engines.
+
+More details of how to use it and its implementation are found in the `Memory Manager Documentation`_.
+
+.. _Memory Manager Documentation: DOCUMENTATION.rst

roxie/roxiemem/roxiemem.cpp

@@ -1173,6 +1173,7 @@ public:
         unsigned base = 0;
         unsigned limit = atomic_read(&freeBase);
         memsize_t running = 0;
+        unsigned runningCount = 0;
         unsigned lastId = 0;
         while (base < limit)
         {
@@ -1187,17 +1188,21 @@ public:
                 if (activityId != lastId)
                 {
                     if (lastId)
-                        map->noteMemUsage(lastId, running);
+                        map->noteMemUsage(lastId, running, runningCount);
                     lastId = activityId;
                     running = chunkSize;
+                    runningCount = 1;
                 }
                 else
+                {
                     running += chunkSize;
+                    runningCount++;
+                }
             }
             base += chunkSize;
         }
         if (lastId)
-            map->noteMemUsage(lastId, running);
+            map->noteMemUsage(lastId, running, runningCount);
     }
 
 private:
@@ -1321,22 +1326,11 @@ public:
     virtual void getPeakActivityUsage(IActivityMemoryUsageMap *map) const
     {
         //This function may not give 100% accurate results if called if there are concurrent allocations/releases
-        unsigned base = 0;
-        unsigned limit = atomic_read(&freeBase);
-        memsize_t running = 0;
-        while (base < limit)
-        {
-            const char *block = data() + base;
-            ChunkHeader * header = (ChunkHeader *)block;
-            unsigned rowCount = atomic_read(&header->count);
-            if (ROWCOUNT(rowCount) != 0)
-                running += chunkSize;
-            base += chunkSize;
-        }
-        if (running)
+        unsigned numAllocs = queryCount()-1;
+        if (numAllocs)
         {
             unsigned activityId = getActivityId(sharedAllocatorId);
-            map->noteMemUsage(activityId, running);
+            map->noteMemUsage(activityId, numAllocs * chunkSize, numAllocs);
         }
     }
 };
@@ -1349,7 +1343,7 @@ class HugeHeaplet : public BigHeapletBase
 protected:
     unsigned allocatorId;
 
-    inline unsigned _sizeInPages() 
+    inline unsigned _sizeInPages() const
     {
         return PAGES(chunkCapacity + dataOffset(), HEAP_ALIGNMENT_SIZE);
     }
@@ -1461,7 +1455,8 @@ public:
     virtual void getPeakActivityUsage(IActivityMemoryUsageMap *map) const 
     {
         unsigned activityId = getActivityId(allocatorId);
-        map->noteMemUsage(activityId, chunkCapacity);
+        map->noteMemUsage(activityId, chunkCapacity, 1);
+        map->noteHeapUsage(chunkCapacity, RHFpacked, _sizeInPages(), chunkCapacity);
     }
 
     virtual void checkHeap() const
@@ -1479,16 +1474,30 @@ struct ActivityEntry
     memsize_t usage;
 };
 
+struct HeapEntry : public CInterface
+{
+public:
+    HeapEntry(memsize_t _allocatorSize, RoxieHeapFlags _heapFlags, memsize_t _numPages, memsize_t _memUsed) :
+        allocatorSize(_allocatorSize), heapFlags(_heapFlags), numPages(_numPages), memUsed(_memUsed)
+    {
+    }
+
+    memsize_t allocatorSize;
+    RoxieHeapFlags heapFlags;
+    memsize_t numPages;
+    memsize_t memUsed;
+};
+
 typedef MapBetween<unsigned, unsigned, ActivityEntry, ActivityEntry> MapActivityToActivityEntry;
 
 class CActivityMemoryUsageMap : public CInterface, implements IActivityMemoryUsageMap
 {
     MapActivityToActivityEntry map;
+    CIArrayOf<HeapEntry> heaps;
     memsize_t maxUsed;
     memsize_t totalUsed;
     unsigned maxActivity;
 
-
 public:
     IMPLEMENT_IINTERFACE;
     CActivityMemoryUsageMap()
@@ -1498,7 +1507,7 @@ public:
         maxActivity = 0;
     }
 
-    virtual void noteMemUsage(unsigned activityId, unsigned memUsed)
+    virtual void noteMemUsage(unsigned activityId, memsize_t memUsed, unsigned numAllocs)
     {
         totalUsed += memUsed;
         ActivityEntry *ret = map.getValue(activityId);
@@ -1506,11 +1515,11 @@ public:
         {
             memUsed += ret->usage;
             ret->usage = memUsed;
-            ret->allocations++;
+            ret->allocations += numAllocs;
         }
         else
         {
-            ActivityEntry e = {activityId, 1, memUsed};
+            ActivityEntry e = {activityId, numAllocs, memUsed};
             map.setValue(activityId, e);
         }
         if (memUsed > maxUsed)
@@ -1520,6 +1529,11 @@ public:
         }
     }
 
+    void noteHeapUsage(memsize_t allocatorSize, RoxieHeapFlags heapFlags, memsize_t numPages, memsize_t memUsed)
+    {
+        heaps.append(*new HeapEntry(allocatorSize, heapFlags, numPages, memUsed));
+    }
+
     static int sortUsage(const void *_l, const void *_r)
     {
         const ActivityEntry *l = *(const ActivityEntry **) _l;
@@ -1547,6 +1561,25 @@ public:
                 j--;
                 logctx.CTXLOG("%"I64F"u bytes allocated by activity %u (%u allocations)", (unsigned __int64) results[j]->usage, getRealActivityId(results[j]->activityId, allocatorCache), results[j]->allocations);
             }
+            logctx.CTXLOG("Heaps:");
+            ForEachItemIn(iHeap, heaps)
+            {
+                HeapEntry & cur = heaps.item(iHeap);
+                StringBuffer flags;
+                if (cur.heapFlags & RHFpacked)
+                    flags.append("P");
+                if (cur.heapFlags & RHFunique)
+                    flags.append("U");
+                if (cur.heapFlags & RHFvariable)
+                    flags.append("V");
+
+                //Should never be called with numPages == 0, but protect against divide by zero in case of race condition etc.
+                unsigned percentUsed = cur.numPages ? (unsigned)((cur.memUsed * 100) / (cur.numPages * HEAP_ALIGNMENT_SIZE)) : 100;
+                unsigned __int64 memReserved = cur.numPages * HEAP_ALIGNMENT_SIZE;
+                logctx.CTXLOG("size: %"I64F"u [%s] reserved: %"I64F"u %u%% (%"I64F"u/%"I64F"u) used",
+                        (unsigned __int64) cur.allocatorSize, flags.str(), (unsigned __int64) cur.numPages, percentUsed, (unsigned __int64) cur.memUsed, (unsigned __int64) memReserved);
+            }
+
             logctx.CTXLOG("------------------ End of snapshot");
             delete [] results;
         }
@@ -1758,16 +1791,23 @@ public:
         return total;
     }
 
-    void getPeakActivityUsage(IActivityMemoryUsageMap * usageMap)
+    void getPeakActivityUsage(IActivityMemoryUsageMap * usageMap) const
     {
         SpinBlock c1(crit);
         BigHeapletBase *finger = active;
+        unsigned numPages = 0;
+        memsize_t numAllocs = 0;
         while (finger)
         {
-            if (finger->queryCount()!=1)
+            unsigned thisCount = finger->queryCount()-1;
+            if (thisCount != 0)
                 finger->getPeakActivityUsage(usageMap);
+            numAllocs += thisCount;
+            numPages++;
             finger = getNext(finger);
         }
+        if (numPages)
+            reportHeapUsage(usageMap, numPages, numAllocs);
     }
 
     inline bool isEmpty() const { return !active; }
@@ -1778,6 +1818,8 @@ public:
     }
 
 protected:
+    virtual void reportHeapUsage(IActivityMemoryUsageMap * usageMap, unsigned numPages, memsize_t numAllocs) const = 0;
+
     inline BigHeapletBase * getNext(const BigHeapletBase * ptr) const { return ptr->next; }
     inline void setNext(BigHeapletBase * ptr, BigHeapletBase * next) const { ptr->next = next; }
 
@@ -1803,6 +1845,11 @@ public:
 
 protected:
     HugeHeaplet * allocateHeaplet(memsize_t _size, unsigned allocatorId);
+
+    virtual void reportHeapUsage(IActivityMemoryUsageMap * usageMap, unsigned numPages, memsize_t numAllocs) const
+    {
+        //Already processed in HugeHeaplet::getPeakActivityUsage(IActivityMemoryUsageMap *map) const
+    }
 };
 
 class CNormalChunkingHeap : public CChunkingHeap
@@ -1815,6 +1862,11 @@ public:
 
     void * doAllocate(unsigned allocatorId);
 
+    virtual void reportHeapUsage(IActivityMemoryUsageMap * usageMap, unsigned numPages, memsize_t numAllocs) const
+    {
+        usageMap->noteHeapUsage(chunkSize, (RoxieHeapFlags)flags, numPages, chunkSize * numAllocs);
+    }
+
 protected:
     inline void * inlineDoAllocate(unsigned allocatorId);
     virtual BigHeapletBase * allocateHeaplet() = 0;
@@ -2137,7 +2189,7 @@ class CChunkingRowManager : public CInterface, implements IRowManager
     friend class CFixedChunkingHeap;
 
     SpinLock crit;
-    SpinLock fixedCrit;  // Should possibly be a ReadWriteLock - better with high contention, worse with low
+    mutable SpinLock fixedCrit;  // Should possibly be a ReadWriteLock - better with high contention, worse with low
     CIArrayOf<CFixedChunkingHeap> normalHeaps;
     CHugeChunkingHeap hugeHeap;
     ITimeLimiter *timeLimit;
@@ -2180,7 +2232,7 @@ public:
             size32_t rounded = roundup(prevSize+1);
             dbgassertex(ROUNDEDHEAP(rounded) == normalHeaps.ordinality());
             size32_t thisSize = ROUNDEDSIZE(rounded);
-            normalHeaps.append(*new CFixedChunkingHeap(this, _logctx, _allocatorCache, thisSize, 0));
+            normalHeaps.append(*new CFixedChunkingHeap(this, _logctx, _allocatorCache, thisSize, RHFvariable));
             prevSize = thisSize;
         }
         pageLimit = (unsigned) PAGES(_memLimit, HEAP_ALIGNMENT_SIZE);
@@ -2338,20 +2390,9 @@ public:
         return (total != 0);
     }
 
-    virtual void getPeakActivityUsage()
+    void getPeakActivityUsage()
     {
-        Owned<IActivityMemoryUsageMap> map = new CActivityMemoryUsageMap;
-        ForEachItemIn(iNormal, normalHeaps)
-            normalHeaps.item(iNormal).getPeakActivityUsage(map);
-        hugeHeap.getPeakActivityUsage(map);
-
-        SpinBlock block(fixedCrit); //Spinblock needed if we can add/remove fixed heaps while allocations are occuring
-        ForEachItemIn(i, fixedHeaps)
-        {
-            CChunkingHeap & fixedHeap = fixedHeaps.item(i);
-            fixedHeap.getPeakActivityUsage(map);
-        }
-
+        Owned<IActivityMemoryUsageMap> map = getActivityUsage();
         SpinBlock c1(crit);
         usageMap.setown(map.getClear());
     }
@@ -2682,6 +2723,7 @@ public:
                     if (numHeapPages == atomic_read(&totalHeapPages))
                     {
                         logctx.CTXLOG("RoxieMemMgr: Memory limit exceeded - current %u, requested %u, limit %u", pageCount, numRequested, pageLimit);
+                        reportMemoryUsage(false);
                         PrintStackReport();
                         throw MakeStringException(ROXIEMM_MEMORY_LIMIT_EXCEEDED, "memory limit exceeded");
                     }
@@ -2746,6 +2788,22 @@ protected:
         return NULL;
     }
 
+    IActivityMemoryUsageMap * getActivityUsage() const
+    {
+        Owned<IActivityMemoryUsageMap> map = new CActivityMemoryUsageMap;
+        ForEachItemIn(iNormal, normalHeaps)
+            normalHeaps.item(iNormal).getPeakActivityUsage(map);
+        hugeHeap.getPeakActivityUsage(map);
+
+        SpinBlock block(fixedCrit); //Spinblock needed if we can add/remove fixed heaps while allocations are occuring
+        ForEachItemIn(i, fixedHeaps)
+        {
+            CChunkingHeap & fixedHeap = fixedHeaps.item(i);
+            fixedHeap.getPeakActivityUsage(map);
+        }
+        return map.getClear();
+    }
+
     CFixedChunkingHeap * createFixedHeap(size32_t size, unsigned activityId, unsigned flags)
     {
         dbgassertex(!(flags & RHFpacked));
@@ -2830,6 +2888,22 @@ protected:
         callbacks.removeRowBuffer(callback);
     }
 
+    virtual void reportMemoryUsage(bool peak) const
+    {
+        if (peak)
+        {
+            if (usageMap)
+                usageMap->report(logctx, allocatorCache);
+        }
+        else
+        {
+            logctx.CTXLOG("RoxieMemMgr: pageLimit=%u peakPages=%u dataBuffs=%u dataBuffPages=%u possibleGoers=%u rowMgr=%p",
+                          pageLimit, peakPages, dataBuffs, dataBuffPages, atomic_read(&possibleGoers), this);
+            Owned<IActivityMemoryUsageMap> map = getActivityUsage();
+            map->report(logctx, allocatorCache);
+        }
+    }
+
     virtual memsize_t getExpectedCapacity(memsize_t size, unsigned heapFlags)
     {
         if (size > FixedSizeHeaplet::maxHeapSize())

roxie/roxiemem/roxiemem.hpp

@@ -382,6 +382,7 @@ enum RoxieHeapFlags
     RHFhasdestructor    = 0x0002,
     RHFunique           = 0x0004,  // create a separate fixed size allocator
     RHFoldfixed         = 0x0008,  // Don't create a special fixed size heap for this
+    RHFvariable         = 0x0010,  // only used for tracing
 };
 
 //This interface is here to allow atomic updates to allocations when they are being resized.  There are a few complications:
@@ -425,6 +426,7 @@ interface IRowManager : extends IInterface
     virtual IVariableRowHeap * createVariableRowHeap(unsigned activityId, unsigned roxieHeapFlags) = 0;            // should this be passed the initial size?
     virtual void addRowBuffer(IBufferedRowCallback * callback) = 0;
     virtual void removeRowBuffer(IBufferedRowCallback * callback) = 0;
+    virtual void reportMemoryUsage(bool peak) const = 0;
     virtual memsize_t getExpectedCapacity(memsize_t size, unsigned heapFlags) = 0; // what is the expected capacity for a given size allocation
     virtual memsize_t getExpectedFootprint(memsize_t size, unsigned heapFlags) = 0; // how much memory will a given size allocation actually use.
 };
@@ -438,7 +440,8 @@ interface ITimeLimiter
 
 interface IActivityMemoryUsageMap : public IInterface
 {
-    virtual void noteMemUsage(unsigned activityId, unsigned memUsed) = 0;
+    virtual void noteMemUsage(unsigned activityId, memsize_t memUsed, unsigned numAllocs) = 0;
+    virtual void noteHeapUsage(memsize_t allocatorSize, RoxieHeapFlags heapFlags, memsize_t memReserved, memsize_t memUsed) = 0;
     virtual void report(const IContextLogger &logctx, const IRowAllocatorCache *allocatorCache) = 0;
 };
 

roxie/roxiemem/sourcedoc.xml

@@ -1,77 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
--->
-<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
-<section>
-    <title>roxie/roxiemem</title>
-
-    <para>
-        The roxie/roxiemem directory contains the sources for the roxie/roxiemem library. This is used by roxie
-        and eclagent (and possibly in the future Thor) to allocate memory for ECL data rows. Note that it is NOT used
-        for allocation of other objects used in queries.
-    </para>
-
-    <para>
-        The roxiemem memory manager's design goals were:
-        <orderedlist>
-            <listitem><para>
-                Allow the memory used by a single query, or by all queries combined, to be limited, with graceful recovery.
-            </para></listitem>
-            <listitem><para>
-                Allow all the memory used by a query to be guaranteed to get freed when the query finishes, thus reducing the
-                possibility of memory leaks.
-            </para></listitem>
-            <listitem><para>
-                Support link-counted rows without having to copy serialized data from slaves.
-            </para></listitem>
-            <listitem><para>
-                Predictable behaviour with no pathogenic cases.
-            </para></listitem>
-            <listitem><para>
-                Be as fast as possible on allocate and deallocate of small rows.
-            </para></listitem>
-        </orderedlist>
-    </para>
-    <para>
-        Note that efficient usage of memory does not appear on that list - the expectation when the memory
-        manager was first designed was that Roxie queries would use minimal amounts of memory and speed was
-        more important.
-    </para>
-    <para>
-        In order to extend roxiemem to support Thor usage too, some changes are needed to these design goals:
-        <orderedlist>
-            <listitem><para>
-                As thor is not executing multiple queries in the same process space, the per-query limiting is
-                not useful.
-            </para></listitem>
-            <listitem><para>
-                Thor cares a lot more than Roxie about efficient use of memory as well as pure speed of allocation.
-            </para></listitem>
-            <listitem><para>
-                Thor typically needs to spill and continue when row memory exceeds available, rather than failing the query.
-                For this to work it needs to be able to tell when memory is running out.
-            </para></listitem>
-        </orderedlist>
-    </para>
-    <para>
-       RoxieMem uses a chunking allocation paradigm to avoid any possibility of fragmentation (which helps keep the behaviour
-       predictable). The patterns of allocation are reasonably well known (and we can use information from the ECL compiler
-       to tell us more - for example - the sizes of record that are going to be allocated by a query). Where variable-size
-       rows are in use this is less predictable though we still know the pattern of sizes used for expanding.
-    </para>
-</section>

roxie/sourcedoc.xml

@@ -27,7 +27,6 @@
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxie/sourcedoc.xml" />
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxieclient/sourcedoc.xml" />
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxieclient/sourcedoc.xml" />
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxiemem/sourcedoc.xml" />
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="roxiepipe/sourcedoc.xml" />
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="udplib/sourcedoc.xml" />
 </section>

testing/ecl/key/topn.xml

@@ -148,3 +148,8 @@
  <Row><_unnamed_cnt_1>2</_unnamed_cnt_1></Row>
  <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
 </Dataset>
+<Dataset name='Result 29'>
+ <Row><i>1</i><num>1</num><children><Row><i>1</i></Row></children></Row>
+ <Row><i>2</i><num>3</num><children><Row><i>3</i></Row><Row><i>20</i></Row><Row><i>100</i></Row></children></Row>
+ <Row><i>4</i><num>2</num><children><Row><i>12</i></Row><Row><i>50</i></Row></children></Row>
+</Dataset>

testing/ecl/topn.ecl

@@ -66,3 +66,18 @@ output(topn(i3, 1, -l, best('d'), local));      // expect {'d',4}
 b1 := topn(g3, 2, l, best('a'));
 output(sort(b1, l, d));     // expect {'a',1},{'a',9'},{'a',6},{'a',8},{'d',11}
 output(table(b1, { count(group) }));        // expect {2,2,1}
+
+
+
+r1 := { unsigned i; };
+r2 := { unsigned i, unsigned num, dataset(r1) children; };
+ds := dataset([
+    { 1, 1, [{1},{2},{3}] },
+    { 2, 3, [{100},{20},{3}] },
+    { 4, 2, [{50},{12},{76}] }], r2);
+r2 t(r2 l) := TRANSFORM
+    SELF.children := TOPN(l.children, l.num, i);
+    SELF := l;
+END;
+p := PROJECT(ds, t(LEFT));
+output(p);					// expect {1,1,{1}} {2,3,{3,20,100}} {4,2,{50}}

thorlcr/graph/thgraph.cpp

@@ -2341,6 +2341,62 @@ public:
 };
 
 ////
+// IContextLogger
+class CThorContextLogger : CSimpleInterface, implements IContextLogger
+{
+    CJobBase &job;
+    unsigned traceLevel;
+public:
+    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+
+    CThorContextLogger(CJobBase &_job) : job(_job)
+    {
+    }
+    virtual void CTXLOG(const char *format, ...) const
+    {
+        va_list args;
+        va_start(args, format);
+        CTXLOGva(format, args);
+        va_end(args);
+    }
+    virtual void CTXLOGva(const char *format, va_list args) const
+    {
+        StringBuffer ss;
+        ss.valist_appendf(format, args);
+        LOG(MCdebugProgress, thorJob, "%s", ss.str());
+    }
+    virtual void logOperatorException(IException *E, const char *file, unsigned line, const char *format, ...) const
+    {
+        va_list args;
+        va_start(args, format);
+        logOperatorExceptionVA(E, file, line, format, args);
+        va_end(args);
+    }
+    virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const
+    {
+        StringBuffer ss;
+        ss.append("ERROR");
+        if (E)
+            ss.append(": ").append(E->errorCode());
+        if (file)
+            ss.appendf(": %s(%d) ", file, line);
+        if (E)
+            E->errorMessage(ss.append(": "));
+        if (format)
+            ss.append(": ").valist_appendf(format, args);
+        LOG(MCoperatorProgress, thorJob, "%s", ss.str());
+    }
+    virtual void noteStatistic(unsigned statCode, unsigned __int64 value, unsigned count) const
+    {
+    }
+    virtual unsigned queryTraceLevel() const
+    {
+        return traceLevel;
+    }
+};
+
+////
+
 CJobBase::CJobBase(const char *_graphName) : graphName(_graphName)
 {
     maxDiskUsage = diskUsage = 0;
@@ -2374,6 +2430,8 @@ void CJobBase::init()
     forceLogGraphIdMin = (graph_id)getWorkUnitValueInt("forceLogGraphIdMin", 0);
     forceLogGraphIdMax = (graph_id)getWorkUnitValueInt("forceLogGraphIdMax", 0);
 
+    logctx.setown(new CThorContextLogger(*this));
+
     // global setting default on, can be overridden by #option
     timeActivities = 0 != getWorkUnitValueInt("timeActivities", globals->getPropBool("@timeActivities", true));
     maxActivityCores = (unsigned)getWorkUnitValueInt("maxActivityCores", globals->getPropInt("@maxActivityCores", 0)); // NB: 0 means system decides
@@ -2388,7 +2446,7 @@ void CJobBase::init()
     bool crcChecking = 0 != getWorkUnitValueInt("THOR_ROWCRC", globals->getPropBool("@THOR_ROWCRC", defaultCrcChecking));
     bool usePackedAllocator = 0 != getWorkUnitValueInt("THOR_PACKEDALLOCATOR", globals->getPropBool("@THOR_PACKEDALLOCATOR", false));
     unsigned memorySpillAt = (unsigned)getWorkUnitValueInt("memorySpillAt", globals->getPropInt("@memorySpillAt", 80));
-    thorAllocator.setown(createThorAllocator(((memsize_t)globalMemorySize)*0x100000, memorySpillAt, crcChecking, usePackedAllocator));
+    thorAllocator.setown(createThorAllocator(((memsize_t)globalMemorySize)*0x100000, memorySpillAt, *logctx, crcChecking, usePackedAllocator));
 
     unsigned defaultMemMB = globalMemorySize*3/4;
     unsigned largeMemSize = getOptUInt("@largeMemSize", defaultMemMB);