Merge remote-tracking branch 'origin/candidate-3.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -281,7 +281,9 @@ else()
     message("WARNING: CMAKE 2.8.1 or later required to create RPMs from this project")
 endif()
 
-set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" )
+if ( UNIX )
+    set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" )
+endif ( UNIX )
 if ( PLATFORM )
 else ( PLATFORM )
     if ( APPLE OR WIN32 )

ecl/hql/hqlfold.cpp

@@ -3096,6 +3096,7 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
     case no_sumlist:
         {
             IHqlExpression * child = expr->queryChild(0);
+            OwnedHqlExpr folded;
             switch (child->getOperator())
             {
             case no_null:
@@ -3122,12 +3123,17 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
                         }
                     }
                     if (ok)
-                    {
-                        return createConstant(sum.getClear());
-                    }
+                        folded.setown(createConstant(sum.getClear()));
                 }
+
                 if (child->numChildren() == 1)
-                    return expr->cloneAllAnnotations(child->queryChild(0));
+                    folded.set(child->queryChild(0));
+
+                if (folded)
+                {
+                    OwnedHqlExpr cast = ensureExprType(folded, expr->queryType());
+                    return expr->cloneAllAnnotations(cast);
+                }
                 break;
             }
             break;

ecl/hqlcpp/hqlcerrors.hpp

@@ -204,7 +204,8 @@
 #define HQLERR_OutsideGroupAggregate            4182
 #define HQLERR_ResourceAddAfterFinalManifest    4183
 #define HQLERR_SkipInsideCreateRow              4184
-#define HQLERR_ScalarOutputWithinApply          4185
+#define HQLERR_KeyedJoinNoRightIndex_X          4185
+#define HQLERR_ScalarOutputWithinApply          4186
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -482,6 +483,7 @@
 #define HQLERR_ResourceAddAfterFinalManifest_Text "%s resource added after manifest was finalized"
 #define HQLERR_SkipInsideCreateRow_Text         "SKIP inside a ROW(<transform>) not supported.  It is only allowed in a DATASET transform."
 #define HQLERR_ScalarOutputWithinApply_Text     "A scalar output within an APPLY is undefined and may fail.  Use OUTPUT(dataset,EXTEND) instead."
+#define HQLERR_KeyedJoinNoRightIndex_X_Text     "Right dataset (%s) for a keyed join isn't a key"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlckey.cpp

@@ -211,15 +211,6 @@ void HqlCppTranslator::buildJoinMatchFunction(BuildCtx & ctx, const char * name,
 
 //--------------------------------------------------------------------------------------------------
 
-IHqlExpression * createKeyFromComplexKey(IHqlExpression * expr)
-{
-    IHqlExpression * base = queryPhysicalRootTable(expr);
-    if (base->getOperator() == no_newkeyindex)
-        return LINK(base);
-    UNIMPLEMENTED_XY("Key", getOpString(base->getOperator()));
-    return NULL;
-}
-
 class KeyedJoinInfo : public CInterface
 {
 public:
@@ -260,6 +251,7 @@ public:
 protected:
     void buildClearRecord(BuildCtx & ctx, RecordSelectIterator & rawIter, RecordSelectIterator & keyIter);
     void buildTransformBody(BuildCtx & ctx, IHqlExpression * transform);
+    IHqlExpression * createKeyFromComplexKey(IHqlExpression * expr);
     IHqlExpression * expandDatasetReferences(IHqlExpression * expr, IHqlExpression * ds);
     IHqlExpression * optimizeTransfer(HqlExprArray & fields, HqlExprArray & values, IHqlExpression * expr, IHqlExpression * leftSelector);
     void optimizeExtractJoinFields();
@@ -345,6 +337,22 @@ KeyedJoinInfo::~KeyedJoinInfo()
 }
 
 
+IHqlExpression * KeyedJoinInfo::createKeyFromComplexKey(IHqlExpression * expr)
+{
+    IHqlExpression * base = queryPhysicalRootTable(expr);
+    if (!base)
+    {
+        translator.throwError1(HQLERR_KeyedJoinNoRightIndex_X, getOpString(expr->getOperator()));
+        return NULL;
+    }
+
+    if (base->getOperator() == no_newkeyindex)
+        return LINK(base);
+
+    translator.throwError1(HQLERR_KeyedJoinNoRightIndex_X, getOpString(base->getOperator()));
+    return NULL;
+}
+
 void KeyedJoinInfo::buildClearRecord(BuildCtx & ctx, RecordSelectIterator & rawIter, RecordSelectIterator & keyIter)
 {
     keyIter.first();
@@ -1430,7 +1438,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
 
 ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx, IHqlExpression * expr)
 {
-    if (!targetThor())
+    if (!targetThor() || insideChildQuery(ctx))
         return buildCachedActivity(ctx, expr->queryChild(0));
 
     HqlExprArray leftSorts, rightSorts;

ecl/hqlcpp/hqlcpp.ipp

@@ -481,7 +481,7 @@ public:
     Linked<IPropertyTree> graph;
 };
 
-enum SubGraphType { SubGraphRoot, SubGraphRemote, SubGraphChild };
+enum SubGraphType { SubGraphRoot, SubGraphRemote, SubGraphChild, SubGraphLoop };
 
 struct SubGraphInfo : public HqlExprAssociation
 {
@@ -1648,7 +1648,7 @@ public:
 
     EvalContext * queryEvalContext(BuildCtx & ctx)          { return (EvalContext *)ctx.queryFirstAssociation(AssocExtractContext); }
     inline unsigned nextActivityId()                        { return ++curActivityId; }
-    bool insideChildGraph(BuildCtx & ctx);
+    bool insideChildOrLoopGraph(BuildCtx & ctx);
     bool insideChildQuery(BuildCtx & ctx);
     bool insideRemoteGraph(BuildCtx & ctx);
     bool isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag);

ecl/hqlcpp/hqlcppds.cpp

@@ -1391,7 +1391,7 @@ unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, bool multiInstance)
     createBuilderAlias(subctx, extractBuilder);
 
     translator.beginExtract(ctx, extractBuilder);
-    translator.doBuildThorSubGraph(ctx, resourced, SubGraphChild, id, represents);
+    translator.doBuildThorSubGraph(ctx, resourced, SubGraphLoop, id, represents);
     translator.endExtract(ctx, extractBuilder);
 
     return id;
@@ -1502,7 +1502,7 @@ unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, bool isParalle
     createBuilderAlias(subctx, extractBuilder);
 
     translator.beginExtract(ctx, extractBuilder);
-    translator.doBuildThorSubGraph(ctx, resourced, SubGraphChild, id, represents);
+    translator.doBuildThorSubGraph(ctx, resourced, SubGraphLoop, id, represents);
     translator.endExtract(ctx, extractBuilder);
 
     return id;

ecl/hthor/CMakeLists.txt

@@ -73,7 +73,7 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DHTHOR_EXPORTS -DSTARTQUERY_EXPORTS )
 
 HPCC_ADD_LIBRARY( hthor SHARED ${SRCS} ${INCLUDES} )
-install ( TARGETS hthor RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION ${ARCHIVE_DIR} )
+install ( TARGETS hthor RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION componentfiles/cl/lib )
 target_link_libraries ( hthor
          jlib
          mp

ecl/regress/issue8437.ecl

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+r1 := { unsigned id, string name; string name2};
+
+r3 := { dataset(r1) children, dataset(r1) children2 };
+r2 := { unsigned id, r3 r };
+
+s := dataset([{0,[{1,'Gavin','Hawthorn'},{2,'Jason','Jones'}],[]}], r2);
+
+r2 t(r2 l) := TRANSFORM
+    SELF.r.children := DISTRIBUTE(l.r.children, HASH(id));
+    SELF := l;
+END;
+
+p := PROJECT(NOFOLD(s), t(LEFT));
+output(p);

ecl/regress/issue8439.ecl

@@ -0,0 +1,30 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+TblDS0RecDef := RECORD 
+    UNSIGNED4 tlength; 
+    UNSIGNED2 num;
+END; 
+
+TblDS0 := DATASET('~infile', TblDS0RecDef,FLAT);
+SelectStruct := RECORD
+ c3 := SUM( GROUP , TblDS0.num );
+END;
+
+DSTable := TABLE( TblDS0( SUM( NUM ) > 90 ), SelectStruct , tLength);
+
+OUTPUT(CHOOSEN(DSTable,100),NAMED('QueryResult'));

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -355,8 +355,9 @@ void CWsEclBinding::getDynNavData(IEspContext &context, IProperties *params, IPr
         const char *setname = params->queryProp("queryset");
         if (!setname || !*setname)
             return;
-
         Owned<IPropertyTree> settree = getQueryRegistry(setname, true);
+        if (!settree)
+            return;
 
         if (params->hasProp("QueryList"))
         {

roxie/ccd/CMakeLists.txt

@@ -80,7 +80,7 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DCCD_EXPORTS -DSTARTQUERY_EXPORTS )
 
 HPCC_ADD_LIBRARY( ccd SHARED ${SRCS} )
-install ( TARGETS ccd RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION ${ARCHIVE_DIR} )
+install ( TARGETS ccd RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION componentfiles/cl/lib )
 
 target_link_libraries ( ccd 
          jlib

rtl/eclrtl/CMakeLists.txt

@@ -58,7 +58,7 @@ else ()
 endif ()
 
 HPCC_ADD_LIBRARY( eclrtl SHARED ${SRCS} )
-install ( TARGETS eclrtl RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION ${ARCHIVE_DIR} )
+install ( TARGETS eclrtl RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION componentfiles/cl/lib )
 target_link_libraries ( eclrtl 
       ${BOOST_REGEX_LIBRARIES}
       ${ICU_LIBRARIES}

rtl/nbcd/CMakeLists.txt

@@ -40,7 +40,7 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DNBCD_EXPORTS )
 
 HPCC_ADD_LIBRARY( nbcd SHARED ${SRCS} )
-install ( TARGETS nbcd RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION ${ARCHIVE_DIR} )
+install ( TARGETS nbcd RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION componentfiles/cl/lib )
 target_link_libraries ( nbcd 
       jlib
       ${CPPUNIT_LIBRARIES}

system/security/LdapSecurity/ldapconnection.cpp

@@ -5089,6 +5089,16 @@ private:
 
         return true;
     }
+
+    bool createUserScope(ISecUser& user)
+    {
+        //Add tempfile scope for given user (spill, paused and checkpoint
+        //files will be created under this user specific scope)
+        StringBuffer resName(queryDfsXmlBranchName(DXB_Internal));
+        resName.append("::").append(user.getName());
+        Owned<ISecResource> resource = new CLdapSecResource(resName.str());
+        return addResource(RT_FILE_SCOPE, user, resource, PT_ADMINISTRATORS_AND_USER, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE));
+    }
 };
 
 int LdapUtils::getServerInfo(const char* ldapserver, int ldapport, StringBuffer& domainDN, LdapServerType& stype, const char* domainname)

system/security/LdapSecurity/ldapconnection.hpp

@@ -187,6 +187,7 @@ interface ILdapClient : extends IInterface
     virtual int countResources(const char* basedn, const char* searchstr, int limit) = 0;
     virtual ILdapConfig* queryConfig() = 0;
     virtual const char* getPasswordStorageScheme() = 0;
+    virtual bool createUserScope(ISecUser& user) = 0;
 };
 
 ILdapClient* createLdapClient(IPropertyTree* cfg);

system/security/LdapSecurity/ldapsecurity.cpp

@@ -1216,6 +1216,19 @@ bool CLdapSecManager::getUserInfo(ISecUser& user, const char* infotype)
     return m_ldap_client->getUserInfo(user, infotype);
 }
 
+bool CLdapSecManager::createUserScopes()
+{
+    Owned<ISecUserIterator> it = getAllUsers();
+    it->first();
+    while(it->isValid())
+    {
+        ISecUser &user = it->get();
+        if (!m_ldap_client->createUserScope(user))
+            PROGLOG("Error creating scope for user '%s'", user.getName());
+        it->next();
+    }
+    return true;
+}
 
 extern "C"
 {

system/security/LdapSecurity/ldapsecurity.ipp

@@ -438,6 +438,7 @@ public:
     {
         return m_passwordExpirationWarningDays;
     }
+    virtual bool createUserScopes();
 };
 
 #endif

system/security/shared/basesecurity.hpp

@@ -284,6 +284,8 @@ public:
         return m_passwordExpirationWarningDays;
     }
 
+    virtual bool createUserScopes() {UNIMPLEMENTED; return false;}
+
 protected:
     const char* getServer(){return m_dbserver.toCharArray();}
     const char* getUser(){return m_dbuser.toCharArray();}

system/security/shared/defaultsecuritymanager.hpp

@@ -79,7 +79,7 @@ public:
 
         return true;
     }
-
+    virtual bool createUserScopes() { return false; }
 };
 
 class CLocalSecurityManager : public CDefaultSecurityManager

system/security/shared/seclib.hpp

@@ -298,6 +298,7 @@ interface ISecManager : extends IInterface
     virtual bool authorizeWorkunitScope(ISecUser & user, ISecResourceList * resources) = 0;
     virtual const char * getDescription() = 0;
     virtual unsigned getPasswordExpirationWarningDays() = 0;
+    virtual bool createUserScopes() = 0;
 };
 
 interface IExtSecurityManager

testing/regress/hpcc/common/error.py

@@ -22,6 +22,8 @@ ERROR = {
     "1001": "Command return code non-zero.",
     "1002": "Command timed out.",
     "2000": "Value must be of Type Suite or ECLFile.",
+    "2001": "ECL Directory does not exist.",
+    "2002": "suiteDir not set. Set in regress.json or use --suiteDir",
     "3000": "Return is null",
     "3001": "Return diff does not match."
 }
@@ -34,6 +36,6 @@ class Error(Exception):
 
     def __str__(self):
         if self.err:
-            return "Error (%s): %s \n %s" % (self.code,
-                                             ERROR[self.code], self.err)
+            return "Error (%s): %s \n %s\n" % (self.code,
+                                               ERROR[self.code], self.err)
         return "Error (%s): %s " % (self.code, ERROR[self.code])

testing/regress/hpcc/common/logger.py

@@ -58,13 +58,13 @@ class Logger(object):
                 prefix = '[Error]' % \
                     record.__dict__
             elif record.__dict__['levelname'] == "DEBUG":
-                prefix = '[TEST]' % \
+                prefix = '[Debug]' % \
                     record.__dict__
             elif record.__dict__['levelname'] == "INFO":
                 prefix = '[Pass]' % \
                     record.__dict__
             elif record.__dict__['levelname'] == "WARNING":
-                prefix = '[Running]' % \
+                prefix = '[Action]' % \
                     record.__dict__
             if self._color:
                 prefix = (self._colors.get(record.levelno, self._normal) +
@@ -77,7 +77,7 @@ class Logger(object):
                 formatted = formatted.rstrip() + "\n" + record.exc_text
             return formatted.replace("\n", "\n    ")
 
-    def addHandler(self, fd, level='debug'):
+    def addHandler(self, fd, level='info'):
         root_logger = logging.getLogger()
         channel = logging.FileHandler(fd)
         channel.setLevel(getattr(logging, level.upper()))
@@ -97,18 +97,9 @@ class Logger(object):
         channel.setFormatter(Logger._LogFormatter(color=color))
         root_logger.addHandler(channel)
 
-    def __init__(self, level='debug'):
+    def setLevel(self, level):
         logging.getLogger().setLevel(getattr(logging, level.upper()))
-        self.enable_pretty_logging()
-
 
-def main():
-    logging.info("INFO")
-    logging.debug("sort.ecl")
-    logging.warn("WARN")
-    logging.error("ERROR")
-    logging.critical("CRITICAL")
-
-if __name__ == '__main__':
-    regLog = Logger('debug')
-    main()
+    def __init__(self, level='info'):
+        self.setLevel(level)
+        self.enable_pretty_logging()

testing/regress/hpcc/common/report.py

@@ -47,7 +47,7 @@ class Report:
         self.report = _dict(report)
         self.name = name
 
-    def display(self):
+    def display(self, log=None):
         reportStr = "\n"
         reportStr += "Results\n"
         reportStr += "-------------------------------------------------\n"
@@ -59,6 +59,9 @@ class Report:
                 reportStr += result.Diff
                 reportStr += "\n"
             reportStr += "-------------------------------------------------\n"
+        if log:
+            reportStr += "Log: %s\n" % str(log)
+            reportStr += "-------------------------------------------------\n"
         logging.warn(reportStr)
 
     def getResult(self, eclfile):

testing/regress/hpcc/common/shell.py

@@ -17,6 +17,7 @@
 ############################################################################ */
 '''
 
+import logging
 import sys
 from subprocess import (
     PIPE,
@@ -42,6 +43,7 @@ class Shell:
 
     def __run(self, *args, **kwargs):
         args = [i for i in args if i is not None]
+        logging.debug("CMD: " + " ". join(args))
         process = Popen(
             args, stdout=kwargs.pop('stdout', PIPE),
             stderr=kwargs.pop('stderr', PIPE),

testing/regress/hpcc/regression/regress.py

@@ -20,8 +20,10 @@
 import logging
 import os
 import sys
+import time
 
 from ..common.config import Config
+from ..common.error import Error
 from ..common.logger import Logger
 from ..common.report import Report, Tee
 from ..regression.suite import Suite
@@ -30,51 +32,70 @@ from ..util.ecl.command import ECLcmd
 
 
 class Regression:
-    def __init__(self, config="regress.json"):
+    def __init__(self, config="regress.json", level='info', suiteDir=None):
         self.config = Config(config).configObj
         self.suites = {}
-        self.log = Logger("DEBUG")
+        self.log = Logger(level)
+        if not suiteDir:
+            self.suiteDir = self.config.suiteDir
+            if not self.suiteDir:
+                raise Error("2002")
+        else:
+            self.suiteDir = suiteDir
+        self.regressionDir = self.config.regressionDir
+        self.logDir = self.config.logDir
+        self.setupDir = os.path.join(self.suiteDir, self.config.setupDir)
+        self.dir_ec = os.path.join(self.suiteDir, self.config.eclDir)
+        self.dir_ex = os.path.join(self.suiteDir, self.config.keyDir)
+        self.dir_a = os.path.join(self.regressionDir, self.config.archiveDir)
+        self.dir_r = os.path.join(self.regressionDir, self.config.resultDir)
+        logging.debug("Suite Dir      : %s", suiteDir)
+        logging.debug("Regression Dir : %s", self.regressionDir)
+        logging.debug("Result Dir     : %s", self.dir_r)
+        logging.debug("Log Dir        : %s", self.logDir)
+        logging.debug("ECL Dir        : %s", self.dir_ec)
+        logging.debug("Key Dir        : %s", self.dir_ex)
+        logging.debug("Setup Dir      : %s", self.setupDir)
+        logging.debug("Archive Dir    : %s", self.dir_a)
+
+    def setLogLevel(self, level):
+        self.log.setLevel(level)
 
     def bootstrap(self):
-        archives = os.path.join(self.config.baseDir, self.config.archiveDir)
-        results = os.path.join(self.config.baseDir, self.config.resultDir)
-        self.createDirectory(archives)
-        self.createDirectory(results)
+        self.createDirectory(self.regressionDir)
+        self.createDirectory(self.dir_a)
+        self.createDirectory(self.dir_r)
+        self.createDirectory(self.logDir)
         self.setup = self.Setup()
         for cluster in self.config.Clusters:
             self.createSuite(cluster)
+        os.chdir(self.regressionDir)
 
     def createDirectory(self, dir_n):
         if not os.path.isdir(dir_n):
             os.makedirs(dir_n)
 
     def createSuite(self, cluster):
-        dir_ec = os.path.join(self.config.baseDir, self.config.eclDir)
-        dir_a = os.path.join(self.config.baseDir, self.config.archiveDir)
-        dir_ex = os.path.join(self.config.baseDir, self.config.keyDir)
-        dir_r = os.path.join(self.config.baseDir, self.config.resultDir)
-        self.suites[cluster] = Suite(cluster, dir_ec, dir_a, dir_ex, dir_r)
+        self.suites[cluster] = Suite(cluster, self.dir_ec,
+                                     self.dir_a, self.dir_ex, self.dir_r)
 
     def Setup(self):
-        setup = os.path.join(self.config.baseDir, self.config.setupDir)
-        dir_a = os.path.join(self.config.baseDir, self.config.archiveDir)
-        dir_ex = os.path.join(self.config.baseDir, self.config.keyDir)
-        dir_r = os.path.join(self.config.baseDir, self.config.resultDir)
-        return Suite('setup', setup, dir_a, dir_ex, dir_r)
+        return Suite('setup', self.setupDir, self.dir_a, self.dir_ex,
+                     self.dir_r)
 
     def runSuite(self, name, suite):
-        logDir = os.path.join(self.config.baseDir, self.config.logDir)
         server = self.config.ip
         report = Report(name)
-        logName = name + ".log"
+        curTime = time.strftime("%y-%m-%d-%H-%M")
+        logName = name + "." + curTime + ".log"
         if name == "setup":
             cluster = 'hthor'
         else:
             cluster = name
-        log = os.path.join(logDir, logName)
+        log = os.path.join(self.logDir, logName)
         self.log.addHandler(log, 'DEBUG')
-        logging.debug("Suite: %s" % name)
-        logging.debug("Queries: %s" % repr(len(suite.getSuite())))
+        logging.warn("Suite: %s" % name)
+        logging.warn("Queries: %s" % repr(len(suite.getSuite())))
         cnt = 1
         for query in suite.getSuite():
             logging.warn("%s. Test: %s" % (repr(cnt), query.ecl))
@@ -97,4 +118,4 @@ class Regression:
                     logging.error("Fail %s" % wuid)
                     logging.error("URL %s" % url)
             cnt += 1
-        report.display()
+        report.display(log)

testing/regress/hpcc/regression/suite.py

@@ -19,6 +19,7 @@
 
 import os
 from ..util.ecl.file import ECLFile
+from ..common.error import Error
 
 
 class Suite:
@@ -33,7 +34,7 @@ class Suite:
 
     def buildSuite(self):
         if not os.path.isdir(self.dir_ec):
-            raise Exception("ECL Directory does not exist.")
+            raise Error("2001", err="Not Found: %s" % self.dir_ec)
         for files in os.listdir(self.dir_ec):
             if files.endswith(".ecl"):
                 ecl = os.path.join(self.dir_ec, files)

testing/regress/hpcc/util/configgen.py

@@ -0,0 +1,104 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+from ..common.shell import Shell
+from ..common.error import Error
+
+
+class ConfigGen(Shell):
+    def __init__(self, environment='/etc/HPCCSystems/environment.xml'):
+        self.defaults = []
+        self.defaults.append('-env')
+        self.defaults.append(environment)
+        self.cmd = self.which('configgen')
+
+    def __ConfigGen(self):
+        return self.command(self.cmd, *self.defaults)
+
+    def section(self):
+        return 'configgen'
+
+    def listall(self):
+        args = []
+        args.append('-listall')
+        try:
+            results = '[' + self.section() + ']\n'
+            data = self.__ConfigGen()(*args)
+            data = [line for line in data.split('\n') if line.strip()]
+            for i in data:
+                val = i.replace(',,', ',None,')
+                if val[-1:] == ',':
+                    val += 'None'
+                ret = val.split(',', 1)
+                results += ret[0] + "=" + ret[1] + "\n"
+            return results
+        except Error as err:
+            return repr(err)
+
+    def list(self, ip='.'):
+        args = []
+        args.append('-ip')
+        args.append(ip)
+        args.append('-list')
+        try:
+            results = '[' + self.section() + ']\n'
+            results += self.__ConfigGen()(*args)
+            return results
+        except Error as err:
+            return repr(err)
+
+    def listdirs(self):
+        args = []
+        args.append('-listdirs')
+        try:
+            results = '[' + self.section() + ']\n'
+            data = self.__ConfigGen()(*args)
+            data = [line for line in data.split('\n') if line.strip()]
+            cnt = 0
+            for i in data:
+                results += "dir" + repr(cnt) + "=" + i
+                cnt += 1
+            return results
+        except Error as err:
+            return repr(err)
+
+    def listcommondirs(self):
+        args = []
+        args.append('-listcommondirs')
+        try:
+            results = '[' + self.section() + ']\n'
+            results += self.__ConfigGen()(*args)
+            return results
+        except Error as err:
+            return repr(err)
+
+    def machines(self):
+        args = []
+        args.append('-machines')
+        try:
+            results = '[' + self.section() + ']\n'
+            data = self.__ConfigGen()(*args)
+            data = [line for line in data.split('\n') if line.strip()]
+            cnt = 0
+            for i in data:
+                results += "machine" + repr(cnt) + "=" + i
+                cnt += 1
+            return results
+        except Error as err:
+            return repr(err)

testing/regress/hpcc/util/ecl/file.py

@@ -103,16 +103,23 @@ class ECLFile:
     def testResults(self):
         d = difflib.Differ()
         try:
+            logging.debug("EXP: " + self.getExpected())
+            logging.debug("REC: " + self.getResults())
+            if not os.path.isfile(self.getExpected()):
+                raise IOError("KEY FILE NOT FOUND. " + self.getExpected())
+            if not os.path.isfile(self.getResults()):
+                raise IOError("RESULT FILE NOT FOUND. " + self.getResults())
             expected = open(self.getExpected(), 'r').readlines()
             recieved = open(self.getResults(), 'r').readlines()
-        except IOError as e:
+            for line in difflib.unified_diff(recieved,
+                                             expected,
+                                             fromfile=self.xml_r,
+                                             tofile=self.xml_e):
+                self.diff += line
+        except Exception as e:
             logging.critical(e)
+            return False
 
-        for line in difflib.unified_diff(recieved,
-                                         expected,
-                                         fromfile=self.xml_r,
-                                         tofile=self.xml_e):
-            self.diff += line
         if not self.diff:
             return True
         return False

testing/regress/regress

@@ -20,9 +20,11 @@
 '''
 
 import argparse
+import logging
 
 from hpcc.regression.regress import Regression
 
+
 if __name__ == "__main__":
     prog = "regress"
     description = 'HPCC Platform Regression suite'
@@ -31,6 +33,11 @@ if __name__ == "__main__":
                         version='%(prog)s 0.0.1')
     parser.add_argument('--config', help="Config file to use.",
                         nargs='?', default="regress.json")
+    parser.add_argument('--loglevel', help="Set the log level.",
+                        nargs='?', default="info",
+                        choices=['info', 'debug'])
+    parser.add_argument('--suiteDir', '-s', help="suiteDir to use.",
+                        nargs='?')
     subparsers = parser.add_subparsers(help='sub-command help')
     parser_list = subparsers.add_parser('list', help='list help')
     parser_list.add_argument('clusters', help="Print clusters from config.",
@@ -39,17 +46,26 @@ if __name__ == "__main__":
     parser_run.add_argument('cluster', help="Run the cluster suite.",
                             nargs='?', default='setup')
     args = parser.parse_args()
-    regress = Regression(args.config)
-    if 'clusters' in args:
-        Clusters = ['setup']
-        for cluster in regress.config.Clusters:
-            Clusters.append(str(cluster))
-        print "Avaliable Clusters: "
-        for i in Clusters:
-            print i
-    if 'cluster' in args:
-        regress.bootstrap()
-        if 'setup' in args.cluster:
-            regress.runSuite('setup', regress.setup)
-        else:
-            regress.runSuite(args.cluster, regress.suites[args.cluster])
+
+    suiteDir = ""
+    if 'suiteDir' in args:
+        suiteDir = args.suiteDir
+    try:
+        regress = Regression(args.config, args.loglevel, suiteDir)
+        if 'clusters' in args:
+            Clusters = ['setup']
+            for cluster in regress.config.Clusters:
+                Clusters.append(str(cluster))
+            print "Avaliable Clusters: "
+            for i in Clusters:
+                print i
+        if 'cluster' in args:
+            regress.bootstrap()
+            if 'setup' in args.cluster:
+                regress.runSuite('setup', regress.setup)
+            else:
+                regress.runSuite(args.cluster, regress.suites[args.cluster])
+    except Exception as e:
+        logging.critical(e)
+    except KeyboardInterrupt:
+        logging.critical("Keyboard Interrupt Caught.")

testing/regress/regress.json

@@ -5,13 +5,14 @@
         "password": "regress",
         "roxie": "127.0.0.1:9876",
         "server": "127.0.0.1:8010",
+        "suiteDir": "",
         "eclDir": "ecl",
         "setupDir": "ecl/setup",
         "keyDir": "ecl/key",
-        "archiveDir": "regression/archives",
-        "resultDir": "regression/results",
-        "logDir": "regression",
-        "baseDir": ".",
+        "archiveDir": "archives",
+        "resultDir": "results",
+        "regressionDir": "/var/lib/HPCCSystems/regression",
+        "logDir": "/var/log/HPCCSystems/regression",
         "Clusters": [
             "hthor",
             "thor",

tools/CMakeLists.txt

@@ -16,6 +16,7 @@
 HPCC_ADD_SUBDIRECTORY (esdl)
 HPCC_ADD_SUBDIRECTORY (hidl)
 HPCC_ADD_SUBDIRECTORY (backupnode "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (addScopes "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (combine "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (dumpkey "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (keydiff "PLATFORM")

tools/addScopes/CMakeLists.txt

@@ -0,0 +1,52 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: addScopes 
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for addScopes
+#####################################################
+
+
+project( addScopes ) 
+
+set (    SRCS 
+         addScopes.cpp 
+    )
+
+include_directories ( 
+         ./../../system/security/LdapSecurity
+         ./../../system/security/shared 
+         ./../../system/jlib 
+         ./../../system/include 
+    )
+
+ADD_DEFINITIONS( -D_CONSOLE )
+
+
+HPCC_ADD_EXECUTABLE ( addScopes ${SRCS} )
+install ( TARGETS addScopes RUNTIME DESTINATION ${EXEC_DIR} )
+
+
+
+target_link_libraries ( addScopes 
+         jlib
+         LdapSecurity 
+    )
+
+

tools/addScopes/addScopes.cpp

@@ -0,0 +1,76 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "seclib.hpp"
+#include "ldapsecurity.hpp"
+#include "jliball.hpp"
+
+#ifndef _WIN32
+#include <unistd.h>
+#endif
+
+int main(int argc, char* argv[])
+{
+    if(argc != 2)
+    {
+        printf("usage: addScopes daliconf.xml\n");
+        printf("\n\n\tCreates user-specific LDAP private file scopes 'hpccinternal::<user>'\n\tand grants users access to their scope. The configuration file\n\tdaliconf.xml is the dali configuration file, typically\n\tfound in /var/lib/HPCCSystems/mydali");
+        return -1;
+    }
+
+    InitModuleObjects();
+
+    try
+    {
+        Owned<IPropertyTree> cfg = createPTreeFromXMLFile(argv[1]);
+        Owned<IPropertyTree> seccfg = cfg->getPropTree(".//ldapSecurity");
+        if(seccfg == NULL)
+        {
+            printf("ldapSecurity not found\n");
+            return -1;
+        }
+#ifdef _NO_LDAP
+        printf("System was built with _NO_LDAP\n");
+        return -1;
+#else
+        Owned<ISecManager> secmgr = newLdapSecManager("addScopes", *LINK(seccfg));
+        if(secmgr == NULL)
+        {
+            printf("Security manager can't be created\n");
+            return -1;
+        }
+        bool ok = secmgr->createUserScopes();
+        if(ok)
+            printf("User scopes added\n");
+        else
+            printf("Some scopes not added\n");
+#endif
+    }
+    catch(IException* e)
+    {
+        StringBuffer errmsg;
+        e->errorMessage(errmsg);
+        printf("%s\n", errmsg.str());
+    }
+    catch(...)
+    {
+        printf("Unknown exception\n");
+    }
+
+    releaseAtoms();
+    return 0;
+}

tools/addScopes/sourcedoc.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>tools/addScopes</title>
+
+    <para>
+        The tools/addScopes directory contains the sources for the tools/addScopes tool.
+    </para>
+</section>