Merge remote-tracking branch 'origin/closedown-4.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workflow.cpp

@@ -748,23 +748,26 @@ bool WorkflowMachine::doExecuteItemDependencies(IRuntimeWorkflowItem & item, uns
     Owned<IWorkflowDependencyIterator> iter = item.getDependencies();
     for(iter->first(); iter->isValid(); iter->next())
     {
-        if (!doExecuteItemDependency(item, iter->query(), scheduledWfid))
+        if (!doExecuteItemDependency(item, iter->query(), scheduledWfid, false))
             return false;
     }
     return true;
 }
 
-bool WorkflowMachine::doExecuteItemDependency(IRuntimeWorkflowItem & item, unsigned dep, unsigned scheduledWfid)
+bool WorkflowMachine::doExecuteItemDependency(IRuntimeWorkflowItem & item, unsigned wfid, unsigned scheduledWfid, bool alwaysEvaluate)
 {
     try
     {
-        return executeItem(dep, scheduledWfid);
+        if (alwaysEvaluate)
+            workflow->queryWfid(wfid).setState(WFStateNull);
+
+        return executeItem(wfid, scheduledWfid);
     }
     catch(WorkflowException * e)
     {
         if(e->queryType() == WorkflowException::ABORT)
             throw;
-        if(!attemptRetry(item, dep, scheduledWfid))
+        if(!attemptRetry(item, wfid, scheduledWfid))
         {
             handleFailure(item, e, true);
             throw;
@@ -819,12 +822,12 @@ bool WorkflowMachine::doExecuteConditionItem(IRuntimeWorkflowItem & item, unsign
     if(iter->next()) wfidFalse = iter->query();
     if(iter->next()) throwUnexpected();
 
-    if (!doExecuteItemDependency(item, wfidCondition, scheduledWfid))
+    if (!doExecuteItemDependency(item, wfidCondition, scheduledWfid, true))
         return false;
     if(condition)
-        return doExecuteItemDependency(item, wfidTrue, scheduledWfid);
+        return doExecuteItemDependency(item, wfidTrue, scheduledWfid, false);
     else if (wfidFalse)
-        return doExecuteItemDependency(item, wfidFalse, scheduledWfid);
+        return doExecuteItemDependency(item, wfidFalse, scheduledWfid, false);
     return true;
 }
 

common/workunit/workflow.hpp

@@ -96,7 +96,7 @@ protected:
 
     // Iterate through dependencies and execute them
     bool doExecuteItemDependencies(IRuntimeWorkflowItem & item, unsigned scheduledWfid);
-    bool doExecuteItemDependency(IRuntimeWorkflowItem & item, unsigned dep, unsigned scheduledWfid);
+    bool doExecuteItemDependency(IRuntimeWorkflowItem & item, unsigned dep, unsigned scheduledWfid, bool alwaysEvaluate);
     // Execute an item (wrapper to deal with exceptions)
     void doExecuteItem(IRuntimeWorkflowItem & item, unsigned scheduledWfid);
     // Actually executes item: calls process->perform()

dali/base/dadfs.cpp

@@ -48,8 +48,8 @@
 #define SDS_SUB_LOCK_TIMEOUT (10000)
 #define SDS_TRANSACTION_RETRY (60000)
 
-#define DFSSERVER_THROTTLE_COUNT 20
-#define DFSSERVER_THROTTLE_TIME 1000
+#define DEFAULT_NUM_DFS_THREADS 30
+#define TIMEOUT_ON_CLOSEDOWN 120000 // On closedown, give up on trying to join a thread in CDaliDFSServer after two minutes
 
 #if _INTERNAL_EDITION == 1
 #ifndef _MSC_VER
@@ -8203,11 +8203,12 @@ bool removeClusterSpares(const char *clusterName, const char *type, SocketEndpoi
 
 
 
-class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements IDaliServer
+class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements IDaliServer, implements IExceptionHandler
 {  // Coven size
     
     bool stopped;
     unsigned defaultTimeout;
+    unsigned numThreads;
 
 public:
 
@@ -8218,6 +8219,8 @@ public:
     {
         stopped = true;
         defaultTimeout = INFINITE; // server uses default
+        numThreads = config->getPropInt("DFS/@numThreads", DEFAULT_NUM_DFS_THREADS);
+        PROGLOG("DFS Server: numThreads=%d", numThreads);
     }
 
     ~CDaliDFSServer()
@@ -8249,29 +8252,24 @@ public:
     int run()
     {
         ICoven &coven=queryCoven();
+        CMessageHandler<CDaliDFSServer> handler("CDaliDFSServer", this, &CDaliDFSServer::processMessage, this, numThreads, TIMEOUT_ON_CLOSEDOWN, INFINITE);
         CMessageBuffer mb;
         stopped = false;
-        unsigned throttlecount = 0;
-        unsigned last;
-        while (!stopped) {
-            try {
+        while (!stopped)
+        {
+            try
+            {
                 mb.clear();
-                if (coven.recv(mb,RANK_ALL,MPTAG_DFS_REQUEST,NULL)) {
-                    if (throttlecount&&(last-msTick()<10))
-                        throttlecount--;
-                    else
-                        throttlecount = DFSSERVER_THROTTLE_COUNT;
-                    processMessage(mb);
-                    if (throttlecount==0) {
-                        WARNLOG("Throttling CDaliDFSServer");
-                        Sleep(DFSSERVER_THROTTLE_TIME);
-                    }
-                    last = msTick();
+                if (coven.recv(mb,RANK_ALL,MPTAG_DFS_REQUEST,NULL))
+                {
+                    handler.handleMessage(mb);
+                    mb.clear(); // ^ has copied mb
                 }   
                 else
                     stopped = true;
             }
-            catch (IException *e) {
+            catch (IException *e)
+            {
                 EXCLOG(e, "CDaliDFSServer");
                 e->Release();
             }
@@ -8625,6 +8623,13 @@ public:
             return ret.append("UNKNOWN");
         }
     }
+    // IExceptionHandler impl.
+    virtual bool fireException(IException *e)
+    {
+        EXCLOG(e, "CDaliDFSServer exception");
+        e->Release();
+        return true;
+    }
 } *daliDFSServer = NULL;
 
 

dali/base/dasds.cpp

@@ -6191,7 +6191,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
     Owned<IRemoteConnection> conn = connect("/", 0, RTM_INTERNAL, INFINITE);
     initializeInternals(conn->queryRoot());
     conn.clear();
-    bool forceGroupUpdate = config.getPropBool("@forceGroupUpdate");
+    bool forceGroupUpdate = config.getPropBool("DFS/@forceGroupUpdate");
     StringBuffer response;
     initClusterGroups(forceGroupUpdate, response, oldEnvironment);
     if (response.length())

ecl/eclagent/eclagent.cpp

@@ -2218,7 +2218,7 @@ void EclAgentWorkflowMachine::doExecutePersistItem(IRuntimeWorkflowItem & item)
         agent.decachePersist(name.str());
     else
         agent.startPersist(name.str());
-    doExecuteItemDependency(item, item.queryPersistWfid(), wfid);
+    doExecuteItemDependency(item, item.queryPersistWfid(), wfid, true);
     if(!persist)
     {
         StringBuffer errmsg;

ecl/eclcc/eclcc.cpp

@@ -34,6 +34,7 @@
 #include "hqlcollect.hpp"
 #include "hqlrepository.hpp"
 #include "hqlerror.hpp"
+#include "hqlcerrors.hpp"
 
 #include "hqlgram.hpp"
 #include "hqltrans.ipp"
@@ -699,9 +700,12 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
         }
         catch (IException * e)
         {
-            StringBuffer exceptionText;
-            e->errorMessage(exceptionText);
-            errs->reportError(ERR_INTERNALEXCEPTION, exceptionText.toCharArray(), queryFullName, 1, 0, 0);
+            if (e->errorCode() != HQLERR_ErrorAlreadyReported)
+            {
+                StringBuffer exceptionText;
+                e->errorMessage(exceptionText);
+                errs->reportError(ERR_INTERNALEXCEPTION, exceptionText.toCharArray(), queryFullName, 1, 0, 0);
+            }
             e->Release();
         }
 

ecl/hql/hql.hpp

@@ -75,6 +75,7 @@ enum object_type
     ob_shared       = 0x0002,
     ob_import       = 0x0004,
     ob_member       = 0x0008,       // is a member of a module
+    ob_virtual      = 0x0010,
 
 //attributes returned from the repository to show the vcs status
     ob_sandbox      = 0x00010000,

ecl/hql/hqlatoms.cpp

@@ -276,7 +276,6 @@ _ATOM ownedAtom;
 _ATOM packedAtom;
 _ATOM parallelAtom;
 _ATOM parameterAtom;
-_ATOM _parameterScopeType_Atom;
 _ATOM partitionAtom;
 _ATOM partitionLeftAtom;
 _ATOM partitionRightAtom;
@@ -392,6 +391,7 @@ _ATOM userMatchFunctionAtom;
 _ATOM valueAtom;
 _ATOM versionAtom;
 _ATOM virtualAtom;
+_ATOM _virtualSeq_Atom;
 _ATOM volatileAtom;
 _ATOM warningAtom;
 _ATOM wholeAtom;
@@ -689,7 +689,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(packed);
     MAKEATOM(parallel);
     MAKEATOM(parameter);
-    MAKESYSATOM(parameterScopeType);
     MAKEATOM(partition);
     partitionLeftAtom = createLowerCaseAtom("partition left");
     partitionRightAtom = createLowerCaseAtom("partition right");
@@ -804,6 +803,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(value);
     MAKEATOM(version);
     MAKEATOM(virtual);
+    MAKESYSATOM(virtualSeq);
     MAKEATOM(volatile);
     MAKEATOM(warning);
     MAKEATOM(whole);

ecl/hql/hqlatoms.hpp

@@ -280,7 +280,6 @@ extern HQL_API _ATOM ownedAtom;
 extern HQL_API _ATOM packedAtom;
 extern HQL_API _ATOM parallelAtom;
 extern HQL_API _ATOM parameterAtom;
-extern HQL_API _ATOM _parameterScopeType_Atom;
 extern HQL_API _ATOM partitionAtom;
 extern HQL_API _ATOM partitionLeftAtom;
 extern HQL_API _ATOM partitionRightAtom;
@@ -397,6 +396,7 @@ extern HQL_API _ATOM userMatchFunctionAtom;
 extern HQL_API _ATOM valueAtom;
 extern HQL_API _ATOM versionAtom;
 extern HQL_API _ATOM virtualAtom;
+extern HQL_API _ATOM _virtualSeq_Atom;
 extern HQL_API _ATOM volatileAtom;
 extern HQL_API _ATOM warningAtom;
 extern HQL_API _ATOM wholeAtom;

ecl/hql/hqlattr.cpp

@@ -427,6 +427,8 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_merge_nomatch:
     case no_namedactual:
     case no_assertconstant:
+    case no_assertconcrete:
+    case no_delayedscope:
 
 //Code generator only - only created once code is being generated.
     case no_postinc:
@@ -564,8 +566,9 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_libraryselect:
     case no_bound_func:
     case no_purevirtual:
-    case no_internalvirtual:
+    case no_internalselect:
     case no_delayedselect:
+    case no_unboundselect:
     case no_libraryscope:
     case no_libraryscopeinstance:
     case no_libraryinput:
@@ -616,7 +619,7 @@ unsigned getOperatorMetaFlags(node_operator op)
 
     case no_unused6:
     case no_unused13: case no_unused14: case no_unused15:
-    case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused28: case no_unused29:
+    case no_unused23: case no_unused24: case no_unused25: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
@@ -1765,8 +1768,9 @@ bool isGroupedActivity(IHqlExpression * expr)
     case no_getgraphloopresult:
     case no_getresult:
     case no_rows:
-    case no_internalvirtual:
+    case no_internalselect:
     case no_delayedselect:
+    case no_unboundselect:
     case no_libraryselect:
     case no_purevirtual:
     case no_libraryinput:
@@ -2860,6 +2864,8 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
     case no_anon:
     case no_nofold:             // assume nothing - to stop subsequent optimizations
     case no_delayedselect:
+    case no_unboundselect:
+    case no_internalselect:
         info.setUnknown(RCMdisk);
         break;
     case no_parse:

ecl/hql/hqlerrors.hpp

@@ -420,6 +420,7 @@
 #define HQLERR_CannotBeGrouped      2389
 #define HQLERR_CannotAccessShared   2390
 #define ERR_PluginNoScripting       2391
+#define ERR_ZERO_SIZE_VIRTUAL       2392
 
 #define ERR_ASSERTION_FAILS         100000
 
@@ -462,6 +463,7 @@
 #define HQLERR_UnexpectedOperator               3125
 #define HQLERR_UnexpectedType                   3126
 #define HQLERR_PayloadMismatch                  3127
+#define HQLERR_MemberXContainsVirtualRef        3128
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contain an illegal cycle/recursive definition %s"
@@ -491,6 +493,7 @@
 #define HQLERR_IncompatibleTypesForField_Text   "Initializer for field %s has the wrong type"
 #define HQLWRN_CouldNotConstantFoldIf_Text      "Could not constant fold the condition on a IFBLOCK for a inline table"
 #define HQLERR_PayloadMismatch_Text             "Mismatched => in inline dictionary definition"
+#define HQLERR_MemberXContainsVirtualRef_Text   "Member %s contains virtual references but not supported as virtual"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hql/hqlexpr.hpp

@@ -119,7 +119,7 @@ enum
 
 // generally applicable start from the top down
     HEFunbound                  = 0x00000010,
-    HEFinternalVirtual          = 0x00000020,
+    HEFinternalSelect          = 0x00000020,
     HEFcontainsDatasetAliasLocally= 0x00000040,
   HEF____unused2____          = 0x00000080,
   HEF____unused3____          = 0x00000100,
@@ -154,7 +154,7 @@ enum
 
 //Combinations used for processing elsewhere:
 //Don't ever inherit the following...
-    HEFalwaysInherit            = HEFunbound|HEFinternalVirtual,
+    HEFalwaysInherit            = HEFunbound|HEFinternalSelect,
     HEFassigninheritFlags       = ~(HEFhousekeeping|HEFalwaysInherit),          // An assign inherits all but this list from the rhs value 
 
 //  HEFcontextDependent         = (HEFgraphDependent|HEFcontainsNlpText|HEFcontainsXmlText|HEFcontainsSkip|HEFcontainsCounter|HEFtransformDependent|HEFtranslated|HEFonFailDependent|HEFcontextDependentException|HEFthrowscalar|HEFthrowds),
@@ -166,7 +166,7 @@ enum
                                    HEFonFailDependent|HEFcontainsActiveDataset|HEFcontainsActiveNonSelector|HEFcontainsDataset|
                                    HEFtranslated|HEFgraphDependent|HEFcontainsNlpText|HEFcontainsXmlText|HEFtransformDependent|
                                    HEFcontainsSkip|HEFcontainsCounter|HEFassertkeyed|HEFcontextDependentException|HEFcontainsAlias|HEFcontainsAliasLocally|
-                                   HEFinternalVirtual|HEFcontainsThisNode|HEFcontainsDatasetAliasLocally),
+                                   HEFinternalSelect|HEFcontainsThisNode|HEFcontainsDatasetAliasLocally),
 
     HEFcontextDependentNoThrow  = (HEFcontextDependent & ~(HEFthrowscalar|HEFthrowds|HEFoldthrows)),
     HEFcontextDependentDataset  = (HEFcontextDependent & ~(HEFthrowscalar)),
@@ -348,9 +348,9 @@ enum _node_operator {
         no_chooseds,
         no_alias,
         no_datasetfromdictionary,
-    no_unused20,
-    no_unused21,
-    no_unused22,
+        no_delayedscope,
+        no_assertconcrete,
+        no_unboundselect,
     no_unused23,
     no_unused24,
         no_dataset_from_transform,
@@ -666,7 +666,7 @@ enum _node_operator {
         no_virtualscope,
         no_concretescope,
         no_purevirtual,
-        no_internalvirtual,
+        no_internalselect,
         no_delayedselect,
         no_pure,
         no_libraryscope,
@@ -997,6 +997,7 @@ enum
     LSFignoreBase   = 0x0002,
     LSFrequired     = 0x0004,
     LSFimport       = 0x0008,
+    LSFfromderived  = 0x0010,
 };
 inline unsigned makeLookupFlags(bool sharedOK, bool ignoreBase, bool required)
 {
@@ -1017,6 +1018,7 @@ interface IHqlScope : public IInterface
     virtual const char * queryFullName() const = 0;
     virtual ISourcePath * querySourcePath() const = 0;
     virtual bool hasBaseClass(IHqlExpression * searchBase) = 0;
+    virtual bool allBasesFullyBound() const = 0;
 
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols) = 0;
     virtual IHqlScope * queryConcreteScope() = 0;
@@ -1180,6 +1182,7 @@ interface IHqlNamedAnnotation : public IHqlAnnotation
     virtual bool isExported() const = 0;
     virtual bool isShared() const = 0;
     virtual bool isPublic() const = 0;
+    virtual bool isVirtual() const = 0;
     virtual int getStartLine() const = 0;
     virtual int getStartColumn() const = 0;
     virtual void setRepositoryFlags(unsigned _flags) = 0;  // To preserve flags like ob_locked, ob_sandbox, etc.
@@ -1336,7 +1339,7 @@ inline bool isNull(IHqlExpression * expr)       { return expr->getOperator() ==
 inline bool isNullAction(IHqlExpression * expr) { return isNull(expr) && expr->isAction(); }
 inline bool isFail(IHqlExpression * expr)       { return expr->getOperator() == no_fail; }
 
-extern HQL_API IHqlExpression * createDelayedReference(node_operator op, IHqlExpression * moduleMarker, IHqlExpression * attr, bool ignoreBase);
+extern HQL_API IHqlExpression * createDelayedReference(node_operator op, IHqlExpression * moduleMarker, IHqlExpression * attr, unsigned lookupFlags, HqlLookupContext & ctx);
 extern HQL_API IHqlExpression * createLibraryInstance(IHqlExpression * scopeFunction, HqlExprArray &operands);
 
 extern HQL_API IHqlExpression* createValue(node_operator op, ITypeInfo *type, HqlExprArray& operands);
@@ -1398,10 +1401,11 @@ extern HQL_API IHqlScope *createScope(node_operator op);
 extern HQL_API IHqlScope *createScope(IHqlScope * scope);
 extern HQL_API IHqlScope *createPrivateScope();
 extern HQL_API IHqlScope *createPrivateScope(IHqlScope * scope);
+extern HQL_API IHqlExpression * createDelayedScope(IHqlExpression * expr);
+extern HQL_API IHqlExpression * createDelayedScope(HqlExprArray &newkids);
 extern HQL_API IHqlRemoteScope *createRemoteScope(_ATOM name, const char * fullName, IEclRepositoryCallback *ds, IProperties* props, IFileContents * _text, bool lazy, IEclSource * eclSource);
 extern HQL_API IHqlExpression * populateScopeAndClose(IHqlScope * scope, const HqlExprArray & children, const HqlExprArray & symbols);
 
-extern HQL_API IHqlScope* createContextScope();
 extern HQL_API IHqlExpression* createTemplateFunctionContext(IHqlExpression* body, IHqlScope* helperScope);
 extern HQL_API IHqlExpression* createFieldMap(IHqlExpression*, IHqlExpression*);
 extern HQL_API IHqlExpression * createNullDataset(IHqlExpression * ds);
@@ -1581,6 +1585,7 @@ extern HQL_API IHqlExpression * queryRecordProperty(IHqlExpression * record, _AT
 extern HQL_API bool isExported(IHqlExpression * expr);
 extern HQL_API bool isShared(IHqlExpression * expr);
 extern HQL_API bool isImport(IHqlExpression * expr);
+extern HQL_API bool isVirtualSymbol(IHqlExpression * expr);
 
 extern HQL_API IECLError * queryAnnotatedWarning(const IHqlExpression * expr);
 
@@ -1669,7 +1674,7 @@ inline bool isContextDependentExceptGraph(IHqlExpression * expr)
 inline bool isGraphDependent(IHqlExpression * expr)     { return (expr->getInfoFlags() & HEFgraphDependent) != 0; }
 inline bool containsTranslated(IHqlExpression * expr)   { return (expr->getInfoFlags() & (HEFtranslated)) != 0; }
 inline bool containsSideEffects(IHqlExpression * expr)  { return (expr->getInfoFlags() & (HEFaction|HEFthrowscalar|HEFthrowds)) != 0; }
-inline bool containsInternalVirtual(IHqlExpression * expr)  { return (expr->getInfoFlags() & (HEFinternalVirtual)) != 0; }
+inline bool containsInternalSelect(IHqlExpression * expr)  { return (expr->getInfoFlags() & (HEFinternalSelect)) != 0; }
 inline bool containsThisNode(IHqlExpression * expr)     { return (expr->getInfoFlags() & (HEFcontainsThisNode)) != 0; }
 
 inline bool containsWorkflow(IHqlExpression * expr)     { return (expr->getInfoFlags2() & (HEF2workflow)) != 0; }

ecl/hql/hqlexpr.ipp

@@ -541,6 +541,7 @@ public:
     virtual bool isExported() const { return (symbolFlags&ob_exported)!=0; };
     virtual bool isShared() const { return (symbolFlags&ob_shared)!=0; };
     virtual bool isPublic() const { return (symbolFlags&(ob_shared|ob_exported))!=0; };
+    virtual bool isVirtual() const { return (symbolFlags & ob_virtual)!=0; };
     virtual void setRepositoryFlags(unsigned _flags) { symbolFlags |= (_flags & ob_registryflags); }
 
 protected:
@@ -945,6 +946,7 @@ public:
     virtual const char * queryFullName() const  { throwUnexpected(); }
     virtual ISourcePath * querySourcePath() const   { throwUnexpected(); }
     virtual bool hasBaseClass(IHqlExpression * searchBase);
+    virtual bool allBasesFullyBound() const { return false; } // Assume the worst
 
     virtual void ensureSymbolsDefined(HqlLookupContext & ctx) { }
 
@@ -955,7 +957,7 @@ public:
 
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols) { throwUnexpected(); }
 
-    virtual IHqlScope * queryConcreteScope() { return this; }
+    virtual IHqlScope * queryConcreteScope();
     virtual IHqlScope * queryResolvedScope(HqlLookupContext * context) { return this; }
 
     virtual IHqlExpression * queryExpression() { return this; }
@@ -974,8 +976,6 @@ protected:
     StringAttr fullName;                //Fully qualified name of this nested module   E.g.: PARENT.CHILD.GRANDCHILD
     SymbolTable symbols;
 
-    IHqlDataset *lookupDataset(_ATOM name);
-
     virtual bool equals(const IHqlExpression & other) const;
 
 public:
@@ -988,7 +988,7 @@ public:
     IHqlScope * cloneAndClose(HqlExprArray & children, HqlExprArray & symbols);
 
 //interface IHqlExpression
-    virtual IHqlScope *queryScope() { return this; };
+    virtual IHqlScope *queryScope() { return this; }
     virtual IHqlExpression *clone(HqlExprArray &newkids);
     virtual void sethash();
 
@@ -1014,7 +1014,6 @@ public:
     virtual void    getSymbols(HqlExprArray& exprs) const;
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols) { throwUnexpected(); }
     virtual bool hasBaseClass(IHqlExpression * searchBase);
-    virtual IHqlScope * queryConcreteScope()    { return this; }
     virtual IHqlScope * queryResolvedScope(HqlLookupContext * context)  { return this; }
     virtual IFileContents * queryDefinitionText() const { return text; }
 
@@ -1099,6 +1098,8 @@ public:
     virtual void defineSymbol(IHqlExpression * expr);
     using CHqlScope::defineSymbol;
 
+    virtual bool allBasesFullyBound() const { return true; }
+    virtual IHqlScope * queryConcreteScope()    { return this; }
     virtual IFileContents * queryDefinitionText() const;
 
     virtual void getSymbols(HqlExprArray& exprs) const;
@@ -1126,7 +1127,8 @@ public:
     virtual void sethash();
 
 //interface IHqlScope
-
+    virtual bool allBasesFullyBound() const { return true; }
+    virtual IHqlScope * queryConcreteScope()    { return this; }
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols);
 };
 
@@ -1136,13 +1138,16 @@ protected:
     Owned<IHqlScope> concrete;
     bool isAbstract;
     bool complete;
-    bool isVirtual;
+    bool containsVirtual;
+    bool allVirtual;
     bool fullyBoundBase;
 
 protected:
     virtual bool equals(const IHqlExpression & other) const;
     IHqlScope * deriveConcreteScope();
-    void ensureVirtual();
+    IHqlExpression * lookupBaseSymbol(IHqlExpression * & definitionModule, _ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
+    void resolveUnboundSymbols();
+    void ensureVirtualSeq();
 
 public:
     CHqlVirtualScope(_ATOM _name, const char * _fullName);
@@ -1154,12 +1159,13 @@ public:
     virtual IHqlExpression *closeExpr();
     virtual void defineSymbol(IHqlExpression * expr);
     virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
+    virtual bool queryForceSymbolVirtual(_ATOM searchName, HqlLookupContext & ctx);
     virtual void sethash();
 
 //interface IHqlScope
-
+    virtual bool allBasesFullyBound() const { return fullyBoundBase; }
     virtual _ATOM   queryName() const {return name;}
-    virtual IHqlScope * queryConcreteScope() { return isVirtual ? concrete.get() : this; }
+    virtual IHqlScope * queryConcreteScope() { return containsVirtual ? concrete.get() : this; }
 };
 
 
@@ -1172,8 +1178,10 @@ public:
 
     virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
     virtual void ensureSymbolsDefined(HqlLookupContext & ctx);
+    virtual bool allBasesFullyBound() const;
     virtual bool isImplicit() const;
     virtual bool isPlugin() const;
+    virtual IHqlScope * queryConcreteScope()    { return this; }
 
 protected:
     CriticalSection cs;
@@ -1181,37 +1189,21 @@ protected:
     bool mergedAll;
 };
 
-/*
-Used for syntax checking an attribute.  It allows specific attributes within a module to be overridden,
-so that syntax check can work as if it is checking a particular attribute
-I suspect it should be done a different way...
-*/
-
-class CHqlSyntaxCheckScope : public CHqlScope
-{
-private:
-    IHqlScope *parent;
-    SymbolTable redefine;
-
-public:
-    CHqlSyntaxCheckScope(IHqlScope *parent, IEclRepository *_ds, const char *attribute, bool clearImportedModule);
-
-    virtual void defineSymbol(_ATOM name, _ATOM _moduleName, IHqlExpression *value, bool exported, bool shared, unsigned symbolFlags, IFileContents *, int lineno, int column, int _startpos, int _bodypos, int _endpos);
-    virtual void defineSymbol(_ATOM name, _ATOM _moduleName, IHqlExpression *value, bool exported, bool shared, unsigned symbolFlags);
-    virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
-};
-
+//Used in the parser to allow symbols to be looked up in a list of multiple independent scopes.
 class CHqlMultiParentScope : public CHqlScope
 {
 protected:
     CopyArray parents;
 
 public:
-    CHqlMultiParentScope(_ATOM, IHqlScope *parent1, ...);
+    CHqlMultiParentScope(_ATOM, ...);
 
     virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
+    virtual IHqlScope * queryConcreteScope() { return this; }
+    virtual bool allBasesFullyBound() const { return true; }
 };
 
+//MORE: I'm not 100% sure why this is different from a CLocalScope... it should be merged
 class CHqlContextScope : public CHqlScope
 {
 protected:
@@ -1228,6 +1220,9 @@ public:
     virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx)
     {  return defined.getLinkedValue(searchName); }
 
+    virtual IHqlScope * queryConcreteScope()    { return this; }
+    virtual bool allBasesFullyBound() const { return false; }
+
 };
 
 class CHqlTemplateFunctionContext : public CHqlExpressionWithType
@@ -1368,16 +1363,20 @@ public:
     virtual bool equals(const IHqlExpression & other) const;
     virtual StringBuffer &toString(StringBuffer &ret);
 
-//IHqlDataset
+//IHqlScope
     virtual void defineSymbol(IHqlExpression * expr)            { throwUnexpected(); }
     virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
 
     virtual void getSymbols(HqlExprArray& exprs) const          { typeScope->getSymbols(exprs); }
+    virtual IHqlScope * queryConcreteScope() { return NULL; }
+    virtual bool allBasesFullyBound() const { return false; }
 
     virtual IHqlExpression * clone(HqlExprArray & children);
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols);
 };
 
+
+//I'm not convinced that this should be derived from CHqlScope.
 class CHqlLibraryInstance : public CHqlScope
 {
 protected:
@@ -1393,16 +1392,64 @@ public:
     virtual bool equals(const IHqlExpression & other) const;
     virtual IHqlExpression * clone(HqlExprArray & children);
 
-//IHqlDataset
+//IHqlScope
     virtual void defineSymbol(IHqlExpression * expr)            { throwUnexpected(); }
     virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
     virtual bool hasBaseClass(IHqlExpression * searchBase)      { return libraryScope->hasBaseClass(searchBase); }
+    virtual bool allBasesFullyBound() const { return false; }
+    virtual IHqlScope * queryConcreteScope()    { return this; }
 
     virtual void getSymbols(HqlExprArray& exprs) const          { libraryScope->getSymbols(exprs); }
 
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols);
 };
 
+class HQL_API CHqlDelayedScope : public CHqlExpressionWithTables, implements IHqlScope
+{
+public:
+    CHqlDelayedScope(HqlExprArray & _ownedOperands);
+    IMPLEMENT_IINTERFACE_USING(CHqlExpression)
+
+//IHqlExpression
+    virtual bool assignableFrom(ITypeInfo * source);
+    virtual bool equals(const IHqlExpression & other) const;
+    virtual IHqlExpression * clone(HqlExprArray & children);
+    virtual ITypeInfo *queryType() const;
+    virtual ITypeInfo *getType();
+    virtual IHqlScope *queryScope() { return this; }
+
+//IHqlScope
+    virtual IHqlExpression * queryExpression() { return this; }
+    virtual IHqlExpression *lookupSymbol(_ATOM searchName, unsigned lookupFlags, HqlLookupContext & ctx);
+
+    virtual void    getSymbols(HqlExprArray& exprs) const;
+    virtual _ATOM   queryName() const { return NULL; }
+    virtual const char * queryFullName() const { return NULL; }
+    virtual ISourcePath * querySourcePath() const { return NULL; }
+    virtual bool hasBaseClass(IHqlExpression * searchBase);
+    virtual bool allBasesFullyBound() const { return false; }
+
+    virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols);
+    virtual IHqlScope * queryConcreteScope();
+    virtual IHqlScope * queryResolvedScope(HqlLookupContext * context);
+    virtual void ensureSymbolsDefined(HqlLookupContext & ctx);
+
+    virtual bool isImplicit() const { return false; }
+    virtual bool isPlugin() const { return false; }
+    virtual int getPropInt(_ATOM, int dft) const { return dft; }
+    virtual bool getProp(_ATOM, StringBuffer &) const { return false; }
+
+//IHqlCreateScope
+    virtual void defineSymbol(_ATOM name, _ATOM moduleName, IHqlExpression *value, bool isExported, bool isShared, unsigned flags, IFileContents *fc, int lineno, int column, int _startpos, int _bodypos, int _endpos) { throwUnexpected(); }
+    virtual void defineSymbol(_ATOM name, _ATOM moduleName, IHqlExpression *value, bool isExported, bool isShared, unsigned flags) { throwUnexpected(); }
+    virtual void defineSymbol(IHqlExpression * expr) { throwUnexpected(); }
+    virtual void removeSymbol(_ATOM name) { throwUnexpected(); }
+
+protected:
+    ITypeInfo * type;
+    IHqlScope * typeScope;
+};
+
 class CHqlVariable : public CHqlExpressionWithType
 {
 protected:
@@ -1625,8 +1672,8 @@ public:
 
 //IHqlExpression
     virtual IHqlExpression *queryFunctionDefinition() const;
-    virtual IHqlScope *queryScope() { return scope; };
-    virtual IHqlSimpleScope *querySimpleScope() { return this; };
+    virtual IHqlScope *queryScope() { return scope; }
+    virtual IHqlSimpleScope *querySimpleScope() { return this; }
     virtual bool equals(const IHqlExpression & other) const;
     virtual IHqlExpression *clone(HqlExprArray &newkids);
     virtual void sethash();
@@ -1705,7 +1752,7 @@ public:
     ~CHqlEnumType();
 
 //IHqlExpression
-    virtual IHqlScope *queryScope() { return scope; };
+    virtual IHqlScope *queryScope() { return scope; }
     virtual bool equals(const IHqlExpression & other) const;
     virtual IHqlExpression *clone(HqlExprArray &newkids);
     virtual void sethash();

ecl/hql/hqlfold.cpp

@@ -5490,8 +5490,9 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_getgraphloopresult:
     case no_getresult:
     case no_rows:
-    case no_internalvirtual:
+    case no_internalselect:
     case no_delayedselect:
+    case no_unboundselect:
     case no_libraryselect:
     case no_purevirtual:
     case no_libraryinput:

ecl/hql/hqlgram.hpp

@@ -804,7 +804,7 @@ protected:
     void expandScopeEntries(HqlExprArrayArray & branches, IHqlExpression * scope);
     void processIfScope(const attribute & errpos, IHqlExpression * cond, IHqlExpression * trueScope, IHqlExpression * falseScope);
 
-    unsigned extraLookupFlags(IHqlScope * scope);
+    unsigned getExtraLookupFlags(IHqlScope * scope);
 
     void appendTransformOption(IHqlExpression * expr) 
     { 

ecl/hql/hqlgram.y

@@ -1283,6 +1283,8 @@ scopeFlag
     : EXPORT            {   $$.setInt(EXPORT_FLAG); $$.setPosition($1); }
     | SHARED            {   $$.setInt(SHARED_FLAG); $$.setPosition($1); }
     | LOCAL             {   $$.setInt(0); $$.setPosition($1); }
+    | EXPORT VIRTUAL    {   $$.setInt(EXPORT_FLAG|VIRTUAL_FLAG); $$.setPosition($1); }
+    | SHARED VIRTUAL    {   $$.setInt(SHARED_FLAG|VIRTUAL_FLAG); $$.setPosition($1); }
     ;
 
 // scopeflags needs to be explicitly included, rather than using an optScopeFlags production, otherwise you get shift reduce errors - since it is the first item on a line.
@@ -6723,6 +6725,13 @@ abstractModule
                             Owned<ITypeInfo> retType = $1.getType();
                             $$.setExpr(parser->leaveLamdaExpression($5), $7);
                         }
+    | IF '(' booleanExpr ',' abstractModule ',' abstractModule ')'
+                        {
+                            OwnedHqlExpr trueExpr = $5.getExpr();
+                            OwnedITypeInfo scopeType = trueExpr->getType();  // actually needs to be the common base class.
+                            OwnedHqlExpr module = createValue(no_if, scopeType.getClear(), $3.getExpr(), LINK(trueExpr), $7.getExpr());
+                            $$.setExpr(createDelayedScope(module.getClear()), $1);
+                        }
     ;
 
 scopeFunctionWithParameters
@@ -8662,9 +8671,6 @@ simpleDataSet
     | SORTED '(' startSortOrder dataSet ')' endSortOrder
                         {
                             OwnedHqlExpr dataset = $4.getExpr();
-                            if (!isKey(dataset))
-                                parser->reportError(ERR_EXPECTED_INDEX, $1, "SORTED(dataset) with no order, can only be used on INDEXes");
-
                             HqlExprArray args, sorted;
                             IHqlExpression * record = dataset->queryRecord();
                             unwindRecordAsSelects(sorted, record, dataset->queryNormalizedSelector());

ecl/hql/hqlgram2.cpp

@@ -2672,6 +2672,7 @@ public:
     virtual const char * queryFullName() const { PSEUDO_UNIMPLEMENTED; return NULL; }
     virtual ISourcePath * querySourcePath() const { PSEUDO_UNIMPLEMENTED; return NULL; }
     virtual bool hasBaseClass(IHqlExpression * searchBase) { return false; }
+    virtual bool allBasesFullyBound() const { return true; }
 
     virtual void ensureSymbolsDefined(HqlLookupContext & ctx) { }
 
@@ -2727,7 +2728,7 @@ void HqlGram::processForwardModuleDefinition(const attribute & errpos)
         return;
 
     IHqlExpression * scopeExpr = queryExpression(scope);
-    if (scopeExpr->hasProperty(virtualAtom))
+    if (scopeExpr->hasProperty(_virtualSeq_Atom))
     {
         reportError(ERR_NO_FORWARD_VIRTUAL, errpos, "Cannot use FORWARD in combination with a VIRTUAL module ");
         return;
@@ -2967,11 +2968,7 @@ bool HqlGram::checkValidBaseModule(const attribute & attr, SharedHqlExpr & expr)
         return true;
     }
 
-    if (op == no_param)
-        reportError(ERR_EXPECTED_MODULE, attr, "Cannot derive a module from a parameter");
-    else
-        reportError(ERR_EXPECTED_MODULE, attr, "Expected the name of a module definition");
-    return false;
+    return true;
 }
 
 
@@ -3018,15 +3015,10 @@ IHqlExpression * HqlGram::implementInterfaceFromModule(const attribute & modpos,
     LinkedHqlExpr projectInterface = _projectInterface;
     if (projectInterface->getOperator() == no_funcdef)
         projectInterface.set(projectInterface->queryChild(0));
-    IHqlScope * implementScope = implementModule->queryScope()->queryConcreteScope();
-    if (!implementScope)
-    {
-        if (projectInterface->queryScope()->queryConcreteScope())
-            reportError(ERR_ABSTRACT_MODULE, modpos, "PROJECT(interface, module) - module is abstract.  (Parameters round the wrong way?)");
-        else
-            reportError(ERR_ABSTRACT_MODULE, modpos, "Cannot PROJECT an abstract module to a new interface");
-        return LINK(projectInterface);
-    }
+
+    OwnedHqlExpr concreteModule = checkCreateConcreteModule(lookupCtx.errs, implementModule, modpos.pos);
+    IHqlScope * scope = concreteModule->queryScope();
+    assertex(scope);
 
     Owned<IHqlScope> newScope = createVirtualScope();
     IHqlExpression * newScopeExpr = queryExpression(newScope);
@@ -3058,7 +3050,7 @@ IHqlExpression * HqlGram::implementInterfaceFromModule(const attribute & modpos,
         {
             IHqlExpression & baseSym = syms.item(iSym);
             _ATOM name = baseSym.queryName();
-            OwnedHqlExpr match  = implementScope->lookupSymbol(name, LSFpublic, lookupCtx);
+            OwnedHqlExpr match  = scope->lookupSymbol(name, LSFpublic, lookupCtx);
             if (match)
             {
                 HqlExprArray parameters;
@@ -3172,7 +3164,7 @@ IHqlExpression *HqlGram::lookupSymbol(IHqlScope * scope, _ATOM searchName)
     return scope->lookupSymbol(searchName, LSFpublic, lookupCtx);
 }
 
-unsigned HqlGram::extraLookupFlags(IHqlScope * scope)
+unsigned HqlGram::getExtraLookupFlags(IHqlScope * scope)
 {
     if (scope == containerScope)
         return LSFsharedOK;
@@ -3339,7 +3331,7 @@ IHqlExpression *HqlGram::lookupSymbol(_ATOM searchName, const attribute& errpos)
         ForEachItemIn(idx2, defaultScopes)
         {
             IHqlScope &plugin = defaultScopes.item(idx2);
-            IHqlExpression *ret = plugin.lookupSymbol(searchName, LSFpublic|extraLookupFlags(&plugin), lookupCtx);
+            IHqlExpression *ret = plugin.lookupSymbol(searchName, LSFpublic|getExtraLookupFlags(&plugin), lookupCtx);
             if (ret)
             {
                 recordLookupInTemplateContext(searchName, ret, templateScope);
@@ -8645,8 +8637,12 @@ bool HqlGram::isVirtualFunction(DefineIdSt * defineid, const attribute & errpos)
             reportError(ERR_BAD_VIRTUAL, errpos, "VIRTUAL can only be used inside a local module definition");
             return false;
         }
-        if (scope && queryExpression(scope)->hasProperty(virtualAtom))
-            return true;
+        if (scope)
+        {
+            IHqlExpression * scopeExpr = scope->queryExpression();
+            if (scopeExpr->hasProperty(interfaceAtom) || scopeExpr->hasProperty(virtualAtom))
+                return true;
+        }
     }
 
     if (defineid->scope & VIRTUAL_FLAG)
@@ -8655,26 +8651,28 @@ bool HqlGram::isVirtualFunction(DefineIdSt * defineid, const attribute & errpos)
 }
 
 //Allow the types to be grouped by different expressions, and sorted by different fields.
-static bool isEquivalentType(ITypeInfo * l, ITypeInfo * r)
+static bool isEquivalentType(ITypeInfo * derivedType, ITypeInfo * baseType)
 {
     loop
     {
-        if (isSameUnqualifiedType(l, r))
+        if (isSameUnqualifiedType(derivedType, baseType))
             return true;
-        if (l->getTypeCode() != r->getTypeCode())
+        if (derivedType->getTypeCode() != baseType->getTypeCode())
             return false;
-        switch (l->getTypeCode())
+        switch (derivedType->getTypeCode())
         {
         case type_table:
         case type_groupedtable:
         case type_row:
             {
-                l = l->queryChildType();
-                r = r->queryChildType();
-                if (!l || !r)
+                derivedType = derivedType->queryChildType();
+                baseType = baseType->queryChildType();
+                if (!derivedType || !baseType)
                     return false;
                 break;
             }
+        case type_scope:
+            return baseType->assignableFrom(derivedType);
         default:
             return false;
         }
@@ -8912,6 +8910,18 @@ void HqlGram::defineSymbolInScope(IHqlScope * scope, DefineIdSt * defineid, IHql
     if (scopeExpr && scopeExpr->getOperator() == no_virtualscope)
         symbolFlags |= ob_member;
 
+    if (defineid->scope & VIRTUAL_FLAG)
+    {
+        symbolFlags |= ob_virtual;
+        ITypeInfo * type = expr->queryType();
+        if (type && type->isScalar() && type->getSize() == 0)
+        {
+            StringBuffer typeText;
+            type->getECLType(typeText);
+            reportError(ERR_ZERO_SIZE_VIRTUAL, idattr, "A VIRTUAL with zero length type %s makes no sense", typeText.str());
+        }
+    }
+
     HqlExprCopyArray activeParameters;
     gatherActiveParameters(activeParameters);
 
@@ -8965,6 +8975,7 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
     else
     {
         expr.setown(createPureVirtual(type));
+        defineid->scope |= VIRTUAL_FLAG;
 
         if (!(defineid->scope & (EXPORT_FLAG|SHARED_FLAG)))
             reportError(ERR_SHOULD_BE_EXPORTED, nameattr, "Pure definitions should be exported or shared");
@@ -9041,6 +9052,12 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
         OwnedHqlExpr anyMatch = localScope->lookupSymbol(name, LSFsharedOK, lookupCtx);
         OwnedHqlExpr localMatch  = localScope->lookupSymbol(name, LSFsharedOK|LSFignoreBase, lookupCtx);
 
+        if (localScopeExpr->hasProperty(virtualAtom) || localScopeExpr->hasProperty(interfaceAtom))
+        {
+            if (canBeVirtual(expr))
+                defineid->scope |= VIRTUAL_FLAG;
+        }
+
         if (!(defineid->scope & (EXPORT_FLAG|SHARED_FLAG)))
         {
             if (anyMatch && !localMatch)
@@ -9058,13 +9075,16 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
         }
         else
         {
-            if (anyMatch && !localScopeExpr->hasProperty(virtualAtom))
+            if (anyMatch && !localScopeExpr->hasProperty(_virtualSeq_Atom))
             {
                 //Not quite right - it is a problem if the place it is defined in isn't virtual
                 reportError(ERR_CANNOT_REDEFINE, nameattr, "Cannot redefine definition %s from a non-virtual MODULE", name->str());
             }
             else if (anyMatch && !localMatch)
             {
+                if (isVirtualSymbol(anyMatch))
+                    defineid->scope |= VIRTUAL_FLAG;
+
                 ITypeInfo * matchType = stripFunctionType(anyMatch->queryType());
                 //check the parameters and return type (if specified) are compatible, promote expression return type to same
                 if (type)
@@ -9337,55 +9357,66 @@ void HqlGram::processIfScope(const attribute & errpos, IHqlExpression * cond, IH
 void HqlGram::cloneInheritedAttributes(IHqlScope * scope, const attribute & errpos)
 {
     IHqlExpression * scopeExpr = queryExpression(scope);
-    AtomArray derived;
-    IHqlExpression * virtualAttr = scopeExpr->queryProperty(virtualAtom);
+    AtomArray inherited;
+    IHqlExpression * virtualSeqAttr = scopeExpr->queryProperty(_virtualSeq_Atom);
     ForEachChild(i, scopeExpr)
     {
-        LinkedHqlExpr cur = scopeExpr->queryChild(i);
-        IHqlScope * base = cur->queryScope();
-        if (cur->getOperator() == no_param)
+        IHqlExpression * cur = scopeExpr->queryChild(i);
+        IHqlScope * curBase = cur->queryScope();
+        if (curBase)
         {
-            cur.setown(base->lookupSymbol(_parameterScopeType_Atom, LSFpublic, lookupCtx));
-            base = cur->queryScope();
-        }
-        if (base)
-        {
-            IHqlExpression * baseVirtualAttr = cur->queryProperty(virtualAtom);
-            IHqlScope * concreteBase = base->queryConcreteScope();
+            IHqlExpression * baseVirtualAttr = cur->queryProperty(_virtualSeq_Atom);
             bool baseIsLibrary = cur->getOperator() == no_libraryscopeinstance;
 
+            //Find all the symbols exported by this base module
             HqlExprArray syms;
-            base->getSymbols(syms);
+            curBase->getSymbols(syms);
             syms.sort(compareSymbolsByName);
+
             ForEachItemIn(iSym, syms)
             {
-                IHqlExpression & baseSym = syms.item(iSym);
-                _ATOM name = baseSym.queryName();
+                _ATOM name = syms.item(iSym).queryName();
+                OwnedHqlExpr baseSym = curBase->lookupSymbol(name, LSFsharedOK|LSFfromderived, lookupCtx);
                 OwnedHqlExpr match  = scope->lookupSymbol(name, LSFsharedOK|LSFignoreBase, lookupCtx);
-                LinkedHqlExpr mapped = &baseSym;
-                if (baseIsLibrary)
-                    mapped.setown(concreteBase->lookupSymbol(name, LSFsharedOK, lookupCtx));        // creates a no_libraryselect
-                else if (baseVirtualAttr)
-                    mapped.setown(quickFullReplaceExpression(&baseSym, baseVirtualAttr, virtualAttr));
+
+                LinkedHqlExpr mapped = baseSym;
+                //Replace any references to the base module attribute with this new module.
+                if (baseVirtualAttr)
+                    mapped.setown(quickFullReplaceExpression(mapped, baseVirtualAttr, virtualSeqAttr));
+
                 if (match)
                 {
-                    if (derived.contains(*name))
+                    if (inherited.contains(*name) && (match->getOperator() != no_purevirtual))
                     {
+                        //Inheriting the definition from more than one base module => check they are compatible.
                         //Ignore differences in the named symbol.  Should think about setting start/end to 0.  What would it break?
                         if (mapped->queryBody() != match->queryBody())
+                        {
+                            //MORE: Could allow it to be ambiguous (by creating a no_purevirtual), but better to require immediate resolution
                             reportError(ERR_AMBIGUOUS_DEF, errpos, "Definition %s must be specified, it has different definitions in base modules", name->str());
+                        }
                     }
                 }
                 else
                 {
+                    //If this base class is unbound then we need a different kind of delayed reference..
+                    if (!curBase->allBasesFullyBound())
+                    {
+                        assertex(virtualSeqAttr || baseIsLibrary);
+                        if (virtualSeqAttr)
+                        {
+                            mapped.setown(createDelayedReference(no_unboundselect, virtualSeqAttr, mapped, LSFsharedOK|LSFfromderived, lookupCtx));
+                        }
+                    }
+
                     scope->defineSymbol(mapped.getClear());
-                    derived.append(*name);
+                    inherited.append(*name);
                 }
             }
         }
     }
 
-    if (virtualAttr)
+    if (virtualSeqAttr)
         scopeExpr->addOperand(errpos.pos.createLocationAttr());
 }
 
@@ -9523,7 +9554,9 @@ IHqlExpression * HqlGram::createLibraryInstance(const attribute & errpos, IHqlEx
                 newValue.setown(createPureVirtual(ds->queryType()));
                 needToMapOutputs = true;
             }
-            newSymbols.append(*cur.cloneAllAnnotations(newValue));
+            OwnedHqlExpr newSymbol = cur.cloneAllAnnotations(newValue);
+            assertex(isVirtualSymbol(newSymbol));
+            newSymbols.append(*newSymbol.getClear());
         }
     }
     unwindChildren(args, body);
@@ -9591,7 +9624,7 @@ IHqlExpression * HqlGram::createEvaluateOutputModule(const attribute & errpos, I
 
 IHqlExpression * HqlGram::createStoredModule(const attribute & errpos, IHqlExpression * scopeExpr)
 {
-    if (!scopeExpr->queryProperty(virtualAtom))
+    if (!scopeExpr->queryProperty(_virtualSeq_Atom))
         reportError(ERR_NOT_INTERFACE, errpos, "Argument must be an interface or virtual module");
     return ::createStoredModule(scopeExpr);
 }

ecl/hql/hqlir.cpp

@@ -269,9 +269,9 @@ const char * getOperatorIRText(node_operator op)
     EXPAND_CASE(no,chooseds);
     EXPAND_CASE(no,alias);
     EXPAND_CASE(no,datasetfromdictionary);
-    EXPAND_CASE(no,unused20);
-    EXPAND_CASE(no,unused21);
-    EXPAND_CASE(no,unused22);
+    EXPAND_CASE(no,delayedscope);
+    EXPAND_CASE(no,assertconcrete);
+    EXPAND_CASE(no,unboundselect);
     EXPAND_CASE(no,unused23);
     EXPAND_CASE(no,unused24);
     EXPAND_CASE(no,dataset_from_transform);
@@ -587,7 +587,7 @@ const char * getOperatorIRText(node_operator op)
     EXPAND_CASE(no,virtualscope);
     EXPAND_CASE(no,concretescope);
     EXPAND_CASE(no,purevirtual);
-    EXPAND_CASE(no,internalvirtual);
+    EXPAND_CASE(no,internalselect);
     EXPAND_CASE(no,delayedselect);
     EXPAND_CASE(no,pure);
     EXPAND_CASE(no,libraryscope);
@@ -743,6 +743,7 @@ inline type_t getRequiredTypeCode(node_operator op)
     case no_attr:
     case no_attr_expr:
     case no_attr_link:
+    case no_service:
         return type_null;
 
     //These must never have the type
@@ -823,6 +824,7 @@ public:
     _ATOM name;
     unsigned __int64 sequence;
     IdArray args;
+    IdArray special;
     IdArray comment;
 };
 
@@ -1276,9 +1278,11 @@ public:
         case annotate_symbol:
             line.append("symbol ").append(info.name);
             if (info.value & ob_exported)
-                line.append("exported");
+                line.append(" exported");
             else if (info.value & ob_shared)
-                line.append("shared");
+                line.append(" shared");
+            if (info.value & ob_virtual)
+                line.append(" virtual");
             break;
         case annotate_location:
             line.append("location '").append(info.name);
@@ -1439,6 +1443,17 @@ protected:
             }
             line.append(")");
         }
+        if (info.special.ordinality())
+        {
+            line.append("[");
+            ForEachItemIn(i, info.special)
+            {
+                if (i)
+                    line.append(",");
+                appendId(info.special.item(i));
+            }
+            line.append("]");
+        }
         type_t tc = getRequiredTypeCode(op);
         if (tc == type_none)
         {
@@ -1852,6 +1867,7 @@ id_t ExpressionIRPlayer::doProcessType(ITypeInfo * type)
                 return target->addCompoundType(tc, info);
             }
         case type_function:
+        case type_feature:
             return target->addUnknownType(tc);
         case type_none:
         case type_ifblock:
@@ -1860,7 +1876,6 @@ id_t ExpressionIRPlayer::doProcessType(ITypeInfo * type)
         case type_pointer:
         case type_class:
         case type_array:
-        case type_feature:
             throwUnexpected();
             break;
         default:
@@ -1953,6 +1968,25 @@ id_t ExpressionIRPlayer::doProcessExpr(IHqlExpression * expr)
     }
 #endif
 
+    switch (op)
+    {
+    case no_externalcall:
+        info.special.append(processExpr(expr->queryExternalDefinition()));
+        break;
+    case no_call:
+        info.special.append(processExpr(expr->queryBody()->queryFunctionDefinition()));
+        break;
+    case no_virtualscope:
+    case no_concretescope:
+        {
+            HqlExprArray scopeSymbols;
+            expr->queryScope()->getSymbols(scopeSymbols);
+            ForEachItemIn(i, scopeSymbols)
+                info.special.append(processExpr(&scopeSymbols.item(i)));
+            break;
+        }
+    }
+
     if (getRequiredTypeCode(op) == type_none)
         info.type = processType(expr->queryType());
     info.sequence = expr->querySequenceExtra();
@@ -2012,6 +2046,8 @@ id_t ExpressionIRPlayer::doProcessAnnotation(IHqlExpression * expr)
             IHqlNamedAnnotation * annotation = static_cast<IHqlNamedAnnotation *>(expr->queryAnnotation());
             info.name = expr->queryName()->str();
             info.value = annotation->isExported() ? ob_exported : annotation->isShared() ? ob_shared : 0;
+            if (annotation->isVirtual())
+                info.value |= ob_virtual;
             break;
         }
     case annotate_location:
@@ -2121,8 +2157,13 @@ extern HQL_API void dump_irn(unsigned n, ...)
     va_start(args, n);
     for (unsigned i=0; i < n;i++)
     {
-        IHqlExpression * expr = va_arg(args, IHqlExpression *);
-        reader.play(expr);
+        IInterface * next = va_arg(args, IInterface *);
+        IHqlExpression * expr = dynamic_cast<IHqlExpression *>(next);
+        ITypeInfo * type = dynamic_cast<ITypeInfo *>(next);
+        if (expr)
+            reader.play(expr);
+        else if (type)
+            reader.play(type);
     }
     va_end(args);
 }
@@ -2135,7 +2176,6 @@ extern HQL_API void dbglogIR(IHqlExpression * expr)
     playIR(output, expr, NULL, NULL);
 }
 
-
 extern HQL_API void dbglogIR(const HqlExprArray & exprs)
 {
     DblgLogIRBuilder output(defaultDumpOptions);

ecl/hql/hqlparse.cpp

@@ -148,6 +148,8 @@ public:
         return parser->lookupSymbol(name, errpos);
     }
 
+    virtual IHqlScope * queryConcreteScope() { return this; }
+    virtual bool allBasesFullyBound() const { return true; }
 };
 
 

ecl/hql/hqlthql.cpp

@@ -669,7 +669,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             s.append('D');
         if (containsAnyDataset(expr))
             s.append('A');
-        if (containsInternalVirtual(expr))
+        if (containsInternalSelect(expr))
             s.append('V');
         if (expr->getInfoFlags() & HEFaction)
             s.append('N');
@@ -1458,8 +1458,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                 else if (expr->querySequenceExtra())
                 {
                     //Not sure any of these should be included
-                    if (expandProcessed || (name != virtualAtom))
-                        s.append("(").append(expr->querySequenceExtra()).append(")");
+                    s.append("(").append(expr->querySequenceExtra()).append(")");
                 }
                 break;
             }
@@ -2640,7 +2639,7 @@ void HqltHql::defaultChildrenToECL(IHqlExpression *expr, StringBuffer &s, bool i
     {
         s.append('(');
         bool needComma = false;
-        if (!xgmmlGraphText || !child0->queryDataset())
+        if (!xgmmlGraphText || (!child0->queryDataset() && !isInternalAttribute(child0)))
         {
             toECL(child0, s, false, inType);
             needComma = true;

ecl/hql/hqltrans.cpp

@@ -885,6 +885,25 @@ IHqlExpression * QuickHqlTransformer::createTransformedBody(IHqlExpression * exp
     case no_libraryscope:
     case no_forwardscope:
         return doCreateTransformedScope(expr);
+    case no_delayedscope:
+        {
+            OwnedHqlExpr newScope = transform(expr->queryChild(0));
+            if (newScope->queryScope())
+                return newScope.getClear();
+            break;
+        }
+    case no_assertconcrete:
+        {
+            OwnedHqlExpr newScope = transform(expr->queryChild(0));
+            IHqlScope * scope = newScope->queryScope();
+            if (scope)
+            {
+                IHqlScope * concrete = scope->queryConcreteScope();
+                if (concrete)
+                    return newScope.getClear();
+            }
+            break;
+        }
     case no_type:
         return transformAlienType(expr);
     case no_enum:
@@ -921,6 +940,20 @@ IHqlExpression * QuickHqlTransformer::createTransformedBody(IHqlExpression * exp
                 return createParameter(expr->queryName(), (unsigned)expr->querySequenceExtra(), newType.getClear(), children);
             break;
         }
+    case no_delayedselect:
+        {
+            IHqlExpression * oldModule = expr->queryChild(1);
+            OwnedHqlExpr newModule = transform(oldModule);
+            if (oldModule != newModule)
+            {
+                _ATOM selectedName = expr->queryChild(3)->queryName();
+                HqlDummyLookupContext dummyctx(errors);
+                IHqlScope * newScope = newModule->queryScope();
+                if (newScope)
+                    return newScope->lookupSymbol(selectedName, makeLookupFlags(true, expr->hasProperty(ignoreBaseAtom), false), dummyctx);
+            }
+            break;
+        }
     }
 
     return completeTransform(expr, children);

ecl/hql/hqlvalid.cpp

@@ -119,9 +119,19 @@ IHqlExpression * checkCreateConcreteModule(IErrorReceiver * errors, IHqlExpressi
         return LINK(queryExpression(concrete));
     }
 
-    if (!areAllBasesFullyBound(expr))
-        return LINK(expr);
+    if (expr->getOperator() == no_delayedscope)
+    {
+        if (expr->queryChild(0)->getOperator() == no_assertconcrete)
+            return LINK(expr);
+    }
+
+    OwnedHqlExpr check = createValue(no_assertconcrete, expr->getType(), LINK(expr), errpos.createLocationAttr());
+    return createDelayedScope(check.getClear());
+}
 
+void reportAbstractModule(IErrorReceiver * errors, IHqlExpression * expr, const ECLlocation & errpos)
+{
+    IHqlScope * scope = expr->queryScope();
     StringBuffer fieldText;
     if (scope)
     {
@@ -146,7 +156,6 @@ IHqlExpression * checkCreateConcreteModule(IErrorReceiver * errors, IHqlExpressi
         reportError(errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE in this context (INTERFACE must be instantiated)");
     else
         reportError(errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE in this context");
-    return LINK(expr);
 }
 
 IHqlExpression * checkCreateConcreteModule(IErrorReceiver * errors, IHqlExpression * expr, const IHqlExpression * locationExpr)

ecl/hql/hqlvalid.hpp

@@ -20,7 +20,8 @@
 #include "hqlexpr.hpp"
 
 //Checking functions
-IHqlExpression * checkCreateConcreteModule(IErrorReceiver * errors, IHqlExpression * expr, const ECLlocation & errpos);
+extern HQL_API void reportAbstractModule(IErrorReceiver * errors, IHqlExpression * expr, const ECLlocation & errpos);
+extern HQL_API IHqlExpression * checkCreateConcreteModule(IErrorReceiver * errors, IHqlExpression * expr, const ECLlocation & errpos);
 extern HQL_API IHqlExpression * checkCreateConcreteModule(IErrorReceiver * errors, IHqlExpression * expr, const IHqlExpression * locationExpr);
 extern HQL_API IHqlExpression * createLocationAttr(ISourcePath * filename, int lineno, int column, int position);
 

ecl/hqlcpp/hqlcpp.cpp

@@ -3264,7 +3264,7 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
         useFunction(tgt.expr);
         return;
     case no_purevirtual:
-    case no_internalvirtual:
+    case no_internalselect:
         {
             //This shouldn't happen we should have an no_checkconcrete wrapper inserted into the tree like checkconstant,
             //but it currently can in obscure library contexts (e.g., library3ie2.xhql)
@@ -3277,6 +3277,7 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
         break;
     }
 
+    EclIR::dbglogIR(expr);
     StringBuffer msg;
     msg.append("Unexpected operator '").append(getOpString(op)).append("' in: HqlCppTranslator::buildExpr(");
     toECL(expr, msg, true);
@@ -5545,7 +5546,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
     else
     {
         IHqlExpression * def = expr->queryBody()->queryFunctionDefinition();
-        assertex(def);
+        assertex(def && def->getOperator() == no_funcdef);
         funcdef.setown(doBuildInternalFunction(def));
     }
 

ecl/hqlcpp/hqlecl.cpp

@@ -392,8 +392,11 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
         }
         catch (IECLError * e)
         {
-            StringBuffer s;
-            errs->reportError(e->errorCode(), e->errorMessage(s).str(), e->getFilename(), e->getLine(), e->getColumn(), e->getPosition());
+            if (e->errorCode() != HQLERR_ErrorAlreadyReported)
+            {
+                StringBuffer s;
+                errs->reportError(e->errorCode(), e->errorMessage(s).str(), e->getFilename(), e->getLine(), e->getColumn(), e->getPosition());
+            }
             e->Release();
             return false;
         }

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5730,10 +5730,13 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
             DEBUG_TIMER("EclServer: peephole optimize", msTick()-time);
         }
     }
-    catch (IException *)
+    catch (IException * e)
     {
         ensureWorkUnitUpdated();
-        throw;
+        if (e->errorCode() != HQLERR_ErrorAlreadyReported)
+            throw;
+        e->Release();
+        return false;
     }
     catch (...)
     {

ecl/hqlcpp/hqlttcpp.cpp

@@ -11863,6 +11863,13 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
             }
             return folded.getClear();
         }
+    case no_assertconcrete:
+        {
+            ECLlocation errpos;
+            errpos.extractLocationAttr(expr->queryChild(1));
+            reportAbstractModule(translator.queryErrors(), expr->queryChild(0), errpos);
+            throw MakeStringException(HQLERR_ErrorAlreadyReported, "%s", "");
+        }
     case no_pat_instance:
         {
             OwnedHqlExpr child = transform(expr->queryChild(0));
@@ -12025,9 +12032,20 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
                 return transformChildrenNoAnnotations(expr);
             break;
         }
-
     case no_call:
-        return transformCall(expr);
+        {
+            IHqlExpression * oldFuncdef = expr->queryFunctionDefinition();
+            if (oldFuncdef->getOperator() == no_delayedselect)
+            {
+                IHqlExpression * module = oldFuncdef->queryChild(1);
+                ECLlocation errpos(module);
+                //errpos.extractLocationAttr(expr->queryChild(1));
+                reportAbstractModule(translator.queryErrors(), module, errpos);
+                throw MakeStringException(HQLERR_ErrorAlreadyReported, "%s", "");
+            }
+            assertex(oldFuncdef->getOperator() == no_funcdef);
+            return transformCall(expr);
+        }
     case no_externalcall:
         //Yuk.... Because we ensure that all records have a name, we need to make sure that external functions that return records
         //also have there return value normalized - otherwise (jtolbert2.xhql) you can create an ambiguity
@@ -12464,7 +12482,7 @@ bool HqlCppTranslator::transformGraphForGeneration(HqlQueryContext & query, Work
     DEBUG_TIMER("EclServer: tree transform: normalize", msTick()-time1);
 
     if (wu()->getDebugValueBool("dumpIR", false))
-        EclIR::dump_ir(exprs);
+        EclIR::dbglogIR(exprs);
 
     checkNormalized(exprs);
 #ifdef PICK_ENGINE_EARLY

ecl/regress/minherit24ae.ecl

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//A variation on local derivation (minherit4)
+//Current problems:
+//1) Bound parameter will be concrete not abstract
+//2) Cloning and overriding would need to be handled in bind phase (ok if common functions extracted)
+//3) Would need a no_param(x, concreteAtom) also bound when a param is bound (or possible concrete(param)), so that uses could be distringuished
+
+
+m1 := MODULE,virtual
+export value1 := 3;
+export value2;
+export f := value1 * value2;
+        END;
+
+
+f(m1 mp) := function
+
+    mBase :=    project(mp, m1,value1);
+
+    mLocal :=   module(mBase)
+        export value1 := 100;
+                end;
+
+    return mLocal.f;
+end;
+
+
+m4 := MODULE(m1)
+export value2 := 21;
+        END;
+
+
+output(f(m4));      // Expected 2100

ecl/regress/minherit28.ecl

@@ -0,0 +1,41 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+int_a := interface
+    export string x;
+end;
+
+mod_a := module(int_a)
+    export string x := 'Gavin';
+end;
+
+mod_b := module(int_a)
+    export string x := 'James';
+end;
+
+
+f(boolean a) := IF(a, mod_a, mod_b);
+
+f(true).x;
+
+g(boolean a) := f(a).x;
+
+g(true);
+
+h(boolean a) := IF(a, mod_a, mod_b).x;
+
+h(true);

ecl/regress/minherit29.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+
+interface1 := module,interface
+export boolean useName;
+export boolean useAvailable;
+export r := { boolean a := useName; boolean b := useAvailable };
+        end;
+
+
+f(interface1 arg) := DATASET(ROW(arg.r));
+
+options1 := module(interface1)
+export boolean useName := true;
+export boolean useAvailable := false;
+    end;
+
+f(options1);

ecl/regress/minherit30.ecl

@@ -0,0 +1,43 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Unusual, but there is some pre-existing code which works along these lines
+//An attribute defines a (virtual) module, and that module contains another module.
+//Values from the contained module are used as constants within that attribute.
+
+//Strictly speaking the nested module isn't complete until the outer module is complete
+//since it might depend on values which are overriden.  However in this case force
+//evaluation if it is non-abstract.
+
+
+IName := interface
+    export string name;
+end;
+
+f(string prefix) := MODULE(IName)
+    SHARED extra := MODULE
+        EXPORT STRING addPrefix := 'true';
+    END;
+
+    SHARED STRING fullPrefix := IF(#expand(extra.addPrefix), prefix + ' ', '');
+
+    EXPORT name := fullPrefix  + 'Halliday';
+END;
+
+
+
+OUTPUT(f('Mr').name);

ecl/regress/minherit30b.ecl

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Unusual, but there is some pre-existing code which works along these lines
+//An attribute defines a (virtual) module, and that module contains another module.
+//Values from the contained module are used as constants within that attribute.
+
+//Strictly speaking the nested module isn't complete until the outer module is complete
+//since it might depend on values which are overriden.  However in this case force
+//evaluation if it is non-abstract.
+
+
+IName := interface
+    export string name;
+end;
+
+f(string prefix) := MODULE(IName)
+
+    SHARED extra1 := MODULE
+        EXPORT STRING addPrefixY := 'true';
+    END;
+
+    //Problematic - addPrefix will contain a self reference by the time the reference is read from the #expand
+    SHARED extra2 := MODULE
+        EXPORT STRING addPrefix := extra1.addPrefixY;
+    END;
+
+    SHARED STRING fullPrefix := IF(#expand(extra2.addPrefix), prefix + ' ', '');
+
+    EXPORT name := fullPrefix  + 'Halliday';
+END;
+
+
+
+OUTPUT(f('Mr').name);

ecl/regress/minherit31.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Unusual, but there is some pre-existing code which works along these lines
+//An attribute defines a (virtual) module, and that module contains another module.
+//Values from the contained module are used as constants within that attribute.
+
+//Strictly speaking the nested module isn't complete until the outer module is complete
+//since it might depend on values which are overriden.  However in this case force
+//evaluation if it is non-abstract.
+
+f(string prefix) := MODULE,VIRTUAL
+
+    SHARED myBase := MODULE,virtual
+        EXPORT addPrefix := true;
+    END;
+
+    EXPORT baseModule(string base) := MODULE(myBase)
+        STRING fullPrefix := IF(addPrefix, prefix + ' ', '');
+        EXPORT name := fullPrefix  + base;
+    END;
+
+    EXPORT myModule := baseModule('Halliday');
+END;
+
+OUTPUT(f('Mr').myModule.name);

ecl/regress/minherit31b.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Unusual, but there is some pre-existing code which works along these lines
+//An attribute defines a (virtual) module, and that module contains another module.
+//Values from the contained module are used as constants within that attribute.
+
+//Strictly speaking the nested module isn't complete until the outer module is complete
+//since it might depend on values which are overriden.  However in this case force
+//evaluation if it is non-abstract.
+
+f(string prefix) := MODULE,VIRTUAL
+
+    SHARED myBase := MODULE,virtual
+        EXPORT addPrefix := true;
+    END;
+
+    EXPORT baseModule(string base) := MODULE(myBase)
+        STRING fullPrefix := IF(addPrefix, prefix + ' ', '');
+        EXPORT name := fullPrefix  + base;
+    END;
+
+    EXPORT myModule := baseModule('Halliday');
+END;
+
+OUTPUT(f('Mr').myModule.name);

ecl/regress/minherit32.ecl

@@ -0,0 +1,43 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Unusual, but there is some pre-existing code which works along these lines
+//An attribute defines a (virtual) module, and that module contains another module.
+//Values from the contained module are used as constants within that attribute.
+
+//Strictly speaking the nested module isn't complete until the outer module is complete
+//since it might depend on values which are overriden.  However in this case force
+//evaluation if it is non-abstract.
+
+f(string prefix) := MODULE,VIRTUAL
+
+    SHARED myBase := MODULE,virtual
+        EXPORT addPrefix := true;
+    END;
+
+    EXPORT baseModule(string base) := MODULE(myBase)
+        EXPORT suffix := '';
+        STRING fullPrefix := IF(addPrefix, prefix + ' ', '');
+        EXPORT name := fullPrefix  + base + suffix;
+    END;
+
+    EXPORT myModule := MODULE(baseModule('Halliday'))
+        EXPORT suffix := ' II';
+    END;
+END;
+
+OUTPUT(f('Mr').myModule.name);

ecl/regress/minherit32_err.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+baseModule := MODULE,VIRTUAL
+    EXPORT VIRTUAL suffix := '';
+    STRING fullPrefix := 'Mr ';
+    EXPORT VIRTUAL name := fullPrefix  + 'Halliday' + suffix;
+END;
+
+myModule := MODULE(baseModule)
+    EXPORT VIRTUAL suffix := ' II';
+END;
+
+OUTPUT(myModule.name);

ecl/regress/minherit32a.ecl

@@ -0,0 +1,35 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+f(string prefix) := MODULE,VIRTUAL
+
+    SHARED myBase := MODULE,virtual
+        EXPORT addPrefix := true;
+    END;
+
+    EXPORT baseModule(string base) := MODULE(myBase),VIRTUAL
+        EXPORT VIRTUAL STRING suffix := '';
+        EXPORT VIRTUAL STRING fullPrefix := IF(addPrefix, prefix + ' ', '');
+        EXPORT VIRTUAL name := fullPrefix  + base + suffix;
+    END;
+
+    EXPORT myModule := MODULE(baseModule('Halliday'))
+        EXPORT VIRTUAL suffix := ' II';
+    END;
+END;
+
+OUTPUT(f('Mr').myModule.name);

ecl/regress/minherit32b.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+f(string prefix) := MODULE,VIRTUAL
+
+    SHARED myBase := MODULE,virtual
+        EXPORT addPrefix := true;
+    END;
+
+    EXPORT baseModule(string base) := MODULE(myBase)
+
+        EXPORT VIRTUAL STRING suffix := '';
+        STRING fullPrefix := IF(addPrefix, prefix + ' ', '');
+        EXPORT VIRTUAL name := fullPrefix  + base + suffix;
+    END;
+
+    EXPORT myModule := MODULE(baseModule('Halliday'))
+        EXPORT suffix := ' II';
+    END;
+END;
+
+OUTPUT(f('Mr').myModule.name);

ecl/regress/minherit32b2.ecl

@@ -0,0 +1,30 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+f := MODULE,VIRTUAL
+
+    EXPORT baseModule := MODULE
+        EXPORT VIRTUAL STRING suffix := '';
+        EXPORT VIRTUAL name := 'Mr Halliday' + suffix;
+    END;
+
+    EXPORT myModule := MODULE(baseModule)
+        EXPORT VIRTUAL suffix := ' II';
+    END;
+END;
+
+OUTPUT(f.myModule.name);

ecl/regress/minherit32b3.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+f := MODULE
+
+    SHARED myBase := MODULE,virtual
+        EXPORT addPrefix := true;
+    END;
+
+    EXPORT baseModule(string base) := MODULE(myBase)
+
+        EXPORT VIRTUAL STRING suffix := '';
+        STRING fullPrefix := IF(addPrefix, 'Mr ', '');
+        EXPORT VIRTUAL name := fullPrefix  + base + suffix;
+    END;
+
+    EXPORT myModule := MODULE(baseModule('Halliday'))
+        EXPORT suffix := ' II';
+    END;
+END;
+
+OUTPUT(f.myModule.name);

ecl/regress/minherit32c.ecl

@@ -0,0 +1,31 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+string prefix := 'Mr';
+
+EXPORT baseModule(string base) := MODULE
+
+    EXPORT VIRTUAL STRING suffix := '';
+    STRING fullPrefix := prefix + ' ';
+    EXPORT VIRTUAL name := fullPrefix  + base + suffix;
+END;
+
+EXPORT myModule := MODULE(baseModule('Halliday'))
+    EXPORT suffix := ' II';
+END;
+
+OUTPUT(myModule.name);

ecl/regress/minherit32d.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+baseModule := MODULE,VIRTUAL
+    EXPORT VIRTUAL STRING suffix := '';
+    STRING fullPrefix := 'Mr ';
+    EXPORT VIRTUAL name := fullPrefix  + 'Halliday' + suffix;
+END;
+
+myModule := MODULE(baseModule)
+    EXPORT VIRTUAL STRING suffix := ' II';
+END;
+
+OUTPUT(myModule.name);

ecl/regress/minherit33.ecl

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+f(string prefix) := FUNCTION
+
+    STRING xx := 'hello';
+
+    g(string y) := FUNCTION
+
+        STRING xx := 'oh no';
+
+        RETURN y + xx;
+    END;
+
+    RETURN prefix + g(xx);
+END;
+
+f('Me');

ecl/regress/minherit4a.ecl

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Local derivation... you really don't want to do this!
+//Current problems:
+//1) Bound parameter will be concrete not abstract
+//2) Cloning and overriding would need to be handled in bind phase (ok if common functions extracted)
+//3) Would need a no_param(x, concreteAtom) also bound when a param is bound (or possible concrete(param)), so that uses could be distringuished
+
+
+m1 := MODULE,virtual
+shared value1 := 1;
+shared value2 := 1;
+export f := value1 * value2;
+        END;
+
+
+f(m1 mp) := function
+
+    mLocal :=   module(mp)
+        shared value1 := 100;
+                end;
+
+    return mLocal.f;
+end;
+
+
+m4 := MODULE(m1)
+shared value2 := 21;
+        END;
+
+
+output(f(m4));      // Expected 2100

esp/files/scripts/GraphPageWidget.js

@@ -16,12 +16,11 @@
 define([
     "dojo/_base/declare",
     "dojo/_base/lang",
-    "dojo/_base/sniff",
     "dojo/_base/array",
     "dojo/dom",
     "dojo/dom-construct",
     "dojo/on",
-    "dojo/has",
+    "dojo/html",
     "dojo/store/Memory",
     "dojo/store/Observable",
 
@@ -55,9 +54,11 @@ define([
     "dijit/Menu",
     "dijit/MenuItem",
     "dijit/MenuSeparator",
+    "dijit/Dialog",
     "dijit/form/TextBox",
+    "dijit/form/SimpleTextarea",
     "dijit/form/DropDownButton"
-], function (declare, lang, sniff, arrayUtil, dom, domConstruct, on, has, Memory, Observable,
+], function (declare, lang, arrayUtil, dom, domConstruct, on, html, Memory, Observable,
             _LayoutWidget, _TemplatedMixin, _WidgetsInTemplateMixin, BorderContainer, TabContainer, ContentPane, registry, Dialog,
             entities,
             OnDemandGrid, Keyboard, Selection, selector, ColumnResizer, DijitRegistry,
@@ -79,6 +80,8 @@ define([
         subgraphsGrid: null,
         verticesGrid: null,
         edgesGrid: null,
+        xgmmlDialog: null,
+        infoDialog: null,
         findField: null,
         findText: "",
         found: [],
@@ -96,6 +99,7 @@ define([
             this.findField = registry.byId(this.id + "FindField");
             this._initGraphControls();
             this._initTimings();
+            this._initDialogs();
         },
 
         startup: function (args) {
@@ -182,6 +186,26 @@ define([
             }
         },
 
+        _initDialogs: function () {
+            var context = this;
+
+            this.infoDialog = registry.byId(this.id + "InfoDialog");
+            on(dom.byId(this.id + "InfoDialogCancel"), "click", function (event) {
+                context.infoDialog.hide();
+            });
+
+            this.xgmmlDialog = registry.byId(this.id + "XGMMLDialog");
+            this.xgmmlTextArea = registry.byId(this.id + "XGMMLTextArea");
+            on(dom.byId(this.id + "XGMMLDialogApply"), "click", function (event) {
+                var xgmml = context.xgmmlTextArea.get("value");
+                context.xgmmlDialog.hide();
+                context.loadGraphFromSource(xgmml);
+            });
+            on(dom.byId(this.id + "XGMMLDialogCancel"), "click", function (event) {
+                context.xgmmlDialog.hide();
+            });
+        },
+
         _initItemGrid: function (grid) {
             var context = this;
             grid.on(".dgrid-row:click", function (evt) {
@@ -276,53 +300,30 @@ define([
             this._doFind(true);
         },
 
-        _showDialog: function(params) {
-            myDialog = new Dialog(params);
-            if (has("chrome")) {
-                this.main.hide();
-                this.overview.hide();
-                this.local.hide();
-
-                var context = this;
-                myDialog.connect(myDialog, "hide", function (e) {
-                    context.main.show();
-                    context.overview.show();
-                    context.local.show();
-                });
-            }
-            myDialog.show();
-        },
-
         _onAbout: function () {
-            this._showDialog({
-                title: "About HPCC Systems Graph Control",
-                content: "Version:  " + this.main.getVersion() + "<br/>" + this.main.getResourceLinks(),
-                style: "width: 320px"
-            });
+            html.set(dom.byId(this.id + "InfoDialogContent"), "<div style='width: 320px; height: 120px; text-align: center;'><p>Version:  " + this.main.getVersion() + "</p><p>" + this.main.getResourceLinks() + "</p>");
+            this.infoDialog.set("title", "About HPCC Systems Graph Control");
+            this.infoDialog.show();
         },
 
         _onGetSVG: function () {
-            this._showDialog({
-                title: "SVG Source",
-                content: entities.encode(this.main.getSVG())
-            });
+            html.set(dom.byId(this.id + "InfoDialogContent"), "<textarea rows='25' cols='80'>" + entities.encode(this.main.getSVG()) + "</textarea>");
+            this.infoDialog.set("title", "SVG Source");
+            this.infoDialog.show();
         },
 
         _onRenderSVG: function () {
             var context = this
             this.main.localLayout(function (svg) {
-                context._showDialog({
-                    title: "Rendered SVG",
-                    content: svg
-                });
+                html.set(dom.byId(context.id + "InfoDialogContent"), "<div style='border: 1px inset grey; width: 640px; height: 480px; overflow : auto; '>" + svg + "</div>");
+                context.infoDialog.set("title", "Rendered SVG");
+                context.infoDialog.show();
             });
         },
 
         _onGetXGMML: function () {
-            this._showDialog({
-                title: "XGMML",
-                content: entities.encode(this.main.getXGMML())
-            });
+            this.xgmmlTextArea.set("value", this.main.getXGMML());
+            this.xgmmlDialog.show();
         },
 
         init: function (params) {
@@ -360,25 +361,29 @@ define([
 
         },
 
+        loadGraphFromSource: function(xgmml, svg) {
+            this.main.setMessage("Loading Data...");
+            this.main.loadXGMML(xgmml);
+            this.overview.loadXGMML(this.main.getLocalisedXGMML([0]));
+            this.loadSubgraphs();
+            this.loadVertices();
+            this.loadEdges();
+            if (svg) {
+                this.main.setMessage("Loading Layout...");
+                if (this.main.mergeSVG(svg)) {
+                    this.main.centerOnItem(0, true);
+                    this.main.setMessage("");
+                    return;
+                }
+            }
+            this.main.setMessage("Performing Layout...");
+            this.main.startLayout("dot");
+        },
+
         loadGraph: function (wu, graphName) {
             var context = this;
             wu.fetchGraphXgmmlByName(graphName, function (xgmml, svg) {
-                context.main.setMessage("Loading Data...");
-                context.main.loadXGMML(xgmml);
-                context.overview.loadXGMML(context.main.getLocalisedXGMML([0]));
-                context.loadSubgraphs();
-                context.loadVertices();
-                context.loadEdges();
-                if (svg) {
-                    context.main.setMessage("Loading Layout...");
-                    if (context.main.mergeSVG(svg)) {
-                        context.main.centerOnItem(0, true);
-                        context.main.setMessage("");
-                        return;
-                    }
-                }
-                context.main.setMessage("Performing Layout...");
-                context.main.startLayout("dot");
+                context.loadGraphFromSource(xgmml, svg);
             });
         },
 

esp/files/templates/GraphPageWidget.html

@@ -49,4 +49,21 @@
             </div>
         </div>
     </div>
+    <div id="${id}XGMMLDialog" title="XGMML" data-dojo-type="dijit.Dialog">
+        <div class="dijitDialogPaneContentArea">
+            <textarea id="${id}XGMMLTextArea" rows="25" cols="80" data-dojo-type="dijit.form.SimpleTextarea">
+            </textarea>
+        </div>
+        <div class="dijitDialogPaneActionBar">
+            <button id="${id}XGMMLDialogApply" type="submit" data-dojo-type="dijit.form.Button" >Apply</button>
+            <button id="${id}XGMMLDialogCancel" type="button" data-dojo-type="dijit.form.Button">Cancel</button>
+        </div>
+    </div>
+    <div id="${id}InfoDialog" title="Info Dialog" data-dojo-type="dijit.Dialog">
+        <div id="${id}InfoDialogContent" class="dijitDialogPaneContentArea">
+        </div>
+        <div class="dijitDialogPaneActionBar">
+            <button id="${id}InfoDialogCancel" type="button" data-dojo-type="dijit.form.Button">Cancel</button>
+        </div>
+    </div>
 </div>

esp/tools/soapplus/xmldiff.cpp

@@ -287,12 +287,19 @@ bool CXmlDiff::cmpAttributes(IPTree* t1, IPTree* t2, const char* xpathFull, bool
     if (a1->isValid() || a2->isValid())
         diff = true;
 
-    if (diff && print)
+    if (diff && print && xpathFull)
     {
         StringBuffer xpathFullBuf(xpathFull);
-        int pos = strrchr(xpathFull, '[') - xpathFull;
-        int len = strrchr(xpathFull, ']') - strrchr(xpathFull, '[');
-        xpathFullBuf.remove(pos, len + 1);
+        const char* r = strrchr(xpathFull, ']');
+        const char* slash = strrchr(xpathFull, '/');
+        if(r > slash) //Only remove the attributes in the last tag
+        {
+            const char* l = strrchr(xpathFull, '[');
+            if(l > slash && l < r)
+            {
+                xpathFullBuf.remove(l - xpathFull, r - l + 1);
+            }
+        }
         StringBuffer attrString;
         getAttrString(t1, attrString);
         printDiff("< %s[%s]\n", xpathFullBuf.str(), attrString.str());
@@ -322,7 +329,7 @@ bool CXmlDiff::cmpPtree(const char* xpath, IPropertyTree* t1, IPropertyTree* t2,
     }
 
     StringBuffer keybuf;
-    keybuf.appendf("%s-%s", xpath, xpathFull);
+    keybuf.appendf("%p-%p", t1, t2);
     std::string key = keybuf.str();
     if(m_compcache.getValue(key.c_str()) != NULL)
     {
@@ -786,6 +793,18 @@ bool CXmlDiff::diffPtree(const char* xpath, IPropertyTree* t1, IPropertyTree* t2
         return false;
     }
 
+    StringBuffer initialPath, initialPathFull;
+    if(!xpath || !*xpath)
+    {
+        initialPath.appendf("%s", name1);
+        xpath = initialPath.str();
+    }
+    if(!xpathFull || !*xpathFull)
+    {
+        initialPathFull.appendf("%s", name1);
+        xpathFull = initialPathFull.str();
+    }
+
     bool isEqual = true;
 
   // compare attrs
@@ -1075,7 +1094,6 @@ bool CXmlDiff::diffPtree(const char* xpath, IPropertyTree* t1, IPropertyTree* t2
     {
         const char* val1 = t1->queryProp(".");
         const char* val2 = t2->queryProp(".");
-        StringBuffer keyBuf;
 
         if((val1 && !val2) || (!val1 && val2) || (val1 && val2 && strcmp(val1, val2) != 0))
         {
@@ -1101,6 +1119,22 @@ void CXmlDiff::printPtree(const char* prefix, const char* xpath, IPropertyTree*
             return;
     }
 
+    StringBuffer initialPath, initialPathFull;
+    if(!xpath || !*xpath)
+    {
+        initialPath.appendf("%s", t->queryName());
+        xpath = initialPath.str();
+    }
+    if(!xpathFull || !*xpathFull)
+    {
+        initialPathFull.appendf("%s", t->queryName());
+        StringBuffer attrString;
+        getAttrString(t, attrString);
+        if(attrString.length())
+            initialPathFull.appendf("[%s]", attrString.str());
+        xpathFull = initialPathFull.str();
+    }
+
     if(t->hasChildren())
     {
         Owned<IPropertyTreeIterator> mi = t->getElements("*");

initfiles/componentfiles/configxml/dali.xsd

@@ -129,6 +129,7 @@
       <xs:attributeGroup ref="Store"/>
       <xs:attributeGroup ref="Backup"/>
       <xs:attributeGroup ref="LDAP"/>
+      <xs:attributeGroup ref="DFS"/>
       <xs:attribute name="build" type="buildType" use="required">
         <xs:annotation>
           <xs:appinfo>
@@ -285,13 +286,20 @@
     </xs:attribute>
   </xs:attributeGroup>
    <xs:attributeGroup name="DFS">
-    <xs:attribute name="forceGroupUpdate" use="optional" default="false">
+    <xs:attribute name="forceGroupUpdate" type="xs:boolean" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>Force group updates on startup, if environment mismatch</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="numThreads" type="xs:nonNegativeInteger" use="optional" default="30">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Number of threads to use to process DFS requests</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
    </xs:attributeGroup>
    <xs:attributeGroup name="LDAP">
     <xs:attribute name="ldapServer" type="ldapServerType" use="optional">

initfiles/componentfiles/configxml/dali.xsl

@@ -172,9 +172,9 @@
           </xsl:call-template>
         </xsl:attribute>
       </xsl:element>
-      <xsl:element name="DFS">
-        <xsl:copy-of select="@forceGroupUpdate"/>
-      </xsl:element>
+      <DFS>
+      <xsl:copy-of select="@forceGroupUpdate | @numThreads"/>
+      </DFS>
       <xsl:element name="Coven">
         <xsl:attribute name="store">dalicoven.xml</xsl:attribute>
         <xsl:element name="Alerts">

system/mp/mputil.hpp

@@ -45,12 +45,12 @@ public:
         return CInterface::Release(); 
     }
 
-    CMessageHandler(const char *_name,PARENT *_parent,void (PARENT::*_handler)(CMessageBuffer &_mb), IExceptionHandler *exceptionHandler=NULL, unsigned maxthreads=40, unsigned timeoutOnRelease=INFINITE)
+    CMessageHandler(const char *_name,PARENT *_parent,void (PARENT::*_handler)(CMessageBuffer &_mb), IExceptionHandler *exceptionHandler=NULL, unsigned maxthreads=40, unsigned timeoutOnRelease=INFINITE, unsigned lowThreadsDelay=1000)
     {
         parent = _parent;
         handler = _handler;
         name = strdup(_name);
-        pool = createThreadPool(name,this,exceptionHandler,maxthreads,1000,0,timeoutOnRelease); // this will cause this to be linked
+        pool = createThreadPool(name,this,exceptionHandler,maxthreads,lowThreadsDelay,0,timeoutOnRelease); // this will cause this to be linked
         hasexceptionhandler = exceptionHandler!=NULL;
     }
     ~CMessageHandler()
@@ -122,7 +122,6 @@ public:
             runname.append(' ').appendhex(*(b++),true);
         pool->start(&mb,runname.str());
     }
-
 };
 
 

testing/ecl/failrestart.ecl

@@ -0,0 +1,35 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//The first time this workunit executes it gets an exception in a conditional workflow item.
+//If it is "recovered" the condition must be re-evaluated (otherwise the false branch is taken the
+//second time.
+
+o0 := OUTPUT('...');
+o1 := FAIL('Oh no!') : independent;
+o2 := OUTPUT('Should not be output') : independent;
+o3 := output('Begin');
+o4 := output('Done');
+
+b1 := SEQUENTIAL(o0, o1);
+b2 := SEQUENTIAL(o0, o2);
+
+cond := true : STORED('cond');
+
+x := IF(cond, b1, b2);
+
+SEQUENTIAL(o3, x, o4);

testing/unittests/dalitests.cpp

@@ -460,6 +460,7 @@ class DaliTests : public CppUnit::TestFixture
         CPPUNIT_TEST(testDFSPromote);
         CPPUNIT_TEST(testDFSDel);
         CPPUNIT_TEST(testDFSRename);
+        CPPUNIT_TEST(testDFSHammer);
     CPPUNIT_TEST_SUITE_END();
 
 #ifndef COMPAT
@@ -608,7 +609,7 @@ class DaliTests : public CppUnit::TestFixture
                 ASSERT(dir.removeEntry(super.str(), user) && "Can't remove super-file");
         }
 
-        logctx.CTXLOG("Creating 'regress::trans' subfiles(1,4)");
+        logctx.CTXLOG("Creating 'regress::trans' subfiles(1,%d)", subsToCreate);
         for (unsigned i=1; i<=subsToCreate; i++) {
             StringBuffer name;
             name.append("sub").append(i);
@@ -1503,6 +1504,66 @@ public:
         dir.renamePhysical("regress::rename::other2", "regress::rename::sub2", user, transaction);
         ASSERT(dir.exists("regress::rename::sub2", user, true, false) && "Renamed from other2 failed");
     }
+
+    void testDFSHammer()
+    {
+        unsigned numFiles = 100;
+        unsigned numReads = 40000;
+        unsigned hammerThreads = 10;
+        setupDFS("hammer", 0, numFiles);
+
+        StringBuffer msg("Reading ");
+        msg.append(numFiles).append(" files").append(numReads).append(" times, on ").append(hammerThreads).append(" threads");
+        logctx.CTXLOG("%s", msg.str());
+
+        class CHammerFactory : public CSimpleInterface, implements IThreadFactory
+        {
+        public:
+            IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+
+            virtual IPooledThread *createNew()
+            {
+                class CHammerThread : public CSimpleInterface, implements IPooledThread
+                {
+                    StringAttr filename;
+                public:
+                    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+
+                    virtual void init(void *param)
+                    {
+                        filename.set((const char *)param);
+                    }
+                    virtual void main()
+                    {
+                        try
+                        {
+                            Owned<IPropertyTree> tree = queryDistributedFileDirectory().getFileTree(filename,user);
+                        }
+                        catch (IException *e)
+                        {
+                            PrintExceptionLog(e, NULL);
+                        }
+                    }
+                    virtual bool stop() { return true; }
+                    virtual bool canReuse() { return true; }
+                };
+                return new CHammerThread();
+            }
+        } poolFactory;
+
+        CTimeMon tm;
+        Owned<IThreadPool> pool = createThreadPool("TSDSTest", &poolFactory, NULL, hammerThreads, 2000);
+        while (numReads--)
+        {
+            StringBuffer filename("regress::hammer::sub");
+            unsigned fn = 1+(getRandom()%numFiles);
+            filename.append(fn);
+            PROGLOG("Hammer file: %s", filename.str());
+            pool->start((void *)filename.str());
+        }
+        pool->joinAll();
+        PROGLOG("Hammer test took: %d ms", tm.elapsed());
+    }
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION( DaliTests );

thorlcr/activities/nsplitter/thnsplitterslave.cpp

@@ -180,6 +180,8 @@ class NSplitterSlaveActivity : public CSlaveActivity
     {
         Owned<CSplitterOutputBase> input;
         Linked<NSplitterSlaveActivity> activity;
+        mutable SpinLock processActiveLock;
+
         unsigned id;
 
     public:
@@ -188,6 +190,7 @@ class NSplitterSlaveActivity : public CSlaveActivity
         CDelayedInput(NSplitterSlaveActivity &_activity) : CThorDataLink(&_activity), activity(&_activity), id(0) { }
         void setInput(CSplitterOutputBase *_input, unsigned _id=0)
         {
+            SpinBlock b(processActiveLock);
             input.setown(_input);
             id = _id;
         }
@@ -222,6 +225,7 @@ class NSplitterSlaveActivity : public CSlaveActivity
         }
         virtual unsigned __int64 queryTotalCycles() const
         {
+            SpinBlock b(processActiveLock);
             if (!input)
                 return 0;
             return input->queryTotalCycles();