Merge remote-tracking branch 'origin/candidate-3.10.x' into candidate-3.10.2

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -3654,13 +3654,16 @@ void XmlSchemaBuilder::addSchemaPrefix()
                     "<xs:element name=\"Row\">"
                         "<xs:complexType>"
                             "<xs:sequence>\n");
+    attributes.append(*new StringBufferItem);
 }
 
 
 void XmlSchemaBuilder::addSchemaSuffix()
 {
-    xml.append(             "</xs:sequence>"
-                        "</xs:complexType>"
+    xml.append(             "</xs:sequence>");
+    xml.append(attributes.tos());
+    attributes.pop();
+    xml.append(         "</xs:complexType>"
                     "</xs:element>\n"
                 "</xs:sequence>"
             "</xs:complexType>"

common/roxiemanager/referencedfilelist.hpp

@@ -21,7 +21,7 @@
 
 #include "jlib.hpp"
 #include "workunit.hpp"
-#include "package.hpp"
+#include "package.h"
 #include "dadfs.hpp"
 #include "dfuutil.hpp"
 

common/workunit/CMakeLists.txt

@@ -32,7 +32,8 @@ set (    SRCS
          workunit.hpp 
          wuerror.hpp
          wujobq.hpp
-         package.hpp
+         package.h
+         pkgimpl.hpp
     )
 
 include_directories ( 

common/workunit/package.cpp

@@ -19,7 +19,7 @@
 #include "platform.h"
 #include "jprop.hpp"
 #include "jptree.hpp"
-#include "package.hpp"
+#include "pkgimpl.hpp"
 
 #include "dadfs.hpp"
 #include "eclrtl.hpp"
@@ -28,289 +28,81 @@
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
-static IPropertyTree * getPackageMapById(const char * id, bool readonly)
+CPackageNode::CPackageNode(IPropertyTree *p)
 {
-    StringBuffer xpath;
-    xpath.append("/PackageMaps/PackageMap[@id=\"").append(id).append("\"]");
-    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
-    if (!conn)
-        return NULL;
-    return conn->getRoot();
-}
-
-static IPropertyTree * getPackageSetById(const char * id, bool readonly)
-{
-    StringBuffer xpath;
-    xpath.append("/PackageSets/PackageSet[@id=\"").append(id).append("\"]");
-    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
-    if (!conn)
-        return NULL;
-    return conn->getRoot();
+    if (p)
+        node.set(p);
+    else
+        node.setown(createPTree("HpccPackages"));
+    StringBuffer xml;
+    toXML(node, xml);
+    hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
 }
 
-static IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly)
+    // Merge base package environment into mergedEnvironment
+void CPackageNode::mergeEnvironment(const CPackageNode *base)
 {
-    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
-    Owned<IPropertyTree> psroot = globalLock->getRoot();
-
-    StringBuffer xpath;
-    xpath.append("PackageSet[@process=\"").append(process).append("\"]");
-    IPropertyTree *ps = psroot->queryPropTree(xpath);
-    if (ps)
-        return getPackageSetById(ps->queryProp("@id"), readonly);
-    Owned<IPropertyTreeIterator> it = psroot->getElements("PackageSet[@process]");
-    ForEach(*it)
+    Owned<IPropertyIterator> envIterator = base->mergedEnvironment->getIterator();
+    ForEach(*envIterator)
     {
-        const char *match = it->query().queryProp("@process");
-        const char *id = it->query().queryProp("@id");
-        if (id && isWildString(match) && WildMatch(process, match))
-            return getPackageSetById(id, readonly);
+        const char *id = envIterator->getPropKey();
+        const char *val = base->mergedEnvironment->queryProp(id);
+        if (id && val && !mergedEnvironment->hasProp(id))
+            mergedEnvironment->setProp(id, val);
     }
-    return NULL;
 }
 
-class CPackageSuperFileArray : public CInterface, implements ISimpleSuperFileEnquiry
-{
-    IArrayOf<IPropertyTree> subFiles;
-public:
-    IMPLEMENT_IINTERFACE;
-    CPackageSuperFileArray(IPropertyTreeIterator &_subs)
-    {
-        ForEach(_subs)
-        {
-            IPropertyTree &sub = _subs.query();
-            sub.Link();
-            subFiles.append(sub);
-        }
-    }
-    virtual unsigned numSubFiles() const
-    {
-        return subFiles.length();
-    }
-    virtual bool getSubFileName(unsigned num, StringBuffer &name) const
-    {
-        if (subFiles.isItem(num))
-        {
-            name.append(subFiles.item(num).queryProp("@value"));
-            return true;
-        }
-        else
-            return false;
-    }
-    virtual unsigned findSubName(const char *subname) const
-    {
-        ForEachItemIn(idx, subFiles)
-        {
-            if (stricmp(subFiles.item(idx).queryProp("@value"), subname))
-                return idx;
-        }
-        return NotFound;
-    }
-    virtual unsigned getContents(StringArray &contents) const
-    {
-        ForEachItemIn(idx, subFiles)
-        {
-            contents.append(subFiles.item(idx).queryProp("@value"));
-        }
-        return subFiles.length();
-    }
-};
-
-
-
-class CHpccPackage : extends CInterface, implements IHpccPackage
+// Use local package and its bases to resolve superfile name list of subfiles via all supported resolvers
+ISimpleSuperFileEnquiry *CPackageNode::resolveSuperFile(const char *superFileName) const
 {
-protected:
-    Owned<IPropertyTree> node;
-    IArrayOf<CHpccPackage> bases;
-    Owned<IProperties> mergedEnvironment;
-    hash64_t hash;
-
-    // Merge base package environment into mergedEnvironment
-    void mergeEnvironment(const CHpccPackage *base)
-    {
-        Owned<IPropertyIterator> envIterator = base->mergedEnvironment->getIterator();
-        ForEach(*envIterator)
-        {
-            const char *id = envIterator->getPropKey();
-            const char *val = base->mergedEnvironment->queryProp(id);
-            if (id && val && !mergedEnvironment->hasProp(id))
-                mergedEnvironment->setProp(id, val);
-        }
-    }
-
-    // Search this package and any bases for an element matching xpath1, then return iterator for its children that match xpath2
-    IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const
-    {
-        IPropertyTree *parentNode = node->queryPropTree(xpath1);
-        if (parentNode)
-            return parentNode->getElements(xpath2);
-        ForEachItemIn(idx, bases)
-        {
-            const CHpccPackage &basePackage = bases.item(idx);
-            IPropertyTreeIterator *it = basePackage.lookupElements(xpath1, xpath2);
-            if (it)
-                return it;
-        }
-        return NULL;
-    }
-
-    inline StringBuffer makeSuperFileXPath(StringBuffer &xpath, const char *superFileName) const
-    {
-        return xpath.append("SuperFile[@id='").appendLower(strlen(superFileName), superFileName).append("']");
-    }
-
-    // Use local package and its bases to resolve superfile name list of subfiles via all supported resolvers
-    ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
+    // Order of resolution:
+    // 1. SuperFiles named in local package
+    // 2. SuperFiles named in bases
+    // There is no dali or local case - a superfile that is resolved in dali must also resolve the subfiles there (and is all done in the resolveLFNusingDali method)
+    if (superFileName && *superFileName && node)
     {
-        // Order of resolution:
-        // 1. SuperFiles named in local package
-        // 2. SuperFiles named in bases
-        // There is no dali or local case - a superfile that is resolved in dali must also resolve the subfiles there (and is all done in the resolveLFNusingDali method)
-        if (superFileName && *superFileName && node)
-        {
-            if (*superFileName=='~')
-                superFileName++;
-            StringBuffer xpath;
-            Owned<IPropertyTreeIterator> subFiles = lookupElements(makeSuperFileXPath(xpath, superFileName), "SubFile");
-            if (subFiles)
-            {
-                Owned<CPackageSuperFileArray> result = new CPackageSuperFileArray(*subFiles);
-                return result.getClear();
-            }
-        }
-        return NULL;
-    }
-
-    inline bool hasProp(const char *xpath) const
-    {
-        return (node) ? node->hasProp(xpath) : false;
-    }
-
-    bool hasSuperFile(const char *superFileName) const
-    {
-        if (!superFileName || !*superFileName || !node)
-            return false;
-
         if (*superFileName=='~')
             superFileName++;
         StringBuffer xpath;
-        if (hasProp(makeSuperFileXPath(xpath, superFileName)))
-            return true;
-        ForEachItemIn(idx, bases)
+        Owned<IPropertyTreeIterator> subFiles = lookupElements(makeSuperFileXPath(xpath, superFileName), "SubFile");
+        if (subFiles)
         {
-            if (bases.item(idx).hasProp(xpath))
-                return true;
+            Owned<CPackageSuperFileArray> result = new CPackageSuperFileArray(*subFiles);
+            return result.getClear();
         }
-
-        return false;
     }
+    return NULL;
+}
 
-    CHpccPackage()
-    {
-        hash = 0;
-    }
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CHpccPackage(IPropertyTree *p)
-    {
-        if (p)
-            node.set(p);
+// Load mergedEnvironment from local XML node
+void CPackageNode::loadEnvironment()
+{
+    mergedEnvironment.setown(createProperties(true));
+    Owned<IPropertyTreeIterator> envIterator = node->getElements("Environment");
+    ForEach(*envIterator)
+    {
+        IPropertyTree &env = envIterator->query();
+        const char *id = env.queryProp("@id");
+        const char *val = env.queryProp("@value");
+        if (!val)
+            val = env.queryProp("@val"); // Historically we used val here - not sure why... other parts of package file used value
+        if (id && val)
+            mergedEnvironment->setProp(id, val);
         else
-            node.setown(createPTree("HpccPackages"));
-        StringBuffer xml;
-        toXML(node, xml);
-        hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
-    }
-
-    ~CHpccPackage()
-    {
-    }
-    // Load mergedEnvironment from local XML node
-    void loadEnvironment()
-    {
-        mergedEnvironment.setown(createProperties(true));
-        Owned<IPropertyTreeIterator> envIterator = node->getElements("Environment");
-        ForEach(*envIterator)
         {
-            IPropertyTree &env = envIterator->query();
-            const char *id = env.queryProp("@id");
-            const char *val = env.queryProp("@value");
-            if (!val)
-                val = env.queryProp("@val"); // Historically we used val here - not sure why... other parts of package file used value
-            if (id && val)
-                mergedEnvironment->setProp(id, val);
-            else
-            {
-                StringBuffer s;
-                toXML(&env, s);
-                throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
-            }
+            StringBuffer s;
+            toXML(&env, s);
+            throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
         }
-        Owned<IAttributeIterator> attrs = node->getAttributes();
-        for(attrs->first(); attrs->isValid(); attrs->next())
-        {
-            StringBuffer s("control:");
-            s.append(attrs->queryName()+1);  // queryName() has a leading @, hence the +1
-            mergedEnvironment->setProp(s.str(), attrs->queryValue());
-        }
-    }
-
-    virtual void resolveBases(IHpccPackageMap *packages)
-    {
-        loadEnvironment();
-        if (packages)
-        {
-            Owned<IPropertyTreeIterator> baseIterator = node->getElements("Base");
-            if (baseIterator->first())
-            {
-                do
-                {
-                    IPropertyTree &baseElem = baseIterator->query();
-                    const char *baseId = baseElem.queryProp("@id");
-                    if (!baseId)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
-                    const IHpccPackage *_base = packages->queryPackage(baseId);
-                    if (_base)
-                    {
-                        const CHpccPackage *base = static_cast<const CHpccPackage *>(_base);
-                        bases.append(const_cast<CHpccPackage &>(*LINK(base)));   // should really be an arrayof<const base> but that would require some fixing in jlib
-                        hash = rtlHash64Data(sizeof(base->hash), &base->hash, hash);
-                        mergeEnvironment(base);
-                    }
-                    else
-                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
-                }
-                while(baseIterator->next());
-            }
-        }
-    }
-
-    virtual const char *queryEnv(const char *varname) const
-    {
-        return mergedEnvironment->queryProp(varname);
-    }
-
-    virtual hash64_t queryHash() const
-    {
-        return hash;
-    }
-
-    virtual const IPropertyTree *queryTree() const
-    {
-        return node;
     }
-
-    virtual IPropertyTree *getQuerySets() const
+    Owned<IAttributeIterator> attrs = node->getAttributes();
+    for(attrs->first(); attrs->isValid(); attrs->next())
     {
-        if (node)
-            return node->getPropTree("QuerySets");
-        else
-            return NULL;
+        StringBuffer s("control:");
+        s.append(attrs->queryName()+1);  // queryName() has a leading @, hence the +1
+        mergedEnvironment->setProp(s.str(), attrs->queryValue());
     }
-};
+}
 
 CHpccPackage *createPackage(IPropertyTree *p)
 {
@@ -321,136 +113,92 @@ CHpccPackage *createPackage(IPropertyTree *p)
 // CPackageMap - an implementation of IPackageMap using a string map
 //================================================================================================
 
-class CHpccPackageMap : public CInterface, implements IHpccPackageMap
-{
-public:
-    MapStringToMyClass<CHpccPackage> packages;
-    StringAttr packageId;
-    StringAttr querySet;
-    bool active;
-    StringArray wildMatches, wildIds;
-public:
-    IMPLEMENT_IINTERFACE;
-    CHpccPackageMap(const char *_packageId, const char *_querySet, bool _active)
-        : packageId(_packageId), querySet(_querySet), active(_active), packages(true)
-    {
-    }
-
-    // IPackageMap interface
-    virtual bool isActive() const
-    {
-        return active;
-    }
-    virtual const IHpccPackage *queryPackage(const char *name) const
-    {
-        return name ? packages.getValue(name) : NULL;
-    }
-    virtual const IHpccPackage *matchPackage(const char *name) const
-    {
-        if (name)
-        {
-            const IHpccPackage *pkg = queryPackage(name);
-            if (pkg)
-                return pkg;
-            ForEachItemIn(idx, wildMatches)
-            {
-                if (WildMatch(name, wildMatches.item(idx), true))
-                    return queryPackage(wildIds.item(idx));
-            }
-        }
-        return NULL;
-    }
-    virtual const char *queryPackageId() const
-    {
-        return packageId;
-    }
-    void load(const char *id)
-    {
-        load(getPackageMapById(id, true));
-    }
-    void load(IPropertyTree *xml)
-    {
-        if (!xml)
-            return;
-        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
-        ForEach(*allpackages)
-        {
-            IPropertyTree &packageTree = allpackages->query();
-            const char *id = packageTree.queryProp("@id");
-            if (!id || !*id)
-                throw MakeStringException(-1, "Invalid package map - Package element missing id attribute");
-            Owned<CHpccPackage> package = createPackage(&packageTree);
-            packages.setValue(id, package.get());
-            const char *queries = packageTree.queryProp("@queries");
-            if (queries && *queries)
-            {
-                wildMatches.append(queries);
-                wildIds.append(id);
-            }
-        }
-        HashIterator it(packages);
-        ForEach (it)
-        {
-            CHpccPackage *pkg = packages.mapToValue(&it.query());
-            if (pkg)
-                pkg->resolveBases(this);
-        }
-    }
-};
-
 //================================================================================================
 // CHpccPackageSet - an implementation of IHpccPackageSet
 //================================================================================================
 
-class CHpccPackageSet : public CInterface, implements IHpccPackageSet
+CHpccPackageSet::CHpccPackageSet(const char *_process) : process(_process)
 {
-    IArrayOf<CHpccPackageMap> packageMaps;
-    StringAttr process;
-public:
-    IMPLEMENT_IINTERFACE;
-    CHpccPackageSet(const char *_process)
-        : process(_process)
-    {
-        Owned<IPropertyTree> ps = resolvePackageSetRegistry(process, true);
-        if (ps)
-            load(ps);
-    }
+    Owned<IPropertyTree> ps = resolvePackageSetRegistry(process, true);
+    if (ps)
+        load(ps);
+}
 
-    void load(IPropertyTree *xml)
+void CHpccPackageSet::load(IPropertyTree *xml)
+{
+    Owned<IPropertyTreeIterator> it = xml->getElements("PackageMap[@active='1']"); //only active for now
+    ForEach(*it)
     {
-        Owned<IPropertyTreeIterator> it = xml->getElements("PackageMap[@active='1']"); //only active for now
-        ForEach(*it)
-        {
-            IPropertyTree &tree = it->query();
-            if (!tree.hasProp("@id"))
-                continue;
-            Owned<CHpccPackageMap> pm = new CHpccPackageMap(tree.queryProp("@id"), tree.queryProp("@querySet"), true);
-            pm->load(tree.queryProp("@id"));
-            packageMaps.append(*pm.getClear());
-        }
+        IPropertyTree &tree = it->query();
+        if (!tree.hasProp("@id"))
+            continue;
+        Owned<CHpccPackageMap> pm = new CHpccPackageMap(tree.queryProp("@id"), tree.queryProp("@querySet"), true);
+        pm->load(tree.queryProp("@id"));
+        packageMaps.append(*pm.getClear());
     }
+}
 
-    virtual const IHpccPackageMap *queryActiveMap(const char *queryset) const
+const IHpccPackageMap *CHpccPackageSet::queryActiveMap(const char *queryset) const
+{
+    ForEachItemIn(i, packageMaps)
     {
-        ForEachItemIn(i, packageMaps)
+        CHpccPackageMap &pm = packageMaps.item(i);
+        StringAttr &match = pm.querySet;
+        if (!match.length())
+            continue;
+        if (isWildString(match))
         {
-            CHpccPackageMap &pm = packageMaps.item(i);
-            StringAttr &match = pm.querySet;
-            if (!match.length())
-                continue;
-            if (isWildString(match))
-            {
-                if (WildMatch(queryset, match))
-                    return &pm;
-            }
-            else if (streq(queryset, match))
+            if (WildMatch(queryset, match))
                 return &pm;
         }
-        return NULL;
+        else if (streq(queryset, match))
+            return &pm;
     }
-};
+    return NULL;
+}
 
 extern WORKUNIT_API IHpccPackageSet *createPackageSet(const char *process)
 {
     return new CHpccPackageSet(process);
 }
+
+extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool readonly)
+{
+    StringBuffer xpath;
+    xpath.append("/PackageMaps/PackageMap[@id=\"").append(id).append("\"]");
+    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return NULL;
+    return conn->getRoot();
+}
+
+extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly)
+{
+    StringBuffer xpath;
+    xpath.append("/PackageSets/PackageSet[@id=\"").append(id).append("\"]");
+    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return NULL;
+    return conn->getRoot();
+}
+
+extern WORKUNIT_API IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly)
+{
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    Owned<IPropertyTree> psroot = globalLock->getRoot();
+
+    StringBuffer xpath;
+    xpath.append("PackageSet[@process=\"").append(process).append("\"]");
+    IPropertyTree *ps = psroot->queryPropTree(xpath);
+    if (ps)
+        return getPackageSetById(ps->queryProp("@id"), readonly);
+    Owned<IPropertyTreeIterator> it = psroot->getElements("PackageSet[@process]");
+    ForEach(*it)
+    {
+        const char *match = it->query().queryProp("@process");
+        const char *id = it->query().queryProp("@id");
+        if (id && isWildString(match) && WildMatch(process, match))
+            return getPackageSetById(id, readonly);
+    }
+    return NULL;
+}

common/workunit/package.hpp

@@ -15,8 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-#ifndef WUPACKAGE_HPP
-#define WUPACKAGE_HPP
+#ifndef WUPACKAGE_H
+#define WUPACKAGE_H
 
 #include "dadfs.hpp"
 #include "workunit.hpp"
@@ -25,12 +25,18 @@ interface IHpccPackage : extends IInterface
 {
     virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const = 0;
     virtual bool hasSuperFile(const char *superFileName) const = 0;
+    virtual const char *queryEnv(const char *varname) const = 0;
+    virtual bool getEnableFieldTranslation() const = 0;
+    virtual const IPropertyTree *queryTree() const = 0;
+    virtual hash64_t queryHash() const = 0;
 };
 
 interface IHpccPackageMap : extends IInterface
 {
-     virtual const IHpccPackage *queryPackage(const char *name) const = 0;
-     virtual const IHpccPackage *matchPackage(const char *name) const = 0;
+    virtual const IHpccPackage *queryPackage(const char *name) const = 0;
+    virtual const IHpccPackage *matchPackage(const char *name) const = 0;
+    virtual const char *queryPackageId() const = 0;
+    virtual bool isActive() const = 0;
 };
 
 interface IHpccPackageSet : extends IInterface
@@ -39,5 +45,9 @@ interface IHpccPackageSet : extends IInterface
 };
 
 extern WORKUNIT_API IHpccPackageSet *createPackageSet(const char *process);
+extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool readonly);
+extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly);
+extern WORKUNIT_API IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly);
+
 
 #endif

common/workunit/pkgimpl.hpp

@@ -0,0 +1,366 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef WUPACKAGE_IMPL_HPP
+#define WUPACKAGE_IMPL_HPP
+
+#include "platform.h"
+#include "jprop.hpp"
+#include "jptree.hpp"
+#include "jregexp.hpp"
+#include "package.h"
+
+class CPackageSuperFileArray : public CInterface, implements ISimpleSuperFileEnquiry
+{
+    IArrayOf<IPropertyTree> subFiles;
+public:
+    IMPLEMENT_IINTERFACE;
+    CPackageSuperFileArray(IPropertyTreeIterator &_subs)
+    {
+        ForEach(_subs)
+        {
+            IPropertyTree &sub = _subs.query();
+            sub.Link();
+            subFiles.append(sub);
+        }
+    }
+    virtual unsigned numSubFiles() const
+    {
+        return subFiles.length();
+    }
+    virtual bool getSubFileName(unsigned num, StringBuffer &name) const
+    {
+        if (subFiles.isItem(num))
+        {
+            name.append(subFiles.item(num).queryProp("@value"));
+            return true;
+        }
+        else
+            return false;
+    }
+    virtual unsigned findSubName(const char *subname) const
+    {
+        ForEachItemIn(idx, subFiles)
+        {
+            if (stricmp(subFiles.item(idx).queryProp("@value"), subname))
+                return idx;
+        }
+        return NotFound;
+    }
+    virtual unsigned getContents(StringArray &contents) const
+    {
+        ForEachItemIn(idx, subFiles)
+        {
+            contents.append(subFiles.item(idx).queryProp("@value"));
+        }
+        return subFiles.length();
+    }
+};
+
+class WORKUNIT_API CPackageNode : extends CInterface, implements IHpccPackage
+{
+protected:
+    Owned<IPropertyTree> node;
+    Owned<IProperties> mergedEnvironment;
+    hash64_t hash;
+
+    void mergeEnvironment(const CPackageNode *base);
+
+    virtual IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const = 0;
+
+    inline StringBuffer makeSuperFileXPath(StringBuffer &xpath, const char *superFileName) const
+    {
+        return xpath.append("SuperFile[@id='").appendLower(strlen(superFileName), superFileName).append("']");
+    }
+
+    ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const;
+
+    inline bool hasProp(const char *xpath) const
+    {
+        return (node) ? node->hasProp(xpath) : false;
+    }
+
+    virtual bool getSysFieldTranslationEnabled() const {return false;}
+    virtual bool getEnableFieldTranslation() const
+    {
+        const char *val = queryEnv("control:enableFieldTranslation");
+        if (val)
+            return strToBool(val);
+        else
+            return getSysFieldTranslationEnabled();
+    }
+
+
+    CPackageNode()
+    {
+        hash = 0;
+    }
+
+    virtual const IHpccPackage *queryRootPackage()
+    {
+        return NULL;
+    }
+
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CPackageNode(IPropertyTree *p);
+
+    ~CPackageNode()
+    {
+    }
+
+    // Load mergedEnvironment from local XML node
+    void loadEnvironment();
+
+    virtual const char *queryEnv(const char *varname) const
+    {
+        return mergedEnvironment->queryProp(varname);
+    }
+
+    virtual hash64_t queryHash() const
+    {
+        return hash;
+    }
+
+    virtual const IPropertyTree *queryTree() const
+    {
+        return node;
+    }
+
+    virtual IPropertyTree *getQuerySets() const
+    {
+        if (!node)
+            return NULL;
+        return node->getPropTree("QuerySets");
+    }
+};
+
+template <class TYPE>
+class CResolvedPackage : public TYPE
+{
+public:
+    typedef CResolvedPackage<TYPE> self;
+    CIArrayOf<self> bases;
+
+    CResolvedPackage<TYPE>(IPropertyTree *p) : TYPE(p) {}
+
+    virtual aindex_t getBaseCount() const {return bases.length();}
+    const self *getResolvedBase(aindex_t pos) const
+    {
+        if (pos < getBaseCount())
+            return &bases.item(pos);
+        return NULL;
+    }
+    virtual const TYPE *getBaseNode(aindex_t pos) const {return (const TYPE *) getResolvedBase(pos);}
+
+    void appendBase(const IHpccPackage *base)
+    {
+        if (base)
+        {
+            const self *p = dynamic_cast<const self *>(base);
+            bases.append(const_cast<self &>(*LINK(p)));   // should really be an arrayof<const base> but that would require some fixing in jlib
+            TYPE::hash = rtlHash64Data(sizeof(p->hash), &p->hash, TYPE::hash);
+            mergeEnvironment(p);
+        }
+    }
+
+    void resolveBases(const IHpccPackageMap *packages)
+    {
+        TYPE::loadEnvironment();
+        if (packages)
+        {
+            Owned<IPropertyTreeIterator> baseIterator = TYPE::node->getElements("Base");
+            if (!baseIterator->first())
+                appendBase(TYPE::queryRootPackage());
+            else
+            {
+                do
+                {
+                    IPropertyTree &baseElem = baseIterator->query();
+                    const char *baseId = baseElem.queryProp("@id");
+                    if (!baseId)
+                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
+                    const IHpccPackage *base = packages->queryPackage(baseId);
+                    if (!base)
+                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
+                    appendBase(base);
+                }
+                while(baseIterator->next());
+            }
+        }
+    }
+
+    // Search this package and any bases for an element matching xpath1, then return iterator for its children that match xpath2
+    IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const
+    {
+        IPropertyTree *parentNode = TYPE::node->queryPropTree(xpath1);
+        if (parentNode)
+            return parentNode->getElements(xpath2);
+        ForEachItemIn(idx, bases)
+        {
+            const self &basePackage = bases.item(idx);
+            IPropertyTreeIterator *it = basePackage.lookupElements(xpath1, xpath2);
+            if (it)
+                return it;
+        }
+        return NULL;
+    }
+
+    bool hasSuperFile(const char *superFileName) const
+    {
+        if (!superFileName || !*superFileName || !TYPE::node)
+            return false;
+
+        if (*superFileName=='~')
+            superFileName++;
+        StringBuffer xpath;
+        if (hasProp(TYPE::makeSuperFileXPath(xpath, superFileName)))
+            return true;
+        ForEachItemIn(idx, bases)
+        {
+            if (bases.item(idx).hasProp(xpath))
+                return true;
+        }
+
+        return false;
+    }
+};
+
+
+typedef CResolvedPackage<CPackageNode> CHpccPackage;
+
+//================================================================================================
+// CPackageMap - an implementation of IPackageMap using a string map
+//================================================================================================
+
+template <class TYPE, class IFACE>
+class CPackageMapOf : public CInterface, implements IHpccPackageMap
+{
+public:
+    typedef CResolvedPackage<TYPE> packageType;
+    MapStringToMyClassViaBase<packageType, IFACE> packages;
+    StringAttr packageId;
+    StringAttr querySet;
+    bool active;
+    StringArray wildMatches, wildIds;
+public:
+    IMPLEMENT_IINTERFACE;
+    CPackageMapOf(const char *_packageId, const char *_querySet, bool _active)
+        : packageId(_packageId), querySet(_querySet), active(_active), packages(true)
+    {
+    }
+
+    // IPackageMap interface
+    virtual bool isActive() const
+    {
+        return active;
+    }
+
+    virtual const packageType *queryResolvedPackage(const char *name) const
+    {
+        return name ? packages.getValue(name) : NULL;
+    }
+
+    virtual const IHpccPackage *queryPackage(const char *name) const
+    {
+        return name ? (IFACE*)packages.getValue(name) : NULL;
+    }
+
+    virtual const char *queryPackageId() const
+    {
+        return packageId;
+    }
+
+    virtual const packageType *matchResolvedPackage(const char *name) const
+    {
+        if (name)
+        {
+            const packageType *pkg = queryResolvedPackage(name);
+            if (pkg)
+                return pkg;
+            ForEachItemIn(idx, wildMatches)
+            {
+                if (WildMatch(name, wildMatches.item(idx), true))
+                    return queryResolvedPackage(wildIds.item(idx));
+            }
+        }
+        return NULL;
+    }
+
+    const IHpccPackage *matchPackage(const char *name) const
+    {
+        return (IFACE *) matchResolvedPackage(name);
+    }
+
+    void load(IPropertyTree *xml)
+    {
+        if (!xml)
+            return;
+        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
+        ForEach(*allpackages)
+        {
+            IPropertyTree &packageTree = allpackages->query();
+            const char *id = packageTree.queryProp("@id");
+            if (!id || !*id)
+                throw MakeStringException(-1, "Invalid package map - Package element missing id attribute");
+            Owned<packageType> package = new packageType(&packageTree);
+            packages.setValue(id, package.get());
+            const char *queries = packageTree.queryProp("@queries");
+            if (queries && *queries)
+            {
+                wildMatches.append(queries);
+                wildIds.append(id);
+            }
+        }
+        HashIterator it(packages);
+        ForEach (it)
+        {
+            packageType *pkg = packages.getValue((const char *)it.query().getKey());
+            if (pkg)
+                pkg->resolveBases(this);
+        }
+    }
+
+    void load(const char *id)
+    {
+        load(getPackageMapById(id, true));
+    }
+};
+
+typedef CPackageMapOf<CPackageNode, IHpccPackage> CHpccPackageMap;
+
+
+//================================================================================================
+// CHpccPackageSet - an implementation of IHpccPackageSet
+//================================================================================================
+
+class WORKUNIT_API CHpccPackageSet : public CInterface, implements IHpccPackageSet
+{
+    IArrayOf<CHpccPackageMap> packageMaps;
+    StringAttr process;
+public:
+    IMPLEMENT_IINTERFACE;
+    CHpccPackageSet(const char *_process);
+
+    void load(IPropertyTree *xml);
+
+    virtual const IHpccPackageMap *queryActiveMap(const char *queryset) const;
+};
+
+#endif

common/workunit/workunit.cpp

@@ -2052,10 +2052,15 @@ public:
     }
     virtual IConstWorkUnit* secOpenWorkUnit(const char *wuid, bool lock, ISecManager *secmgr=NULL, ISecUser *secuser=NULL)
     {
+        StringBuffer wuidStr(wuid);
+        wuidStr.trim();
+        if (!wuidStr.length())
+            return NULL;
+
         if (workUnitTraceLevel > 1)
-            PrintLog("openWorkUnit %s", wuid);
+            PrintLog("openWorkUnit %s", wuidStr.str());
         StringBuffer wuRoot;
-        getXPath(wuRoot, wuid);
+        getXPath(wuRoot, wuidStr.str());
         IRemoteConnection* conn = sdsManager->connect(wuRoot.str(), session, lock ? RTM_LOCK_READ|RTM_LOCK_SUB : 0, SDS_LOCK_TIMEOUT);
         if (conn)
         {
@@ -2073,7 +2078,7 @@ public:
         else
         {
             if (workUnitTraceLevel > 0)
-                PrintLog("openWorkUnit %s not found", wuid);
+                PrintLog("openWorkUnit %s not found", wuidStr.str());
             return NULL;
         }
     }

esp/eclwatch/ws_XSLT/result_lib.xslt

@@ -421,6 +421,9 @@
                         </xsl:when>
                         <xsl:otherwise>
                             <data>
+                                <xsl:if test="number($escapeResults) and not(starts-with($rowSchema/@name, '__html__'))">
+                                    <xsl:attribute name="escape">1</xsl:attribute>
+                                </xsl:if>
                                 <xsl:for-each select="$matchingData|$matchingData2">
                                     <xsl:value-of select="."/>
                                     <xsl:if test="position()!=last()">, </xsl:if>

esp/files/scripts/configmgr/configmgr.js

@@ -1920,7 +1920,7 @@ function createEnvXmlView(allrows, compName, subRecordIndex) {
       }
 
      var aMenuItemsX = [ { text: "Delete",        onclick: { fn: onContextMenuBeforeShowDeleteContextMenu} },
-                          { text: "Add Attribute", onclick: { fn: onContextMenuBeforeAddAttribute}, }
+                          { text: "Add Attribute", onclick: { fn: onContextMenuBeforeAddAttribute} }
                         ];
       top.document.rightDT = dt;
       top.document.rightDT.expandRecord = function(id) {

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1241,7 +1241,12 @@ bool CWsDfuEx::DFUDeleteFiles(IEspContext &context, IEspDFUArrayActionRequest &r
                         returnStr.appendf("<Message><Value>%s</Value></Message>",errorStr.str());
                         DBGLOG("%s", errorStr.str());
                     }
-                    else 
+                    else if (!deleted)
+                    {
+                        returnStr.appendf("<Message><Value>Logical File %s not deleted. No error message.</Value></Message>",logicalFileName.str());
+                        DBGLOG("Logical File %s not deleted. No error message.\n",logicalFileName.str());
+                    }
+                    else
                     {
                         PrintLog("Deleted Logical File: %s\n",logicalFileName.str());
                         returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>",logicalFileName.str());
@@ -1252,7 +1257,15 @@ bool CWsDfuEx::DFUDeleteFiles(IEspContext &context, IEspDFUArrayActionRequest &r
                     df.clear(); 
                     deleted = queryDistributedFileDirectory().removeEntry(logicalFileName.str(),userdesc, REMOVE_FILE_SDS_CONNECT_TIMEOUT); // this can remove clusters also
                     if (deleted)
+                    {
+                        PrintLog("Detached File: %s\n",logicalFileName.str());
                         returnStr.appendf("<Message><Value>Detached File %s</Value></Message>", logicalFileName.str());
+                    }
+                    else
+                    {
+                        returnStr.appendf("<Message><Value>File %s not detached.</Value></Message>",logicalFileName.str());
+                        DBGLOG("File %s not detached.\n",logicalFileName.str());
+                    }
                 }
 
                 if (!deleted)

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -121,7 +121,7 @@ void ensureWsWorkunitAccess(IEspContext& context, const char* wuid, SecAccessFla
     Owned<IWorkUnitFactory> wf = getWorkUnitFactory(context.querySecManager(), context.queryUser());
     Owned<IConstWorkUnit> cw = wf->openWorkUnit(wuid, false);
     if (!cw)
-        throw MakeStringException(ECLWATCH_ECL_WU_ACCESS_DENIED, "Failed to open workunit %s when ensuring workunit access", wuid);
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT, "Failed to open workunit %s when ensuring workunit access", wuid);
     ensureWsWorkunitAccess(context, *cw, minAccess);
 }
 

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -40,7 +40,7 @@
 #include "thorplugin.hpp"
 #include "roxiecontrol.hpp"
 
-#include "package.hpp"
+#include "package.h"
 
 #ifdef _USE_ZLIB
 #include "zcrypt.hpp"
@@ -1713,36 +1713,37 @@ bool CWsWorkunitsEx::onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEsp
             getArchivedWUInfo(context, wuid.str(), resp);
         else
         {
-            //The access is checked here because getArchivedWUInfo() has its own access check.
-            ensureWsWorkunitAccess(context, wuid.str(), SecAccess_Read);
-
-            unsigned flags=0;
-            if (req.getTruncateEclTo64k())
-                flags|=WUINFO_TruncateEclTo64k;
-            if (req.getIncludeExceptions())
-                flags|=WUINFO_IncludeExceptions;
-            if (req.getIncludeGraphs())
-                flags|=WUINFO_IncludeGraphs;
-            if (req.getIncludeSourceFiles())
-                flags|=WUINFO_IncludeSourceFiles;
-            if (req.getIncludeResults())
-                flags|=WUINFO_IncludeResults;
-            if (req.getIncludeVariables())
-                flags|=WUINFO_IncludeVariables;
-            if (req.getIncludeTimers())
-                flags|=WUINFO_IncludeTimers;
-            if (req.getIncludeDebugValues())
-                flags|=WUINFO_IncludeDebugValues;
-            if (req.getIncludeApplicationValues())
-                flags|=WUINFO_IncludeApplicationValues;
-            if (req.getIncludeWorkflows())
-                flags|=WUINFO_IncludeWorkflows;
-            if (!req.getSuppressResultSchemas())
-                flags|=WUINFO_IncludeEclSchemas;
-            if (req.getIncludeXmlSchemas())
-                flags|=WUINFO_IncludeXmlSchema;
             try
             {
+                //The access is checked here because getArchivedWUInfo() has its own access check.
+                ensureWsWorkunitAccess(context, wuid.str(), SecAccess_Read);
+
+                unsigned flags=0;
+                if (req.getTruncateEclTo64k())
+                    flags|=WUINFO_TruncateEclTo64k;
+                if (req.getIncludeExceptions())
+                    flags|=WUINFO_IncludeExceptions;
+                if (req.getIncludeGraphs())
+                    flags|=WUINFO_IncludeGraphs;
+                if (req.getIncludeSourceFiles())
+                    flags|=WUINFO_IncludeSourceFiles;
+                if (req.getIncludeResults())
+                    flags|=WUINFO_IncludeResults;
+                if (req.getIncludeVariables())
+                    flags|=WUINFO_IncludeVariables;
+                if (req.getIncludeTimers())
+                    flags|=WUINFO_IncludeTimers;
+                if (req.getIncludeDebugValues())
+                    flags|=WUINFO_IncludeDebugValues;
+                if (req.getIncludeApplicationValues())
+                    flags|=WUINFO_IncludeApplicationValues;
+                if (req.getIncludeWorkflows())
+                    flags|=WUINFO_IncludeWorkflows;
+                if (!req.getSuppressResultSchemas())
+                    flags|=WUINFO_IncludeEclSchemas;
+                if (req.getIncludeXmlSchemas())
+                    flags|=WUINFO_IncludeXmlSchema;
+
                 WsWuInfo winfo(context, wuid.str());
                 winfo.getInfo(resp.updateWorkunit(), flags);
 

initfiles/componentfiles/configxml/dali.xsl

@@ -158,7 +158,7 @@
             </xsl:attribute>
           </xsl:if>
           <xsl:attribute name="backupComputer">
-            <xsl:value-of select="@backupComputer"/>
+            <xsl:value-of select="/Environment/Hardware/Computer[@name=current()/@backupComputer]/@netAddress"/>
           </xsl:attribute>
         </xsl:if>
         <xsl:attribute name="asyncBackup">

initfiles/componentfiles/thor/run_thor

@@ -51,7 +51,8 @@ while [ 1 ]; do
     if [ "$localthor" = "true" ]; then
         $deploydir/start_slaves $THORMASTER thorslave${LCR} $THORMASTER $THORMASTERPORT $logdir $instancedir $THORNAME $PATH_PRE $logredirect
     else
-        $deploydir/frunssh $instancedir/uslaves.start "/bin/sh -c '$deploydir/start_slaves %a thorslave${LCR} $THORMASTER $THORMASTERPORT $logdir $instancedir $THORNAME $PATH_PRE $logredirect'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries 2>&1
+        nslaves=`cat $instancedir/uslaves.start | wc -l`
+        $deploydir/frunssh $instancedir/uslaves.start "/bin/sh -c '$deploydir/start_slaves %a thorslave${LCR} $THORMASTER $THORMASTERPORT $logdir $instancedir $THORNAME $PATH_PRE $logredirect'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$nslaves 2>&1
     fi
 
     echo thormaster cmd : $instancedir/thormaster_$THORNAME MASTER=$THORMASTER:$THORMASTERPORT

initfiles/componentfiles/thor/stop_thor

@@ -90,7 +90,8 @@ else
         fi
     fi
     sort $instancedir/slaves | uniq > $instancedir/uslaves
-    $deploydir/frunssh $instancedir/uslaves "/bin/sh -c '$deploydir/stop_slaves ${comp_base} $PATH_PRE $THORMASTER:$THORMASTERPORT'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries 2>&1 | egrep -v "no process killed"
+    nslaves=`cat $instancedir/uslaves.start | wc -l`
+    $deploydir/frunssh $instancedir/uslaves "/bin/sh -c '$deploydir/stop_slaves ${comp_base} $PATH_PRE $THORMASTER:$THORMASTERPORT'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$nslaves 2>&1 | egrep -v "no process killed"
     echo slaves stopped
 fi
 

roxie/ccd/ccdquery.cpp

@@ -787,7 +787,7 @@ public:
             return NULL;
     }
 
-    static hash64_t getQueryHash(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+    static hash64_t getQueryHash(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
     {
         hash64_t hashValue = rtlHash64VStr(dll->queryDll()->queryName(), package.queryHash());
         hashValue = rtlHash64VStr(id, hashValue);
@@ -1310,7 +1310,7 @@ public:
     }
 };
 
-extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
 {
     CriticalBlock b(CQueryFactory::queryCreateLock);
     hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo);
@@ -1320,7 +1320,7 @@ extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *
         ::Release(dll);
         return cached;
     }
-    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, package, hashValue);
+    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue);
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }
@@ -1331,7 +1331,7 @@ extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu)
     if (!dll)
         return NULL;
     SCMStringBuffer wuid;
-    return createServerQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), NULL); // MORE - if use a constant for id might cache better?
+    return createServerQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootRoxiePackage(), NULL); // MORE - if use a constant for id might cache better?
 }
 
 //==============================================================================================================================================
@@ -1559,7 +1559,7 @@ public:
     }
 };
 
-IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned channel, const IPropertyTree *stateInfo)
+IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, unsigned channel, const IPropertyTree *stateInfo)
 {
     CriticalBlock b(CQueryFactory::queryCreateLock);
     hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo);
@@ -1569,7 +1569,7 @@ IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, con
         ::Release(dll);
         return cached;
     }
-    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, package, hashValue, channel);
+    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, channel);
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }
@@ -1580,7 +1580,7 @@ extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned
     if (!dll)
         return NULL;
     SCMStringBuffer wuid;
-    return createSlaveQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), channelNo, NULL);  // MORE - if use a constant for id might cache better?
+    return createSlaveQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootRoxiePackage(), channelNo, NULL);  // MORE - if use a constant for id might cache better?
 }
 
 IRecordLayoutTranslator * createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta)

roxie/ccd/ccdquery.hpp

@@ -30,6 +30,7 @@
 #include "ccddali.hpp"
 #include "thorcommon.ipp"
 #include "roxierow.hpp"
+#include "package.h"
 
 class TranslatorArray : public CInterface, implements IInterface
 {
@@ -228,8 +229,8 @@ extern const IQueryDll *createExeQueryDll(const char *exeName);
 extern const IQueryDll *createWuQueryDll(IConstWorkUnit *wu);
 
 extern IRecordLayoutTranslator *createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta);
-extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo);
-extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned _channelNo, const IPropertyTree *stateInfo);
+extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo);
+extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, unsigned _channelNo, const IPropertyTree *stateInfo);
 extern IQueryFactory *getQueryFactory(hash64_t hashvalue, unsigned channel);
 extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu);
 extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned channelNo);

roxie/ccd/ccdserver.cpp

@@ -30017,7 +30017,7 @@ public:
         CriticalBlock b(daliUpdateCrit);
         if (!dynamicPackage)
         {
-            dynamicPackage.setown(createPackage(NULL));
+            dynamicPackage.setown(createRoxiePackage(NULL, NULL));
         }
         return dynamicPackage->lookupFileName(filename, isOpt, true, workUnit);
     }
@@ -30027,7 +30027,7 @@ public:
         CriticalBlock b(daliUpdateCrit);
         if (!dynamicPackage)
         {
-            dynamicPackage.setown(createPackage(NULL));
+            dynamicPackage.setown(createRoxiePackage(NULL, NULL));
         }
         return dynamicPackage->createFileName(filename, overwrite, extend, clusters, workUnit);
     }
@@ -32445,7 +32445,7 @@ protected:
 
     void init()
     {
-        package.setown(createPackage(NULL));
+        package.setown(createPackage(NULL, NULL));
         ctx.setown(createSlaveContext(NULL, logctx, 0, 50*1024*1024, NULL));
         queryDll.setown(createExeQueryDll("roxie"));
         queryFactory.setown(createServerQueryFactory("test", queryDll.getLink(), *package, NULL));

roxie/ccd/ccdstate.cpp

@@ -36,6 +36,8 @@
 #include "dafdesc.hpp"
 #include "dautils.hpp"
 
+#include "pkgimpl.hpp"
+
 //-------------------------------------------------------------------------------------------
 // class CRoxiePluginCtx - provide the environments for plugins loaded by roxie. 
 // Base class handles making sure memory allocation comes from the right heap. 
@@ -179,19 +181,41 @@ public:
  * </PackageMaps>
  */
 
-class CRoxiePackage : extends CInterface, implements IRoxiePackage
+class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
 {
 protected:
-    Owned<IPropertyTree> node;
-    IArrayOf<CRoxiePackage> bases;
     Owned<IRoxieDaliHelper> daliHelper;
-    Owned<IProperties> mergedEnvironment;
-    hash64_t hash;
-    bool compulsory;  // This concept may well disappear...
 
     mutable CriticalSection cacheLock;
     mutable CopyMapStringToMyClass<IResolvedFile> fileCache;
 
+    virtual aindex_t getBaseCount() const = 0;
+    virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
+
+    //map ambiguous IHpccPackage
+    virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
+    {
+        return CPackageNode::resolveSuperFile(superFileName);
+    }
+    virtual const char *queryEnv(const char *varname) const
+    {
+        return CPackageNode::queryEnv(varname);
+    }
+    virtual bool getEnableFieldTranslation() const
+    {
+        return CPackageNode::getEnableFieldTranslation();
+    }
+    virtual const IPropertyTree *queryTree() const
+    {
+        return CPackageNode::queryTree();
+    }
+    virtual hash64_t queryHash() const
+    {
+        return CPackageNode::queryHash();
+    }
+
+    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
+
     // Add a filename and the corresponding IResolvedFile to the cache
     void addCache(const char *filename, const IResolvedFile *file) const
     {
@@ -224,82 +248,7 @@ protected:
         }
         return NULL;
     }
-    // Load mergedEnvironment from local XML node
-    void loadEnvironment()
-    {
-        mergedEnvironment.setown(createProperties(true));
-        Owned<IPropertyTreeIterator> envIterator = node->getElements("Environment");
-        ForEach(*envIterator)
-        {
-            IPropertyTree &env = envIterator->query();
-            const char *id = env.queryProp("@id");
-            const char *val = env.queryProp("@value");
-            if (!val)
-                val = env.queryProp("@val"); // Historically we used val here - not sure why... other parts of package file used value
-            if (id && val)
-                mergedEnvironment->setProp(id, val);
-            else
-            {
-                StringBuffer s;
-                toXML(&env, s);
-                throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
-            }
-        }
-        Owned<IAttributeIterator> attrs = node->getAttributes();
-        for(attrs->first(); attrs->isValid(); attrs->next())
-        {
-            StringBuffer s("control:");
-            s.append(attrs->queryName()+1);  // queryName() has a leading @, hence the +1
-            mergedEnvironment->setProp(s.str(), attrs->queryValue());
-        }
-    }
-    // Merge base package environment into mergedEnvironment
-    void mergeEnvironment(const CRoxiePackage *base)
-    {
-        Owned<IPropertyIterator> envIterator = base->mergedEnvironment->getIterator();
-        ForEach(*envIterator)
-        {
-            const char *id = envIterator->getPropKey();
-            const char *val = base->mergedEnvironment->queryProp(id);
-            if (id && val && !mergedEnvironment->hasProp(id))
-                mergedEnvironment->setProp(id, val);
-        }
-    }
-    // Search this package and any bases for an element matching xpath1, then return iterator for its children that match xpath2
-    IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const
-    {
-        IPropertyTree *parentNode = node->queryPropTree(xpath1);
-        if (parentNode)
-            return parentNode->getElements(xpath2);
-        ForEachItemIn(idx, bases)
-        {
-            const CRoxiePackage &basePackage = bases.item(idx);
-            IPropertyTreeIterator *it = basePackage.lookupElements(xpath1, xpath2);
-            if (it)
-                return it;
-        }
-        return NULL;
-    }
-    // Use local package and its bases to resolve superfile name list of subfiles via all supported resolvers
-    const ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
-    {
-        // Order of resolution: 
-        // 1. SuperFiles named in local package
-        // 2. SuperFiles named in bases
-        // There is no dali or local case - a superfile that is resolved in dali must also resolve the subfiles there (and is all done in the resolveLFNusingDali method)
-        if (node)
-        {
-            StringBuffer superFileXPath;
-            superFileXPath.append("SuperFile[@id='").append(superFileName).append("']");
-            Owned<IPropertyTreeIterator> subFiles = lookupElements(superFileXPath, "SubFile");
-            if (subFiles)
-            {
-                Owned<CSimpleSuperFileArray> result = new CSimpleSuperFileArray(*subFiles);
-                return result.getClear();
-            }
-        }
-        return NULL;
-    }
+
     // Use local package file only to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingPackage(const char *fileName) const
     {
@@ -316,6 +265,7 @@ protected:
         }
         return NULL;
     }
+
     // Use dali to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingDali(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate) const
     {
@@ -423,10 +373,13 @@ protected:
                 addCache(fileName, result);
             return result;
         }
-        ForEachItemIn(idx, bases)
+        aindex_t count = getBaseCount();
+        for (aindex_t i = 0; i < count; i++)
         {
-            const CRoxiePackage &basePackage = bases.item(idx);
-            IResolvedFile *result = basePackage.lookupFile(fileName, cache, writeAccess, alwaysCreate);
+            const CRoxiePackageNode *basePackage = getBaseNode(i);
+            if (!basePackage)
+                continue;
+            IResolvedFile *result = basePackage->lookupFile(fileName, cache, writeAccess, alwaysCreate);
             if (result)
                 return result;
         }
@@ -434,98 +387,27 @@ protected:
     }
 
     // default constructor for derived class use
-    CRoxiePackage()
+    CRoxiePackageNode()
     {
-        hash = 0;
-        compulsory = false;
     }
 
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxiePackage(IPropertyTree *p)
+    CRoxiePackageNode(IPropertyTree *p) : CPackageNode(p)
     {
-        if (p)
-            node.set(p);
-        else
-            node.setown(createPTree("RoxiePackages"));
-        StringBuffer xml;
-        toXML(node, xml);
-        hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
-        compulsory = false;
         daliHelper.setown(connectToDali()); // MORE - should make this conditional
         if (!daliHelper.get() || !daliHelper->connected())
             node->setPropBool("@localFiles", true);
     }
 
-    ~CRoxiePackage()
+    ~CRoxiePackageNode()
     {
         assertex(fileCache.count()==0);
         // If it's possible for cached objects to outlive the cache I think there is a problem...
         // we could set the cache field to null here for any objects still in cache but there would be a race condition
     }
 
-    virtual void resolveBases(IPackageMap *packages)
-    {
-        loadEnvironment();
-        if (packages)
-        {
-            Owned<IPropertyTreeIterator> baseIterator = node->getElements("Base");
-            if (baseIterator->first())
-            {
-                do
-                {
-                    IPropertyTree &baseElem = baseIterator->query();
-                    const char *baseId = baseElem.queryProp("@id");
-                    if (!baseId)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
-                    const IRoxiePackage *_base = packages->queryPackage(baseId);
-                    if (_base)
-                    {
-                        const CRoxiePackage *base = static_cast<const CRoxiePackage *>(_base);
-                        bases.append(const_cast<CRoxiePackage &>(*LINK(base)));   // should really be an arrayof<const base> but that would require some fixing in jlib
-                        hash = rtlHash64Data(sizeof(base->hash), &base->hash, hash);
-                        mergeEnvironment(base);
-                    }
-                    else
-                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
-                }
-                while(baseIterator->next());
-            }
-            else 
-            {
-                const IRoxiePackage &rootPackage = queryRootPackage();
-                const CRoxiePackage &base = static_cast<const CRoxiePackage &>(rootPackage);
-                base.Link();
-                bases.append(const_cast<CRoxiePackage &>(base));   // should really be an arryof<const base> but that would require some fixing in jlib
-                hash = rtlHash64Data(sizeof(base.hash), &base.hash, hash);
-                mergeEnvironment(&base);
-            }
-        }
-        const char * val = queryEnv("control:compulsory");
-        if (val)
-            compulsory=strToBool(val);
-    }
-
-    virtual bool getEnableFieldTranslation() const
-    {
-        const char *val = queryEnv("control:enableFieldTranslation");
-        if (val)
-            return strToBool(val);
-        else
-            return fieldTranslationEnabled;
-    }
-
-    virtual const char *queryEnv(const char *varname) const
-    {
-        return mergedEnvironment->queryProp(varname);
-    }
-
-    virtual hash64_t queryHash() const 
-    {
-        return hash;
-    }
-
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const 
     {
         StringBuffer xpath;
@@ -578,11 +460,6 @@ public:
         return createRoxieWriteHandler(daliHelper, ldFile.getClear(), clusters);
     }
 
-    virtual const IPropertyTree *queryTree() const
-    {
-        return node;
-    }
-
     virtual IPropertyTree *getQuerySets() const
     {
         if (node)
@@ -592,84 +469,66 @@ public:
     }
 };
 
-IRoxiePackage *createPackage(IPropertyTree *p)
+typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;
+
+IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages)
 {
-    return new CRoxiePackage(p);
+    Owned<CRoxiePackage> pkg = new CRoxiePackage(p);
+    if (packages)
+        pkg->resolveBases(packages);
+    return pkg.getClear();
 }
 
 //================================================================================================
 // CPackageMap - an implementation of IPackageMap using a string map
 //================================================================================================
 
-class CPackageMap : public CInterface, implements IPackageMap
+class CRoxiePackageMap : public CPackageMapOf<CRoxiePackageNode, IRoxiePackage>, implements IRoxiePackageMap
 {
-    MapStringToMyClass<IRoxiePackage> packages;
-    StringAttr packageId;
-    bool active;
-    StringArray wildMatches, wildIds;
 public:
     IMPLEMENT_IINTERFACE;
-    CPackageMap(const char *_packageId, bool _active)
-        : packageId(_packageId), active(_active), packages(true)
+
+    typedef CPackageMapOf<CRoxiePackageNode, IRoxiePackage> BASE;
+
+    CRoxiePackageMap(const char *_packageId, const char *_querySet, bool _active)
+        : BASE(_packageId, _querySet, _active)
     {
     }
 
-    // IPackageMap interface
-    virtual bool isActive() const
+    //map ambiguous IHpccPackageMap interface
+    virtual const IHpccPackage *queryPackage(const char *name) const
     {
-        return active;
+        return BASE::queryPackage(name);
     }
-    virtual const IRoxiePackage *queryPackage(const char *name) const
+    virtual const IHpccPackage *matchPackage(const char *name) const
     {
-        return name ? packages.getValue(name) : NULL;
+        return BASE::matchPackage(name);
     }
-    virtual const IRoxiePackage *matchPackage(const char *name) const
+    virtual const char *queryPackageId() const
     {
-        if (name)
-        {
-            ForEachItemIn(idx, wildMatches)
-            {
-                if (WildMatch(name, wildMatches.item(idx), true))
-                    return queryPackage(wildIds.item(idx));
-            }
-        }
-        return NULL;
+        return BASE::queryPackageId();
     }
-    virtual const char *queryPackageId() const
+    virtual bool isActive() const
     {
-        return packageId;
+        return BASE::isActive();
     }
-    void load(IPropertyTree *xml)
+
+    virtual const IRoxiePackage *queryRoxiePackage(const char *name) const
     {
-        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
-        ForEach(*allpackages)
-        {
-            IPropertyTree &packageTree = allpackages->query();
-            const char *id = packageTree.queryProp("@id");
-            if (id && *id)
-            {
-                Owned<IRoxiePackage> package = createPackage(&packageTree);
-                package->resolveBases(this);
-                packages.setValue(id, package);
-                const char *queries = packageTree.queryProp("@queries");
-                if (queries && *queries)
-                {
-                    wildMatches.append(queries);
-                    wildIds.append(id);
-                }
-            }
-            else
-                throw MakeStringException(ROXIE_UNKNOWN_PACKAGE, "Invalid package map - Package element missing id attribute");
-        }
+        return queryResolvedPackage(name);
+    }
+    virtual const IRoxiePackage *matchRoxiePackage(const char *name) const
+    {
+        return matchResolvedPackage(name);
     }
 };
 
-static CPackageMap *emptyPackageMap;
+static CRoxiePackageMap *emptyPackageMap;
 static CRoxiePackage *rootPackage;
 static SpinLock emptyPackageMapCrit;
 static IRoxieDebugSessionManager *debugSessionManager;
 
-extern const IRoxiePackage &queryRootPackage()
+extern const IRoxiePackage &queryRootRoxiePackage()
 {
     SpinBlock b(emptyPackageMapCrit);
     if (!rootPackage)
@@ -681,11 +540,11 @@ extern const IRoxiePackage &queryRootPackage()
     return *rootPackage;
 }
 
-extern const IPackageMap &queryEmptyPackageMap()
+extern const IRoxiePackageMap &queryEmptyRoxiePackageMap()
 {
     SpinBlock b(emptyPackageMapCrit);
     if (!emptyPackageMap)
-        emptyPackageMap = new CPackageMap("<none>", true);
+        emptyPackageMap = new CRoxiePackageMap("<none>", NULL, true);
     return *emptyPackageMap;
 }
 
@@ -765,7 +624,7 @@ protected:
             throw MakeStringException(ROXIE_INTERNAL_ERROR, "Invalid parameters to addAlias");
     }
 
-    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo) = 0;
+    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo) = 0;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -785,7 +644,7 @@ public:
         return active;
     }
 
-    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash)
+    virtual void load(const IPropertyTree *querySet, const IRoxiePackageMap &packages, hash64_t &hash)
     {
         Owned<IPropertyTreeIterator> queryNames = querySet->getElements("Query");
         ForEach (*queryNames)
@@ -798,7 +657,7 @@ public:
                 if (!id || !*id || !dllName || !*dllName)
                     throw MakeStringException(ROXIE_QUERY_MODIFICATION, "dll and id must be specified");
                 Owned<const IQueryDll> queryDll = createQueryDll(dllName);
-                const IRoxiePackage *package = NULL;
+                const IHpccPackage *package = NULL;
                 const char *packageName = query.queryProp("@package");
                 if (packageName && *packageName)
                 {
@@ -810,7 +669,7 @@ public:
                 {
                     package = packages.queryPackage(id);  // Look for an exact match, then a fuzzy match, using query name as the package id
                     if(!package) package = packages.matchPackage(id);
-                    if (!package) package = &queryRootPackage();
+                    if (!package) package = &queryRootRoxiePackage();
                 }
                 assertex(package);
                 addQuery(id, loadQueryFromDll(id, queryDll.getClear(), *package, &query), hash);
@@ -932,7 +791,7 @@ public:
     {
     }
 
-    virtual IQueryFactory * loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+    virtual IQueryFactory * loadQueryFromDll(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
     {
         return createServerQueryFactory(id, dll, package, stateInfo);
     }
@@ -956,7 +815,7 @@ public:
         channelNo = _channelNo;
     }
 
-    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
     {
         return createSlaveQueryFactory(id, dll, package, channelNo, stateInfo);
     }
@@ -997,7 +856,7 @@ public:
         return managers[idx];
     }
 
-    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash)
+    virtual void load(const IPropertyTree *querySets, const IRoxiePackageMap &packages, hash64_t &hash)
     {
         for (unsigned channel = 0; channel < numChannels; channel++)
             if (managers[channel])
@@ -1079,7 +938,7 @@ class CRoxieQueryPackageManager : public CInterface
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxieQueryPackageManager(unsigned _numChannels, const char *_querySet, const IPackageMap *_packages)
+    CRoxieQueryPackageManager(unsigned _numChannels, const char *_querySet, const IRoxiePackageMap *_packages)
         : numChannels(_numChannels), packages(_packages), querySet(_querySet)
     {
         queryHash = 0;
@@ -1196,7 +1055,7 @@ protected:
     Owned<CRoxieSlaveQuerySetManagerSet> slaveManagers;
     Owned<IRoxieQuerySetManager> serverManager;
 
-    Owned<const IPackageMap> packages;
+    Owned<const IRoxiePackageMap> packages;
     unsigned numChannels;
     hash64_t queryHash;
     StringAttr querySet;
@@ -1229,7 +1088,7 @@ class CRoxieDaliQueryPackageManager : public CRoxieQueryPackageManager, implemen
 
 public:
     IMPLEMENT_IINTERFACE;
-    CRoxieDaliQueryPackageManager(unsigned _numChannels, const IPackageMap *_packages, const char *_querySet)
+    CRoxieDaliQueryPackageManager(unsigned _numChannels, const IRoxiePackageMap *_packages, const char *_querySet)
         : CRoxieQueryPackageManager(_numChannels, _querySet, _packages)
     {
         daliHelper.setown(connectToDali());
@@ -1274,7 +1133,7 @@ class CStandaloneQueryPackageManager : public CRoxieQueryPackageManager
 public:
     IMPLEMENT_IINTERFACE;
 
-    CStandaloneQueryPackageManager(unsigned _numChannels, const char *_querySet, const IPackageMap *_packages, IPropertyTree *_standaloneDll)
+    CStandaloneQueryPackageManager(unsigned _numChannels, const char *_querySet, const IRoxiePackageMap *_packages, IPropertyTree *_standaloneDll)
         : CRoxieQueryPackageManager(_numChannels, _querySet, _packages), standaloneDll(_standaloneDll)
     {
         assertex(standaloneDll);
@@ -2282,7 +2141,7 @@ private:
         throw MakeStringException(ROXIE_INVALID_INPUT, "Badly formated control query");
     }
 
-    void createQueryPackageManager(unsigned numChannels, const IPackageMap *packageMap, const char *querySet)
+    void createQueryPackageManager(unsigned numChannels, const IRoxiePackageMap *packageMap, const char *querySet)
     {
         // Called from reload inside write lock block
         Owned<CRoxieQueryPackageManager> qpm = new CRoxieDaliQueryPackageManager(numChannels, packageMap, querySet);
@@ -2331,7 +2190,7 @@ private:
                             DBGLOG("Loading package map %s, active %s", packageMapId, isActive ? "true" : "false");
                         try
                         {
-                            Owned<CPackageMap> packageMap = new CPackageMap(packageMapId, isActive);
+                            Owned<CRoxiePackageMap> packageMap = new CRoxiePackageMap(packageMapId, packageMapFilter, isActive);
                             Owned<IPropertyTree> xml = daliHelper->getPackageMap(packageMapId);
                             packageMap->load(xml);
                             createQueryPackageManager(numChannels, packageMap.getLink(), querySet);
@@ -2352,7 +2211,7 @@ private:
         {
             if (traceLevel)
                 DBGLOG("Loading empty package for QuerySet %s", querySet);
-            createQueryPackageManager(numChannels, LINK(&queryEmptyPackageMap()), querySet);
+            createQueryPackageManager(numChannels, LINK(&queryEmptyRoxiePackageMap()), querySet);
         }
         if (traceLevel)
             DBGLOG("Loaded packages");
@@ -2365,7 +2224,7 @@ private:
         standAloneDllTree.setown(createPTree("Query"));
         standAloneDllTree->setProp("@id", "roxie");
         standAloneDllTree->setProp("@dll", standAloneDll->queryDll()->queryName());
-        Owned<CRoxieQueryPackageManager> qpm = new CStandaloneQueryPackageManager(numChannels, querySet, LINK(&queryEmptyPackageMap()), standAloneDllTree.getClear());
+        Owned<CRoxieQueryPackageManager> qpm = new CStandaloneQueryPackageManager(numChannels, querySet, LINK(&queryEmptyRoxiePackageMap()), standAloneDllTree.getClear());
         qpm->load();
         stateHash = rtlHash64Data(sizeof(stateHash), &stateHash, qpm->getHash());
         allQueryPackages.append(*qpm.getClear());

roxie/ccd/ccdstate.hpp

@@ -29,6 +29,7 @@
 #include "thorcommon.hpp"
 #include "ccddali.hpp"
 #include "thorcommon.ipp"
+#include "package.h"
 
 interface IFilePartMap : public IInterface
 {
@@ -43,43 +44,29 @@ interface IFilePartMap : public IInterface
 extern IFilePartMap *createFilePartMap(const char *fileName, IFileDescriptor &fdesc);
 
 interface IRoxiePackage;
-interface IPackageMap : public IInterface
+interface IRoxiePackageMap : public IHpccPackageMap
 {
-    // Lookup package using exact id
-    virtual const IRoxiePackage *queryPackage(const char *name) const = 0;
-    // Lookup package using fuzzy id
-    virtual const IRoxiePackage *matchPackage(const char *name) const = 0;
-    virtual const char *queryPackageId() const = 0;
-    virtual bool isActive() const = 0;
+    virtual const IRoxiePackage *queryRoxiePackage(const char *name) const = 0;
+    virtual const IRoxiePackage *matchRoxiePackage(const char *name) const = 0;
 };
-extern const IRoxiePackage &queryRootPackage();
-extern const IPackageMap &queryEmptyPackageMap();
+extern const IRoxiePackage &queryRootRoxiePackage();
+extern const IRoxiePackageMap &queryEmptyRoxiePackageMap();
 
-interface IRoxiePackage : extends IInterface 
+interface IRoxiePackage : public IHpccPackage
 {
-    // Complete the setup of a package by resolving base package references
-    virtual void resolveBases(IPackageMap *packages) = 0;
-    // Lookup package environment variable
-    virtual const char *queryEnv(const char *varname) const = 0;
-    // Lookup package environment variable controlling field translation
-    virtual bool getEnableFieldTranslation() const = 0;
-    // Return entire XML tree for package
-    virtual const IPropertyTree *queryTree() const = 0;
     // Lookup information in package to resolve existing logical file name
     virtual const IResolvedFile *lookupFileName(const char *fileName, bool opt, bool cacheDaliResults, IConstWorkUnit *wu) const = 0;
     // Lookup information in package to create new logical file name
     virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const = 0;
-    // Retrieve hash for the package
-    virtual hash64_t queryHash() const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTree *getQuerySets() const = 0;
     // Remove a resolved file from the cache 
     virtual void removeCache(const IResolvedFile *goer) const = 0; // note that this is const as cache is considered mutable
 };
 
-extern IRoxiePackage *createPackage(IPropertyTree *p);
+extern IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages);
 
 interface ISlaveDynamicFileCache : extends IInterface
 {
@@ -103,7 +90,7 @@ interface IRoxieQuerySetManager : extends IInterface
 {
     virtual bool isActive() const = 0;
     virtual IQueryFactory *getQuery(const char *id, const IRoxieContextLogger &ctx) const = 0;
-    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash) = 0;
+    virtual void load(const IPropertyTree *querySet, const IRoxiePackageMap &packages, hash64_t &hash) = 0;
     virtual void getStats(const char *queryName, const char *graphName, StringBuffer &reply, const IRoxieContextLogger &logctx) const = 0;
     virtual void resetQueryTimings(const char *queryName, const IRoxieContextLogger &logctx) = 0;
     virtual void resetAllQueryTimings() = 0;
@@ -120,7 +107,7 @@ interface IRoxieDebugSessionManager : extends IInterface
 
 interface IRoxieQuerySetManagerSet : extends IInterface
 {
-    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash) = 0;
+    virtual void load(const IPropertyTree *querySets, const IRoxiePackageMap &packages, hash64_t &hash) = 0;
 };
 
 interface IRoxieQueryPackageManagerSet : extends IInterface

rtl/eclrtl/eclrtl.hpp

@@ -50,7 +50,6 @@
 #endif
 
 typedef unsigned char byte;
-typedef unsigned __int64 hash64_t;
 interface IEngineRowAllocator;
 interface IOutputMetaData;
 interface IOutputRowSerializer;

system/include/platform.h

@@ -439,6 +439,7 @@ typedef int socklen_t;
 
 #define strtok(a,b)   j_strtok_deprecated(a,b)  // will disappear at some point
 
+typedef unsigned __int64 hash64_t;
 typedef unsigned __int64 __uint64;
 typedef __uint64 offset_t;
 typedef unsigned char byte;