HPCC-7950 Use common HpccPackage classes in roxie

For consistency package logic should be centralized and shared between
components.

Create new template based classes so that roxie can extend them with
support for file caching without impacting efficency.

Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

common/roxiemanager/referencedfilelist.hpp

@@ -21,7 +21,7 @@
 
 #include "jlib.hpp"
 #include "workunit.hpp"
-#include "package.hpp"
+#include "package.h"
 #include "dadfs.hpp"
 #include "dfuutil.hpp"
 

common/workunit/CMakeLists.txt

@@ -32,7 +32,8 @@ set (    SRCS
          workunit.hpp 
          wuerror.hpp
          wujobq.hpp
-         package.hpp
+         package.h
+         pkgimpl.hpp
     )
 
 include_directories ( 

common/workunit/package.cpp

@@ -19,7 +19,7 @@
 #include "platform.h"
 #include "jprop.hpp"
 #include "jptree.hpp"
-#include "package.hpp"
+#include "pkgimpl.hpp"
 
 #include "dadfs.hpp"
 #include "eclrtl.hpp"
@@ -28,289 +28,81 @@
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
-static IPropertyTree * getPackageMapById(const char * id, bool readonly)
+CPackageNode::CPackageNode(IPropertyTree *p)
 {
-    StringBuffer xpath;
-    xpath.append("/PackageMaps/PackageMap[@id=\"").append(id).append("\"]");
-    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
-    if (!conn)
-        return NULL;
-    return conn->getRoot();
-}
-
-static IPropertyTree * getPackageSetById(const char * id, bool readonly)
-{
-    StringBuffer xpath;
-    xpath.append("/PackageSets/PackageSet[@id=\"").append(id).append("\"]");
-    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
-    if (!conn)
-        return NULL;
-    return conn->getRoot();
+    if (p)
+        node.set(p);
+    else
+        node.setown(createPTree("HpccPackages"));
+    StringBuffer xml;
+    toXML(node, xml);
+    hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
 }
 
-static IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly)
+    // Merge base package environment into mergedEnvironment
+void CPackageNode::mergeEnvironment(const CPackageNode *base)
 {
-    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
-    Owned<IPropertyTree> psroot = globalLock->getRoot();
-
-    StringBuffer xpath;
-    xpath.append("PackageSet[@process=\"").append(process).append("\"]");
-    IPropertyTree *ps = psroot->queryPropTree(xpath);
-    if (ps)
-        return getPackageSetById(ps->queryProp("@id"), readonly);
-    Owned<IPropertyTreeIterator> it = psroot->getElements("PackageSet[@process]");
-    ForEach(*it)
+    Owned<IPropertyIterator> envIterator = base->mergedEnvironment->getIterator();
+    ForEach(*envIterator)
     {
-        const char *match = it->query().queryProp("@process");
-        const char *id = it->query().queryProp("@id");
-        if (id && isWildString(match) && WildMatch(process, match))
-            return getPackageSetById(id, readonly);
+        const char *id = envIterator->getPropKey();
+        const char *val = base->mergedEnvironment->queryProp(id);
+        if (id && val && !mergedEnvironment->hasProp(id))
+            mergedEnvironment->setProp(id, val);
     }
-    return NULL;
 }
 
-class CPackageSuperFileArray : public CInterface, implements ISimpleSuperFileEnquiry
-{
-    IArrayOf<IPropertyTree> subFiles;
-public:
-    IMPLEMENT_IINTERFACE;
-    CPackageSuperFileArray(IPropertyTreeIterator &_subs)
-    {
-        ForEach(_subs)
-        {
-            IPropertyTree &sub = _subs.query();
-            sub.Link();
-            subFiles.append(sub);
-        }
-    }
-    virtual unsigned numSubFiles() const
-    {
-        return subFiles.length();
-    }
-    virtual bool getSubFileName(unsigned num, StringBuffer &name) const
-    {
-        if (subFiles.isItem(num))
-        {
-            name.append(subFiles.item(num).queryProp("@value"));
-            return true;
-        }
-        else
-            return false;
-    }
-    virtual unsigned findSubName(const char *subname) const
-    {
-        ForEachItemIn(idx, subFiles)
-        {
-            if (stricmp(subFiles.item(idx).queryProp("@value"), subname))
-                return idx;
-        }
-        return NotFound;
-    }
-    virtual unsigned getContents(StringArray &contents) const
-    {
-        ForEachItemIn(idx, subFiles)
-        {
-            contents.append(subFiles.item(idx).queryProp("@value"));
-        }
-        return subFiles.length();
-    }
-};
-
-
-
-class CHpccPackage : extends CInterface, implements IHpccPackage
+// Use local package and its bases to resolve superfile name list of subfiles via all supported resolvers
+ISimpleSuperFileEnquiry *CPackageNode::resolveSuperFile(const char *superFileName) const
 {
-protected:
-    Owned<IPropertyTree> node;
-    IArrayOf<CHpccPackage> bases;
-    Owned<IProperties> mergedEnvironment;
-    hash64_t hash;
-
-    // Merge base package environment into mergedEnvironment
-    void mergeEnvironment(const CHpccPackage *base)
-    {
-        Owned<IPropertyIterator> envIterator = base->mergedEnvironment->getIterator();
-        ForEach(*envIterator)
-        {
-            const char *id = envIterator->getPropKey();
-            const char *val = base->mergedEnvironment->queryProp(id);
-            if (id && val && !mergedEnvironment->hasProp(id))
-                mergedEnvironment->setProp(id, val);
-        }
-    }
-
-    // Search this package and any bases for an element matching xpath1, then return iterator for its children that match xpath2
-    IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const
-    {
-        IPropertyTree *parentNode = node->queryPropTree(xpath1);
-        if (parentNode)
-            return parentNode->getElements(xpath2);
-        ForEachItemIn(idx, bases)
-        {
-            const CHpccPackage &basePackage = bases.item(idx);
-            IPropertyTreeIterator *it = basePackage.lookupElements(xpath1, xpath2);
-            if (it)
-                return it;
-        }
-        return NULL;
-    }
-
-    inline StringBuffer makeSuperFileXPath(StringBuffer &xpath, const char *superFileName) const
-    {
-        return xpath.append("SuperFile[@id='").appendLower(strlen(superFileName), superFileName).append("']");
-    }
-
-    // Use local package and its bases to resolve superfile name list of subfiles via all supported resolvers
-    ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
+    // Order of resolution:
+    // 1. SuperFiles named in local package
+    // 2. SuperFiles named in bases
+    // There is no dali or local case - a superfile that is resolved in dali must also resolve the subfiles there (and is all done in the resolveLFNusingDali method)
+    if (superFileName && *superFileName && node)
     {
-        // Order of resolution:
-        // 1. SuperFiles named in local package
-        // 2. SuperFiles named in bases
-        // There is no dali or local case - a superfile that is resolved in dali must also resolve the subfiles there (and is all done in the resolveLFNusingDali method)
-        if (superFileName && *superFileName && node)
-        {
-            if (*superFileName=='~')
-                superFileName++;
-            StringBuffer xpath;
-            Owned<IPropertyTreeIterator> subFiles = lookupElements(makeSuperFileXPath(xpath, superFileName), "SubFile");
-            if (subFiles)
-            {
-                Owned<CPackageSuperFileArray> result = new CPackageSuperFileArray(*subFiles);
-                return result.getClear();
-            }
-        }
-        return NULL;
-    }
-
-    inline bool hasProp(const char *xpath) const
-    {
-        return (node) ? node->hasProp(xpath) : false;
-    }
-
-    bool hasSuperFile(const char *superFileName) const
-    {
-        if (!superFileName || !*superFileName || !node)
-            return false;
-
         if (*superFileName=='~')
             superFileName++;
         StringBuffer xpath;
-        if (hasProp(makeSuperFileXPath(xpath, superFileName)))
-            return true;
-        ForEachItemIn(idx, bases)
+        Owned<IPropertyTreeIterator> subFiles = lookupElements(makeSuperFileXPath(xpath, superFileName), "SubFile");
+        if (subFiles)
         {
-            if (bases.item(idx).hasProp(xpath))
-                return true;
+            Owned<CPackageSuperFileArray> result = new CPackageSuperFileArray(*subFiles);
+            return result.getClear();
         }
-
-        return false;
     }
+    return NULL;
+}
 
-    CHpccPackage()
-    {
-        hash = 0;
-    }
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CHpccPackage(IPropertyTree *p)
-    {
-        if (p)
-            node.set(p);
+// Load mergedEnvironment from local XML node
+void CPackageNode::loadEnvironment()
+{
+    mergedEnvironment.setown(createProperties(true));
+    Owned<IPropertyTreeIterator> envIterator = node->getElements("Environment");
+    ForEach(*envIterator)
+    {
+        IPropertyTree &env = envIterator->query();
+        const char *id = env.queryProp("@id");
+        const char *val = env.queryProp("@value");
+        if (!val)
+            val = env.queryProp("@val"); // Historically we used val here - not sure why... other parts of package file used value
+        if (id && val)
+            mergedEnvironment->setProp(id, val);
         else
-            node.setown(createPTree("HpccPackages"));
-        StringBuffer xml;
-        toXML(node, xml);
-        hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
-    }
-
-    ~CHpccPackage()
-    {
-    }
-    // Load mergedEnvironment from local XML node
-    void loadEnvironment()
-    {
-        mergedEnvironment.setown(createProperties(true));
-        Owned<IPropertyTreeIterator> envIterator = node->getElements("Environment");
-        ForEach(*envIterator)
         {
-            IPropertyTree &env = envIterator->query();
-            const char *id = env.queryProp("@id");
-            const char *val = env.queryProp("@value");
-            if (!val)
-                val = env.queryProp("@val"); // Historically we used val here - not sure why... other parts of package file used value
-            if (id && val)
-                mergedEnvironment->setProp(id, val);
-            else
-            {
-                StringBuffer s;
-                toXML(&env, s);
-                throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
-            }
+            StringBuffer s;
+            toXML(&env, s);
+            throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
         }
-        Owned<IAttributeIterator> attrs = node->getAttributes();
-        for(attrs->first(); attrs->isValid(); attrs->next())
-        {
-            StringBuffer s("control:");
-            s.append(attrs->queryName()+1);  // queryName() has a leading @, hence the +1
-            mergedEnvironment->setProp(s.str(), attrs->queryValue());
-        }
-    }
-
-    virtual void resolveBases(IHpccPackageMap *packages)
-    {
-        loadEnvironment();
-        if (packages)
-        {
-            Owned<IPropertyTreeIterator> baseIterator = node->getElements("Base");
-            if (baseIterator->first())
-            {
-                do
-                {
-                    IPropertyTree &baseElem = baseIterator->query();
-                    const char *baseId = baseElem.queryProp("@id");
-                    if (!baseId)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
-                    const IHpccPackage *_base = packages->queryPackage(baseId);
-                    if (_base)
-                    {
-                        const CHpccPackage *base = static_cast<const CHpccPackage *>(_base);
-                        bases.append(const_cast<CHpccPackage &>(*LINK(base)));   // should really be an arrayof<const base> but that would require some fixing in jlib
-                        hash = rtlHash64Data(sizeof(base->hash), &base->hash, hash);
-                        mergeEnvironment(base);
-                    }
-                    else
-                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
-                }
-                while(baseIterator->next());
-            }
-        }
-    }
-
-    virtual const char *queryEnv(const char *varname) const
-    {
-        return mergedEnvironment->queryProp(varname);
-    }
-
-    virtual hash64_t queryHash() const
-    {
-        return hash;
-    }
-
-    virtual const IPropertyTree *queryTree() const
-    {
-        return node;
     }
-
-    virtual IPropertyTree *getQuerySets() const
+    Owned<IAttributeIterator> attrs = node->getAttributes();
+    for(attrs->first(); attrs->isValid(); attrs->next())
     {
-        if (node)
-            return node->getPropTree("QuerySets");
-        else
-            return NULL;
+        StringBuffer s("control:");
+        s.append(attrs->queryName()+1);  // queryName() has a leading @, hence the +1
+        mergedEnvironment->setProp(s.str(), attrs->queryValue());
     }
-};
+}
 
 CHpccPackage *createPackage(IPropertyTree *p)
 {
@@ -321,136 +113,92 @@ CHpccPackage *createPackage(IPropertyTree *p)
 // CPackageMap - an implementation of IPackageMap using a string map
 //================================================================================================
 
-class CHpccPackageMap : public CInterface, implements IHpccPackageMap
-{
-public:
-    MapStringToMyClass<CHpccPackage> packages;
-    StringAttr packageId;
-    StringAttr querySet;
-    bool active;
-    StringArray wildMatches, wildIds;
-public:
-    IMPLEMENT_IINTERFACE;
-    CHpccPackageMap(const char *_packageId, const char *_querySet, bool _active)
-        : packageId(_packageId), querySet(_querySet), active(_active), packages(true)
-    {
-    }
-
-    // IPackageMap interface
-    virtual bool isActive() const
-    {
-        return active;
-    }
-    virtual const IHpccPackage *queryPackage(const char *name) const
-    {
-        return name ? packages.getValue(name) : NULL;
-    }
-    virtual const IHpccPackage *matchPackage(const char *name) const
-    {
-        if (name)
-        {
-            const IHpccPackage *pkg = queryPackage(name);
-            if (pkg)
-                return pkg;
-            ForEachItemIn(idx, wildMatches)
-            {
-                if (WildMatch(name, wildMatches.item(idx), true))
-                    return queryPackage(wildIds.item(idx));
-            }
-        }
-        return NULL;
-    }
-    virtual const char *queryPackageId() const
-    {
-        return packageId;
-    }
-    void load(const char *id)
-    {
-        load(getPackageMapById(id, true));
-    }
-    void load(IPropertyTree *xml)
-    {
-        if (!xml)
-            return;
-        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
-        ForEach(*allpackages)
-        {
-            IPropertyTree &packageTree = allpackages->query();
-            const char *id = packageTree.queryProp("@id");
-            if (!id || !*id)
-                throw MakeStringException(-1, "Invalid package map - Package element missing id attribute");
-            Owned<CHpccPackage> package = createPackage(&packageTree);
-            packages.setValue(id, package.get());
-            const char *queries = packageTree.queryProp("@queries");
-            if (queries && *queries)
-            {
-                wildMatches.append(queries);
-                wildIds.append(id);
-            }
-        }
-        HashIterator it(packages);
-        ForEach (it)
-        {
-            CHpccPackage *pkg = packages.mapToValue(&it.query());
-            if (pkg)
-                pkg->resolveBases(this);
-        }
-    }
-};
-
 //================================================================================================
 // CHpccPackageSet - an implementation of IHpccPackageSet
 //================================================================================================
 
-class CHpccPackageSet : public CInterface, implements IHpccPackageSet
+CHpccPackageSet::CHpccPackageSet(const char *_process) : process(_process)
 {
-    IArrayOf<CHpccPackageMap> packageMaps;
-    StringAttr process;
-public:
-    IMPLEMENT_IINTERFACE;
-    CHpccPackageSet(const char *_process)
-        : process(_process)
-    {
-        Owned<IPropertyTree> ps = resolvePackageSetRegistry(process, true);
-        if (ps)
-            load(ps);
-    }
+    Owned<IPropertyTree> ps = resolvePackageSetRegistry(process, true);
+    if (ps)
+        load(ps);
+}
 
-    void load(IPropertyTree *xml)
+void CHpccPackageSet::load(IPropertyTree *xml)
+{
+    Owned<IPropertyTreeIterator> it = xml->getElements("PackageMap[@active='1']"); //only active for now
+    ForEach(*it)
     {
-        Owned<IPropertyTreeIterator> it = xml->getElements("PackageMap[@active='1']"); //only active for now
-        ForEach(*it)
-        {
-            IPropertyTree &tree = it->query();
-            if (!tree.hasProp("@id"))
-                continue;
-            Owned<CHpccPackageMap> pm = new CHpccPackageMap(tree.queryProp("@id"), tree.queryProp("@querySet"), true);
-            pm->load(tree.queryProp("@id"));
-            packageMaps.append(*pm.getClear());
-        }
+        IPropertyTree &tree = it->query();
+        if (!tree.hasProp("@id"))
+            continue;
+        Owned<CHpccPackageMap> pm = new CHpccPackageMap(tree.queryProp("@id"), tree.queryProp("@querySet"), true);
+        pm->load(tree.queryProp("@id"));
+        packageMaps.append(*pm.getClear());
     }
+}
 
-    virtual const IHpccPackageMap *queryActiveMap(const char *queryset) const
+const IHpccPackageMap *CHpccPackageSet::queryActiveMap(const char *queryset) const
+{
+    ForEachItemIn(i, packageMaps)
     {
-        ForEachItemIn(i, packageMaps)
+        CHpccPackageMap &pm = packageMaps.item(i);
+        StringAttr &match = pm.querySet;
+        if (!match.length())
+            continue;
+        if (isWildString(match))
         {
-            CHpccPackageMap &pm = packageMaps.item(i);
-            StringAttr &match = pm.querySet;
-            if (!match.length())
-                continue;
-            if (isWildString(match))
-            {
-                if (WildMatch(queryset, match))
-                    return &pm;
-            }
-            else if (streq(queryset, match))
+            if (WildMatch(queryset, match))
                 return &pm;
         }
-        return NULL;
+        else if (streq(queryset, match))
+            return &pm;
     }
-};
+    return NULL;
+}
 
 extern WORKUNIT_API IHpccPackageSet *createPackageSet(const char *process)
 {
     return new CHpccPackageSet(process);
 }
+
+extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool readonly)
+{
+    StringBuffer xpath;
+    xpath.append("/PackageMaps/PackageMap[@id=\"").append(id).append("\"]");
+    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return NULL;
+    return conn->getRoot();
+}
+
+extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly)
+{
+    StringBuffer xpath;
+    xpath.append("/PackageSets/PackageSet[@id=\"").append(id).append("\"]");
+    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return NULL;
+    return conn->getRoot();
+}
+
+extern WORKUNIT_API IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly)
+{
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    Owned<IPropertyTree> psroot = globalLock->getRoot();
+
+    StringBuffer xpath;
+    xpath.append("PackageSet[@process=\"").append(process).append("\"]");
+    IPropertyTree *ps = psroot->queryPropTree(xpath);
+    if (ps)
+        return getPackageSetById(ps->queryProp("@id"), readonly);
+    Owned<IPropertyTreeIterator> it = psroot->getElements("PackageSet[@process]");
+    ForEach(*it)
+    {
+        const char *match = it->query().queryProp("@process");
+        const char *id = it->query().queryProp("@id");
+        if (id && isWildString(match) && WildMatch(process, match))
+            return getPackageSetById(id, readonly);
+    }
+    return NULL;
+}

common/workunit/package.hpp

@@ -15,8 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-#ifndef WUPACKAGE_HPP
-#define WUPACKAGE_HPP
+#ifndef WUPACKAGE_H
+#define WUPACKAGE_H
 
 #include "dadfs.hpp"
 #include "workunit.hpp"
@@ -25,12 +25,18 @@ interface IHpccPackage : extends IInterface
 {
     virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const = 0;
     virtual bool hasSuperFile(const char *superFileName) const = 0;
+    virtual const char *queryEnv(const char *varname) const = 0;
+    virtual bool getEnableFieldTranslation() const = 0;
+    virtual const IPropertyTree *queryTree() const = 0;
+    virtual hash64_t queryHash() const = 0;
 };
 
 interface IHpccPackageMap : extends IInterface
 {
-     virtual const IHpccPackage *queryPackage(const char *name) const = 0;
-     virtual const IHpccPackage *matchPackage(const char *name) const = 0;
+    virtual const IHpccPackage *queryPackage(const char *name) const = 0;
+    virtual const IHpccPackage *matchPackage(const char *name) const = 0;
+    virtual const char *queryPackageId() const = 0;
+    virtual bool isActive() const = 0;
 };
 
 interface IHpccPackageSet : extends IInterface
@@ -39,5 +45,9 @@ interface IHpccPackageSet : extends IInterface
 };
 
 extern WORKUNIT_API IHpccPackageSet *createPackageSet(const char *process);
+extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool readonly);
+extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly);
+extern WORKUNIT_API IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly);
+
 
 #endif

common/workunit/pkgimpl.hpp

@@ -0,0 +1,366 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef WUPACKAGE_IMPL_HPP
+#define WUPACKAGE_IMPL_HPP
+
+#include "platform.h"
+#include "jprop.hpp"
+#include "jptree.hpp"
+#include "jregexp.hpp"
+#include "package.h"
+
+class CPackageSuperFileArray : public CInterface, implements ISimpleSuperFileEnquiry
+{
+    IArrayOf<IPropertyTree> subFiles;
+public:
+    IMPLEMENT_IINTERFACE;
+    CPackageSuperFileArray(IPropertyTreeIterator &_subs)
+    {
+        ForEach(_subs)
+        {
+            IPropertyTree &sub = _subs.query();
+            sub.Link();
+            subFiles.append(sub);
+        }
+    }
+    virtual unsigned numSubFiles() const
+    {
+        return subFiles.length();
+    }
+    virtual bool getSubFileName(unsigned num, StringBuffer &name) const
+    {
+        if (subFiles.isItem(num))
+        {
+            name.append(subFiles.item(num).queryProp("@value"));
+            return true;
+        }
+        else
+            return false;
+    }
+    virtual unsigned findSubName(const char *subname) const
+    {
+        ForEachItemIn(idx, subFiles)
+        {
+            if (stricmp(subFiles.item(idx).queryProp("@value"), subname))
+                return idx;
+        }
+        return NotFound;
+    }
+    virtual unsigned getContents(StringArray &contents) const
+    {
+        ForEachItemIn(idx, subFiles)
+        {
+            contents.append(subFiles.item(idx).queryProp("@value"));
+        }
+        return subFiles.length();
+    }
+};
+
+class WORKUNIT_API CPackageNode : extends CInterface, implements IHpccPackage
+{
+protected:
+    Owned<IPropertyTree> node;
+    Owned<IProperties> mergedEnvironment;
+    hash64_t hash;
+
+    void mergeEnvironment(const CPackageNode *base);
+
+    virtual IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const = 0;
+
+    inline StringBuffer makeSuperFileXPath(StringBuffer &xpath, const char *superFileName) const
+    {
+        return xpath.append("SuperFile[@id='").appendLower(strlen(superFileName), superFileName).append("']");
+    }
+
+    ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const;
+
+    inline bool hasProp(const char *xpath) const
+    {
+        return (node) ? node->hasProp(xpath) : false;
+    }
+
+    virtual bool getSysFieldTranslationEnabled() const {return false;}
+    virtual bool getEnableFieldTranslation() const
+    {
+        const char *val = queryEnv("control:enableFieldTranslation");
+        if (val)
+            return strToBool(val);
+        else
+            return getSysFieldTranslationEnabled();
+    }
+
+
+    CPackageNode()
+    {
+        hash = 0;
+    }
+
+    virtual const IHpccPackage *queryRootPackage()
+    {
+        return NULL;
+    }
+
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CPackageNode(IPropertyTree *p);
+
+    ~CPackageNode()
+    {
+    }
+
+    // Load mergedEnvironment from local XML node
+    void loadEnvironment();
+
+    virtual const char *queryEnv(const char *varname) const
+    {
+        return mergedEnvironment->queryProp(varname);
+    }
+
+    virtual hash64_t queryHash() const
+    {
+        return hash;
+    }
+
+    virtual const IPropertyTree *queryTree() const
+    {
+        return node;
+    }
+
+    virtual IPropertyTree *getQuerySets() const
+    {
+        if (!node)
+            return NULL;
+        return node->getPropTree("QuerySets");
+    }
+};
+
+template <class TYPE>
+class CResolvedPackage : public TYPE
+{
+public:
+    typedef CResolvedPackage<TYPE> self;
+    CIArrayOf<self> bases;
+
+    CResolvedPackage<TYPE>(IPropertyTree *p) : TYPE(p) {}
+
+    virtual aindex_t getBaseCount() const {return bases.length();}
+    const self *getResolvedBase(aindex_t pos) const
+    {
+        if (pos < getBaseCount())
+            return &bases.item(pos);
+        return NULL;
+    }
+    virtual const TYPE *getBaseNode(aindex_t pos) const {return (const TYPE *) getResolvedBase(pos);}
+
+    void appendBase(const IHpccPackage *base)
+    {
+        if (base)
+        {
+            const self *p = dynamic_cast<const self *>(base);
+            bases.append(const_cast<self &>(*LINK(p)));   // should really be an arrayof<const base> but that would require some fixing in jlib
+            TYPE::hash = rtlHash64Data(sizeof(p->hash), &p->hash, TYPE::hash);
+            mergeEnvironment(p);
+        }
+    }
+
+    void resolveBases(const IHpccPackageMap *packages)
+    {
+        TYPE::loadEnvironment();
+        if (packages)
+        {
+            Owned<IPropertyTreeIterator> baseIterator = TYPE::node->getElements("Base");
+            if (!baseIterator->first())
+                appendBase(TYPE::queryRootPackage());
+            else
+            {
+                do
+                {
+                    IPropertyTree &baseElem = baseIterator->query();
+                    const char *baseId = baseElem.queryProp("@id");
+                    if (!baseId)
+                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
+                    const IHpccPackage *base = packages->queryPackage(baseId);
+                    if (!base)
+                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
+                    appendBase(base);
+                }
+                while(baseIterator->next());
+            }
+        }
+    }
+
+    // Search this package and any bases for an element matching xpath1, then return iterator for its children that match xpath2
+    IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const
+    {
+        IPropertyTree *parentNode = TYPE::node->queryPropTree(xpath1);
+        if (parentNode)
+            return parentNode->getElements(xpath2);
+        ForEachItemIn(idx, bases)
+        {
+            const self &basePackage = bases.item(idx);
+            IPropertyTreeIterator *it = basePackage.lookupElements(xpath1, xpath2);
+            if (it)
+                return it;
+        }
+        return NULL;
+    }
+
+    bool hasSuperFile(const char *superFileName) const
+    {
+        if (!superFileName || !*superFileName || !TYPE::node)
+            return false;
+
+        if (*superFileName=='~')
+            superFileName++;
+        StringBuffer xpath;
+        if (hasProp(TYPE::makeSuperFileXPath(xpath, superFileName)))
+            return true;
+        ForEachItemIn(idx, bases)
+        {
+            if (bases.item(idx).hasProp(xpath))
+                return true;
+        }
+
+        return false;
+    }
+};
+
+
+typedef CResolvedPackage<CPackageNode> CHpccPackage;
+
+//================================================================================================
+// CPackageMap - an implementation of IPackageMap using a string map
+//================================================================================================
+
+template <class TYPE, class IFACE>
+class CPackageMapOf : public CInterface, implements IHpccPackageMap
+{
+public:
+    typedef CResolvedPackage<TYPE> packageType;
+    MapStringToMyClassViaBase<packageType, IFACE> packages;
+    StringAttr packageId;
+    StringAttr querySet;
+    bool active;
+    StringArray wildMatches, wildIds;
+public:
+    IMPLEMENT_IINTERFACE;
+    CPackageMapOf(const char *_packageId, const char *_querySet, bool _active)
+        : packageId(_packageId), querySet(_querySet), active(_active), packages(true)
+    {
+    }
+
+    // IPackageMap interface
+    virtual bool isActive() const
+    {
+        return active;
+    }
+
+    virtual const packageType *queryResolvedPackage(const char *name) const
+    {
+        return name ? packages.getValue(name) : NULL;
+    }
+
+    virtual const IHpccPackage *queryPackage(const char *name) const
+    {
+        return name ? (IFACE*)packages.getValue(name) : NULL;
+    }
+
+    virtual const char *queryPackageId() const
+    {
+        return packageId;
+    }
+
+    virtual const packageType *matchResolvedPackage(const char *name) const
+    {
+        if (name)
+        {
+            const packageType *pkg = queryResolvedPackage(name);
+            if (pkg)
+                return pkg;
+            ForEachItemIn(idx, wildMatches)
+            {
+                if (WildMatch(name, wildMatches.item(idx), true))
+                    return queryResolvedPackage(wildIds.item(idx));
+            }
+        }
+        return NULL;
+    }
+
+    const IHpccPackage *matchPackage(const char *name) const
+    {
+        return (IFACE *) matchResolvedPackage(name);
+    }
+
+    void load(IPropertyTree *xml)
+    {
+        if (!xml)
+            return;
+        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
+        ForEach(*allpackages)
+        {
+            IPropertyTree &packageTree = allpackages->query();
+            const char *id = packageTree.queryProp("@id");
+            if (!id || !*id)
+                throw MakeStringException(-1, "Invalid package map - Package element missing id attribute");
+            Owned<packageType> package = new packageType(&packageTree);
+            packages.setValue(id, package.get());
+            const char *queries = packageTree.queryProp("@queries");
+            if (queries && *queries)
+            {
+                wildMatches.append(queries);
+                wildIds.append(id);
+            }
+        }
+        HashIterator it(packages);
+        ForEach (it)
+        {
+            packageType *pkg = packages.getValue((const char *)it.query().getKey());
+            if (pkg)
+                pkg->resolveBases(this);
+        }
+    }
+
+    void load(const char *id)
+    {
+        load(getPackageMapById(id, true));
+    }
+};
+
+typedef CPackageMapOf<CPackageNode, IHpccPackage> CHpccPackageMap;
+
+
+//================================================================================================
+// CHpccPackageSet - an implementation of IHpccPackageSet
+//================================================================================================
+
+class WORKUNIT_API CHpccPackageSet : public CInterface, implements IHpccPackageSet
+{
+    IArrayOf<CHpccPackageMap> packageMaps;
+    StringAttr process;
+public:
+    IMPLEMENT_IINTERFACE;
+    CHpccPackageSet(const char *_process);
+
+    void load(IPropertyTree *xml);
+
+    virtual const IHpccPackageMap *queryActiveMap(const char *queryset) const;
+};
+
+#endif

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -40,7 +40,7 @@
 #include "thorplugin.hpp"
 #include "roxiecontrol.hpp"
 
-#include "package.hpp"
+#include "package.h"
 
 #ifdef _USE_ZLIB
 #include "zcrypt.hpp"

roxie/ccd/ccdquery.cpp

@@ -787,7 +787,7 @@ public:
             return NULL;
     }
 
-    static hash64_t getQueryHash(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+    static hash64_t getQueryHash(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
     {
         hash64_t hashValue = rtlHash64VStr(dll->queryDll()->queryName(), package.queryHash());
         hashValue = rtlHash64VStr(id, hashValue);
@@ -1310,7 +1310,7 @@ public:
     }
 };
 
-extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
 {
     CriticalBlock b(CQueryFactory::queryCreateLock);
     hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo);
@@ -1320,7 +1320,7 @@ extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *
         ::Release(dll);
         return cached;
     }
-    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, package, hashValue);
+    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue);
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }
@@ -1331,7 +1331,7 @@ extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu)
     if (!dll)
         return NULL;
     SCMStringBuffer wuid;
-    return createServerQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), NULL); // MORE - if use a constant for id might cache better?
+    return createServerQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootRoxiePackage(), NULL); // MORE - if use a constant for id might cache better?
 }
 
 //==============================================================================================================================================
@@ -1559,7 +1559,7 @@ public:
     }
 };
 
-IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned channel, const IPropertyTree *stateInfo)
+IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, unsigned channel, const IPropertyTree *stateInfo)
 {
     CriticalBlock b(CQueryFactory::queryCreateLock);
     hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo);
@@ -1569,7 +1569,7 @@ IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, con
         ::Release(dll);
         return cached;
     }
-    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, package, hashValue, channel);
+    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, channel);
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }
@@ -1580,7 +1580,7 @@ extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned
     if (!dll)
         return NULL;
     SCMStringBuffer wuid;
-    return createSlaveQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), channelNo, NULL);  // MORE - if use a constant for id might cache better?
+    return createSlaveQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootRoxiePackage(), channelNo, NULL);  // MORE - if use a constant for id might cache better?
 }
 
 IRecordLayoutTranslator * createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta)

roxie/ccd/ccdquery.hpp

@@ -30,6 +30,7 @@
 #include "ccddali.hpp"
 #include "thorcommon.ipp"
 #include "roxierow.hpp"
+#include "package.h"
 
 class TranslatorArray : public CInterface, implements IInterface
 {
@@ -228,8 +229,8 @@ extern const IQueryDll *createExeQueryDll(const char *exeName);
 extern const IQueryDll *createWuQueryDll(IConstWorkUnit *wu);
 
 extern IRecordLayoutTranslator *createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta);
-extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo);
-extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned _channelNo, const IPropertyTree *stateInfo);
+extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo);
+extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IHpccPackage &package, unsigned _channelNo, const IPropertyTree *stateInfo);
 extern IQueryFactory *getQueryFactory(hash64_t hashvalue, unsigned channel);
 extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu);
 extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned channelNo);

roxie/ccd/ccdserver.cpp

@@ -30017,7 +30017,7 @@ public:
         CriticalBlock b(daliUpdateCrit);
         if (!dynamicPackage)
         {
-            dynamicPackage.setown(createPackage(NULL));
+            dynamicPackage.setown(createRoxiePackage(NULL, NULL));
         }
         return dynamicPackage->lookupFileName(filename, isOpt, true, workUnit);
     }
@@ -30027,7 +30027,7 @@ public:
         CriticalBlock b(daliUpdateCrit);
         if (!dynamicPackage)
         {
-            dynamicPackage.setown(createPackage(NULL));
+            dynamicPackage.setown(createRoxiePackage(NULL, NULL));
         }
         return dynamicPackage->createFileName(filename, overwrite, extend, clusters, workUnit);
     }
@@ -32445,7 +32445,7 @@ protected:
 
     void init()
     {
-        package.setown(createPackage(NULL));
+        package.setown(createPackage(NULL, NULL));
         ctx.setown(createSlaveContext(NULL, logctx, 0, 50*1024*1024, NULL));
         queryDll.setown(createExeQueryDll("roxie"));
         queryFactory.setown(createServerQueryFactory("test", queryDll.getLink(), *package, NULL));

roxie/ccd/ccdstate.cpp

@@ -36,6 +36,8 @@
 #include "dafdesc.hpp"
 #include "dautils.hpp"
 
+#include "pkgimpl.hpp"
+
 //-------------------------------------------------------------------------------------------
 // class CRoxiePluginCtx - provide the environments for plugins loaded by roxie. 
 // Base class handles making sure memory allocation comes from the right heap. 
@@ -179,19 +181,41 @@ public:
  * </PackageMaps>
  */
 
-class CRoxiePackage : extends CInterface, implements IRoxiePackage
+class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
 {
 protected:
-    Owned<IPropertyTree> node;
-    IArrayOf<CRoxiePackage> bases;
     Owned<IRoxieDaliHelper> daliHelper;
-    Owned<IProperties> mergedEnvironment;
-    hash64_t hash;
-    bool compulsory;  // This concept may well disappear...
 
     mutable CriticalSection cacheLock;
     mutable CopyMapStringToMyClass<IResolvedFile> fileCache;
 
+    virtual aindex_t getBaseCount() const = 0;
+    virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
+
+    //map ambiguous IHpccPackage
+    virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
+    {
+        return CPackageNode::resolveSuperFile(superFileName);
+    }
+    virtual const char *queryEnv(const char *varname) const
+    {
+        return CPackageNode::queryEnv(varname);
+    }
+    virtual bool getEnableFieldTranslation() const
+    {
+        return CPackageNode::getEnableFieldTranslation();
+    }
+    virtual const IPropertyTree *queryTree() const
+    {
+        return CPackageNode::queryTree();
+    }
+    virtual hash64_t queryHash() const
+    {
+        return CPackageNode::queryHash();
+    }
+
+    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
+
     // Add a filename and the corresponding IResolvedFile to the cache
     void addCache(const char *filename, const IResolvedFile *file) const
     {
@@ -224,82 +248,7 @@ protected:
         }
         return NULL;
     }
-    // Load mergedEnvironment from local XML node
-    void loadEnvironment()
-    {
-        mergedEnvironment.setown(createProperties(true));
-        Owned<IPropertyTreeIterator> envIterator = node->getElements("Environment");
-        ForEach(*envIterator)
-        {
-            IPropertyTree &env = envIterator->query();
-            const char *id = env.queryProp("@id");
-            const char *val = env.queryProp("@value");
-            if (!val)
-                val = env.queryProp("@val"); // Historically we used val here - not sure why... other parts of package file used value
-            if (id && val)
-                mergedEnvironment->setProp(id, val);
-            else
-            {
-                StringBuffer s;
-                toXML(&env, s);
-                throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
-            }
-        }
-        Owned<IAttributeIterator> attrs = node->getAttributes();
-        for(attrs->first(); attrs->isValid(); attrs->next())
-        {
-            StringBuffer s("control:");
-            s.append(attrs->queryName()+1);  // queryName() has a leading @, hence the +1
-            mergedEnvironment->setProp(s.str(), attrs->queryValue());
-        }
-    }
-    // Merge base package environment into mergedEnvironment
-    void mergeEnvironment(const CRoxiePackage *base)
-    {
-        Owned<IPropertyIterator> envIterator = base->mergedEnvironment->getIterator();
-        ForEach(*envIterator)
-        {
-            const char *id = envIterator->getPropKey();
-            const char *val = base->mergedEnvironment->queryProp(id);
-            if (id && val && !mergedEnvironment->hasProp(id))
-                mergedEnvironment->setProp(id, val);
-        }
-    }
-    // Search this package and any bases for an element matching xpath1, then return iterator for its children that match xpath2
-    IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const
-    {
-        IPropertyTree *parentNode = node->queryPropTree(xpath1);
-        if (parentNode)
-            return parentNode->getElements(xpath2);
-        ForEachItemIn(idx, bases)
-        {
-            const CRoxiePackage &basePackage = bases.item(idx);
-            IPropertyTreeIterator *it = basePackage.lookupElements(xpath1, xpath2);
-            if (it)
-                return it;
-        }
-        return NULL;
-    }
-    // Use local package and its bases to resolve superfile name list of subfiles via all supported resolvers
-    const ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
-    {
-        // Order of resolution: 
-        // 1. SuperFiles named in local package
-        // 2. SuperFiles named in bases
-        // There is no dali or local case - a superfile that is resolved in dali must also resolve the subfiles there (and is all done in the resolveLFNusingDali method)
-        if (node)
-        {
-            StringBuffer superFileXPath;
-            superFileXPath.append("SuperFile[@id='").append(superFileName).append("']");
-            Owned<IPropertyTreeIterator> subFiles = lookupElements(superFileXPath, "SubFile");
-            if (subFiles)
-            {
-                Owned<CSimpleSuperFileArray> result = new CSimpleSuperFileArray(*subFiles);
-                return result.getClear();
-            }
-        }
-        return NULL;
-    }
+
     // Use local package file only to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingPackage(const char *fileName) const
     {
@@ -316,6 +265,7 @@ protected:
         }
         return NULL;
     }
+
     // Use dali to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingDali(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate) const
     {
@@ -423,10 +373,13 @@ protected:
                 addCache(fileName, result);
             return result;
         }
-        ForEachItemIn(idx, bases)
+        aindex_t count = getBaseCount();
+        for (aindex_t i = 0; i < count; i++)
         {
-            const CRoxiePackage &basePackage = bases.item(idx);
-            IResolvedFile *result = basePackage.lookupFile(fileName, cache, writeAccess, alwaysCreate);
+            const CRoxiePackageNode *basePackage = getBaseNode(i);
+            if (!basePackage)
+                continue;
+            IResolvedFile *result = basePackage->lookupFile(fileName, cache, writeAccess, alwaysCreate);
             if (result)
                 return result;
         }
@@ -434,98 +387,27 @@ protected:
     }
 
     // default constructor for derived class use
-    CRoxiePackage()
+    CRoxiePackageNode()
     {
-        hash = 0;
-        compulsory = false;
     }
 
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxiePackage(IPropertyTree *p)
+    CRoxiePackageNode(IPropertyTree *p) : CPackageNode(p)
     {
-        if (p)
-            node.set(p);
-        else
-            node.setown(createPTree("RoxiePackages"));
-        StringBuffer xml;
-        toXML(node, xml);
-        hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
-        compulsory = false;
         daliHelper.setown(connectToDali()); // MORE - should make this conditional
         if (!daliHelper.get() || !daliHelper->connected())
             node->setPropBool("@localFiles", true);
     }
 
-    ~CRoxiePackage()
+    ~CRoxiePackageNode()
     {
         assertex(fileCache.count()==0);
         // If it's possible for cached objects to outlive the cache I think there is a problem...
         // we could set the cache field to null here for any objects still in cache but there would be a race condition
     }
 
-    virtual void resolveBases(IPackageMap *packages)
-    {
-        loadEnvironment();
-        if (packages)
-        {
-            Owned<IPropertyTreeIterator> baseIterator = node->getElements("Base");
-            if (baseIterator->first())
-            {
-                do
-                {
-                    IPropertyTree &baseElem = baseIterator->query();
-                    const char *baseId = baseElem.queryProp("@id");
-                    if (!baseId)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
-                    const IRoxiePackage *_base = packages->queryPackage(baseId);
-                    if (_base)
-                    {
-                        const CRoxiePackage *base = static_cast<const CRoxiePackage *>(_base);
-                        bases.append(const_cast<CRoxiePackage &>(*LINK(base)));   // should really be an arrayof<const base> but that would require some fixing in jlib
-                        hash = rtlHash64Data(sizeof(base->hash), &base->hash, hash);
-                        mergeEnvironment(base);
-                    }
-                    else
-                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
-                }
-                while(baseIterator->next());
-            }
-            else 
-            {
-                const IRoxiePackage &rootPackage = queryRootPackage();
-                const CRoxiePackage &base = static_cast<const CRoxiePackage &>(rootPackage);
-                base.Link();
-                bases.append(const_cast<CRoxiePackage &>(base));   // should really be an arryof<const base> but that would require some fixing in jlib
-                hash = rtlHash64Data(sizeof(base.hash), &base.hash, hash);
-                mergeEnvironment(&base);
-            }
-        }
-        const char * val = queryEnv("control:compulsory");
-        if (val)
-            compulsory=strToBool(val);
-    }
-
-    virtual bool getEnableFieldTranslation() const
-    {
-        const char *val = queryEnv("control:enableFieldTranslation");
-        if (val)
-            return strToBool(val);
-        else
-            return fieldTranslationEnabled;
-    }
-
-    virtual const char *queryEnv(const char *varname) const
-    {
-        return mergedEnvironment->queryProp(varname);
-    }
-
-    virtual hash64_t queryHash() const 
-    {
-        return hash;
-    }
-
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const 
     {
         StringBuffer xpath;
@@ -578,11 +460,6 @@ public:
         return createRoxieWriteHandler(daliHelper, ldFile.getClear(), clusters);
     }
 
-    virtual const IPropertyTree *queryTree() const
-    {
-        return node;
-    }
-
     virtual IPropertyTree *getQuerySets() const
     {
         if (node)
@@ -592,84 +469,66 @@ public:
     }
 };
 
-IRoxiePackage *createPackage(IPropertyTree *p)
+typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;
+
+IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages)
 {
-    return new CRoxiePackage(p);
+    Owned<CRoxiePackage> pkg = new CRoxiePackage(p);
+    if (packages)
+        pkg->resolveBases(packages);
+    return pkg.getClear();
 }
 
 //================================================================================================
 // CPackageMap - an implementation of IPackageMap using a string map
 //================================================================================================
 
-class CPackageMap : public CInterface, implements IPackageMap
+class CRoxiePackageMap : public CPackageMapOf<CRoxiePackageNode, IRoxiePackage>, implements IRoxiePackageMap
 {
-    MapStringToMyClass<IRoxiePackage> packages;
-    StringAttr packageId;
-    bool active;
-    StringArray wildMatches, wildIds;
 public:
     IMPLEMENT_IINTERFACE;
-    CPackageMap(const char *_packageId, bool _active)
-        : packageId(_packageId), active(_active), packages(true)
+
+    typedef CPackageMapOf<CRoxiePackageNode, IRoxiePackage> BASE;
+
+    CRoxiePackageMap(const char *_packageId, const char *_querySet, bool _active)
+        : BASE(_packageId, _querySet, _active)
     {
     }
 
-    // IPackageMap interface
-    virtual bool isActive() const
+    //map ambiguous IHpccPackageMap interface
+    virtual const IHpccPackage *queryPackage(const char *name) const
     {
-        return active;
+        return BASE::queryPackage(name);
     }
-    virtual const IRoxiePackage *queryPackage(const char *name) const
+    virtual const IHpccPackage *matchPackage(const char *name) const
     {
-        return name ? packages.getValue(name) : NULL;
+        return BASE::matchPackage(name);
     }
-    virtual const IRoxiePackage *matchPackage(const char *name) const
+    virtual const char *queryPackageId() const
     {
-        if (name)
-        {
-            ForEachItemIn(idx, wildMatches)
-            {
-                if (WildMatch(name, wildMatches.item(idx), true))
-                    return queryPackage(wildIds.item(idx));
-            }
-        }
-        return NULL;
+        return BASE::queryPackageId();
     }
-    virtual const char *queryPackageId() const
+    virtual bool isActive() const
     {
-        return packageId;
+        return BASE::isActive();
     }
-    void load(IPropertyTree *xml)
+
+    virtual const IRoxiePackage *queryRoxiePackage(const char *name) const
     {
-        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
-        ForEach(*allpackages)
-        {
-            IPropertyTree &packageTree = allpackages->query();
-            const char *id = packageTree.queryProp("@id");
-            if (id && *id)
-            {
-                Owned<IRoxiePackage> package = createPackage(&packageTree);
-                package->resolveBases(this);
-                packages.setValue(id, package);
-                const char *queries = packageTree.queryProp("@queries");
-                if (queries && *queries)
-                {
-                    wildMatches.append(queries);
-                    wildIds.append(id);
-                }
-            }
-            else
-                throw MakeStringException(ROXIE_UNKNOWN_PACKAGE, "Invalid package map - Package element missing id attribute");
-        }
+        return queryResolvedPackage(name);
+    }
+    virtual const IRoxiePackage *matchRoxiePackage(const char *name) const
+    {
+        return matchResolvedPackage(name);
     }
 };
 
-static CPackageMap *emptyPackageMap;
+static CRoxiePackageMap *emptyPackageMap;
 static CRoxiePackage *rootPackage;
 static SpinLock emptyPackageMapCrit;
 static IRoxieDebugSessionManager *debugSessionManager;
 
-extern const IRoxiePackage &queryRootPackage()
+extern const IRoxiePackage &queryRootRoxiePackage()
 {
     SpinBlock b(emptyPackageMapCrit);
     if (!rootPackage)
@@ -681,11 +540,11 @@ extern const IRoxiePackage &queryRootPackage()
     return *rootPackage;
 }
 
-extern const IPackageMap &queryEmptyPackageMap()
+extern const IRoxiePackageMap &queryEmptyRoxiePackageMap()
 {
     SpinBlock b(emptyPackageMapCrit);
     if (!emptyPackageMap)
-        emptyPackageMap = new CPackageMap("<none>", true);
+        emptyPackageMap = new CRoxiePackageMap("<none>", NULL, true);
     return *emptyPackageMap;
 }
 
@@ -765,7 +624,7 @@ protected:
             throw MakeStringException(ROXIE_INTERNAL_ERROR, "Invalid parameters to addAlias");
     }
 
-    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo) = 0;
+    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo) = 0;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -785,7 +644,7 @@ public:
         return active;
     }
 
-    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash)
+    virtual void load(const IPropertyTree *querySet, const IRoxiePackageMap &packages, hash64_t &hash)
     {
         Owned<IPropertyTreeIterator> queryNames = querySet->getElements("Query");
         ForEach (*queryNames)
@@ -798,7 +657,7 @@ public:
                 if (!id || !*id || !dllName || !*dllName)
                     throw MakeStringException(ROXIE_QUERY_MODIFICATION, "dll and id must be specified");
                 Owned<const IQueryDll> queryDll = createQueryDll(dllName);
-                const IRoxiePackage *package = NULL;
+                const IHpccPackage *package = NULL;
                 const char *packageName = query.queryProp("@package");
                 if (packageName && *packageName)
                 {
@@ -810,7 +669,7 @@ public:
                 {
                     package = packages.queryPackage(id);  // Look for an exact match, then a fuzzy match, using query name as the package id
                     if(!package) package = packages.matchPackage(id);
-                    if (!package) package = &queryRootPackage();
+                    if (!package) package = &queryRootRoxiePackage();
                 }
                 assertex(package);
                 addQuery(id, loadQueryFromDll(id, queryDll.getClear(), *package, &query), hash);
@@ -932,7 +791,7 @@ public:
     {
     }
 
-    virtual IQueryFactory * loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+    virtual IQueryFactory * loadQueryFromDll(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
     {
         return createServerQueryFactory(id, dll, package, stateInfo);
     }
@@ -956,7 +815,7 @@ public:
         channelNo = _channelNo;
     }
 
-    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
+    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IHpccPackage &package, const IPropertyTree *stateInfo)
     {
         return createSlaveQueryFactory(id, dll, package, channelNo, stateInfo);
     }
@@ -997,7 +856,7 @@ public:
         return managers[idx];
     }
 
-    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash)
+    virtual void load(const IPropertyTree *querySets, const IRoxiePackageMap &packages, hash64_t &hash)
     {
         for (unsigned channel = 0; channel < numChannels; channel++)
             if (managers[channel])
@@ -1079,7 +938,7 @@ class CRoxieQueryPackageManager : public CInterface
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxieQueryPackageManager(unsigned _numChannels, const char *_querySet, const IPackageMap *_packages)
+    CRoxieQueryPackageManager(unsigned _numChannels, const char *_querySet, const IRoxiePackageMap *_packages)
         : numChannels(_numChannels), packages(_packages), querySet(_querySet)
     {
         queryHash = 0;
@@ -1196,7 +1055,7 @@ protected:
     Owned<CRoxieSlaveQuerySetManagerSet> slaveManagers;
     Owned<IRoxieQuerySetManager> serverManager;
 
-    Owned<const IPackageMap> packages;
+    Owned<const IRoxiePackageMap> packages;
     unsigned numChannels;
     hash64_t queryHash;
     StringAttr querySet;
@@ -1229,7 +1088,7 @@ class CRoxieDaliQueryPackageManager : public CRoxieQueryPackageManager, implemen
 
 public:
     IMPLEMENT_IINTERFACE;
-    CRoxieDaliQueryPackageManager(unsigned _numChannels, const IPackageMap *_packages, const char *_querySet)
+    CRoxieDaliQueryPackageManager(unsigned _numChannels, const IRoxiePackageMap *_packages, const char *_querySet)
         : CRoxieQueryPackageManager(_numChannels, _querySet, _packages)
     {
         daliHelper.setown(connectToDali());
@@ -1274,7 +1133,7 @@ class CStandaloneQueryPackageManager : public CRoxieQueryPackageManager
 public:
     IMPLEMENT_IINTERFACE;
 
-    CStandaloneQueryPackageManager(unsigned _numChannels, const char *_querySet, const IPackageMap *_packages, IPropertyTree *_standaloneDll)
+    CStandaloneQueryPackageManager(unsigned _numChannels, const char *_querySet, const IRoxiePackageMap *_packages, IPropertyTree *_standaloneDll)
         : CRoxieQueryPackageManager(_numChannels, _querySet, _packages), standaloneDll(_standaloneDll)
     {
         assertex(standaloneDll);
@@ -2282,7 +2141,7 @@ private:
         throw MakeStringException(ROXIE_INVALID_INPUT, "Badly formated control query");
     }
 
-    void createQueryPackageManager(unsigned numChannels, const IPackageMap *packageMap, const char *querySet)
+    void createQueryPackageManager(unsigned numChannels, const IRoxiePackageMap *packageMap, const char *querySet)
     {
         // Called from reload inside write lock block
         Owned<CRoxieQueryPackageManager> qpm = new CRoxieDaliQueryPackageManager(numChannels, packageMap, querySet);
@@ -2331,7 +2190,7 @@ private:
                             DBGLOG("Loading package map %s, active %s", packageMapId, isActive ? "true" : "false");
                         try
                         {
-                            Owned<CPackageMap> packageMap = new CPackageMap(packageMapId, isActive);
+                            Owned<CRoxiePackageMap> packageMap = new CRoxiePackageMap(packageMapId, packageMapFilter, isActive);
                             Owned<IPropertyTree> xml = daliHelper->getPackageMap(packageMapId);
                             packageMap->load(xml);
                             createQueryPackageManager(numChannels, packageMap.getLink(), querySet);
@@ -2352,7 +2211,7 @@ private:
         {
             if (traceLevel)
                 DBGLOG("Loading empty package for QuerySet %s", querySet);
-            createQueryPackageManager(numChannels, LINK(&queryEmptyPackageMap()), querySet);
+            createQueryPackageManager(numChannels, LINK(&queryEmptyRoxiePackageMap()), querySet);
         }
         if (traceLevel)
             DBGLOG("Loaded packages");
@@ -2365,7 +2224,7 @@ private:
         standAloneDllTree.setown(createPTree("Query"));
         standAloneDllTree->setProp("@id", "roxie");
         standAloneDllTree->setProp("@dll", standAloneDll->queryDll()->queryName());
-        Owned<CRoxieQueryPackageManager> qpm = new CStandaloneQueryPackageManager(numChannels, querySet, LINK(&queryEmptyPackageMap()), standAloneDllTree.getClear());
+        Owned<CRoxieQueryPackageManager> qpm = new CStandaloneQueryPackageManager(numChannels, querySet, LINK(&queryEmptyRoxiePackageMap()), standAloneDllTree.getClear());
         qpm->load();
         stateHash = rtlHash64Data(sizeof(stateHash), &stateHash, qpm->getHash());
         allQueryPackages.append(*qpm.getClear());

roxie/ccd/ccdstate.hpp

@@ -29,6 +29,7 @@
 #include "thorcommon.hpp"
 #include "ccddali.hpp"
 #include "thorcommon.ipp"
+#include "package.h"
 
 interface IFilePartMap : public IInterface
 {
@@ -43,43 +44,29 @@ interface IFilePartMap : public IInterface
 extern IFilePartMap *createFilePartMap(const char *fileName, IFileDescriptor &fdesc);
 
 interface IRoxiePackage;
-interface IPackageMap : public IInterface
+interface IRoxiePackageMap : public IHpccPackageMap
 {
-    // Lookup package using exact id
-    virtual const IRoxiePackage *queryPackage(const char *name) const = 0;
-    // Lookup package using fuzzy id
-    virtual const IRoxiePackage *matchPackage(const char *name) const = 0;
-    virtual const char *queryPackageId() const = 0;
-    virtual bool isActive() const = 0;
+    virtual const IRoxiePackage *queryRoxiePackage(const char *name) const = 0;
+    virtual const IRoxiePackage *matchRoxiePackage(const char *name) const = 0;
 };
-extern const IRoxiePackage &queryRootPackage();
-extern const IPackageMap &queryEmptyPackageMap();
+extern const IRoxiePackage &queryRootRoxiePackage();
+extern const IRoxiePackageMap &queryEmptyRoxiePackageMap();
 
-interface IRoxiePackage : extends IInterface 
+interface IRoxiePackage : public IHpccPackage
 {
-    // Complete the setup of a package by resolving base package references
-    virtual void resolveBases(IPackageMap *packages) = 0;
-    // Lookup package environment variable
-    virtual const char *queryEnv(const char *varname) const = 0;
-    // Lookup package environment variable controlling field translation
-    virtual bool getEnableFieldTranslation() const = 0;
-    // Return entire XML tree for package
-    virtual const IPropertyTree *queryTree() const = 0;
     // Lookup information in package to resolve existing logical file name
     virtual const IResolvedFile *lookupFileName(const char *fileName, bool opt, bool cacheDaliResults, IConstWorkUnit *wu) const = 0;
     // Lookup information in package to create new logical file name
     virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const = 0;
-    // Retrieve hash for the package
-    virtual hash64_t queryHash() const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTree *getQuerySets() const = 0;
     // Remove a resolved file from the cache 
     virtual void removeCache(const IResolvedFile *goer) const = 0; // note that this is const as cache is considered mutable
 };
 
-extern IRoxiePackage *createPackage(IPropertyTree *p);
+extern IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages);
 
 interface ISlaveDynamicFileCache : extends IInterface
 {
@@ -103,7 +90,7 @@ interface IRoxieQuerySetManager : extends IInterface
 {
     virtual bool isActive() const = 0;
     virtual IQueryFactory *getQuery(const char *id, const IRoxieContextLogger &ctx) const = 0;
-    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash) = 0;
+    virtual void load(const IPropertyTree *querySet, const IRoxiePackageMap &packages, hash64_t &hash) = 0;
     virtual void getStats(const char *queryName, const char *graphName, StringBuffer &reply, const IRoxieContextLogger &logctx) const = 0;
     virtual void resetQueryTimings(const char *queryName, const IRoxieContextLogger &logctx) = 0;
     virtual void resetAllQueryTimings() = 0;
@@ -120,7 +107,7 @@ interface IRoxieDebugSessionManager : extends IInterface
 
 interface IRoxieQuerySetManagerSet : extends IInterface
 {
-    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash) = 0;
+    virtual void load(const IPropertyTree *querySets, const IRoxiePackageMap &packages, hash64_t &hash) = 0;
 };
 
 interface IRoxieQueryPackageManagerSet : extends IInterface

rtl/eclrtl/eclrtl.hpp

@@ -50,7 +50,6 @@
 #endif
 
 typedef unsigned char byte;
-typedef unsigned __int64 hash64_t;
 interface IEngineRowAllocator;
 interface IOutputMetaData;
 interface IOutputRowSerializer;

system/include/platform.h

@@ -439,6 +439,7 @@ typedef int socklen_t;
 
 #define strtok(a,b)   j_strtok_deprecated(a,b)  // will disappear at some point
 
+typedef unsigned __int64 hash64_t;
 typedef unsigned __int64 __uint64;
 typedef __uint64 offset_t;
 typedef unsigned char byte;