Merge remote-tracking branch 'origin/candidate-3.10.x'

Conflicts:
	ecl/hql/hqlutil.cpp

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/csvsplitter.cpp

@@ -238,7 +238,7 @@ done:
             unsigned match = matcher.getMatch((size32_t)(end-cur), (const char *)cur, matchLen);
             if ((match & 255) == ESCAPE)
             {
-                ptrdiff_t restLen = end-cur+matchLen;
+                ptrdiff_t restLen = end-(cur+matchLen);
                 memmove(cur, cur+matchLen, restLen);
                 end -= matchLen;
                 lengths[curColumn] -= matchLen;

common/workunit/package.cpp

@@ -98,7 +98,7 @@ void CPackageNode::loadEnvironment()
         {
             StringBuffer s;
             toXML(&env, s);
-            throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
+            throw MakeStringException(PACKAGE_MISSING_ID, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
         }
     }
     Owned<IAttributeIterator> attrs = node->getAttributes();
@@ -110,6 +110,31 @@ void CPackageNode::loadEnvironment()
     }
 }
 
+bool CPackageNode::validate(StringArray &warn, StringArray &err) const
+{
+    if (!node)
+        return true;
+    StringAttr packageId = node->queryProp("@id");
+    if (packageId.isEmpty())
+        err.append("Package has no id attribute");
+    Owned<IPropertyTreeIterator> files = node->getElements("SuperFile");
+    ForEach(*files)
+    {
+        IPropertyTree &super = files->query();
+        StringAttr superId = super.queryProp("@id");
+        if (superId.isEmpty())
+            err.append("SuperFile has no id attribute");
+
+        if (!super.hasProp("SubFile"))
+        {
+            VStringBuffer msg("Package['%s']/SuperFile['%s'] has no SubFiles defined", packageId.sget(), superId.sget());
+            warn.append(msg.str());
+        }
+    }
+    return true;
+}
+
+
 CHpccPackage *createPackage(IPropertyTree *p)
 {
     return new CHpccPackage(p);
@@ -119,6 +144,20 @@ CHpccPackage *createPackage(IPropertyTree *p)
 // CPackageMap - an implementation of IPackageMap using a string map
 //================================================================================================
 
+IHpccPackageMap *createPackageMapFromPtree(IPropertyTree *t, const char *queryset, const char *id)
+{
+    Owned<CHpccPackageMap> pm = new CHpccPackageMap(id, queryset, true);
+    pm->load(t);
+    return pm.getClear();
+}
+
+IHpccPackageMap *createPackageMapFromXml(const char *xml, const char *queryset, const char *id)
+{
+    Owned<IPropertyTree> t = createPTreeFromXMLString(xml);
+    return createPackageMapFromPtree(t, queryset, id);
+}
+
+
 //================================================================================================
 // CHpccPackageSet - an implementation of IHpccPackageSet
 //================================================================================================

common/workunit/package.h

@@ -18,9 +18,17 @@
 #ifndef WUPACKAGE_H
 #define WUPACKAGE_H
 
+#include "errorlist.h"
 #include "dadfs.hpp"
 #include "workunit.hpp"
 
+// error codes
+#define PACKAGE_TARGET_NOT_FOUND      PACKAGE_ERROR_START
+#define PACKAGE_MISSING_ID            PACKAGE_ERROR_START+1
+#define PACKAGE_NO_SUBFILES           PACKAGE_ERROR_START+2
+#define PACKAGE_NOT_FOUND             PACKAGE_ERROR_START+3
+
+
 interface IHpccPackage : extends IInterface
 {
     virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const = 0;
@@ -37,6 +45,7 @@ interface IHpccPackageMap : extends IInterface
     virtual const IHpccPackage *matchPackage(const char *name) const = 0;
     virtual const char *queryPackageId() const = 0;
     virtual bool isActive() const = 0;
+    virtual bool validate(StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages) const = 0;
 };
 
 interface IHpccPackageSet : extends IInterface
@@ -44,6 +53,9 @@ interface IHpccPackageSet : extends IInterface
      virtual const IHpccPackageMap *queryActiveMap(const char *queryset) const = 0;
 };
 
+extern WORKUNIT_API IHpccPackageMap *createPackageMapFromXml(const char *xml, const char *queryset, const char *id);
+extern WORKUNIT_API IHpccPackageMap *createPackageMapFromPtree(IPropertyTree *t, const char *queryset, const char *id);
+
 extern WORKUNIT_API IHpccPackageSet *createPackageSet(const char *process);
 extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool readonly);
 extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly);

common/workunit/pkgimpl.hpp

@@ -115,7 +115,6 @@ protected:
         return NULL;
     }
 
-
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -149,6 +148,7 @@ public:
             return NULL;
         return node->getPropTree("QuerySets");
     }
+    virtual bool validate(StringArray &warn, StringArray &err) const;
 };
 
 enum baseResolutionState
@@ -207,10 +207,10 @@ public:
                     IPropertyTree &baseElem = baseIterator->query();
                     const char *baseId = baseElem.queryProp("@id");
                     if (!baseId)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
+                        throw MakeStringException(PACKAGE_MISSING_ID, "PACKAGE_ERROR: base element missing id attribute");
                     const IHpccPackage *base = packages->queryPackage(baseId);
                     if (!base)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
+                        throw MakeStringException(PACKAGE_NOT_FOUND, "PACKAGE_ERROR: base package %s not found", baseId);
                     CResolvedPackage<TYPE> *rebase = dynamic_cast<CResolvedPackage<TYPE> *>(const_cast<IHpccPackage *>(base));
                     rebase->resolveBases(packages);
                     appendBase(base);
@@ -255,6 +255,11 @@ public:
 
         return false;
     }
+
+    virtual bool validate(StringArray &warn, StringArray &err) const
+    {
+        return TYPE::validate(warn, err);
+    }
 };
 
 
@@ -333,7 +338,7 @@ public:
             IPropertyTree &packageTree = allpackages->query();
             const char *id = packageTree.queryProp("@id");
             if (!id || !*id)
-                throw MakeStringException(-1, "Invalid package map - Package element missing id attribute");
+                throw MakeStringException(PACKAGE_MISSING_ID, "Invalid package map - Package element missing id attribute");
             Owned<packageType> package = new packageType(&packageTree);
             packages.setValue(id, package.get());
             package->loadEnvironment();
@@ -357,6 +362,62 @@ public:
     {
         load(getPackageMapById(id, true));
     }
+
+    virtual bool validate(StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages) const
+    {
+        bool isValid = true;
+        MapStringTo<bool> referencedPackages;
+        Owned<IPropertyTree> qs = getQueryRegistry(querySet, true);
+        if (!qs)
+            throw MakeStringException(PACKAGE_TARGET_NOT_FOUND, "Target %s not found", querySet.sget());
+        Owned<IPropertyTreeIterator> queries = qs->getElements("Query");
+        HashIterator it(packages);
+        ForEach (it)
+        {
+            const char *packageId = (const char *)it.query().getKey();
+            packageType *pkg = packages.getValue(packageId);
+            if (!pkg)
+                continue;
+            if (!pkg->validate(warn, err))
+                isValid = false;
+            Owned<IPropertyTreeIterator> baseNodes = pkg->queryTree()->getElements("Base");
+            ForEach(*baseNodes)
+            {
+                const char *baseId = baseNodes->query().queryProp("@id");
+                if (!baseId || !*baseId)
+                {
+                    VStringBuffer msg("Package '%s' contains Base element with no id attribute", packageId);
+                    err.append(msg.str());
+                    continue;
+                }
+                if (!referencedPackages.getValue(baseId))
+                    referencedPackages.setValue(baseId, true);
+            }
+        }
+        ForEach(*queries)
+        {
+            const char *queryid = queries->query().queryProp("@id");
+            if (queryid && *queryid)
+            {
+                const IHpccPackage *matched = matchPackage(queryid);
+                if (matched)
+                {
+                    const char *matchId = matched->queryTree()->queryProp("@id");
+                    if (!referencedPackages.getValue(matchId))
+                        referencedPackages.setValue(matchId, true);
+                }
+                else
+                    unmatchedQueries.append(queryid);
+            }
+        }
+        ForEach (it)
+        {
+            const char *packageId = (const char *)it.query().getKey();
+            if (!referencedPackages.getValue(packageId))
+                unusedPackages.append(packageId);
+        }
+        return isValid;
+    }
 };
 
 typedef CPackageMapOf<CPackageNode, IHpccPackage> CHpccPackageMap;

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -779,7 +779,7 @@
 
                     <mediaobject>
                       <imageobject>
-                        <imagedata fileref="images/GS_1311.JPG"
+                        <imagedata fileref="images/GS_1311.jpg"
                                    vendor="eclwatchSS" />
                       </imageobject>
                     </mediaobject>
@@ -913,13 +913,13 @@
           use the eclcc compiler locally to create an archive to be sent to
           the eclccServer:</para>
 
-          <para><programlisting>eclcc hello.ecl -E | ecl run --target=thor --server=&lt;IP Address of the ESP&gt;:8010</programlisting></para>
+          <para><programlisting>eclcc hello.ecl -E | ecl run - --target=thor --server=&lt;IP Address of the ESP&gt;:8010</programlisting></para>
 
           <para>The target parameter must name a valid target cluster name as
           listed in your environment's topology section.</para>
         </sect3>
 
-        <sect3>
+        <sect3 role="brk">
           <title>Running a basic ECL program from the ECL IDE</title>
 
           <para><orderedlist>

ecl/ecl-package/ecl-package.cpp

@@ -30,6 +30,21 @@
 
 //=========================================================================================
 
+IClientWsPackageProcess *getWsPackageSoapService(const char *server, const char *port, const char *username, const char *password)
+{
+    if(server == NULL)
+        throw MakeStringException(-1, "Server url not specified");
+
+    VStringBuffer url("http://%s:%s/WsPackageProcess", server, port);
+
+    IClientWsPackageProcess *packageProcessClient = createWsPackageProcessClient();
+    packageProcessClient->addServiceUrl(url.str());
+    packageProcessClient->setUsernameToken(username, password, NULL);
+
+    return packageProcessClient;
+}
+
+
 class EclCmdPackageActivate : public EclCmdCommon
 {
 public:
@@ -263,8 +278,8 @@ public:
             {
                 IConstPackageListMapData& req = pkgMapInfo.item(i);
                 printf("\nPackage Name = %s  active = %d\n", req.getId(), req.getActive());
-                IArrayOf<IConstPackageListData> &pkgInfo = req.getPkgListData();
 
+                IArrayOf<IConstPackageListData> &pkgInfo = req.getPkgListData();
                 unsigned int numPkgs = pkgInfo.ordinality();
                 for (unsigned int j = 0; j <numPkgs; j++)
                 {
@@ -580,6 +595,131 @@ private:
     StringAttr optDaliIP;
 };
 
+class EclCmdPackageValidate : public EclCmdCommon
+{
+public:
+    EclCmdPackageValidate()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optTarget.isEmpty())
+                    optTarget.set(arg);
+                else if (optFileName.isEmpty())
+                    optFileName.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optFileName.isEmpty())
+            err.append("\n ... Missing package file name\n\n");
+        else if (optTarget.isEmpty())
+            err.append("\n ... Specify a cluster name\n\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+        StringBuffer pkgInfo;
+        pkgInfo.loadFile(optFileName);
+
+        fprintf(stdout, "\nvalidating packagemap file %s\n\n", optFileName.sget());
+
+        Owned<IClientValidatePackageRequest> request = packageProcessClient->createValidatePackageRequest();
+        request->setInfo(pkgInfo);
+        request->setTarget(optTarget);
+
+        Owned<IClientValidatePackageResponse> resp = packageProcessClient->ValidatePackage(request);
+        if (resp->getExceptions().ordinality()>0)
+            outputMultiExceptions(resp->getExceptions());
+        StringArray &errors = resp->getErrors();
+        if (errors.ordinality()==0)
+            fputs("   No errors found\n", stdout);
+        else
+        {
+            fputs("   Error(s):\n", stderr);
+            ForEachItemIn(i, errors)
+                fprintf(stderr, "      %s\n", errors.item(i));
+        }
+        StringArray &warnings = resp->getWarnings();
+        if (warnings.ordinality()==0)
+            fputs("   No warnings found\n", stdout);
+        else
+        {
+            fputs("   Warning(s):\n", stderr);
+            ForEachItemIn(i, warnings)
+                fprintf(stderr, "      %s\n", warnings.item(i));
+        }
+        StringArray &unmatchedQueries = resp->getQueries().getUnmatched();
+        if (unmatchedQueries.ordinality()>0)
+        {
+            fputs("\n   Queries without matching package:\n", stderr);
+            ForEachItemIn(i, unmatchedQueries)
+                fprintf(stderr, "      %s\n", unmatchedQueries.item(i));
+        }
+        StringArray &unusedPackages = resp->getPackages().getUnmatched();
+        if (unusedPackages.ordinality()>0)
+        {
+            fputs("\n   Packages without matching queries:\n", stderr);
+            ForEachItemIn(i, unusedPackages)
+                fprintf(stderr, "      %s\n", unusedPackages.item(i));
+        }
+
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+                    "\n"
+                    "The 'validate' command will checkout the contents of the package map file \n"
+                    "\n"
+                    "ecl packagemap validate <target> <filename>\n"
+                    " Options:\n"
+                    "   <target>                    name of target to use when adding package map information\n"
+                    "   <filename>                  name of file containing package map information\n",
+                    stdout);
+
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optFileName;
+    StringAttr optTarget;
+};
 
 IEclCommand *createPackageSubCommand(const char *cmdname)
 {
@@ -597,7 +737,9 @@ IEclCommand *createPackageSubCommand(const char *cmdname)
         return new EclCmdPackageInfo();
     if (strieq(cmdname, "list"))
         return new EclCmdPackageList();
-    return NULL;
+    if (strieq(cmdname, "validate"))
+        return new EclCmdPackageValidate();
+return NULL;
 }
 
 //=========================================================================================
@@ -621,6 +763,7 @@ public:
             "      deactivate   deactivate a package map (package map will not get loaded)\n"
             "      list         list loaded package map names\n"
             "      info         return active package map information\n"
+            "      validate     validate information in the package map file \n"
         );
     }
 };

ecl/hql/hqlexpr.cpp

@@ -15189,7 +15189,9 @@ unsigned firstPayloadField(IHqlExpression * record, unsigned cnt)
 IHqlExpression * createSelector(node_operator op, IHqlExpression * ds, IHqlExpression * seq)
 {
 //  OwnedHqlExpr record = getUnadornedExpr(ds->queryRecord());
-    IHqlExpression * record = ds->queryRecord()->queryBody();
+    IHqlExpression * dsRecord = ds->queryRecord();
+    assertex(dsRecord);
+    IHqlExpression * record = dsRecord->queryBody();
 
     switch (op)
     {
@@ -15197,11 +15199,11 @@ IHqlExpression * createSelector(node_operator op, IHqlExpression * ds, IHqlExpre
     case no_right:
     case no_top:
         assertex(seq && seq->isAttribute());
-        return createRow(op, LINK(record->queryBody()), LINK(seq));
+        return createRow(op, LINK(record), LINK(seq));
     case no_self:
         {
             //seq is set when generating code unique target selectors
-            return createRow(op, LINK(record->queryBody()), LINK(seq));
+            return createRow(op, LINK(record), LINK(seq));
         }
     case no_none:
         return LINK(ds);

ecl/hql/hqlgram.y

@@ -1281,6 +1281,7 @@ knownFunction1
 scopeFlag
     : EXPORT            {   $$.setInt(EXPORT_FLAG); $$.setPosition($1); }
     | SHARED            {   $$.setInt(SHARED_FLAG); $$.setPosition($1); }
+    | LOCAL             {   $$.setInt(0); $$.setPosition($1); }
     ;
 
 // scopeflags needs to be explicitly included, rather than using an optScopeFlags production, otherwise you get shift reduce errors - since it is the first item on a line.

ecl/hql/hqlutil.cpp

@@ -4609,6 +4609,10 @@ static bool splitDatasetAttribute(SharedHqlExpr & dataset, SharedHqlExpr & attri
         if (leftOp == no_selectmap)
             return false;
 
+        //If this is a selection from an inscope dataset then this must not be assumed to be an input dataset.
+        if (expr->isDataset() && !expr->hasProperty(newAtom))
+            return false;
+
         IHqlExpression * lhs = LINK(left);
         IHqlExpression * field = expr->queryChild(1);
         if (lhs->isDataset()) lhs = createRow(no_activerow, lhs);

ecl/hqlcpp/hqlcse.cpp

@@ -108,7 +108,18 @@ bool canCreateTemporary(IHqlExpression * expr)
     case no_loopbody:
         return false;
     }
-    return !expr->isAction() && !expr->isTransform();
+    ITypeInfo * type = expr->queryType();
+    if (!type)
+        return false;
+    switch (type->getTypeCode())
+    {
+    case type_transform:
+    case type_null:
+    case type_void:
+        return false;
+    default:
+        return true;
+    }
 }
 
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -11141,6 +11141,14 @@ static void gatherPotentialSelectors(HqlExprArray & args, IHqlExpression * expr)
 }
 
 
+IHqlExpression * HqlTreeNormalizer::transformChildrenNoAnnotations(IHqlExpression * expr)
+{
+    HqlExprArray args;
+    ForEachChild(i, expr)
+        args.append(*transform(expr->queryChild(i)->queryBody()));
+    return completeTransform(expr, args);
+}
+
 //The following symbol removal code works, but I'm not sure I want to do it at the moment because of the changes to the HOLe queries
 //Remove as many named symbols as we can - try and keep for datasets and statements so can go in the tree.
 IHqlExpression * HqlTreeNormalizer::createTransformed(IHqlExpression * expr)
@@ -11915,6 +11923,14 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
                 }
             }
 #endif
+
+            //If a named symbol is used as an argument to maxlength you can end up with a situation
+            //where records are identical except for that named symbol.
+            //If f(a + b) is then optimized to b this can lead to incompatible selectors, since
+            //a and b are "compatible", but not identical.
+            //This then causes chaos, so strip them as a precaution... but it is only a partial solution.
+            if (name == maxLengthAtom)
+                return transformChildrenNoAnnotations(expr);
             break;
         }
 

ecl/hqlcpp/hqlttcpp.ipp

@@ -1167,6 +1167,7 @@ protected:
     IHqlExpression * validateKeyedJoin(IHqlExpression * expr);
 
 protected:
+    IHqlExpression * transformChildrenNoAnnotations(IHqlExpression * expr);
     IHqlExpression * makeRecursiveName(_ATOM searchModule, _ATOM searchName);
     HqlTreeNormalizerInfo * queryCommonExtra(IHqlExpression * expr);
     IHqlExpression * transformSimpleConst(IHqlExpression * expr)

ecl/regress/issue8819.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    This program is free software: you can redistribute it and/or modify
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+myfunc(num) := functionmacro
+    local numPlus := num + 1;
+    return numPlus;
+endmacro;
+
+numPlus := 'this will cause syntax error';
+myfunc(5);

ecl/regress/issue8857.eclxml

@@ -0,0 +1,167 @@
+<Archive build="community_3.8.6-1" eclVersion="3.6.1" legacyMode="0">
+ <Query attributePath="_local_directory_.xyz"/>
+ <Module key="_local_directory_" name="_local_directory_">
+  <Attribute key="xyz" name="xyz" sourcePath="C:\xyz.ecl">
+import red.source.yyy.xxx;
+#stored(&apos;frequency&apos;, &apos;daily&apos;);
+
+xxx.zzz.files_spray.web_user_curr_ds;
+  </Attribute>
+ </Module>
+ <Module key="red" name="red"/>
+ <Module key="red.source" name="red.source"/>
+ <Module key="red.source.yyy" name="red.source.yyy"/>
+ <Module key="red.source.yyy.xxx" name="red.source.yyy.xxx">
+  <Attribute key="zzz" name="zzz" sourcePath="C:\development\source\yyy\xxx\zzz.ecl">
+   import red.source.yyy.xxx;
+import red.common;
+import std;
+
+EXPORT zzz := module
+
+  shared COMMA_DELIM := &apos;,&apos;;
+	shared PIPE_DELIM  := &apos;||&apos;;
+	
+	shared string FREQUENCY       :=  trim(STD.str.ToLowerCase(common.stored_frequency), all); 
+	shared string NAME_SUFFIX     := if(frequency = &apos;daily&apos;, &apos;_prelim&apos;, &apos;_actual&apos;);  	
+	
+	shared String SPRAY_PREFIX		:= xxx.common.constants.spray_prefix;
+	shared String STG_PREFIX			:= xxx.common.constants.stg_prefix;
+	
+  getCSVDelimiter := function
+		return if (FREQUENCY = &apos;daily&apos;, COMMA_DELIM, PIPE_DELIM);
+	end;
+
+	getDailySprayDSName(String baseFileName, boolean prelimFlag=False) := function
+		newBaseFileName := if (FREQUENCY = &apos;DAILY&apos;, baseFileName + &apos;_prelim&apos;, baseFileName);
+		return  if (prelimFlag, SPRAY_PREFIX + &apos;::&apos; + newBaseFileName, SPRAY_PREFIX + &apos;::&apos; + baseFileName);
+	end;
+
+Export files_spray := module
+		export web_user_curr_ds 		:= dataset(getDailySprayDSName(&apos;web_user_curr&apos;),xxx.layouts.web_user_curr,   csv(terminator(&apos;\n&apos;), separator(getCSVDelimiter), quote(&apos;&quot;&apos;)));
+End;
+
+	
+End;
+  </Attribute>
+ </Module>
+ <Module key="red.common" name="red.common">
+  <Attribute key="stored_frequency" name="stored_frequency" sourcePath="C:\development\common\stored_frequency.ecl">
+   EXPORT string stored_frequency := &apos;daily&apos; : stored(&apos;frequency&apos;);&#32;
+  </Attribute>
+ </Module>
+ <Module key="std" name="std">
+  <Attribute key="str" name="str" sourcePath="C:\Program Files (x86)\HPCC Systems\HPCC\bin\ver_3_6\ecllibrary\std\Str.ecl">
+   /*##############################################################################
+## Copyright (c) 2011 HPCC Systems.  All rights reserved.
+############################################################################## */
+
+EXPORT Str := MODULE
+
+/*
+  Since this is primarily a wrapper for a plugin, all the definitions for this standard library
+  module are included in a single file.  Generally I would expect them in individual files.
+  */
+
+IMPORT lib_stringlib;
+
+/**
+ * Returns the argument string with all upper case characters converted to lower case.
+ * 
+ * @param src           The string that is being converted.
+ */
+
+EXPORT STRING ToLowerCase(STRING src) := lib_stringlib.StringLib.StringToLowerCase(src);
+
+
+END;&#13;&#10;
+  </Attribute>
+ </Module>
+ <Module flags="5"
+         fullname="C:\Program Files (x86)\HPCC Systems\HPCC\bin\ver_3_6\plugins\stringlib.dll"
+         key="lib_stringlib"
+         name="lib_stringlib"
+         plugin="stringlib.dll"
+         sourcePath="lib_stringlib"
+         version="STRINGLIB 1.1.14">
+  <Text>export StringLib := SERVICE
+  string StringFilterOut(const string src, const string _within) : c, pure,entrypoint=&apos;slStringFilterOut&apos;; 
+  string StringFilter(const string src, const string _within) : c, pure,entrypoint=&apos;slStringFilter&apos;; 
+  string StringSubstituteOut(const string src, const string _within, const string _newchar) : c, pure,entrypoint=&apos;slStringSubsOut&apos;; 
+  string StringSubstitute(const string src, const string _within, const string _newchar) : c, pure,entrypoint=&apos;slStringSubs&apos;; 
+  string StringRepad(const string src, unsigned4 size) : c, pure,entrypoint=&apos;slStringRepad&apos;; 
+  string StringTranslate(const string src, const string _within, const string _mapping) : c, pure,entrypoint=&apos;slStringTranslate&apos;; 
+  unsigned integer4 StringFind(const string src, const string tofind, unsigned4 instance ) : c, pure,entrypoint=&apos;slStringFind&apos;; 
+  unsigned integer4 StringUnboundedUnsafeFind(const string src, const string tofind ) : c, pure,entrypoint=&apos;slStringFind2&apos;; 
+  unsigned integer4 StringFindCount(const string src, const string tofind) : c, pure,entrypoint=&apos;slStringFindCount&apos;; 
+  unsigned integer4 EbcdicStringFind(const ebcdic string src, const ebcdic string tofind , unsigned4 instance ) : c,pure,entrypoint=&apos;slStringFind&apos;; 
+  unsigned integer4 EbcdicStringUnboundedUnsafeFind(const ebcdic string src, const ebcdic string tofind ) : c,pure,entrypoint=&apos;slStringFind2&apos;; 
+  string StringExtract(const string src, unsigned4 instance) : c,pure,entrypoint=&apos;slStringExtract&apos;; 
+  string8 GetDateYYYYMMDD() : c,once,entrypoint=&apos;slGetDateYYYYMMDD2&apos;;
+  varstring GetBuildInfo() : c,once,entrypoint=&apos;slGetBuildInfo&apos;;
+  string Data2String(const data src) : c,pure,entrypoint=&apos;slData2String&apos;;
+  data String2Data(const string src) : c,pure,entrypoint=&apos;slString2Data&apos;;
+  string StringToLowerCase(const string src) : c,pure,entrypoint=&apos;slStringToLowerCase&apos;;
+  string StringToUpperCase(const string src) : c,pure,entrypoint=&apos;slStringToUpperCase&apos;;
+  string StringToProperCase(const string src) : c,pure,entrypoint=&apos;slStringToProperCase&apos;;
+  string StringToCapitalCase(const string src) : c,pure,entrypoint=&apos;slStringToCapitalCase&apos;;
+  string StringToTitleCase(const string src) : c,pure,entrypoint=&apos;slStringToTitleCase&apos;;
+  integer4 StringCompareIgnoreCase(const string src1, string src2) : c,pure,entrypoint=&apos;slStringCompareIgnoreCase&apos;;
+  string StringReverse(const string src) : c,pure,entrypoint=&apos;slStringReverse&apos;;
+  string StringFindReplace(const string src, const string stok, const string rtok) : c,pure,entrypoint=&apos;slStringFindReplace&apos;;
+  string StringCleanSpaces(const string src) : c,pure,entrypoint=&apos;slStringCleanSpaces&apos;; 
+  boolean StringWildMatch(const string src, const string _pattern, boolean _noCase) : c, pure,entrypoint=&apos;slStringWildMatch&apos;; 
+  boolean StringWildExactMatch(const string src, const string _pattern, boolean _noCase) : c, pure,entrypoint=&apos;slStringWildExactMatch&apos;; 
+  boolean StringContains(const string src, const string _pattern, boolean _noCase) : c, pure,entrypoint=&apos;slStringContains&apos;; 
+  string StringExtractMultiple(const string src, unsigned8 mask) : c,pure,entrypoint=&apos;slStringExtractMultiple&apos;; 
+  unsigned integer4 EditDistance(const string l, const string r) : c, pure,entrypoint=&apos;slEditDistance&apos;; 
+  boolean EditDistanceWithinRadius(const string l, const string r, unsigned4 radius) : c,pure,entrypoint=&apos;slEditDistanceWithinRadius&apos;; 
+  unsigned integer4 EditDistanceV2(const string l, const string r) : c, pure,entrypoint=&apos;slEditDistanceV2&apos;; 
+  boolean EditDistanceWithinRadiusV2(const string l, const string r, unsigned4 radius) : c,pure,entrypoint=&apos;slEditDistanceWithinRadiusV2&apos;; 
+  string StringGetNthWord(const string src, unsigned4 n) : c, pure,entrypoint=&apos;slStringGetNthWord&apos;; 
+  unsigned4 StringWordCount(const string src) : c, pure,entrypoint=&apos;slStringWordCount&apos;; 
+  unsigned4 CountWords(const string src, const string _separator, BOOLEAN allow_blanks) : c, pure,entrypoint=&apos;slCountWords&apos;; 
+  SET OF STRING SplitWords(const string src, const string _separator, BOOLEAN allow_blanks) : c, pure,entrypoint=&apos;slSplitWords&apos;; 
+  STRING CombineWords(set of string src, const string _separator) : c, pure,entrypoint=&apos;slCombineWords&apos;; 
+  UNSIGNED4 StringToDate(const string src, const varstring format) : c, pure,entrypoint=&apos;slStringToDate&apos;; 
+  UNSIGNED4 MatchDate(const string src, set of varstring formats) : c, pure,entrypoint=&apos;slMatchDate&apos;; 
+  STRING FormatDate(UNSIGNED4 date, const varstring format) : c, pure,entrypoint=&apos;slFormatDate&apos;; 
+END;</Text>
+ </Module>
+ <Module key="red.source.yyy.xxx.common" name="red.source.yyy.xxx.common"/>
+ <Module key="red.source.yyy.xxx.common.constants" name="red.source.yyy.xxx.common.constants">
+  <Attribute key="spray_prefix" name="spray_prefix" sourcePath="C:\development\source\yyy\xxx\common\constants\spray_prefix.ecl">
+   import red.dm;
+import red.common;
+import red.source.yyy.xxx;
+
+Export spray_prefix 	 := case (common.stored_frequency,
+																&apos;daily&apos; =&gt; &apos;~thor::red::spray::xxx::daily&apos; ,
+																&apos;monthly&apos; =&gt; &apos;~thor::red::spray::xxx::monthly&apos;,
+																 &apos;&apos;);&#13;&#10;
+  </Attribute>
+  <Attribute key="stg_prefix" name="stg_prefix" sourcePath="C:\development\source\yyy\xxx\common\constants\stg_prefix.ecl">
+   Export stg_prefix      	:= &apos;~thor::red::stg::xxx&apos;;&#13;&#10;&#9;&#13;&#10;&#13;&#10;&#9;
+  </Attribute>
+ </Module>
+ <Module key="red.dm" name="red.dm"/>
+ <Module key="red.source.yyy.xxx.layouts" name="red.source.yyy.xxx.layouts">
+  <Attribute key="web_user_curr" name="web_user_curr" sourcePath="C:\development\source\yyy\xxx\layouts\web_user_curr.ecl">
+   export web_user_curr := record
+integer 	web_user_id;
+string10 	dw_start_dt;
+string10 	dw_end_dt;
+string11 	sub_acct_id;
+string25 	user_signon_id;
+string64 	web_user_last_name;
+string64 	web_user_first_name;
+string1 	web_user_mid_init;
+string2 	web_user_type_cd;
+string50 	web_user_type_descr;
+string10 	web_id_cancel_dt;
+string10 	create_dt;
+string2 	actv_ind;
+end;
+  </Attribute>
+ </Module>
+</Archive>

esp/scm/ws_packageprocess.ecm

@@ -110,6 +110,31 @@ ESPresponse [exceptions_inline] ListPackageResponse
     ESParray<ESPstruct PackageListMapData> PkgListMapData;
 };
 
+ESPrequest ValidatePackageRequest
+{
+    string Info;
+    string Target;
+};
+
+ESPstruct ValidatePackageInfo
+{
+    ESParray<string> Unmatched;
+};
+
+ESPstruct ValidatePackageQueries
+{
+    ESParray<string> Unmatched;
+};
+
+ESPresponse [exceptions_inline] ValidatePackageResponse
+{
+    ESPstruct BasePackageStatus status;
+    ESParray<string> Warnings;
+    ESParray<string> Errors;
+    ESPstruct ValidatePackageInfo packages;
+    ESPstruct ValidatePackageQueries queries;
+};
+
 ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
@@ -119,6 +144,7 @@ ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("
     ESPmethod DeActivatePackage(DeActivatePackageRequest, DeActivatePackageResponse);
     ESPmethod ListPackage(ListPackageRequest, ListPackageResponse);
     ESPmethod GetPackage(GetPackageRequest, GetPackageResponse);
+    ESPmethod ValidatePackage(ValidatePackageRequest, ValidatePackageResponse);
 };
 
 SCMexportdef(WsPackageProcess);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -26,6 +26,7 @@
 #include "ws_workunits.hpp"
 #include "packageprocess_errors.h"
 #include "referencedfilelist.hpp"
+#include "package.h"
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
@@ -525,3 +526,20 @@ bool CWsPackageProcessEx::onGetPackage(IEspContext &context, IEspGetPackageReque
     resp.setInfo(info);
     return true;
 }
+
+bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp)
+{
+    StringArray warnings;
+    StringArray errors;
+    StringArray unmatchedQueries;
+    StringArray unusedPackages;
+
+    Owned<IHpccPackageMap> map = createPackageMapFromXml(req.getInfo(), req.getTarget(), NULL);
+    map->validate(warnings, errors, unmatchedQueries, unusedPackages);
+
+    resp.setWarnings(warnings);
+    resp.setErrors(errors);
+    resp.updateQueries().setUnmatched(unmatchedQueries);
+    resp.updatePackages().setUnmatched(unusedPackages);
+    return true;
+}

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -51,6 +51,7 @@ public:
     virtual bool onDeActivatePackage(IEspContext &context, IEspDeActivatePackageRequest &req, IEspDeActivatePackageResponse &resp);
     virtual bool onListPackage(IEspContext &context, IEspListPackageRequest &req, IEspListPackageResponse &resp);
     virtual bool onGetPackage(IEspContext &context, IEspGetPackageRequest &req, IEspGetPackageResponse &resp);
+    virtual bool onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp);
 };
 
 #endif //_ESPWIZ_ws_packageprocess_HPP__

roxie/ccd/ccdfile.cpp

@@ -2085,7 +2085,15 @@ public:
                             }
                         }
                         if (part)
-                            keyset->addIndex(createKeyIndex(part->queryFilename(), crc, *part, false, false)); 
+                        {
+                            if (lazyOpen)
+                            {
+                                // We pass the IDelayedFile interface to createKeyIndex, so that it does not open the file immediately
+                                keyset->addIndex(createKeyIndex(part->queryFilename(), crc, *QUERYINTERFACE(part.get(), IDelayedFile), false, false));
+                            }
+                            else
+                                keyset->addIndex(createKeyIndex(part->queryFilename(), crc, *part.get(), false, false));
+                        }
                         else
                             keyset->addIndex(NULL);
                     }
@@ -2116,7 +2124,13 @@ public:
                     pdesc->getCrc(crc);
                     StringBuffer pname;
                     pdesc->getPath(pname);
-                    key.setown(createKeyIndex(pname.str(), crc, *keyFile, numParts>1, false));
+                    if (lazyOpen)
+                    {
+                        // We pass the IDelayedFile interface to createKeyIndex, so that it does not open the file immediately
+                        key.setown(createKeyIndex(pname.str(), crc, *QUERYINTERFACE(keyFile.get(), IDelayedFile), numParts>1, false));
+                    }
+                    else
+                        key.setown(createKeyIndex(pname.str(), crc, *keyFile.get(), numParts>1, false));
                     keyset->addIndex(LINK(key->queryPart(0)));
                 }
                 else

roxie/ccd/ccdstate.cpp

@@ -193,28 +193,6 @@ protected:
     virtual aindex_t getBaseCount() const = 0;
     virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
 
-    //map ambiguous IHpccPackage
-    virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
-    {
-        return CPackageNode::resolveSuperFile(superFileName);
-    }
-    virtual const char *queryEnv(const char *varname) const
-    {
-        return CPackageNode::queryEnv(varname);
-    }
-    virtual bool getEnableFieldTranslation() const
-    {
-        return CPackageNode::getEnableFieldTranslation();
-    }
-    virtual const IPropertyTree *queryTree() const
-    {
-        return CPackageNode::queryTree();
-    }
-    virtual hash64_t queryHash() const
-    {
-        return CPackageNode::queryHash();
-    }
-
     virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
 
     // Add a filename and the corresponding IResolvedFile to the cache
@@ -468,6 +446,28 @@ public:
         else
             return NULL;
     }
+
+    //map ambiguous IHpccPackage
+    virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
+    {
+        return CPackageNode::resolveSuperFile(superFileName);
+    }
+    virtual const char *queryEnv(const char *varname) const
+    {
+        return CPackageNode::queryEnv(varname);
+    }
+    virtual bool getEnableFieldTranslation() const
+    {
+        return CPackageNode::getEnableFieldTranslation();
+    }
+    virtual const IPropertyTree *queryTree() const
+    {
+        return CPackageNode::queryTree();
+    }
+    virtual hash64_t queryHash() const
+    {
+        return CPackageNode::queryHash();
+    }
 };
 
 typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;
@@ -513,6 +513,10 @@ public:
     {
         return BASE::isActive();
     }
+    virtual bool validate(StringArray &wrn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages) const
+    {
+        return BASE::validate(wrn, err, unmatchedQueries, unusedPackages);
+    }
 
     virtual const IRoxiePackage *queryRoxiePackage(const char *name) const
     {

system/include/errorlist.h

@@ -45,6 +45,9 @@
 #define WORKUNIT_ERROR_START    5000
 #define WORKUNIT_ERROR_END      5099
 
+#define PACKAGE_ERROR_START    5200
+#define PACKAGE_ERROR_END      5299
+
 #define WUWEB_ERROR_START       5500
 #define WUWEB_ERROR_END         5599
 

system/jhtree/ctfile.cpp

@@ -365,7 +365,7 @@ bool CWriteNode::add(offset_t pos, const void *indata, size32_t insize, unsigned
     }
 
     if (insize>keyLen)
-        throw MakeStringException(0, "key+payload exceeds max length");
+        throw MakeStringException(0, "key+payload (%u) exceeds max length (%u)", insize, keyLen);
     memcpy(lastKeyValue, indata, insize);
     lastSequence = sequence;
     hdr.numKeys++;