Merge remote-tracking branch 'origin/closedown-4.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dasess.cpp

@@ -923,8 +923,8 @@ public:
         obj.set(_obj); 
 #ifndef _NO_DALIUSER_STACKTRACE
         StringBuffer sb;
-        if (udesc)
-            udesc->getUserName(sb);
+        if (_udesc)
+            _udesc->getUserName(sb);
         if (sb.length()==0)
         {
             DBGLOG("UNEXPECTED USER (NULL) in dasess.cpp CLdapWorkItem::start() line %d",__LINE__);

ecl/hql/hqltrans.cpp

@@ -3104,25 +3104,25 @@ void NewSelectorReplacingTransformer::initSelectorMapping(IHqlExpression * oldDa
     if (oldDataset->isDatarow() || op == no_activetable || op == no_self || op == no_selfref)
     {
         if (isAlwaysActiveRow(newDataset) || newDataset->isDatarow())
-            setRootMapping(oldDataset, newDataset);         // A row, so Don't change any new references to the dataset
+            setRootMapping(oldDataset, newDataset, false);         // A row, so Don't change any new references to the dataset
         else
         {
             OwnedHqlExpr newActive = ensureActiveRow(newDataset);
-            setRootMapping(oldDataset, newActive);          // A row, so Don't change any new references to the dataset
+            setRootMapping(oldDataset, newActive, false);          // A row, so Don't change any new references to the dataset
         }
     }
     else
     {
-        setRootMapping(oldDataset, oldDataset);         // Don't change any new references to the dataset
+        setRootMapping(oldDataset, oldDataset, false);         // Don't change any new references to the dataset
     }
-    setSelectorMapping(oldDataset, newSelector);
+    setRootMapping(oldDataset, newSelector, true);
 
     if (op == no_left || op == no_right)
         oldSelector.set(oldDataset);
 }
 
 
-void NewSelectorReplacingTransformer::setNestedMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlSimpleScope * oldScope, IHqlExpression * newRecord)
+void NewSelectorReplacingTransformer::setNestedMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlSimpleScope * oldScope, IHqlExpression * newRecord, bool isSelector)
 {
     ForEachChild(i, newRecord)
     {
@@ -3130,10 +3130,10 @@ void NewSelectorReplacingTransformer::setNestedMapping(IHqlExpression * oldSel,
         switch (cur->getOperator())
         {
         case no_record:
-            setNestedMapping(oldSel, newSel, oldScope, cur);
+            setNestedMapping(oldSel, newSel, oldScope, cur, isSelector);
             break;
         case no_ifblock:
-            setNestedMapping(oldSel, newSel, oldScope, cur->queryChild(1));
+            setNestedMapping(oldSel, newSel, oldScope, cur->queryChild(1), isSelector);
             break;
         case no_field:
             {
@@ -3143,29 +3143,33 @@ void NewSelectorReplacingTransformer::setNestedMapping(IHqlExpression * oldSel,
                 {
                     OwnedHqlExpr oldSelected = createSelectExpr(LINK(oldSel), LINK(oldField));
                     OwnedHqlExpr newSelected = createSelectExpr(LINK(newSel), LINK(cur));
-                    setRootMapping(oldSelected, newSelected, oldField->queryRecord());
+                    setRootMapping(oldSelected, newSelected, oldField->queryRecord(), isSelector);
                 }
             }
         }
     }
 }
 
-void NewSelectorReplacingTransformer::setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * oldRecord)
+void NewSelectorReplacingTransformer::setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * oldRecord, bool isSelector)
 {
-    setMappingOnly(oldSel, newSel);
+    if (isSelector)
+        setSelectorMapping(oldSel, newSel);
+    else
+        setMappingOnly(oldSel, newSel);
+
     IHqlExpression * newRecord = newSel->queryRecord();
     if (oldRecord != newRecord)
     {
         if (oldRecord != queryNullRecord() && newRecord != queryNullRecord())
         {
-            setNestedMapping(oldSel, newSel, oldRecord->querySimpleScope(), newRecord);
+            setNestedMapping(oldSel, newSel, oldRecord->querySimpleScope(), newRecord, isSelector);
         }
     }
 }
 
-void NewSelectorReplacingTransformer::setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel)
+void NewSelectorReplacingTransformer::setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, bool isSelector)
 {
-    setRootMapping(oldSel, newSel, oldSel->queryRecord());
+    setRootMapping(oldSel, newSel, oldSel->queryRecord(), isSelector);
 }
 
 IHqlExpression * NewSelectorReplacingTransformer::createTransformed(IHqlExpression * expr)
@@ -3274,7 +3278,7 @@ IHqlExpression * updateMappedFields(IHqlExpression * expr, IHqlExpression * oldS
     NewSelectorReplacingTransformer transformer;
     if (oldSelector != newSelector)
         transformer.initSelectorMapping(oldSelector, newSelector);
-    transformer.setRootMapping(newSelector, newSelector, oldSelector->queryRecord());
+    transformer.setRootMapping(newSelector, newSelector, oldSelector->queryRecord(), false);
     bool same = true;
     for (; i < max; i++)
     {

ecl/hql/hqltrans.ipp

@@ -1034,11 +1034,11 @@ public:
 
     inline bool foundAmbiguity() const { return introducesAmbiguity; }
 
-    void setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * record);
+    void setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * record, bool isSelector);
 
 protected:
-    void setNestedMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlSimpleScope * oldScope, IHqlExpression * newRecord);
-    void setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel);
+    void setNestedMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlSimpleScope * oldScope, IHqlExpression * newRecord, bool isSelector);
+    void setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, bool isSelector);
 
 protected:
     OwnedHqlExpr oldSelector;

ecl/hql/hqlutil.cpp

@@ -2363,8 +2363,13 @@ protected:
     void checkSelect(IHqlExpression * expr)
     {
         IHqlExpression * ds = expr->queryChild(0);
+        if (ds->getOperator() == no_activetable)
+            return;
+
         IHqlExpression * field = expr->queryChild(1);
-        IHqlSimpleScope * scope = ds->queryRecord()->querySimpleScope();
+        IHqlExpression * record = ds->queryRecord();
+        assertex(record);
+        IHqlSimpleScope * scope = record->querySimpleScope();
         OwnedHqlExpr match = scope->lookupSymbol(field->queryName());
         if (match != field)
         {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9044,6 +9044,7 @@ IHqlExpression * HqlCppTranslator::optimizeGraphPostResource(IHqlExpression * ex
     LinkedHqlExpr resourced = expr;
     // Second attempt to spot compound disk reads - this time of spill files for thor.
     resourced.setown(optimizeCompoundSource(resourced, csfFlags));
+    checkNormalized(resourced);
     //insert projects after compound created...
     if (options.optimizeResourcedProjects)
     {
@@ -9051,6 +9052,7 @@ IHqlExpression * HqlCppTranslator::optimizeGraphPostResource(IHqlExpression * ex
         OwnedHqlExpr optimized = insertImplicitProjects(*this, resourced.get(), options.optimizeSpillProject);
         DEBUG_TIMER("EclServer: implicit projects", msTick()-time);
         traceExpression("AfterResourcedImplicit", resourced);
+        checkNormalized(optimized);
 
         if (optimized != resourced)
             resourced.setown(optimizeCompoundSource(optimized, csfFlags));

ecl/hqlcpp/hqliproj.cpp

@@ -75,7 +75,11 @@ NestedField * UsedFieldSet::addNested(IHqlExpression * field)
         assertex(originalFields->contains(*field));
 #endif
         NestedField * original = originalFields->findNested(field);
-        assertex(original);
+        if (!original)
+        {
+            EclIR::dump_ir(field, originalFields->findNestedByName(field)->field);
+            throwUnexpected();
+        }
         match = new NestedField(field, &original->used);
         appendNested(*LINK(field), match);
     }
@@ -254,6 +258,17 @@ NestedField * UsedFieldSet::findNested(IHqlExpression * field) const
     return NULL;
 }
 
+NestedField * UsedFieldSet::findNestedByName(IHqlExpression * field) const
+{
+    ForEachItemIn(i2, nested)
+    {
+        NestedField & cur = nested.item(i2);
+        if (cur.field->queryName() == field->queryName())
+            return &cur;
+    }
+    return NULL;
+}
+
 IHqlExpression * UsedFieldSet::createFilteredAssign(IHqlExpression * field, IHqlExpression * value, IHqlExpression * newSelf, const UsedFieldSet * exceptions) const
 {
     if (!contains(*field))

ecl/hqlcpp/hqliproj.ipp

@@ -130,6 +130,7 @@ public:
     IHqlExpression * createFilteredTransform(IHqlExpression * transform, const UsedFieldSet * exceptions) const;
     void calcFinalRecord(bool canPack, bool ignoreIfEmpty);
     NestedField * findNested(IHqlExpression * field) const;
+    NestedField * findNestedByName(IHqlExpression * field) const;
     void gatherTransformValuesUsed(HqlExprArray * selfSelects, HqlExprArray * parentSelects, HqlExprArray * values, IHqlExpression * selector, IHqlExpression * transform);
     void getText(StringBuffer & s) const;
     void intersectFields(const UsedFieldSet & source);

ecl/regress/nested13.ecl

@@ -0,0 +1,78 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('targetClusterType', 'thorlcr');
+#option ('allowScopeMigrate', false);
+
+level1Rec := RECORD
+    string1 a;
+    string1 b;
+    string1 c;
+END;
+
+level2Rec := RECORD
+    unsigned1 nogrouping;
+    level1Rec r;
+END;
+
+dsraw := DATASET('ds1', level2Rec, thor);
+
+ds1 := PROJECT(dsraw, TRANSFORM(level2Rec, SELF.nogrouping := 0; SELF := LEFT));
+
+r1 := RECORD
+  unsigned8 nogrouping;
+  unsigned8 countgroup;
+ END;
+
+
+ds0 := DATASET([{0,(unsigned8) (COUNT(ds1))}], r1);
+
+layout_uniques := RECORD
+  unsigned8 countgroup;
+  unsigned8 nogrouping;
+  unsigned8 r_a_unique;
+  unsigned8 r_b_unique;
+  unsigned8 r_c_unique;
+ END;
+
+da := ds1(r.a != '');
+dax := TABLE(da, { nogrouping, string1 r_a_unique := r.a });
+dax1l := DISTRIBUTE(dax, RANDOM());
+dax2l := TABLE(dax1l, {  nogrouping, r_a_unique }, nogrouping, r_a_unique, local); 
+dax1g := DISTRIBUTE(dax2l, RANDOM());
+dax2g := TABLE(dax1g, {  nogrouping, r_a_unique }, nogrouping, r_a_unique); 
+daxc := TABLE(dax2g, { nogrouping, unsigned8 cnt := COUNT(GROUP); }, nogrouping, few);
+
+
+db := ds1(r.b != '');
+dbx := TABLE(db, { nogrouping, string1 r_b_unique := r.b });
+dbx1l := DISTRIBUTE(dbx, RANDOM());
+dbx2l := TABLE(dbx1l, {  nogrouping, r_b_unique }, nogrouping, r_b_unique, local); 
+dbx1g := DISTRIBUTE(dbx2l, RANDOM());
+dbx2g := TABLE(dbx1g, {  nogrouping, r_b_unique }, nogrouping, r_b_unique); 
+dbxc := TABLE(dbx2g, { nogrouping, unsigned8 cnt := COUNT(GROUP); }, nogrouping, few);
+
+layout_uniques t(r1 l) := TRANSFORM
+    SELF.r_a_unique := daxc(nogrouping = l.nogrouping)[1].cnt;
+    SELF.r_b_unique := dbxc(nogrouping = l.nogrouping)[1].cnt;
+    SELF := l;
+    SELF := [];
+END;
+
+p := PROJECT(ds0, t(LEFT));
+
+output(p);

ecllibrary/std/system/Email.ecl

@@ -2,6 +2,8 @@
 ## HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.  All rights reserved.
 ############################################################################## */
 
+import lib_fileservices;
+
 RETURN MODULE
 
 /*

esp/services/ws_dfu/ws_dfuService.cpp

@@ -5057,7 +5057,7 @@ int CWsDfuEx::GetIndexData(IEspContext &context, bool bSchemaOnly, const char* i
 
     Owned<IUserDescriptor> udesc;
     ISecUser * secUser = context.queryUser();
-    if(secUser->getName() && *secUser->getName())
+    if(secUser && secUser->getName() && *secUser->getName())
     {
         udesc.setown(createUserDescriptor());
         udesc->set(secUser->getName(), secUser->credentials().getPassword());

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -197,7 +197,7 @@ void addPackageMapInfo(IPropertyTree *pkgSetRegistry, const char *target, const
 
 
     mapTree = root->addPropTree("PackageMap", createPTree());
-    mapTree->addProp("@id", packageMapName);
+    mapTree->addProp("@id", lcName);
 
     StringArray fileNames;
     Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);

roxie/ccd/ccdstate.cpp

@@ -399,7 +399,8 @@ public:
     {
         StringBuffer fileName;
         expandLogicalFilename(fileName, _fileName, wu, false);
-        DBGLOG("lookupFileName %s", fileName.str());
+        if (traceLevel > 5)
+            DBGLOG("lookupFileName %s", fileName.str());
 
         const IResolvedFile *result = lookupFile(fileName, cache, false, false);
         if (!result)