Merge branch 'candidate-5.6.0' into candidate-6.0.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -896,6 +896,7 @@ CLocalEnvironment::CLocalEnvironment(const char* environmentFile)
    }
 
    machineCacheBuilt = false;
+   dropZoneCacheBuilt = false;
 }
 
 CLocalEnvironment::CLocalEnvironment(IRemoteConnection *_conn, IPropertyTree* root/*=NULL*/, 
@@ -910,6 +911,7 @@ CLocalEnvironment::CLocalEnvironment(IRemoteConnection *_conn, IPropertyTree* ro
       p.setown(conn->getRoot());
 
     machineCacheBuilt = false;
+    dropZoneCacheBuilt = false;
 }
 
 CLocalEnvironment::~CLocalEnvironment()
@@ -1329,6 +1331,7 @@ void CLocalEnvironment::clearCache()
     }
     cache.kill();
     machineCacheBuilt = false;
+    dropZoneCacheBuilt = false;
     resetPasswordsFromSDS();
 }
 

docs/ECLLanguageReference/ECLR_mods/BltInFunc-LOOP.xml

@@ -31,13 +31,13 @@
   role="bold"></emphasis></para>
 
   <para><emphasis role="bold">LOOP(</emphasis><emphasis>
-  dataset,</emphasis><emphasis role="bold">
-  </emphasis><emphasis>loopcondition, loopbody </emphasis><emphasis
-  role="bold">)</emphasis><emphasis role="bold"></emphasis></para>
+  dataset, rowfilter,
+  loopcondition, loopbody </emphasis><emphasis role="bold">)</emphasis></para>
 
   <para><emphasis role="bold">LOOP(</emphasis><emphasis>
-  dataset,</emphasis><emphasis role="bold"> </emphasis><emphasis>rowfilter,
-  loopcondition, loopbody </emphasis><emphasis role="bold">)</emphasis></para>
+  dataset,</emphasis><emphasis role="bold"> </emphasis><emphasis>loopfilter,
+  loopcondition, loopbody </emphasis><emphasis
+  role="bold">)</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">

ecl/hql/hqlerror.cpp

@@ -230,7 +230,7 @@ class HQL_API ThrowingErrorReceiver : public ErrorReceiverSink
 
 void ThrowingErrorReceiver::report(IError* error)
 {
-    throw error;
+    throw LINK(error);
 }
 
 IErrorReceiver * createThrowingErrorReceiver()

ecl/hqlcpp/hqlhtcpp.cpp

@@ -8034,7 +8034,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
 
 }
 
-void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * rightRecord, IHqlExpression * leftSelector, IHqlExpression * rightSelector, StringBuffer & selectorText, bool isCount)
+void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * leftSelector, IHqlExpression * rightRecord, IHqlExpression * rightSelector, StringBuffer & selectorText, bool isCount)
 {
     switch (expr->getOperator())
     {

ecl/regress/rowdiff.ecl

@@ -26,16 +26,18 @@ string20            middle;
                 end;
             end;
 
-
-in1rec :=   record
+idRec := record
 unsigned    id;
+END;
+
+in1rec :=   record(idRec)
 complexName name;
 unsigned    age;
 string      title;
         end;
 
 in2rec := record
-unsigned    id;
+idRec;
 complexName name;
 real8       age;
 boolean     dead;

esp/services/ws_fs/ws_fsService.cpp

@@ -2711,6 +2711,8 @@ bool CFileSprayEx::onFileList(IEspContext &context, IEspFileListRequest &req, IE
 
         double version = context.getClientVersion();
         const char* netaddr = req.getNetaddr();
+        if (!netaddr || !*netaddr)
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Network address not specified.");
         const char* mask = req.getMask();
         bool directoryOnly = req.getDirectoryOnly();
 

plugins/cassandra/cassandraembed.cpp

@@ -1114,7 +1114,12 @@ public:
     {
         if (isAll)
             UNSUPPORTED("SET(ALL)");
-        collection.setown(new CassandraCollection(cass_collection_new(CASS_COLLECTION_TYPE_SET, numElements)));
+        // We don't know whether the corresponding field in Cassandra is a list or a set. Try binding a dummy list to tell which.
+        CassandraCollection temp(cass_collection_new(CASS_COLLECTION_TYPE_LIST, 0));
+        if (cass_statement_bind_collection(stmtInfo->queryStatement(), thisParam, temp) == CASS_OK)
+            collection.setown(new CassandraCollection(cass_collection_new(CASS_COLLECTION_TYPE_LIST, numElements)));
+        else
+            collection.setown(new CassandraCollection(cass_collection_new(CASS_COLLECTION_TYPE_SET, numElements)));
         return true;
     }
     virtual bool processBeginDataset(const RtlFieldInfo * field, unsigned numRows)

plugins/cassandra/cpp-driver

@@ -1 +1 @@
-Subproject commit d7bbad34db39a51f209c6fadd07c9ec3a0bb86b7
+Subproject commit b4bb435129bab533612fa2caf194555fa943f925

plugins/pyembed/pyembed.cpp

@@ -345,6 +345,9 @@ public:
         }
 
         const RtlFieldInfo * const *fields = type->queryFields();
+        if (!fields && type->queryChildType())
+            fields = type->queryChildType()->queryFields();
+        assertex(fields);
         StringBuffer names;
         while (*fields)
         {
@@ -842,7 +845,7 @@ public:
     {
         // Expect to see a tuple here, or possibly (if the ECL record has a single field), an arbitrary scalar object
         // If it's a tuple, we push it onto our stack as the active object
-        nextField(NULL);  // MORE - should it be passing field?
+        nextField(field);
         if (!PyTuple_Check(elem))
         {
             if (countFields(field->type->queryFields())==1)
@@ -1043,11 +1046,13 @@ protected:
     void push()
     {
         stack.append(args.getClear());
+        args.setown(PyList_New(0));
     }
     void pop()
     {
-        addArg(args.getClear());
+        OwnedPyObject arg = args.getClear();
         args.setown((PyObject *) stack.popGet());
+        addArg(arg.getClear());
     }
     void addArg(PyObject *arg)
     {

rtl/eclrtl/rtlfield.cpp

@@ -219,8 +219,8 @@ size32_t RtlSwapIntTypeInfo::build(ARowBuilder &builder, size32_t offset, const
 {
     builder.ensureCapacity(length+offset, str(field->name));
     __int64 val = isUnsigned() ? (__int64) source.getUnsignedResult(field) : source.getSignedResult(field);
-    // NOTE - we assume that the value returned from the source is already a swapped int
-    rtlWriteInt(builder.getSelf() + offset, val, length);
+    // NOTE - we assume that the value returned from the source is NOT already a swapped int - source doesn;t know that we are going to store it swapped
+    rtlWriteSwapInt(builder.getSelf() + offset, val, length);
     offset += length;
     return offset;
 }
@@ -298,14 +298,12 @@ size32_t RtlStringTypeInfo::build(ARowBuilder &builder, size32_t offset, const R
         builder.ensureCapacity(offset+size+sizeof(size32_t), str(field->name));
         byte *dest = builder.getSelf()+offset;
         rtlWriteInt4(dest, size);
-#if 0
-        // NOTE - you might argue that we should convert the incoming data to EBCDIC. But it seems more useful to
-        // define the semantics as being that the IFieldSource should return EBCDIC if you have declared the matching field as EBCDIC
-        // (otherwise, why did you bother?)
+        // NOTE - it has been the subject of debate whether we should convert the incoming data to EBCDIC, or expect the IFieldSource to have already returned ebcdic
+        // In order to be symmetrical with the passing of ecl data to a IFieldProcessor the former interpretation is preferred.
+        // Expecting source.getStringResult to somehow "know" that EBCDIC was expected seems odd.
         if (isEbcdic())
             rtlStrToEStr(size, (char *) dest+sizeof(size32_t), size, (char *)value);
         else
-#endif
             memcpy(dest+sizeof(size32_t), value, size);
         offset += size+sizeof(size32_t);
     }
@@ -313,12 +311,9 @@ size32_t RtlStringTypeInfo::build(ARowBuilder &builder, size32_t offset, const R
     {
         builder.ensureCapacity(offset+length, str(field->name));
         byte *dest = builder.getSelf()+offset;
-#if 0
-        // See above...
         if (isEbcdic())
             rtlStrToEStr(length, (char *) dest, size, (char *) value);
         else
-#endif
             rtlStrToStr(length, dest, size, value);
         offset += length;
     }

system/jhtree/jhtree.cpp

@@ -609,6 +609,11 @@ public:
                 throw e;
             }
         }
+        else
+        {
+            keySize = 0;
+            keyedSize = 0;
+        }
     }
 
     virtual void reset(bool crappyHack)
@@ -2545,7 +2550,11 @@ public:
             numkeys = _keyset->numParts();
         }
         else
+        {
+            keySize = 0;
+            keyedSize = 0;
             numkeys = 0;
+        }
         killBuffers();
     }
 
@@ -2694,7 +2703,8 @@ public:
         if (!started)
         {
             started = true;
-            segs.checkSize(keyedSize, "[merger]"); //PG: not sure what keyname to use here
+            if (keyedSize)
+                segs.checkSize(keyedSize, "[merger]"); //PG: not sure what keyname to use here
         }
         if (!crappyHack)
         {

testing/regress/ecl/key/rowdiff.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><id>1</id><diff1>                                   </diff1><diff2>                                   </diff2></Row>
+ <Row><id>2</id><diff1>name.surname,name.middle           </diff1><diff2>surname,middle                     </diff2></Row>
+ <Row><id>3</id><diff1>age                                </diff1><diff2>                                   </diff2></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>1</id><diff1>0,0,0,0,0                          </diff1><diff2>0,0,0                              </diff2></Row>
+ <Row><id>2</id><diff1>0,0,1,1,0                          </diff1><diff2>0,1,1                              </diff2></Row>
+ <Row><id>3</id><diff1>0,0,0,0,1                          </diff1><diff2>0,0,0                              </diff2></Row>
+</Dataset>

testing/regress/ecl/key/streame.xml

@@ -23,3 +23,6 @@
 <Dataset name='Result 5'>
  <Row><Result_5>Yo</Result_5></Row>
 </Dataset>
+<Dataset name='Result 6'>
+ <Row><name1>Gavin</name1><name2>Halliday  </name2><childnames><Row><name>a</name><value>1</value></Row><Row><name>b</name><value>2</value></Row><Row><name>c</name><value>3</value></Row></childnames><childdict><Row><name>aa</name><value>11</value></Row></childdict><r><name>aaa</name><value>111</value></r><val1>250</val1><val2>-1</val2><u1>là</u1><u2>là</u2><u3>là      </u3><val3>1</val3><d>4141</d><b>false</b><ss1><Item>1</Item><Item>2</Item></ss1></Row>
+</Dataset>

testing/regress/ecl/rowdiff.ecl

@@ -0,0 +1,83 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+
+complexName :=
+            record
+string30        forename;
+string20        surname;
+                ifblock(self.surname <> 'Windsor')
+string20            middle;
+                end;
+            end;
+
+idRec := record
+unsigned    id;
+END;
+
+in1rec :=   record(idRec)
+complexName name;
+unsigned    age;
+string      title;
+        end;
+
+in2rec := record
+idRec;
+complexName name;
+real8       age;
+boolean     dead;
+        end;
+
+
+in1 := dataset([
+        {1,'Gavin','Hawthorn','',33,'Mr'},
+        {2,'Mia','Hawthorn','',33,'Dr'},
+        {3,'Elizabeth','Windsor',99,'Queen'}
+        ], in1rec);
+
+
+in2 := dataset([
+        {1,'Gavin','Hawthorn','',33,false},
+        {2,'Mia','','Jean',33,false},
+        {3,'Elizabeth','Windsor',99.1,false}
+        ], in2rec);
+
+outrec :=
+        record
+unsigned        id;
+string35        diff1;
+string35        diff2;
+        end;
+
+outrec t1(in1 l, in2 r) := transform
+//      self.id := if(l = r, SKIP, l.id);
+        self.id := l.id;
+        self.diff1 := rowdiff(l, r);
+        self.diff2 := rowdiff(l.name, r.name);
+    end;
+
+output(join(in1, in2, left.id = right.id, t1(left, right)));
+
+
+outrec t2(in1 l, in2 r) := transform
+        self.id := l.id;
+        self.diff1 := rowdiff(l, r, count);
+        self.diff2 := rowdiff(l.name, r.name, count);
+    end;
+
+output(join(in1, in2, left.id = right.id, t2(left, right)));

testing/regress/ecl/streame.ecl

@@ -50,8 +50,8 @@ ENDEMBED;
 
 dataset(namesRecord) streamedNames(data d, utf8 u) := EMBED(Python)
   return [  \
-     ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1,  U'là',  U'là',  U'là', 0x01000000, d, False, set(["1","2"])), \
-     ("John", "Smith", [], [], ("c", 3), 250, -1,  U'là',  U'là',  u, 0x02000000, d, True, []) \
+     ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1,  U'là',  U'là',  U'là', 1, d, False, set(["1","2"])), \
+     ("John", "Smith", [], [], ("c", 3), 250, -1,  U'là',  U'là',  u, 2, d, True, []) \
      ]
 ENDEMBED;
 
@@ -80,3 +80,13 @@ childrec tnamed(string s) := EMBED(Python)
 ENDEMBED;
 
 output(tnamed('Yo').name);
+
+// Test passing records into Python
+
+dataset(namesRecord) streamInOut(dataset(namesRecord) recs) := EMBED(Python)
+  for rec in recs:
+    if rec.name1 == 'Gavin':
+       yield rec
+ENDEMBED;
+
+output(streamInOut(streamedNames(d'AA', u'là')));