Merge pull request #8161 from richardkchapman/python-stream-set

HPCC-14835 Can't pass a dataset that includes a set/child dataset to Python

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

plugins/pyembed/pyembed.cpp

@@ -345,6 +345,9 @@ public:
         }
 
         const RtlFieldInfo * const *fields = type->queryFields();
+        if (!fields && type->queryChildType())
+            fields = type->queryChildType()->queryFields();
+        assertex(fields);
         StringBuffer names;
         while (*fields)
         {
@@ -842,7 +845,7 @@ public:
     {
         // Expect to see a tuple here, or possibly (if the ECL record has a single field), an arbitrary scalar object
         // If it's a tuple, we push it onto our stack as the active object
-        nextField(NULL);  // MORE - should it be passing field?
+        nextField(field);
         if (!PyTuple_Check(elem))
         {
             if (countFields(field->type->queryFields())==1)
@@ -1043,11 +1046,13 @@ protected:
     void push()
     {
         stack.append(args.getClear());
+        args.setown(PyList_New(0));
     }
     void pop()
     {
-        addArg(args.getClear());
+        OwnedPyObject arg = args.getClear();
         args.setown((PyObject *) stack.popGet());
+        addArg(arg.getClear());
     }
     void addArg(PyObject *arg)
     {

rtl/eclrtl/rtlfield.cpp

@@ -219,8 +219,8 @@ size32_t RtlSwapIntTypeInfo::build(ARowBuilder &builder, size32_t offset, const
 {
     builder.ensureCapacity(length+offset, field->name->str());
     __int64 val = isUnsigned() ? (__int64) source.getUnsignedResult(field) : source.getSignedResult(field);
-    // NOTE - we assume that the value returned from the source is already a swapped int
-    rtlWriteInt(builder.getSelf() + offset, val, length);
+    // NOTE - we assume that the value returned from the source is NOT already a swapped int - source doesn;t know that we are going to store it swapped
+    rtlWriteSwapInt(builder.getSelf() + offset, val, length);
     offset += length;
     return offset;
 }
@@ -298,14 +298,12 @@ size32_t RtlStringTypeInfo::build(ARowBuilder &builder, size32_t offset, const R
         builder.ensureCapacity(offset+size+sizeof(size32_t), field->name->str());
         byte *dest = builder.getSelf()+offset;
         rtlWriteInt4(dest, size);
-#if 0
-        // NOTE - you might argue that we should convert the incoming data to EBCDIC. But it seems more useful to
-        // define the semantics as being that the IFieldSource should return EBCDIC if you have declared the matching field as EBCDIC
-        // (otherwise, why did you bother?)
+        // NOTE - it has been the subject of debate whether we should convert the incoming data to EBCDIC, or expect the IFieldSource to have already returned ebcdic
+        // In order to be symmetrical with the passing of ecl data to a IFieldProcessor the former interpretation is preferred.
+        // Expecting source.getStringResult to somehow "know" that EBCDIC was expected seems odd.
         if (isEbcdic())
             rtlStrToEStr(size, (char *) dest+sizeof(size32_t), size, (char *)value);
         else
-#endif
             memcpy(dest+sizeof(size32_t), value, size);
         offset += size+sizeof(size32_t);
     }
@@ -313,12 +311,9 @@ size32_t RtlStringTypeInfo::build(ARowBuilder &builder, size32_t offset, const R
     {
         builder.ensureCapacity(offset+length, field->name->str());
         byte *dest = builder.getSelf()+offset;
-#if 0
-        // See above...
         if (isEbcdic())
             rtlStrToEStr(length, (char *) dest, size, (char *) value);
         else
-#endif
             rtlStrToStr(length, dest, size, value);
         offset += length;
     }

testing/regress/ecl/key/streame.xml

@@ -23,3 +23,6 @@
 <Dataset name='Result 5'>
  <Row><Result_5>Yo</Result_5></Row>
 </Dataset>
+<Dataset name='Result 6'>
+ <Row><name1>Gavin</name1><name2>Halliday  </name2><childnames><Row><name>a</name><value>1</value></Row><Row><name>b</name><value>2</value></Row><Row><name>c</name><value>3</value></Row></childnames><childdict><Row><name>aa</name><value>11</value></Row></childdict><r><name>aaa</name><value>111</value></r><val1>250</val1><val2>-1</val2><u1>là</u1><u2>là</u2><u3>là      </u3><val3>1</val3><d>4141</d><b>false</b><ss1><Item>1</Item><Item>2</Item></ss1></Row>
+</Dataset>

testing/regress/ecl/streame.ecl

@@ -48,8 +48,8 @@ ENDEMBED;
 
 dataset(namesRecord) streamedNames(data d, utf8 u) := EMBED(Python)
   return [  \
-     ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1,  U'là',  U'là',  U'là', 0x01000000, d, False, set(["1","2"])), \
-     ("John", "Smith", [], [], ("c", 3), 250, -1,  U'là',  U'là',  u, 0x02000000, d, True, []) \
+     ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1,  U'là',  U'là',  U'là', 1, d, False, set(["1","2"])), \
+     ("John", "Smith", [], [], ("c", 3), 250, -1,  U'là',  U'là',  u, 2, d, True, []) \
      ]
 ENDEMBED;
 
@@ -78,3 +78,13 @@ childrec tnamed(string s) := EMBED(Python)
 ENDEMBED;
 
 output(tnamed('Yo').name);
+
+// Test passing records into Python
+
+dataset(namesRecord) streamInOut(dataset(namesRecord) recs) := EMBED(Python)
+  for rec in recs:
+    if rec.name1 == 'Gavin':
+       yield rec
+ENDEMBED;
+
+output(streamInOut(streamedNames(d'AA', u'là')));