HPCC-10457 Streamed dataset support for Python

Minor fixes in response to code review

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclagent/eclagent.cpp

@@ -2089,9 +2089,6 @@ void EclAgent::runProcess(IEclProcess *process)
         rowManager->getMemoryUsage();//Causes statistics to be written to logfile
     }
 
-#ifdef _DEBUG_LEAKS
-    rowManager.clear();//Early release of rowManager, so activity IDs of leaked blocks are available
-#endif
     rowManager.clear(); // Must go before the allocatorCache
     allocatorMetaCache.clear(); //release meta before libraries unloaded
     queryLibraries.kill();

ecl/hql/hqlexpr.hpp

@@ -1862,6 +1862,5 @@ extern HQL_API bool getBoolAttribute(IHqlExpression * expr, IAtom * name, bool d
 extern HQL_API void setLegacyEclSemantics(bool _value);
 extern HQL_API bool queryLegacyEclSemantics();
 void exportSymbols(IPropertyTree* data, IHqlScope * scope, HqlLookupContext & ctx);
-inline bool isCall(node_operator op) { return (op == no_call || op == no_externalcall); }
 
 #endif

ecl/hqlcpp/hqlcpp.cpp

@@ -11630,11 +11630,11 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
             bindFunc = bindDataParamId;
             break;
         case type_row:
-            bindFunc = bindRowParamId; // more
+            bindFunc = bindRowParamId;
             break;
         case type_table:
         case type_groupedtable:
-            bindFunc = bindDatasetParamId; // more
+            bindFunc = bindDatasetParamId;
             break;
         case type_set:
         {
@@ -11731,8 +11731,6 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
         funcctx.addGroupPass(pass);
     }
     expandFunctionPrototype(proto, funcdef);
-//    BoundRow * row = bindSelf(deserializectx, dataset, "crSelf");
-
 
     if (bodyCode->getOperator() == no_embedbody)
     {

plugins/pyembed/CMakeLists.txt

@@ -23,7 +23,7 @@
 #    Cmake Input File for pyembed
 #####################################################
 
-set ( debug_python On )   # A lot slower but can assist in debugging...
+set ( debug_python Off )   # A lot slower but can assist in debugging...
 SET ( DEBUG_PYTHON_LIBRARY "/usr/lib/libpython2.7_d.so" )
 
 project( pyembed )

plugins/pyembed/pyembed.cpp

@@ -870,13 +870,9 @@ public:
     {
         // You don't really know what size Py_UNICODE is (varies from system to system), so go via utf8
         unsigned unicodeChars;
-        char *unicode;
-        rtlUnicodeToUtf8X(unicodeChars, unicode, len, value);
-        size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode);
-        PyObject *vval = PyUnicode_FromStringAndSize(unicode, sizeBytes);   // NOTE - requires size in bytes not chars
-        checkPythonError();
-        addArg( vval);
-        rtlFree(unicode);  // MORE - this will leak on exceptions
+        rtlDataAttr unicode;
+        rtlUnicodeToUtf8X(unicodeChars, unicode.refstr(), len, value);
+        processUtf8(unicodeChars, unicode.getstr(), field);
     }
     virtual void processQString(unsigned len, const char *value, const RtlFieldInfo * field)
     {
@@ -1293,7 +1289,6 @@ protected:
         Py_DECREF(arg);
         checkPythonError();
     }
-    StringBuffer delayedScript;
 };
 
 class Python27EmbedImportContext : public Python27EmbedContextBase

rtl/eclrtl/rtlfield.cpp

@@ -447,11 +447,11 @@ size32_t RtlVarStringTypeInfo::build(ARowBuilder &builder, size32_t offset, cons
     size32_t size;
     char *value;
     source.getStringResult(field, size, value);
-    byte *dest = builder.getSelf()+offset;
     if (!isFixedSize())
     {
         builder.ensureCapacity(offset+size+1, field->name->str());
         // See notes re EBCDIC conversion in RtlStringTypeInfo code
+        byte *dest = builder.getSelf()+offset;
         memcpy(dest, value, size);
         dest[size] = '\0';
         offset += size+1;
@@ -459,6 +459,7 @@ size32_t RtlVarStringTypeInfo::build(ARowBuilder &builder, size32_t offset, cons
     else
     {
         builder.ensureCapacity(offset+length, field->name->str());
+        byte *dest = builder.getSelf()+offset;
         rtlStrToVStr(length, dest, size, value);
         offset += length;
     }
@@ -526,7 +527,6 @@ size32_t RtlQStringTypeInfo::build(ARowBuilder &builder, size32_t offset, const
     size32_t size;
     char *value;
     source.getStringResult(field, size, value);
-    byte *dest = builder.getSelf()+offset;
     if (!isFixedSize())
     {
         size32_t sizeInBytes = rtlQStrSize(size) + sizeof(size32_t);
@@ -887,7 +887,7 @@ inline size32_t processFields(const RtlFieldInfo * const * cur, const byte * sel
     return offset;
 }
 
-inline size32_t processFields(const RtlFieldInfo * const * cur, ARowBuilder &builder, size32_t offset, IFieldSource &source)
+inline size32_t buildFields(const RtlFieldInfo * const * cur, ARowBuilder &builder, size32_t offset, IFieldSource &source)
 {
     loop
     {
@@ -952,7 +952,7 @@ size32_t RtlRecordTypeInfo::toXML(const byte * self, const byte * selfrow, const
 size32_t RtlRecordTypeInfo::build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const
 {
     source.processBeginRow(field);
-    offset = processFields(fields, builder, offset, source);
+    offset = buildFields(fields, builder, offset, source);
     source.processEndRow(field);
     return offset;
 }

testing/ecl/streame.ecl

@@ -64,3 +64,12 @@ ENDEMBED;
 
 output(streamedNames(d'AA', u'là'));
 output (testGenerator(10));
+
+// Test Python code returning named tuples
+childrec tnamed(string s) := EMBED(Python)
+  import collections;
+  childrec = collections.namedtuple("childrec", "value,name")
+  return childrec(1,s)
+ENDEMBED;
+
+output(tnamed('Yo').name);