Merge pull request #8776 from richardkchapman/project-embed-variable

HPCC-14988 Projecting fields into embeds

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -58,9 +58,11 @@ IIdAtom * maxSizeId;
 IIdAtom * __optionsId;
 IIdAtom * outputId;
 IIdAtom * physicalLengthId;
+IIdAtom * __queryId;
 IIdAtom * selfId;
 IIdAtom * sharedId;
 IIdAtom * storeId;
+IIdAtom * substituteEmbeddedScriptId;
 IIdAtom * supportsImportId;
 IIdAtom * supportsScriptId;
 IIdAtom * syntaxCheckId;
@@ -328,6 +330,7 @@ IAtom * prefetchAtom;
 IAtom * preloadAtom;
 IAtom * priorityAtom;
 IAtom * privateAtom;
+IAtom * projectedAtom;
 IAtom * _propAligned_Atom;
 IAtom * _propRecordCount_Atom;
 IAtom * _propSize_Atom;
@@ -500,9 +503,11 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEID(__options);
     MAKEID(output);
     MAKEID(physicalLength);
+    MAKEID(__query);
     MAKEID(self);
     MAKEID(shared);
     MAKEID(store);
+    MAKEID(substituteEmbeddedScript);
     MAKEID(supportsImport);
     MAKEID(supportsScript);
     MAKEID(syntaxCheck);
@@ -771,6 +776,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(preload);
     MAKEATOM(priority);
     MAKEATOM(private);
+    MAKEATOM(projected);
     MAKESYSATOM(propAligned);
     MAKESYSATOM(propRecordCount);
     MAKESYSATOM(propSize);

ecl/hql/hqlatoms.hpp

@@ -60,9 +60,11 @@ extern HQL_API IIdAtom * maxSizeId;
 extern HQL_API IIdAtom * __optionsId;
 extern HQL_API IIdAtom * outputId;
 extern HQL_API IIdAtom * physicalLengthId;
+extern HQL_API IIdAtom * __queryId;
 extern HQL_API IIdAtom * selfId;
 extern HQL_API IIdAtom * sharedId;
 extern HQL_API IIdAtom * storeId;
+extern HQL_API IIdAtom * substituteEmbeddedScriptId;
 extern HQL_API IIdAtom * supportsImportId;
 extern HQL_API IIdAtom * supportsScriptId;
 extern HQL_API IIdAtom * syntaxCheckId;
@@ -333,6 +335,7 @@ extern HQL_API IAtom * prefetchAtom;
 extern HQL_API IAtom * preloadAtom;
 extern HQL_API IAtom * priorityAtom;
 extern HQL_API IAtom * privateAtom;
+extern HQL_API IAtom * projectedAtom;
 extern HQL_API IAtom * _propAligned_Atom;
 extern HQL_API IAtom * _propRecordCount_Atom;
 extern HQL_API IAtom * _propSize_Atom;

ecl/hql/hqlerrors.hpp

@@ -429,6 +429,7 @@
 #define ERR_DUPLICATE_FILENAME      2398
 #define ERR_DUPLICATE_SOURCE        2399
 #define ERR_PROBABILITY_RANGE       2400
+#define ERR_EMBEDPROJECT_INVALID    2401
 
 #define ERR_CPP_COMPILE_ERROR       2999
 

ecl/hql/hqlexpr.cpp

@@ -2710,6 +2710,7 @@ IHqlExpression * queryNewColumnProvider(IHqlExpression * expr)
     switch (op)
     {
     case no_alias:
+    case no_call:
         return expr->queryRecord();
     case no_createrow:
     case no_typetransfer:
@@ -15061,6 +15062,10 @@ extern HQL_API bool hasUnknownTransform(IHqlExpression * expr)
         if (expr->hasAttribute(mergeTransformAtom))
             return true;
         break;
+    case no_call:
+        if (isProjectableCall(expr))
+            return false;
+        break;
     case no_inlinetable:
         {
             IHqlExpression * transforms = expr->queryChild(0);

ecl/hql/hqlgram.y

@@ -1085,12 +1085,13 @@ embedBody
                             OwnedHqlExpr embedText = $2.getExpr();
                             $$.setExpr(parser->processEmbedBody($2, embedText, NULL, attrs), $1);
                         }
-    | EMBED '(' abstractModule ',' expression ')'
+    | EMBED '(' abstractModule ',' expression attribs ')'
                         {
-                            parser->normalizeExpression($5, type_stringorunicode, true);
+                            parser->normalizeExpression($5, type_stringorunicode, false);
                             OwnedHqlExpr language = $3.getExpr();
                             OwnedHqlExpr embedText = $5.getExpr();
-                            $$.setExpr(parser->processEmbedBody($5, embedText, language, NULL), $1);
+                            OwnedHqlExpr attribs = $6.getExpr();
+                            $$.setExpr(parser->processEmbedBody($5, embedText, language, attribs), $1);
                         }
     | IMPORT '(' abstractModule ',' expression attribs ')'
                         {

ecl/hql/hqlgram2.cpp

@@ -907,6 +907,19 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
 {
     HqlExprArray args;
     embedText->unwindList(args, no_comma);
+    Linked<ITypeInfo> type = current_type;
+    if (!type)
+        type.setown(makeVoidType());
+
+    if (type->getTypeCode() == type_record)
+        type.setown(makeRowType(LINK(type)));
+
+    IHqlExpression * record = queryOriginalRecord(type);
+
+    if (!checkAllowed(errpos, "cpp", "Embedded code"))
+        args.append(*createExprAttribute(_disallowed_Atom));
+    if (attribs)
+        attribs->unwindList(args, no_comma);
     if (language)
     {
         IHqlScope *pluginScope = language->queryScope();
@@ -926,19 +939,38 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
             // MORE - create an expression that calls it, and const fold it, I guess....
         }
         args.append(*createExprAttribute(languageAtom, getEmbedContextFunc.getClear()));
+        IHqlExpression *projectedAttr = queryAttribute(projectedAtom, args);
+        if (projectedAttr)
+        {
+            IHqlExpression *projectedSearch = projectedAttr->queryChild(0);
+            if (!projectedSearch || !(isStringType(projectedSearch->queryType()) || isUnicodeType(projectedSearch->queryType())))
+                reportError(ERR_EMBEDPROJECT_INVALID, errpos, "PROJECTED attribute requires a string parameter");
+            else
+            {
+                IValue *projectedSearchValue = projectedSearch->queryValue();
+                if (!projectedSearchValue )
+                {
+                    // To relax this we'd need to pass the value as a hidden parameter as we do for the other options and the embed text,
+                    // But I really can't see it being useful
+                    reportError(ERR_EMBEDPROJECT_INVALID, errpos, "PROJECTED attribute requires a constant string parameter");
+                }
+                IValue *queryText = embedText->queryValue();
+                if (queryText)
+                {
+                    StringBuffer origQueryText;
+                    queryText->getUTF8Value(origQueryText);
+                    StringBuffer search;
+                    projectedSearchValue->getUTF8Value(search);
+                    if (!strstr(origQueryText, search))
+                        reportError(ERR_EMBEDPROJECT_INVALID, errpos, "PROJECTED attribute value %s does not appear in the embed text", search.str());
+                }
+            }
+            if (!record)
+                reportError(ERR_EMBEDPROJECT_INVALID, errpos, "PROJECTED should only be used when returning a record type");
+            else if (!isSimpleRecord(record))
+                reportError(ERR_EMBEDPROJECT_INVALID, errpos, "PROJECTED requires a simple output record (no nested records or IFBLOCKs)");
+        }
     }
-    if (!checkAllowed(errpos, "cpp", "Embedded code"))
-        args.append(*createExprAttribute(_disallowed_Atom));
-    if (attribs)
-        attribs->unwindList(args, no_comma);
-    Linked<ITypeInfo> type = current_type;
-    if (!type)
-        type.setown(makeVoidType());
-
-    if (type->getTypeCode() == type_record)
-        type.setown(makeRowType(LINK(type)));
-
-    IHqlExpression * record = queryOriginalRecord(type);
     OwnedHqlExpr result;
     if (record)
     {

ecl/hql/hqlutil.cpp

@@ -53,6 +53,7 @@ static IHqlExpression * cacheAlignedAttr;
 static IHqlExpression * cacheEmbeddedAttr;
 static IHqlExpression * cacheInlineAttr;
 static IHqlExpression * cacheLinkCountedAttr;
+static IHqlExpression * cacheProjectedAttr;
 static IHqlExpression * cacheReferenceAttr;
 static IHqlExpression * cacheStreamedAttr;
 static IHqlExpression * cacheUnadornedAttr;
@@ -82,6 +83,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     cacheEmbeddedAttr = createAttribute(embeddedAtom);
     cacheInlineAttr = createAttribute(inlineAtom);
     cacheLinkCountedAttr = createAttribute(_linkCounted_Atom);
+    cacheProjectedAttr = createAttribute(projectedAtom);
     cacheReferenceAttr = createAttribute(referenceAtom);
     cacheStreamedAttr = createAttribute(streamedAtom);
     cacheUnadornedAttr = createAttribute(_propUnadorned_Atom);
@@ -109,6 +111,7 @@ MODULE_EXIT()
     cacheEmbeddedAttr->Release();
     cacheInlineAttr->Release();
     cacheLinkCountedAttr->Release();
+    cacheProjectedAttr->Release();
     cacheReferenceAttr->Release();
     cacheStreamedAttr->Release();
     cacheUnadornedAttr->Release();
@@ -212,10 +215,18 @@ extern HQL_API IHqlExpression * queryLinkCountedAttr()
 {
     return cacheLinkCountedAttr;
 }
+extern HQL_API IHqlExpression * queryProjectedAttr()
+{
+    return cacheProjectedAttr;
+}
 extern HQL_API IHqlExpression * getLinkCountedAttr()
 {
     return LINK(cacheLinkCountedAttr);
 }
+extern HQL_API IHqlExpression * getProjectedAttr()
+{
+    return LINK(cacheProjectedAttr);
+}
 extern HQL_API IHqlExpression * getStreamedAttr()
 {
     return LINK(cacheStreamedAttr);
@@ -2173,7 +2184,6 @@ unsigned getFieldCount(IHqlExpression * expr)
     }
 }
 
-
 IHqlExpression * queryChildActivity(IHqlExpression * expr, unsigned index)
 {
     unsigned firstActivityIndex = 0;
@@ -2244,6 +2254,86 @@ unsigned isEmptyRecord(IHqlExpression * record)
     return true;
 }
 
+void getSimpleFields(HqlExprArray &out, IHqlExpression *record)
+{
+    ForEachChild(i, record)
+    {
+        IHqlExpression * cur = record->queryChild(i);
+        switch (cur->getOperator())
+        {
+        case no_attr:
+        case no_attr_expr:
+            break;
+        case no_field:
+            switch (cur->queryType()->getTypeCode())
+            {
+            case type_record:
+            case type_row:
+                {
+                    IHqlExpression *nested = cur->queryRecord();
+                    if (nested)
+                        getSimpleFields(out, nested);
+                    break;
+                }
+            case type_table:
+            case type_groupedtable:
+            case type_alien:
+            case type_any:
+            case type_dictionary:
+                throwUnexpected();
+            default:
+                out.append(*LINK(cur));
+                break;
+            }
+            break;
+        case no_record:
+            getSimpleFields(out, cur);
+            break;
+        default:
+            throwUnexpected();
+        }
+    }
+}
+
+unsigned isSimpleRecord(IHqlExpression * record)
+{
+    ForEachChild(i, record)
+    {
+        IHqlExpression * cur = record->queryChild(i);
+        switch (cur->getOperator())
+        {
+        case no_attr:
+        case no_attr_expr:
+            break;
+        case no_field:
+            switch (cur->queryType()->getTypeCode())
+            {
+            case type_record:
+            case type_row:
+                {
+                    IHqlExpression *nested = cur->queryRecord();
+                    if (nested && !isSimpleRecord(nested))
+                        return false;
+                    break;
+                }
+            case type_table:
+            case type_groupedtable:
+            case type_alien:
+            case type_any:
+            case type_dictionary:
+                return false;
+            }
+            break;
+        case no_record:
+            if (!isSimpleRecord(cur))
+                return false;
+            break;
+        default:
+            return false;
+        }
+    }
+    return record->numChildren()>0;
+}
 
 bool isTrivialSelectN(IHqlExpression * expr)
 {
@@ -5553,6 +5643,23 @@ bool isConstantDictionary(IHqlExpression * expr)
     return false;
 }
 
+bool isProjectableCall(IHqlExpression *expr)
+{
+    if (expr->getOperator() != no_call)
+        return false;
+    IHqlExpression * funcdef = expr->queryBody()->queryFunctionDefinition();
+    assertex(funcdef);
+    IHqlExpression * body = funcdef->queryChild(0);
+    assertex(body);
+    if ((funcdef->getOperator() == no_funcdef) && (body->getOperator() == no_outofline))
+    {
+        IHqlExpression * bodycode = body->queryChild(0);
+        if (bodycode->getOperator() == no_embedbody && bodycode->hasAttribute(projectedAtom))
+            return true;
+    }
+    return false;
+}
+
 inline bool iseol(char c) { return c == '\r' || c == '\n'; }
 
 static unsigned skipSpace(unsigned start, unsigned len, const char * buffer)

ecl/hql/hqlutil.hpp

@@ -91,6 +91,9 @@ extern HQL_API bool hasActiveTopDataset(IHqlExpression * expr);
 extern HQL_API unsigned getFieldCount(IHqlExpression * expr);
 extern HQL_API unsigned getFlatFieldCount(IHqlExpression * expr);
 extern HQL_API unsigned isEmptyRecord(IHqlExpression * record);
+extern HQL_API unsigned isSimpleRecord(IHqlExpression * record);
+extern HQL_API void getSimpleFields(HqlExprArray &out, IHqlExpression *record);
+
 extern HQL_API bool isTrivialSelectN(IHqlExpression * expr);
 
 extern HQL_API IHqlExpression * queryConvertChoosenNSort(IHqlExpression * expr, unsigned __int64 topNlimit);
@@ -198,6 +201,8 @@ extern HQL_API IHqlExpression * queryUncastExpr(IHqlExpression * expr);
 extern HQL_API bool areConstant(const HqlExprArray & args);
 extern HQL_API bool getFoldedConstantText(StringBuffer& ret, IHqlExpression * expr);
 
+extern HQL_API bool isProjectableCall(IHqlExpression *expr);
+
 extern HQL_API IHqlExpression * createTransformForField(IHqlExpression * field, IHqlExpression * value);
 extern HQL_API IHqlExpression * convertScalarToRow(IHqlExpression * value, ITypeInfo * fieldType);
 extern HQL_API bool splitResultValue(SharedHqlExpr & dataset, SharedHqlExpr & attribute, IHqlExpression * value);
@@ -596,6 +601,7 @@ extern HQL_API IHqlExpression * queryDefaultMaxRecordLengthExpr();
 extern HQL_API IHqlExpression * getFixedSizeAttr(unsigned size);
 extern HQL_API IHqlExpression * queryAlignedAttr();
 extern HQL_API IHqlExpression * queryLinkCountedAttr();
+extern HQL_API IHqlExpression * queryProjectedAttr();
 extern HQL_API IHqlExpression * queryUnadornedAttr();
 extern HQL_API IHqlExpression * queryNlpParsePseudoTable();
 extern HQL_API IHqlExpression * queryXmlParsePseudoTable();
@@ -606,6 +612,7 @@ extern HQL_API IHqlExpression * getInlineAttr();
 extern HQL_API IHqlExpression * getReferenceAttr();
 
 extern HQL_API IHqlExpression * getLinkCountedAttr();
+extern HQL_API IHqlExpression * getProjectedAttr();
 extern HQL_API IHqlExpression * getStreamedAttr();
 
 extern HQL_API IHqlExpression * getGlobalSequenceNumber();

ecl/hqlcpp/hqlcpp.cpp

@@ -11770,13 +11770,19 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         createParam.append("|EFnoreturn");
 
     IHqlExpression *optionsParam = nullptr;
-    if (formals->numChildren())
-    {
-        optionsParam = formals->queryChild(formals->numChildren()-1);
-        if (optionsParam->queryId() != __optionsId)
-            optionsParam = nullptr;
+    IHqlExpression *queryParam = nullptr;
+    unsigned numRealParams = 0;
+    ForEachChild(formalIdx, formals)
+    {
+        IHqlExpression *formal = formals->queryChild(formalIdx);
+        if (formal->queryId()==__optionsId)
+            optionsParam = formal;
+        else if (formal->queryId()==__queryId)
+            queryParam = formal;
+        else
+            numRealParams++;
     }
-    if (formals->numChildren()==(optionsParam ? 1 : 0))
+    if (!numRealParams)
         createParam.append("|EFnoparams");
 
     if (optionsParam)
@@ -11795,12 +11801,65 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
 
     HqlExprArray scriptArgs;
     scriptArgs.append(*LINK(ctxVar));
-    scriptArgs.append(*LINK(bodyCode->queryChild(0)));
+    if (bodyCode->hasAttribute(projectedAtom))
+    {
+        assertex(!isImport);
+        // Generate the field list from the output record
+        StringBuffer fieldlist;
+        IHqlExpression *outRec = bodyCode->queryChild(1);
+        assertex(outRec->queryRecordType());
+        HqlExprArray fields;
+        getSimpleFields(fields, outRec);
+        ForEachItemIn(idx, fields)
+        {
+            IIdAtom *fieldName = fields.item(idx).queryId();
+            assertex(fieldName);
+            fieldlist.append(',').append(fieldName->queryStr());
+        }
+        assertex(fieldlist.length());
+        LinkedHqlExpr substSearch = queryAttributeChild(bodyCode, projectedAtom, 0);
+        assertex (substSearch);
+        IValue *substValue = substSearch->queryValue();
+        if (queryParam || !substValue)
+        {
+            HqlExprArray args;
+            if (queryParam)
+                args.append(*createActualFromFormal(queryParam));
+            else
+                args.append(*LINK(bodyCode->queryChild(0)));
+            args.append(*createConstant(createUtf8Value(fieldlist.length()-1, fieldlist+1, makeUtf8Type(UNKNOWN_LENGTH, NULL))));
+            args.append(*LINK(substSearch));
+            scriptArgs.append(*bindFunctionCall(substituteEmbeddedScriptId, args,makeUtf8Type(UNKNOWN_LENGTH, NULL)));
+        }
+        else
+        {
+            IValue *query = bodyCode->queryChild(0)->queryValue();
+            assertex(query);
+            StringBuffer origBody;
+            query->getUTF8Value(origBody);
+            StringBuffer search;
+            substValue->getUTF8Value(search);
+            rtlDataAttr result;
+            unsigned resultLen;
+            rtlSubstituteEmbeddedScript(resultLen, result.refstr(), origBody.length(), origBody.str(), fieldlist.length()-1, fieldlist.str()+1, search.length(), search.str());
+            scriptArgs.append(*createConstant(createUtf8Value(resultLen, result.getstr(), makeUtf8Type(resultLen, NULL))));
+        }
+    }
+    else
+    {
+        if (queryParam)
+        {
+            OwnedHqlExpr query = createActualFromFormal(queryParam);
+            scriptArgs.append(*query.getClear());
+        }
+        else
+            scriptArgs.append(*LINK(bodyCode->queryChild(0)));
+    }
     buildFunctionCall(funcctx, isImport ? importId : compileEmbeddedScriptId, scriptArgs);
     ForEachChild(i, formals)
     {
         IHqlExpression * param = formals->queryChild(i);
-        if (param == optionsParam)
+        if (param == optionsParam || param==queryParam)
             continue;
         HqlExprArray args;
         args.append(*LINK(ctxVar));

ecl/hqlcpp/hqlcppsys.ecl

@@ -884,6 +884,7 @@ const char * cppSystemText[]  = {
     "   set of any getSetResult(integer4 typeCode, unsigned4 elemSize) : method,entrypoint='getSetResult';",
 
     "   compileEmbeddedScript(const utf8 script) : method,entrypoint='compileEmbeddedScript';",
+    "   utf8 substituteEmbeddedScript(const utf8 script, const utf8 fields, const utf8 substitute) : eclrtl,include,pure,entrypoint='rtlSubstituteEmbeddedScript';",
     "   import(const utf8 script) : method,entrypoint='importFunction';",
     "   END;",
     NULL };

ecl/hqlcpp/hqliproj.cpp

@@ -421,14 +421,14 @@ IHqlExpression * UsedFieldSet::createRowTransform(IHqlExpression * row, const Us
 }
 
 
-void UsedFieldSet::calcFinalRecord(bool canPack, bool ignoreIfEmpty)
+void UsedFieldSet::calcFinalRecord(bool canPack, bool ignoreIfEmpty, bool disallowEmpty)
 {
     assertex(originalFields);
     if (finalRecord)
         return;
 
     ForEachItemIn(i1, nested)
-        nested.item(i1).used.calcFinalRecord(canPack, true);
+        nested.item(i1).used.calcFinalRecord(canPack, true, false);
 
     IHqlExpression * originalRecord = queryOriginalRecord();
     if (checkAllFieldsUsed())
@@ -479,7 +479,10 @@ void UsedFieldSet::calcFinalRecord(bool canPack, bool ignoreIfEmpty)
     {
         if (ignoreIfEmpty)
             return;
-        recordFields.append(*createAttribute(_nonEmpty_Atom));
+        if (disallowEmpty)
+            recordFields.append(*LINK(queryOriginalRecord()->queryChild(0)));
+        else
+            recordFields.append(*createAttribute(_nonEmpty_Atom));
     }
 
     finalRecord.setown(createRecord(recordFields));
@@ -1085,13 +1088,13 @@ IHqlExpression * ComplexImplicitProjectInfo::createOutputProject(IHqlExpression
 }
 
 
-void ComplexImplicitProjectInfo::finalizeOutputRecord()
+void ComplexImplicitProjectInfo::finalizeOutputRecord(bool disallowEmpty)
 {
     //MORE: Create them in the same order as the original record + don't change if numOutputFields = numOriginalOutputFields
     if (!queryOutputRecord())
     {
         bool canPack = (safeToReorderOutput() && okToOptimize());
-        outputFields.calcFinalRecord(canPack, false);
+        outputFields.calcFinalRecord(canPack, false, disallowEmpty);
     }
 }
 
@@ -1595,6 +1598,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
         case IterateTransformActivity:
         case DenormalizeActivity:
         case CreateRecordSourceActivity:
+        case CreateNonEmptyRecordSourceActivity:
             if (hasUnknownTransform(expr))
                 complexExtra->preventOptimization();
             break;
@@ -1615,6 +1619,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
         case CompoundActivity:
         case CompoundableActivity:
         case CreateRecordSourceActivity:
+        case CreateNonEmptyRecordSourceActivity:
         case AnyTypeActivity:
             break;
         case RollupTransformActivity:
@@ -2169,7 +2174,12 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_call:
     case no_externalcall:
         if (hasActivityType(expr))
-            return SourceActivity;
+        {
+            if (isProjectableCall(expr))
+                return CreateNonEmptyRecordSourceActivity;
+            else
+                return SourceActivity;
+        }
         //MORE: What about parameters??
         return NonActivity;
     case no_commonspill:
@@ -2423,7 +2433,7 @@ void ImplicitProjectTransformer::calculateFieldsUsed(IHqlExpression * expr)
         {
             //output will now be whatever fields are required by the output fields.
             //input will be the same as the output fields, since it is just a wrapper node.
-            extra->finalizeOutputRecord();
+            extra->finalizeOutputRecord(false);
             //MORE: Not sure this is neededextra->leftFieldsRequired.clone(extra->outputFields);
             extra->insertProject = true;
             assertex(extra->inputs.ordinality() == 0);
@@ -2437,14 +2447,15 @@ void ImplicitProjectTransformer::calculateFieldsUsed(IHqlExpression * expr)
                 extra->inputs.item(0).stopOptimizeCompound(true);
 
             if (extra->okToOptimize())
-                extra->finalizeOutputRecord();
+                extra->finalizeOutputRecord(false);
             break;
         }
+    case CreateNonEmptyRecordSourceActivity:
     case CreateRecordSourceActivity:
     case AnyTypeActivity:
         {
             if (extra->okToOptimize())
-                extra->finalizeOutputRecord();
+                extra->finalizeOutputRecord(extra->activityKind() == CreateNonEmptyRecordSourceActivity);
             break;
         }
     case PassThroughActivity:
@@ -2880,6 +2891,43 @@ IHqlExpression * ImplicitProjectTransformer::createTransformed(IHqlExpression *
             }
             break;
         }
+    case CreateNonEmptyRecordSourceActivity:
+        {
+            assertex(expr->getOperator() == no_call);
+            //Always reduce things that create a new record so they only project the fields they need to
+            if (complexExtra->outputChanged())
+            {
+                HqlExprArray args;
+                IHqlExpression * funcdef = expr->queryBody()->queryFunctionDefinition();
+                assertex(funcdef);
+                IHqlExpression * body = funcdef->queryChild(0);
+                assertex(body);
+                if ((funcdef->getOperator() == no_funcdef) && (body->getOperator() == no_outofline))
+                {
+                    IHqlExpression * bodycode = body->queryChild(0);
+                    if (bodycode->getOperator() == no_embedbody)
+                    {
+                        OwnedHqlExpr newBodyCode = replaceChild(bodycode, 1, complexExtra->queryOutputRecord());
+                        OwnedHqlExpr newBody = replaceChild(body, 0, newBodyCode);
+                        OwnedHqlExpr newFuncdef = replaceChild(funcdef, 0, newBody);
+                        unwindChildren(args, expr, 0);
+                        transformed.setown(createBoundFunction(NULL, newFuncdef, args, NULL, DEFAULT_EXPAND_CALL));
+
+                        logChange("Auto project embed", expr, complexExtra->outputFields);
+                        return transformed.getClear();
+                    }
+                }
+                throwUnexpected();
+                break;
+            }
+            else
+            {
+                transformed.setown(createParentTransformed(expr));
+                //MORE: Need to replace left/right with their transformed varieties because the record may have changed format
+                transformed.setown(updateSelectors(transformed, expr));
+            }
+            break;
+        }
     case CompoundActivity:
         {
             transformed.setown(createParentTransformed(expr));
@@ -3028,8 +3076,9 @@ void ImplicitProjectTransformer::finalizeFields(IHqlExpression * expr)
     case CompoundActivity:
     case CompoundableActivity:
     case CreateRecordSourceActivity:
+    case CreateNonEmptyRecordSourceActivity:
     case AnyTypeActivity:
-        extra->finalizeOutputRecord();
+        extra->finalizeOutputRecord(extra->activityKind() == CreateNonEmptyRecordSourceActivity);
         break;
     case DenormalizeActivity:
     case RollupTransformActivity:
@@ -3047,7 +3096,7 @@ void ImplicitProjectTransformer::finalizeFields(IHqlExpression * expr)
                 extra->fieldsToBlank.getText(fieldText);
                 DBGLOG("ImplicitProject: Fields %s for %s not required by outputs - so blank in transform", fieldText.str(), opString);
             }
-            extra->finalizeOutputRecord();
+            extra->finalizeOutputRecord(false);
             break;
         }
     case FixedInputActivity:
@@ -3074,7 +3123,7 @@ void ImplicitProjectTransformer::finalizeFields(IHqlExpression * expr)
                         ComplexImplicitProjectInfo & cur = extra->inputs.item(i2);
                         extra->outputFields.intersectFields(cur.outputFields);
                     }
-                    extra->finalizeOutputRecord();
+                    extra->finalizeOutputRecord(false);
                     anyProjected = true;
                     break;
                 }
@@ -3090,7 +3139,7 @@ void ImplicitProjectTransformer::finalizeFields(IHqlExpression * expr)
         if (extra->insertProject && requiresFewerFields(extra->leftFieldsRequired, extra->inputs.item(0)))
         {
             extra->outputFields.set(extra->leftFieldsRequired);
-            extra->finalizeOutputRecord();
+            extra->finalizeOutputRecord(false);
         }
         else
             extra->setMatchingOutput(&extra->inputs.item(0));

ecl/hqlcpp/hqliproj.ipp

@@ -46,6 +46,7 @@ enum ProjectExprKind
                                         //in output (not nesc evaluated in the input)
     SinkActivity,                       // a sink, but that doesn't necessarily use all input fields.
     CreateRecordSourceActivity,         // a source activity containing a transform i.e., inline table
+    CreateNonEmptyRecordSourceActivity, // as above but can't handle empty output record
     ComplexNonActivity,                 
     AnyTypeActivity,                    // can be created any type.
 };
@@ -128,7 +129,7 @@ public:
     int compareOrder(IHqlExpression * left, IHqlExpression * right) const;
     void createDifference(const UsedFieldSet & left, const UsedFieldSet & right);
     IHqlExpression * createFilteredTransform(IHqlExpression * transform, const UsedFieldSet * exceptions) const;
-    void calcFinalRecord(bool canPack, bool ignoreIfEmpty);
+    void calcFinalRecord(bool canPack, bool ignoreIfEmpty, bool disallowEmpty);
     NestedField * findNested(IHqlExpression * field) const;
     NestedField * findNestedByName(IHqlExpression * field) const;
     void gatherTransformValuesUsed(HqlExprArray * selfSelects, HqlExprArray * parentSelects, HqlExprArray * values, IHqlExpression * selector, IHqlExpression * transform);
@@ -299,7 +300,7 @@ public:
 
     void addAllOutputs();
     IHqlExpression * createOutputProject(IHqlExpression * ds);
-    void finalizeOutputRecord();
+    void finalizeOutputRecord(bool disallowEmpty);
     void inheritRequiredFields(const UsedFieldSet & requiredList);
     bool safeToReorderInput();
     bool safeToReorderOutput();

ecl/hqlcpp/hqlttcpp.cpp

@@ -6200,6 +6200,11 @@ IHqlExpression * WorkflowTransformer::extractCommonWorkflow(IHqlExpression * exp
     return getValue.getClear();
 }
 
+static bool isInternalEmbedAttr(IAtom *name)
+{
+    return name == languageAtom || name == projectedAtom || name == streamedAtom || name == _linkCounted_Atom ||name == importAtom;
+}
+
 IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression * newFuncDef)
 {
     IHqlExpression * body = newFuncDef->queryChild(0);
@@ -6236,7 +6241,7 @@ IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression *
         ForEachChild(idx, bodyCode)
         {
             IHqlExpression *child = bodyCode->queryChild(idx);
-            if (child->isAttribute() && child->queryName() != languageAtom && child->queryName() != importAtom)
+            if (child->isAttribute() && !isInternalEmbedAttr(child->queryName()))
             {
                 StringBuffer attrParam;
                 if (attrArgs.ordinality())
@@ -6281,6 +6286,13 @@ IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression *
             newDefaults.append(*createOmittedValue());
         newDefaults.append(*folded.getClear());
 
+        IHqlExpression *query = bodyCode->queryChild(0);
+        if (!query->queryValue())
+        {
+            newFormals.append(*createParameter(__queryId, newFormals.length(), LINK(unknownUtf8Type), attrs));
+            newDefaults.append(*LINK(query));
+        }
+
         funcdefArgs.append(*formals->clone(newFormals));
         funcdefArgs.append(*createValueSafe(no_sortlist, makeSortListType(NULL), newDefaults));
         unwindChildren(funcdefArgs, newFuncDef, 3);
@@ -6319,8 +6331,11 @@ IHqlExpression * WorkflowTransformer::transformInternalCall(IHqlExpression * tra
         IHqlExpression * ecl = body->queryChild(0);
         if (ecl->getOperator() == no_embedbody && ecl->hasAttribute(languageAtom))
         {
-            // Copy the new default value into the end of the parameters array
-            parameters.append(*LINK(newFuncDef->queryChild(2)->queryChild(parameters.length())));
+            // Copy the new default value(s) into the end of the parameters array
+            IHqlExpression *formals = newFuncDef->queryChild(1);
+            IHqlExpression *defaults = newFuncDef->queryChild(2);
+            while (parameters.length() < formals->numChildren())
+                parameters.append(*LINK(defaults->queryChild(parameters.length())));
         }
     }
 

rtl/eclrtl/eclrtl.cpp

@@ -5302,6 +5302,19 @@ void rtlAddExceptionTag(StringBuffer & errorText, const char * tag, const char *
 
 //---------------------------------------------------------------------------
 
+void rtlSubstituteEmbeddedScript(size32_t &__lenResult, char * &__result, size32_t scriptChars, const char *script, size32_t outFieldsChars, const char *outFields, size32_t searchChars, const char *search)
+{
+    StringBuffer result;
+    result.append(rtlUtf8Size(scriptChars, script), script);
+    StringBuffer outFieldsX(rtlUtf8Size(outFieldsChars, outFields), outFields);
+    StringBuffer searchX(rtlUtf8Size(searchChars, search), search);
+    result.replaceString(searchX.str(), outFieldsX.str());
+    __lenResult = result.length();
+    __result = result.detach();
+}
+
+//---------------------------------------------------------------------------
+
 void rtlRowBuilder::forceAvailable(size32_t size)
 {
     const size32_t chunkSize = 64;

rtl/eclrtl/eclrtl.hpp

@@ -838,4 +838,6 @@ interface IEmbedContext : extends IInterface
 
 typedef IEmbedContext * (* GetEmbedContextFunction)();
 
+ECLRTL_API void rtlSubstituteEmbeddedScript(size32_t &__lenResult, char * &__result, size32_t scriptChars, const char *script, size32_t outFieldsChars, const char *outFields, size32_t searchChars, const char *search);
+
 #endif

testing/regress/ecl/key/mysqlembed.xml

@@ -5,43 +5,52 @@
  <Row><name>utf8test</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>2019-02-01 23:59:59</dt></Row>
 </Dataset>
 <Dataset name='Result 2'>
- <Row><Result_2>name1</Result_2></Row>
+ <Row><Result_2>2</Result_2></Row>
 </Dataset>
 <Dataset name='Result 3'>
- <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>1963-11-22 12:30:00</dt></Row>
+ <Row><name>name1</name></Row>
+ <Row><name>name2</name></Row>
+ <Row><name>nulls</name></Row>
+ <Row><name>utf8test</name></Row>
 </Dataset>
 <Dataset name='Result 4'>
- <Row><Result_4>utf8test</Result_4></Row>
+ <Row><Result_4>name1</Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><name>utf8test</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>2019-02-01 23:59:59</dt></Row>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>1963-11-22 12:30:00</dt></Row>
 </Dataset>
 <Dataset name='Result 6'>
- <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>1963-11-22 12:30:00</dt></Row>
- <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2><dt>2015-12-25 01:23:45</dt></Row>
+ <Row><Result_6>utf8test</Result_6></Row>
 </Dataset>
 <Dataset name='Result 7'>
- <Row><Result_7>4</Result_7></Row>
+ <Row><name>utf8test</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>2019-02-01 23:59:59</dt></Row>
 </Dataset>
 <Dataset name='Result 8'>
- <Row><Result_8>true</Result_8></Row>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>1963-11-22 12:30:00</dt></Row>
+ <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2><dt>2015-12-25 01:23:45</dt></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>5.6</Result_9></Row>
+ <Row><Result_9>4</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>7.800000190734863</Result_10></Row>
+ <Row><Result_10>true</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>6161353561613535</Result_11></Row>
+ <Row><Result_11>5.6</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>Straße</Result_12></Row>
+ <Row><Result_12>7.800000190734863</Result_12></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><Result_13>Straße  </Result_13></Row>
+ <Row><Result_13>6161353561613535</Result_13></Row>
 </Dataset>
 <Dataset name='Result 14'>
+ <Row><Result_14>Straße</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>Straße  </Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
  <Row><dt1>19631122123000</dt1><dt2>1963-11-22 12:30:00</dt2></Row>
  <Row><dt1>20151225012345</dt1><dt2>2015-12-25 01:23:45</dt2></Row>
  <Row><dt1>0</dt1><dt2>                   </dt2></Row>

testing/regress/ecl/mysqlembed.ecl

@@ -24,8 +24,11 @@ myServer := 'localhost' : stored('myServer');
 myUser := 'rchapman' : stored('myUser');
 myDb := 'test' : stored('myDb');
 
-childrec := RECORD
-   string name,
+stringrec := RECORD
+   string name
+END;
+
+childrec := RECORD(stringrec)
    integer4 value { default(99999) },
    boolean boolval { default(true) },
    real8 r8 {default(99.99)},
@@ -33,13 +36,12 @@ childrec := RECORD
    DATA d {default (D'999999')},
    DECIMAL10_2 ddd {default(9.99)},
    UTF8 u1 {default(U'9999 ß')},
-   UNICODE8 u2 {default(U'9999 ßßßß')},
+   record
+     UNICODE8 u2 {default(U'9999 ßßßß')},
+   end;
    STRING19 dt {default('1963-11-22 12:30:00')},
 END;
 
-stringrec := RECORD
-   string name
-END;
 
 stringrec extractName(childrec l) := TRANSFORM
   SELF := l;
@@ -68,10 +70,18 @@ initializeUtf8() := EMBED(mysql : server(myServer),user(myUser),database(myDB))
   INSERT INTO tbl1 values ('utf8test', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße', '2019-02-01 23:59:59');
 ENDEMBED;
 
-dataset(childrec) testMySQLDS() := EMBED(mysql : server(myServer),user(myUser),database(myDB))
-  SELECT * from tbl1;
+dataset(childrec) testMySQLDS() := EMBED(mysql : server(myServer),user(myUser),database(myDB),PROJECTED('OUTPUTFIELDS'))
+  SELECT OUTPUTFIELDS from tbl1;
+ENDEMBED;
+
+dataset(childrec) testMySQLDS2() := EMBED(mysql : server(myServer),user(myUser),database(myDB),PROJECTED('[]'))
+  SELECT [] from tbl1 where u1='Straße';
 ENDEMBED;
 
+ds3query := u'SELECT ** FROM tbl1;' : STORED('ds3query');
+
+dataset(childrec) testMySQLDS3() := EMBED(mysql, ds3query : server(myServer),user(myUser),database(myDB),PROJECTED(u'**'));
+
 childrec testMySQLRow() := EMBED(mysql : server(myServer),user(myUser),database(myDB))
   SELECT * from tbl1 LIMIT 1;
 ENDEMBED;
@@ -152,6 +162,8 @@ sequential (
   initializeUtf8(),
   PARALLEL (
   OUTPUT(testMySQLDS()),
+  COUNT(testMySQLDS2()),
+  OUTPUT(testMySQLDS3(), {name}),
   OUTPUT(testMySQLRow().name),
   OUTPUT(testMySQLParms('name1', 1, true, 1.2, 3.4, D'aa55aa55', U'Straße', U'Straße')),
   OUTPUT(testMySQLString()),