HPCC-10230 FILEPOSITION(FALSE) on index removes implicit payload field.

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlerrors.hpp

@@ -472,6 +472,7 @@
 #define HQLERR_PrefixJoinRequiresEquality       3132
 #define HQLERR_AtmostFollowUnknownSubstr        3133
 #define HQLERR_AtmostLegacyMismatch             3134
+#define HQLERR_PropertyArgumentNotConstant      3135
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contains an illegal cycle/recursive definition %s"
@@ -508,6 +509,7 @@
 #define HQLERR_PrefixJoinRequiresEquality_Text  "Global JOIN with no required equalities requires ALL"
 #define HQLERR_AtmostFollowUnknownSubstr_Text   "ATMOST [1..*] on an unknown length string must be last in the optional list"
 #define HQLERR_AtmostLegacyMismatch_Text        "Legacy JOIN condition on field[1..*] should be included in the optional fields"
+#define HQLERR_PropertyArgumentNotConstant_Text "The argument to attribute '%s' must be a constant"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hql/hqlexpr.cpp

@@ -14840,7 +14840,9 @@ unsigned numPayloadFields(IHqlExpression * index)
     IHqlExpression * payloadAttr = index->queryAttribute(_payload_Atom);
     if (payloadAttr)
         return (unsigned)getIntValue(payloadAttr->queryChild(0));
-    return 1;
+    if (getBoolAttribute(index, filepositionAtom, true))
+        return 1;
+    return 0;
 }
 
 unsigned numKeyedFields(IHqlExpression * index)
@@ -15341,19 +15343,42 @@ bool isKeyedCountAggregate(IHqlExpression * aggregate)
 }
 
 
+static bool getBoolAttributeValue(IHqlExpression * attr)
+{
+    IHqlExpression * value = attr->queryChild(0);
+    //No argument implies true
+    if (!value)
+        return true;
+
+    //If it is a constant return it.
+    if (value->queryValue())
+        return getBoolValue(value, true);
+
+    //Not a constant => fold th expression
+    OwnedHqlExpr folded = foldHqlExpression(value);
+    if (folded->queryValue())
+        return getBoolValue(folded, true);
+
+    throwError1(HQLERR_PropertyArgumentNotConstant, attr->queryName()->str());
+}
+
 bool getBoolAttribute(IHqlExpression * expr, IAtom * name, bool dft)
 {
     if (!expr)
         return dft;
-    IHqlExpression * prop = expr->queryAttribute(name);
-    if (!prop)
+    IHqlExpression * attr = expr->queryAttribute(name);
+    if (!attr)
         return dft;
-    IHqlExpression * value = prop->queryChild(0);
-    if (!value)
-        return true;
-    return getBoolValue(value, true);
+    return getBoolAttributeValue(attr);
 }
 
+bool getBoolAttributeInList(IHqlExpression * expr, IAtom * search, bool dft)
+{
+    IHqlExpression * match = queryAttributeInList(search, expr);
+    if (!match)
+        return dft;
+    return getBoolAttributeValue(match);
+}
 
 IHqlExpression * queryOriginalRecord(IHqlExpression * expr)
 {

ecl/hql/hqlexpr.hpp

@@ -1860,6 +1860,7 @@ extern HQL_API IPropertyTree * createAttributeArchive();
 extern HQL_API void ensureSymbolsDefined(IHqlExpression * scope, HqlLookupContext & ctx);
 extern HQL_API void ensureSymbolsDefined(IHqlScope * scope, HqlLookupContext & ctx);
 extern HQL_API bool getBoolAttribute(IHqlExpression * expr, IAtom * name, bool dft=false);
+extern HQL_API bool getBoolAttributeInList(IHqlExpression * expr, IAtom * name, bool dft);
 
 extern HQL_API void setLegacyEclSemantics(bool _value);
 extern HQL_API bool queryLegacyEclSemantics();

ecl/hql/hqlgram.hpp

@@ -454,7 +454,7 @@ public:
     void checkWorkflowMultiples(IHqlExpression * currentWorkflow, IHqlExpression * newWorkflow, attribute& errpos);
     void checkJoinFlags(const attribute & err, IHqlExpression * joinExpr);
     void checkLoopFlags(const attribute & err, IHqlExpression * loopExpr);
-    IHqlExpression * checkIndexRecord(IHqlExpression * record, const attribute & errpos);
+    IHqlExpression * checkIndexRecord(IHqlExpression * record, const attribute & errpos, OwnedHqlExpr & indexAttrs);
     void checkIndexFieldType(IHqlExpression * cur, bool isPayload, bool insideNestedRecord, const attribute & errpos);
     void checkIndexRecordType(IHqlExpression * record, unsigned numPayloadFields, bool insideNestedRecord, const attribute & errpos);
     void checkIndexRecordTypes(IHqlExpression * index, const attribute & errpos);

ecl/hql/hqlgram.y

@@ -2886,6 +2886,10 @@ buildFlag
                         {
                             $$.setExpr(createAttribute(dedupAtom), $1);
                         }
+    | FILEPOSITION optConstBoolArg
+                        {
+                            $$.setExpr(createExprAttribute(filepositionAtom, $2.getExpr()), $1);
+                        }
     ;
 
 localAttribute
@@ -3084,6 +3088,10 @@ indexFlag
                             $$.setPosition($1);
                         }
     | UNORDERED         {   $$.setExpr(createAttribute(unorderedAtom)); $$.setPosition($1); }
+    | FILEPOSITION optConstBoolArg
+                        {
+                            $$.setExpr(createExprAttribute(filepositionAtom, $2.getExpr()), $1);
+                        }
     | commonAttribute
     ;
 
@@ -4976,6 +4984,15 @@ optExpression
     | expression
     ;
 
+optConstBoolArg
+    :                   {   $$.setNullExpr(); }
+    | '(' expression ')'
+                        {
+                            parser->normalizeExpression($2, type_boolean, true);
+                            $$.inherit($2);
+                        }
+    ;
+
 booleanExpr
     : expression        {
                             parser->normalizeExpression($1, type_boolean, false);
@@ -8382,7 +8399,8 @@ simpleDataSet
 
                             parser->inheritRecordMaxLength(dataset, record);
 
-                            record.setown(parser->checkIndexRecord(record, $5));
+                            bool hasFileposition = getBoolAttributeInList(extra, filepositionAtom, true);
+                            record.setown(parser->checkIndexRecord(record, $5, extra));
                             if (transform)
                             {
                                 if (!recordTypesMatch(dataset, transform))

ecl/hql/hqlgram2.cpp

@@ -49,6 +49,7 @@
 #include "hqlrepository.hpp"
 #include "hqlir.hpp"
 
+#define ADD_IMPLICIT_FILEPOS_FIELD_TO_INDEX         TRUE
 #define FAST_FIND_FIELD
 //#define USE_WHEN_FOR_SIDEEFFECTS
 #define MANYFIELDS_THRESHOLD                        2000
@@ -978,10 +979,16 @@ IHqlExpression * HqlGram::processIndexBuild(attribute & indexAttr, attribute * r
         }
         else
         {
-            checkIndexRecordType(record, 1, false, *recordAttr);
+            bool hasFileposition = getBoolAttributeInList(flags, filepositionAtom, true);
+            unsigned numPayloadFields = hasFileposition ? 1 : 0;
+
+            checkIndexRecordType(record, numPayloadFields, false, *recordAttr);
         }
+
+        //Recalculated because it might be updated in modifyIndexPayloadRecord() above
+        bool hasFileposition = getBoolAttributeInList(flags, filepositionAtom, true);
         record.setown(checkBuildIndexRecord(record.getClear(), *recordAttr));
-        record.setown(checkIndexRecord(record, *recordAttr));
+        record.setown(checkIndexRecord(record, *recordAttr, flags));
         inputDataset.setown(createDatasetF(no_selectfields, LINK(dataset), LINK(record), NULL));
         warnIfRecordPacked(inputDataset, *recordAttr);
     }
@@ -6660,6 +6667,9 @@ IHqlExpression * HqlGram::createBuildIndexFromIndex(attribute & indexAttr, attri
     IHqlExpression * payload = index->queryAttribute(_payload_Atom);
     if (payload)
         args.append(*LINK(payload));
+    IHqlExpression * fileposition = index->queryAttribute(filepositionAtom);
+    if (fileposition)
+        args.append(*LINK(fileposition));
     if (distribution)
         args.append(*distribution.getClear());
 
@@ -6819,22 +6829,22 @@ void HqlGram::checkSoapRecord(attribute & errpos)
 }
 
 
-IHqlExpression * HqlGram::checkIndexRecord(IHqlExpression * record, const attribute & errpos)
+IHqlExpression * HqlGram::checkIndexRecord(IHqlExpression * record, const attribute & errpos, OwnedHqlExpr & indexAttrs)
 {
     unsigned numFields = record->numChildren();
-    if (numFields)
+    if (numFields && getBoolAttributeInList(indexAttrs, filepositionAtom, true))
     {
         // if not, implies some error (already reported)
         if (numFields == 1)
-            reportError(ERR_INDEX_COMPONENTS, errpos, "Record for index should have at least one component and a fileposition");
+        {
+            indexAttrs.setown(createComma(indexAttrs.getClear(), createExprAttribute(filepositionAtom, createConstant(false))));
+        }
         else
         {
             IHqlExpression * lastField = record->queryChild(numFields-1);
             ITypeInfo * fileposType = lastField->queryType();
             if (!isIntegralType(fileposType))
-                reportError(ERR_INDEX_FILEPOS_EXPECTED_LAST, errpos, "Expected last field to be an integral fileposition field");
-//          else if (fileposType->getSize() != 8)
-//              reportWarning(ERR_INDEX_FILEPOS_UNEXPECTED_SIZE, errpos.pos, "Expected fileposition field to be 8 bytes");
+                indexAttrs.setown(createComma(indexAttrs.getClear(), createExprAttribute(filepositionAtom, createConstant(false))));
         }
     }
     return LINK(record);
@@ -7223,14 +7233,15 @@ IHqlExpression * HqlGram::createRecordExcept(IHqlExpression * left, IHqlExpressi
 }
 
 
-IHqlExpression * HqlGram::createIndexFromRecord(IHqlExpression * record, IHqlExpression * attr, const attribute & errpos)
+IHqlExpression * HqlGram::createIndexFromRecord(IHqlExpression * record, IHqlExpression * attrs, const attribute & errpos)
 {
     IHqlExpression * ds = createDataset(no_null, LINK(record), NULL);
-    OwnedHqlExpr finalRecord = checkIndexRecord(record, errpos);
+    OwnedHqlExpr newAttrs = LINK(attrs);
+    OwnedHqlExpr finalRecord = checkIndexRecord(record, errpos, newAttrs);
     finalRecord.setown(cleanIndexRecord(finalRecord));
 
     OwnedHqlExpr transform = createClearTransform(finalRecord, errpos);
-    return createDataset(no_newkeyindex, ds, createComma(LINK(finalRecord), transform.getClear(), LINK(attr)));
+    return createDataset(no_newkeyindex, ds, createComma(LINK(finalRecord), transform.getClear(), newAttrs.getClear()));
 }
 
 
@@ -7993,28 +8004,33 @@ void HqlGram::modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & p
         payloadCount = fields.ordinality() - oldFields;
     }
     //This needs to be here until filepositions are no longer special cased.
-    if (!lastFieldType || !lastFieldType->isInteger())
-    {
-        IHqlSimpleScope * payloadScope = payload ? payload->querySimpleScope() : NULL;
-        IIdAtom * implicitFieldName;
-        for (unsigned suffix =1;;suffix++)
-        {
-            StringBuffer name;
-            name.append("__internal_fpos");
-            if (suffix > 1)
-                name.append(suffix);
-            name.append("__");
-            implicitFieldName = createIdAtom(name);
-            OwnedHqlExpr resolved = scope->lookupSymbol(implicitFieldName);
-            if (!resolved && payloadScope)
-                resolved.setown(payloadScope->lookupSymbol(implicitFieldName));
-            if (!resolved)
-                break;
-        }
+    if (!lastFieldType || !lastFieldType->isInteger() && getBoolAttributeInList(extra, filepositionAtom, true))
+    {
+        if (ADD_IMPLICIT_FILEPOS_FIELD_TO_INDEX)
+        {
+            IHqlSimpleScope * payloadScope = payload ? payload->querySimpleScope() : NULL;
+            IIdAtom * implicitFieldName;
+            for (unsigned suffix =1;;suffix++)
+            {
+                StringBuffer name;
+                name.append("__internal_fpos");
+                if (suffix > 1)
+                    name.append(suffix);
+                name.append("__");
+                implicitFieldName = createIdAtom(name);
+                OwnedHqlExpr resolved = scope->lookupSymbol(implicitFieldName);
+                if (!resolved && payloadScope)
+                    resolved.setown(payloadScope->lookupSymbol(implicitFieldName));
+                if (!resolved)
+                    break;
+            }
 
-        ITypeInfo * fileposType = makeIntType(8, false);
-        fields.append(*createField(implicitFieldName, fileposType, createConstant(I64C(0)), createAttribute(_implicitFpos_Atom)));
-        payloadCount++;
+            ITypeInfo * fileposType = makeIntType(8, false);
+            fields.append(*createField(implicitFieldName, fileposType, createConstant(I64C(0)), createAttribute(_implicitFpos_Atom)));
+            payloadCount++;
+        }
+        else
+            extra.setown(createComma(extra.getClear(), createExprAttribute(filepositionAtom, createConstant(false))));
     }
 
     extra.setown(createComma(extra.getClear(), createAttribute(_payload_Atom, createConstant((__int64) payloadCount))));

ecl/hql/hqlmeta.cpp

@@ -2508,7 +2508,8 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
                 if (expr->queryAttribute(sort_KeyedAtom))
                 {
                     IHqlExpression * payloadAttr = expr->queryAttribute(_payload_Atom);
-                    unsigned payloadCount = payloadAttr ? (unsigned)getIntValue(payloadAttr->queryChild(0), 1) : 1;
+                    bool hasFileposition = getBoolAttribute(expr, filepositionAtom, true);
+                    unsigned payloadCount = payloadAttr ? (unsigned)getIntValue(payloadAttr->queryChild(0), 1) : hasFileposition ? 1 : 0;
                     unsigned payloadIndex = firstPayloadField(record, payloadCount);
                     unwindRecordAsSelects(sortExprs, record, queryActiveTableSelector(), payloadIndex);
                 }

ecl/hql/hqlusage.cpp

@@ -428,16 +428,23 @@ IPropertyTree * SourceFieldUsage::createReport(bool includeFieldDetail, const IP
         if (!original)
             original = source;
         IHqlExpression * lastField = queryLastField(original->queryRecord());
-        if (usedFilepos || !lastField->hasAttribute(_implicitFpos_Atom))
+        if (getBoolAttribute(source, filepositionAtom, true))
         {
-            numFields++;
-            if (usedFilepos || usedAll)
+            if (usedFilepos || !lastField->hasAttribute(_implicitFpos_Atom))
             {
-                if (includeFieldDetail)
-                    addSelect(entry, lastField, true);
-                numFieldsUsed++;
+                numFields++;
+                if (usedFilepos || usedAll)
+                {
+                    if (includeFieldDetail)
+                        addSelect(entry, lastField, true);
+                    numFieldsUsed++;
+                }
             }
         }
+        else
+        {
+            assertex(!usedFilepos);
+        }
     }
 
     entry->setPropInt("@numFields", numFields);

ecl/hqlcpp/hqlckey.cpp

@@ -284,6 +284,7 @@ protected:
     OwnedHqlExpr    extractJoinFieldsTransform;
     bool            canOptimizeTransfer;
     bool            hasComplexIndex;
+    bool            keyHasFileposition;
 };
 
 KeyedJoinInfo::KeyedJoinInfo(HqlCppTranslator & _translator, IHqlExpression * _expr, bool _canOptimizeTransfer) : translator(_translator)
@@ -299,6 +300,7 @@ KeyedJoinInfo::KeyedJoinInfo(HqlCppTranslator & _translator, IHqlExpression * _e
         key.set(keyed->queryChild(0));
         if (right->getOperator() == no_keyed)
             right = right->queryChild(0);
+        assertex(getBoolAttribute(right, filepositionAtom, true));
         file.set(right);
         IHqlExpression * rightTable = queryPhysicalRootTable(right);
         if (!rightTable || rightTable->queryNormalizedSelector() != right->queryNormalizedSelector())
@@ -324,6 +326,8 @@ KeyedJoinInfo::KeyedJoinInfo(HqlCppTranslator & _translator, IHqlExpression * _e
             translator.throwError1(HQLERR_KeyedJoinNoRightIndex_X, getOpString(right->getOperator()));
     }
 
+    keyHasFileposition = getBoolAttribute(key, filepositionAtom, true);
+
     if (!originalKey)
         originalKey.set(key);
     expandedKey.setown(translator.convertToPhysicalIndex(key));
@@ -687,7 +691,7 @@ void KeyedJoinInfo::buildTransformBody(BuildCtx & ctx, IHqlExpression * transfor
     if (extractJoinFieldsTransform)
     {
         IHqlExpression * fileposField = isFullJoin() ? queryVirtualFileposField(file->queryRecord()) : queryLastField(key->queryRecord());
-        if (fileposField && (expr->getOperator() != no_denormalizegroup))
+        if (keyHasFileposition && fileposField && (expr->getOperator() != no_denormalizegroup))
         {
             HqlMapTransformer fileposMapper;
             OwnedHqlExpr select = createSelectExpr(LINK(serializedRight), LINK(fileposField));
@@ -997,6 +1001,7 @@ void KeyedJoinInfo::optimizeExtractJoinFields()
         {
 //          unwindFields(fieldsAccessed, key->queryRecord());
             extractedRecord.set(key->queryRecord());
+            if (keyHasFileposition)
                 fileposField = queryLastField(key->queryRecord());
         }
     }
@@ -1017,7 +1022,7 @@ void KeyedJoinInfo::optimizeExtractJoinFields()
         {
             IHqlExpression * keyRecord = key->queryRecord();
             IHqlExpression * filepos = queryLastField(keyRecord);
-            if (filepos)
+            if (filepos && keyHasFileposition)
                 fieldsAccessed.zap(*filepos);
 
             if (translator.getTargetClusterType() != HThorCluster)
@@ -1142,7 +1147,6 @@ bool KeyedJoinInfo::processFilter()
     //Now need to transform the index into its real representation so
     //the hozed transforms take place.
     unsigned payload = numPayloadFields(key);
-    assertex(payload);          // don't use rawindex once payload can be 0
     TableProjectMapper mapper(expandedKey);
     OwnedHqlExpr rightSelect = createSelector(no_right, key, joinSeq);
     OwnedHqlExpr newFilter = mapper.expandFields(keyedKeyFilter, rightSelect, rawKey, rawKey);

ecl/hqlcpp/hqlcpp.ipp

@@ -1646,7 +1646,7 @@ public:
     void buildSkewThresholdMembers(BuildCtx & ctx, IHqlExpression * expr);
     void doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, const HqlExprArray & left, const HqlExprArray & right);
     void buildSlidingMatchFunction(BuildCtx & ctx, const HqlExprArray & leftEq, const HqlExprArray & rightEq, const HqlExprArray & slidingMatches, const char * funcname, unsigned childIndex, const DatasetReference & datasetL, const DatasetReference & datasetR);
-    void doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpression * record, SharedHqlExpr & rawRecord);
+    void doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpression * record, SharedHqlExpr & rawRecord, bool hasFileposition);
 
     void buildKeyedJoinExtra(ActivityInstance & instance, IHqlExpression * expr, KeyedJoinInfo * joinKey);
     void buildKeyJoinIndexReadHelper(ActivityInstance & instance, IHqlExpression * expr, KeyedJoinInfo * joinKey);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9718,6 +9718,8 @@ void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name,
 {
     IHqlExpression * payload = expr ? expr->queryAttribute(_payload_Atom) : NULL;
     OwnedHqlExpr exprToCrc = getSerializedForm(dataset->queryRecord(), diskAtom);
+
+    //MORE: Should this take FILEPOSITION(FALSE) into account?
     unsigned payloadSize = 1;
     if (payload)
         payloadSize = (unsigned)getIntValue(payload->queryChild(0)) + payloadDelta;
@@ -9729,10 +9731,10 @@ void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name,
     doBuildUnsignedFunction(ctx, name, crc);
 }
 
-static void createOutputIndexRecord(HqlMapTransformer & mapper, HqlExprArray & fields, IHqlExpression * record, bool isMainRecord, bool allowTranslate)
+static void createOutputIndexRecord(HqlMapTransformer & mapper, HqlExprArray & fields, IHqlExpression * record, bool hasFileposition, bool allowTranslate)
 {
     unsigned numFields = record->numChildren();
-    unsigned max = isMainRecord ? numFields-1 : numFields;
+    unsigned max = hasFileposition ? numFields-1 : numFields;
     for (unsigned idx=0; idx < max; idx++)
     {
         IHqlExpression * cur = record->queryChild(idx);
@@ -9799,10 +9801,10 @@ static void createOutputIndexRecord(HqlMapTransformer & mapper, HqlExprArray & f
 }
 
 
-static void createOutputIndexTransform(HqlExprArray & assigns, IHqlExpression * self, IHqlExpression * tgtRecord, IHqlExpression * srcRecord, IHqlExpression* srcDataset, bool isMainRecord, bool allowTranslate)
+static void createOutputIndexTransform(HqlExprArray & assigns, IHqlExpression * self, IHqlExpression * tgtRecord, IHqlExpression * srcRecord, IHqlExpression* srcDataset, bool hasFileposition, bool allowTranslate)
 {
     unsigned numFields = srcRecord->numChildren();
-    unsigned max = isMainRecord ? numFields-1 : numFields;
+    unsigned max = hasFileposition ? numFields-1 : numFields;
     for (unsigned idx=0; idx < max; idx++)
     {
         IHqlExpression * cur = srcRecord->queryChild(idx);
@@ -9847,19 +9849,19 @@ static void createOutputIndexTransform(HqlExprArray & assigns, IHqlExpression *
 }
 
 
-void HqlCppTranslator::doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpression * record, SharedHqlExpr & rawRecord)
+void HqlCppTranslator::doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpression * record, SharedHqlExpr & rawRecord, bool hasFileposition)
 {
     OwnedHqlExpr srcDataset = createDataset(no_anon, LINK(record));
 
     HqlExprArray fields;
     HqlExprArray assigns;
     HqlMapTransformer mapper;
-    createOutputIndexRecord(mapper, fields, record, true, true);
+    createOutputIndexRecord(mapper, fields, record, hasFileposition, true);
 
     OwnedHqlExpr newRecord = createRecord(fields);
     rawRecord.set(newRecord);
     OwnedHqlExpr self = getSelf(newRecord);
-    createOutputIndexTransform(assigns, self, newRecord, record, srcDataset, true, true);
+    createOutputIndexTransform(assigns, self, newRecord, record, srcDataset, hasFileposition, true);
 
     OwnedHqlExpr tgtDataset = createDataset(no_anon, newRecord.getLink());
     OwnedHqlExpr transform = createValue(no_newtransform, makeTransformType(newRecord->getType()), assigns);
@@ -9876,8 +9878,12 @@ void HqlCppTranslator::doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpressio
     doTransform(subctx, transform, selfCursor);
 
     OwnedHqlExpr fposVar = createVariable("filepos", makeIntType(8, false));
-    OwnedHqlExpr fposField = createSelectExpr(LINK(srcDataset), LINK(queryLastField(record)));
-    buildAssignToTemp(subctx, fposVar, fposField);
+    OwnedHqlExpr fposValue;
+    if (hasFileposition)
+        fposValue.setown(createSelectExpr(LINK(srcDataset), LINK(queryLastField(record))));
+    else
+        fposValue.setown(getSizetConstant(0));
+    buildAssignToTemp(subctx, fposVar, fposValue);
 
     buildReturnRecordSize(subctx, selfCursor);
 
@@ -10031,6 +10037,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
     IHqlExpression * compressAttr = expr->queryAttribute(compressedAtom);
     IHqlExpression * widthExpr = queryAttributeChild(expr, widthAtom, 0);
     bool hasTLK = !expr->hasAttribute(noRootAtom);
+    bool hasFileposition = getBoolAttribute(expr, filepositionAtom, true);
     bool singlePart = expr->hasAttribute(fewAtom);
     if (matchesConstantValue(widthExpr, 1))
     {
@@ -10115,7 +10122,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
     }
 
     OwnedHqlExpr rawRecord;
-    doBuildIndexOutputTransform(instance->startctx, record, rawRecord);
+    doBuildIndexOutputTransform(instance->startctx, record, rawRecord, hasFileposition);
     buildFormatCrcFunction(instance->classctx, "getFormatCrc", rawRecord, expr, 0);
 
     if (compressAttr && compressAttr->hasAttribute(rowAtom))

ecl/hqlcpp/hqlsource.cpp

@@ -372,7 +372,7 @@ static IHqlExpression * nextDiskField(IHqlExpression * diskRecord, unsigned & di
 }
 
 
-void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression * self, IHqlExpression * diskRecord, IHqlExpression * record, IHqlExpression * diskDataset, bool allowTranslate, unsigned fileposIndex)
+static void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression * self, IHqlExpression * diskRecord, IHqlExpression * record, IHqlExpression * diskDataset, bool allowTranslate, unsigned fileposIndex)
 {
     unsigned numFields = record->numChildren();
     unsigned diskIndex = 0;
@@ -423,11 +423,12 @@ void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression * self,
 }
 
 
-void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression * diskDataset, IHqlExpression * tableExpr)
+static void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression * diskDataset, IHqlExpression * tableExpr, bool hasFilePosition)
 {
     IHqlExpression * record = tableExpr->queryRecord();
+    unsigned fileposIndex = (hasFilePosition ? record->numChildren() - 1 : NotFound);
     OwnedHqlExpr self = getSelf(record);
-    createPhysicalLogicalAssigns(assigns, self, diskDataset->queryRecord(), record, diskDataset, true, record->numChildren()-1);
+    createPhysicalLogicalAssigns(assigns, self, diskDataset->queryRecord(), record, diskDataset, true, fileposIndex);
 }
 
 
@@ -457,10 +458,10 @@ static IHqlExpression * mapIfBlock(HqlMapTransformer & mapper, IHqlExpression *
 }
 
 
-static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpression * tableExpr, IHqlExpression * record, bool isMainRecord, bool allowTranslate)
+static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpression * tableExpr, IHqlExpression * record, bool hasWeirdFileposition, bool allowTranslate)
 {
     HqlExprArray physicalFields;
-    unsigned max = record->numChildren() - (isMainRecord ? 1 : 0);
+    unsigned max = record->numChildren() - (hasWeirdFileposition ? 1 : 0);
     for (unsigned idx=0; idx < max; idx++)
     {
         IHqlExpression * cur = record->queryChild(idx);
@@ -540,22 +541,24 @@ IHqlExpression * HqlCppTranslator::convertToPhysicalIndex(IHqlExpression * table
     IHqlExpression * record = tableExpr->queryRecord();
 
     HqlMapTransformer mapper;
-    IHqlExpression * diskRecord = createPhysicalIndexRecord(mapper, tableExpr, record, true, true);
+    bool hasFileposition = getBoolAttribute(tableExpr, filepositionAtom, true);
+    IHqlExpression * diskRecord = createPhysicalIndexRecord(mapper, tableExpr, record, hasFileposition, true);
 
-    __int64 payload = numPayloadFields(tableExpr);
-    assertex(payload);
+    unsigned payload = numPayloadFields(tableExpr);
+    assertex(payload || !hasFileposition);
+    unsigned newPayload = hasFileposition ? payload-1 : payload;
     HqlExprArray args;
     unwindChildren(args, tableExpr);
     args.replace(*diskRecord, 1);
     removeAttribute(args, _payload_Atom);
-    args.append(*createAttribute(_payload_Atom, createConstant(payload-1)));
+    args.append(*createAttribute(_payload_Atom, getSizetConstant(newPayload)));
     args.append(*createAttribute(_original_Atom, LINK(tableExpr)));
 
     //remove the preload attribute and replace with correct value
     IHqlExpression * newDataset = createDataset(tableExpr->getOperator(), args);
 
     HqlExprArray assigns;
-    createPhysicalLogicalAssigns(assigns, newDataset, tableExpr);
+    createPhysicalLogicalAssigns(assigns, newDataset, tableExpr, hasFileposition);
     OwnedHqlExpr projectedTable = createDataset(no_newusertable, newDataset, createComma(LINK(record), createValue(no_newtransform, makeTransformType(record->getType()), assigns)));
     physicalIndexCache.setValue(tableExpr, projectedTable);
     return projectedTable.getClear();

ecl/regress/issue10230a.ecl

@@ -0,0 +1,20 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+d := dataset('~people', { string15 name, unsigned8 filepos{virtual(fileposition)} }, flat);
+i := index(d, { name }, 'peopleIndex'); // implicit FILEPOSITION(FALSE), previously gave an error
+output(i(name = 'Gavin'));

ecl/regress/issue10230b.ecl

@@ -0,0 +1,20 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+d := dataset('~people', { string15 name, unsigned8 dob }, flat);
+i := index(d, { name, dob }, 'peopleIndex'); // dob goes into the fileposition field
+output(i(name = 'Gavin' and dob != 0));

ecl/regress/issue10230c.ecl

@@ -0,0 +1,20 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+d := dataset('~people', { string15 name, unsigned8 dob }, flat);
+i := index(d, { name, dob }, 'peopleIndex', FILEPOSITION(false)); // dob goes into the keyed portion, not filepos
+output(i(name = 'Gavin' and dob != 0));

ecl/regress/issue10230d.ecl

@@ -0,0 +1,23 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+boolean hasFileposition := 100 > 200;
+
+d := dataset('~people', { string15 name, unsigned8 dob }, flat);
+// The parameter can be any compile-time constant?  It should be supported, but isn't currently
+i := index(d, { name, dob }, 'peopleIndex', FILEPOSITION(hasFileposition)); // dob goes into the keyed portion, not filepos
+output(i(name = 'Gavin' and dob != 0));

ecl/regress/issue10230e.ecl

@@ -0,0 +1,23 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+boolean hasFileposition := 100 < 200;
+
+d := dataset('~people', { string15 name, unsigned8 dob }, flat);
+// The parameter can be any compile-time constant?  It should be supported, but isn't currently
+i := index(d, { name, dob }, 'peopleIndex', FILEPOSITION(hasFileposition)); // dob goes into the filepos portion
+output(i(name = 'Gavin' and dob != 0));

ecl/regress/issue10230f.ecl

@@ -0,0 +1,22 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+d := dataset('~people', { string15 name, unsigned8 dob }, flat);
+
+// The parameter can be any compile-time constant?  It should be supported, but isn't currently
+i(boolean hasFileposition) := index(d, { name, dob }, 'peopleIndex', FILEPOSITION(hasFileposition)); // dob goes into the keyed portion, not filepos
+output(i(false)(name = 'Gavin' and dob != 0));

ecl/regress/issue10230g.ecl

@@ -0,0 +1,22 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+boolean hasFileposition := 100 < 200;
+
+ds := dataset('~people', { string15 name, unsigned8 dob }, flat);
+
+BUILD(ds, { name, dob }, 'peopleIndex', FILEPOSITION(hasFileposition)); // dob goes into the filepos portion

ecl/regress/issue10230h.ecl

@@ -0,0 +1,22 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+boolean hasFileposition := 100 > 200;
+
+ds := dataset('~people', { string15 name, unsigned8 dob }, flat);
+
+BUILD(ds, { name, dob }, 'peopleIndex', FILEPOSITION(hasFileposition)); // dob goes into the filepos portion