HPCC-17537

Configuration option to enable file restriction

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>

common/fileview2/fvwugen.cpp

@@ -112,7 +112,7 @@ IHqlExpression * buildDiskFileViewerEcl(const char * logicalName, IHqlExpression
     OwnedHqlExpr newRecord = createRecord(fields);
     newRecord.setown(createSymbol(createIdAtom("_SourceRecord_"), newRecord.getLink(), ob_private));
 
-    OwnedHqlExpr dataset = createNewDataset(createConstant(logicalName), newRecord.getLink(), createValue(no_thor), NULL, NULL, NULL, NULL);
+    OwnedHqlExpr dataset = createNewDataset(createConstant(logicalName), newRecord.getLink(), createValue(no_thor), NULL, NULL, NULL);
     OwnedHqlExpr filtered = addFilter(dataset, filepos);
     OwnedHqlExpr projected = addSimplifyProject(filtered);
     OwnedHqlExpr output = addOutput(projected);
@@ -123,7 +123,7 @@ IHqlExpression * buildDiskFileViewerEcl(const char * logicalName, IHqlExpression
 
 IHqlExpression * buildDiskOutputEcl(const char * logicalName, IHqlExpression * record)
 {
-    OwnedHqlExpr dataset = createNewDataset(createConstant(logicalName), LINK(record), createValue(no_thor), NULL, NULL, NULL, NULL);
+    OwnedHqlExpr dataset = createNewDataset(createConstant(logicalName), LINK(record), createValue(no_thor), NULL, NULL, NULL);
     return addOutput(dataset);
 }
 

ecl/hql/hqlexpr.cpp

@@ -12336,7 +12336,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
     return ret;
 }
 
-extern IHqlExpression *createNewDataset(IHqlExpression *name, IHqlExpression *recorddef, IHqlExpression *mode, IHqlExpression *parent, IHqlExpression *joinCondition, IHqlExpression * signature, IHqlExpression * options)
+extern IHqlExpression *createNewDataset(IHqlExpression *name, IHqlExpression *recorddef, IHqlExpression *mode, IHqlExpression *parent, IHqlExpression *joinCondition, IHqlExpression * options)
 {
     HqlExprArray args;
     args.append(*name);
@@ -12353,8 +12353,6 @@ extern IHqlExpression *createNewDataset(IHqlExpression *name, IHqlExpression *re
         options->unwindList(args, no_comma);
         options->Release();
     }
-    if (signature)
-        args.append(*signature);
     return createDataset(no_table, args);
 }
 

ecl/hql/hqlexpr.hpp

@@ -1258,7 +1258,7 @@ extern HQL_API IHqlExpression *createDatasetF(node_operator op, ...);
 extern HQL_API IHqlExpression *createDictionary(node_operator op, IHqlExpression *initializer, IHqlExpression *recordDef);
 extern HQL_API IHqlExpression *createDictionary(node_operator op, IHqlExpression *dictionary);
 extern HQL_API IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms);
-extern HQL_API IHqlExpression *createNewDataset(IHqlExpression *name, IHqlExpression *recorddef, IHqlExpression *mode, IHqlExpression *parent, IHqlExpression *joinCondition, IHqlExpression * signature, IHqlExpression * options);
+extern HQL_API IHqlExpression *createNewDataset(IHqlExpression *name, IHqlExpression *recorddef, IHqlExpression *mode, IHqlExpression *parent, IHqlExpression *joinCondition, IHqlExpression * options);
 extern HQL_API IHqlExpression *createRow(node_operator op, IHqlExpression *Dataset, IHqlExpression *element = NULL);
 extern HQL_API IHqlExpression *createRow(node_operator op, HqlExprArray & args);
 extern HQL_API IHqlExpression *createRowF(node_operator op, ...);

ecl/hql/hqlgram.hpp

@@ -478,6 +478,8 @@ public:
     void checkExportedModule(const attribute & errpos, IHqlExpression * scopeExpr);
     bool checkCompatibleSymbol(const attribute & errpos, IHqlExpression * prevValue, IHqlExpression * newValue);
     bool checkAllowed(const attribute & errpos, const char *category, const char *description);
+    void saveDiskAccessInformation(const attribute & errpos, HqlExprArray & options);
+    void saveDiskAccessInformation(const attribute & errpos, OwnedHqlExpr & options);
     IHqlExpression * createAveList(const attribute & errpos, IHqlExpression * list);
     IHqlExpression * createIff(attribute & condAttr, attribute & leftAttr, attribute & rightAttr);
     IHqlExpression * createListFromExpressionList(attribute & attr);
@@ -576,7 +578,7 @@ public:
     void beginDefineId(IIdAtom * name, ITypeInfo * type);
 
     IHqlExpression * processAlienType(const attribute & errpos);
-    IHqlExpression * processIndexBuild(attribute & indexAttr, attribute * recordAttr, attribute * payloadAttr, attribute & filenameAttr, attribute & flagsAttr);
+    IHqlExpression * processIndexBuild(const attribute &err, attribute & indexAttr, attribute * recordAttr, attribute * payloadAttr, attribute & filenameAttr, attribute & flagsAttr);
     IHqlExpression * processCompoundFunction(attribute & result, bool outOfLine);
     IHqlExpression * processEmbedBody(const attribute & errpos, IHqlExpression * embedText, IHqlExpression * language, IHqlExpression *attribs);
     IHqlExpression * getGpgSignature();

ecl/hql/hqlgram.y

@@ -2352,17 +2352,17 @@ actionStmt
                         }
     | BUILD '(' startTopFilter ',' ',' thorFilenameOrList optBuildFlags ')' endTopFilter
                         {
-                            $$.setExpr(parser->processIndexBuild($3, NULL, NULL, $6, $7), $1);
+                            $$.setExpr(parser->processIndexBuild($1, $3, NULL, NULL, $6, $7), $1);
                             parser->processUpdateAttr($$);
                         }
     | BUILD '(' startTopFilter ',' recordDef ',' thorFilenameOrList optBuildFlags ')' endTopFilter
                         {
-                            $$.setExpr(parser->processIndexBuild($3, &$5, NULL, $7, $8), $1);
+                            $$.setExpr(parser->processIndexBuild($1, $3, &$5, NULL, $7, $8), $1);
                             parser->processUpdateAttr($$);
                         }
     | BUILD '(' startTopFilter ',' recordDef ',' nullRecordDef ',' thorFilenameOrList optBuildFlags ')' endTopFilter
                         {
-                            $$.setExpr(parser->processIndexBuild($3, &$5, &$7, $9, $10), $1);
+                            $$.setExpr(parser->processIndexBuild($1, $3, &$5, &$7, $9, $10), $1);
                             parser->processUpdateAttr($$);
                         }
     | BUILD '(' startTopFilter optBuildFlags ')' endTopFilter
@@ -2697,7 +2697,10 @@ actionStmt
                             if (!recordTypesMatch($3.queryExpr(), $5.queryExpr()))
                                 parser->reportError(ERR_TYPEMISMATCH_RECORD, $4, "Indexes must have the same structure");
                             parser->normalizeExpression($7, type_string, false);
-                            $$.setExpr(createValueFromCommaList(no_keydiff, makeVoidType(), createComma($3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr())));
+
+                            OwnedHqlExpr options = createComma($3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr());
+                            parser->saveDiskAccessInformation($1, options);
+                            $$.setExpr(createValueFromCommaList(no_keydiff, makeVoidType(), options.getClear()));
                             $$.setPosition($1);
                         }
     | KEYPATCH '(' dataSet ',' expression ',' expression keyDiffFlags ')'
@@ -2706,7 +2709,10 @@ actionStmt
                                 parser->reportError(ERR_EXPECTED_INDEX,$3,"Expected an index");
                             parser->normalizeExpression($5, type_string, false);
                             parser->normalizeExpression($7, type_string, false);
-                            $$.setExpr(createValueFromCommaList(no_keypatch, makeVoidType(), createComma($3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr())));
+                            OwnedHqlExpr options = createComma($3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr());
+
+                            parser->saveDiskAccessInformation($1, options);
+                            $$.setExpr(createValueFromCommaList(no_keypatch, makeVoidType(), options.getClear()));
                             $$.setPosition($1);
                         }
     | EVALUATE '(' expression ')'
@@ -8608,8 +8614,9 @@ simpleDataSet
                                 counter = createAttribute(_countProject_Atom, counter);
 
                             //MORE: This should require local otherwise it needs to do a full join type thing.
-                            IHqlExpression * extra = createComma($5.getExpr(), $7.getExpr(), transform, createComma($12.getExpr(), counter, $14.getExpr()));
-                            $$.setExpr(createDataset(no_denormalize, ds, extra));
+                            OwnedHqlExpr extra = createComma($5.getExpr(), $7.getExpr(), transform, createComma($12.getExpr(), counter, $14.getExpr()));
+                            parser->saveDiskAccessInformation($1, extra);
+                            $$.setExpr(createDataset(no_denormalize, ds, extra.getClear()));
                             $$.setPosition($1);
                             parser->checkJoinFlags($1, $$.queryExpr());
                         }
@@ -8618,8 +8625,9 @@ simpleDataSet
                             parser->normalizeExpression($7, type_boolean, false);
                             IHqlExpression * ds = $3.getExpr();
                             IHqlExpression * transform = $13.getExpr();
-                            IHqlExpression * extra = createComma($5.getExpr(), $7.getExpr(), transform, createComma($14.getExpr(), $16.getExpr(), $17.getExpr()));
-                            $$.setExpr(createDataset(no_denormalizegroup, ds, extra));
+                            OwnedHqlExpr extra = createComma($5.getExpr(), $7.getExpr(), transform, createComma($14.getExpr(), $16.getExpr(), $17.getExpr()));
+                            parser->saveDiskAccessInformation($1, extra);
+                            $$.setExpr(createDataset(no_denormalizegroup, ds, extra.getClear()));
                             $$.setPosition($1);
                             $11.release();
                             parser->checkJoinFlags($1, $$.queryExpr());
@@ -8816,6 +8824,8 @@ simpleDataSet
 
                             bool hasFileposition = getBoolAttributeInList(extra, filepositionAtom, true);
                             record.setown(parser->checkIndexRecord(record, $5, extra));
+
+                            parser->saveDiskAccessInformation($1, extra);
                             if (transform)
                             {
                                 if (!recordTypesMatch(dataset, transform))
@@ -8841,6 +8851,7 @@ simpleDataSet
                             OwnedHqlExpr record = $3.getExpr();
                             OwnedHqlExpr extra = $4.getExpr();
                             parser->extractIndexRecordAndExtra(record, extra);
+                            parser->saveDiskAccessInformation($1, extra);
                             IHqlExpression *index = parser->createIndexFromRecord(record, extra, $3);
                             parser->checkValidLookupFlag(index, index->queryChild(3), $1);
                             parser->checkIndexRecordTypes(index, $1);
@@ -8862,6 +8873,9 @@ simpleDataSet
                                 else
                                     args.replace(*newName.getLink(), 3);
                                 $6.unwindCommaList(args);
+                                OwnedHqlExpr options;
+                                parser->saveDiskAccessInformation($1, options);
+                                if (options) args.append(*options.getClear());
                                 dataset.setown(createDataset(keyOp, args));
                                 parser->checkValidLookupFlag(dataset, newName, $5);
                             }
@@ -8899,7 +8913,8 @@ simpleDataSet
                                 filename.setown(createValue(no_assertconstant, filename->getType(), LINK(filename->queryChild(0))));
                                 options.setown(createComma(options.getClear(), createAttribute(localUploadAtom)));
                             }
-                            IHqlExpression * dataset = createNewDataset(filename.getLink(), $6.getExpr(), mode.getClear(), NULL, NULL, parser->getGpgSignature(), options.getClear());
+                            parser->saveDiskAccessInformation($1, options);
+                            IHqlExpression * dataset = createNewDataset(filename.getLink(), $6.getExpr(), mode.getClear(), NULL, NULL, options.getClear());
                             parser->checkValidRecordMode(dataset, $4, $9);
                             parser->checkValidLookupFlag(dataset, filename, $3);
                             $$.setExpr(dataset);
@@ -8929,7 +8944,7 @@ simpleDataSet
                             default:
                                 parser->reportError(ERR_EXPECTED, $1, "Expected STRING/UTF8/UNICODE");
                             }
-                            
+
                             OwnedHqlExpr empty = createList(no_list, makeSetType(NULL), NULL);
                             OwnedHqlExpr quoted = createExprAttribute(quoteAtom, LINK(empty));
                             OwnedHqlExpr separator = createExprAttribute(separatorAtom, LINK(empty));
@@ -8943,7 +8958,8 @@ simpleDataSet
                                 filename.setown(createValue(no_assertconstant, filename->getType(), LINK(filename->queryChild(0))));
                                 options.setown(createComma(options.getClear(), createAttribute(localUploadAtom)));
                             }
-                            IHqlExpression * dataset = createNewDataset(filename.getLink(), record.getClear(), mode.getClear(), NULL, NULL, parser->getGpgSignature(), options.getClear());
+                            parser->saveDiskAccessInformation($1, options);
+                            IHqlExpression * dataset = createNewDataset(filename.getLink(), record.getClear(), mode.getClear(), NULL, NULL, options.getClear());
                             parser->checkValidRecordMode(dataset, $4, $9);
                             parser->checkValidLookupFlag(dataset, filename, $3);
                             $$.setExpr(dataset, $1);
@@ -8957,8 +8973,10 @@ simpleDataSet
                             IHqlExpression * origin = $3.getExpr();
                             OwnedHqlExpr filename = $5.getExpr();
                             IHqlExpression * mode = $7.getExpr();
-                            IHqlExpression * attrs = createComma(createAttribute(_origin_Atom, origin), $8.getExpr());
-                            IHqlExpression * dataset = createNewDataset(filename.getLink(), LINK(origin->queryRecord()), mode, NULL, NULL, parser->getGpgSignature(), attrs);
+                            OwnedHqlExpr attrs = createComma(createAttribute(_origin_Atom, origin), $8.getExpr());
+                            parser->saveDiskAccessInformation($1, attrs);
+
+                            IHqlExpression * dataset = createNewDataset(filename.getLink(), LINK(origin->queryRecord()), mode, NULL, NULL, attrs.getClear());
 
                             parser->checkValidRecordMode(dataset, $3, $7);
                             parser->checkValidLookupFlag(dataset, filename, $3);
@@ -9059,7 +9077,7 @@ simpleDataSet
                                 attrs.setown(createComma(attrs.getClear(), createAttribute(_disallowed_Atom)));
                             parser->normalizeExpression($3, type_string, false);
                             parser->checkValidPipeRecord($5, $5.queryExpr(), attrs, NULL);
-                            $$.setExpr(createNewDataset(createBlankString(), $5.getExpr(), createValue(no_pipe, makeNullType(), $3.getExpr()), NULL, NULL, NULL, LINK(attrs)));
+                            $$.setExpr(createNewDataset(createBlankString(), $5.getExpr(), createValue(no_pipe, makeNullType(), $3.getExpr()), NULL, NULL, LINK(attrs)));
                             $$.setPosition($1);
                         }
     | PIPE '(' startTopFilter ',' expression optPipeOptions endTopFilter ')'

ecl/hql/hqlgram2.cpp

@@ -1076,7 +1076,7 @@ IHqlExpression * HqlGram::processUserAggregate(const attribute & mainPos, attrib
     return ret.getClear();
 }
 
-IHqlExpression * HqlGram::processIndexBuild(attribute & indexAttr, attribute * recordAttr, attribute * payloadAttr, attribute & filenameAttr, attribute & flagsAttr)
+IHqlExpression * HqlGram::processIndexBuild(const attribute &err, attribute & indexAttr, attribute * recordAttr, attribute * payloadAttr, attribute & filenameAttr, attribute & flagsAttr)
 {
     if (!recordAttr)
         warnIfRecordPacked(indexAttr);
@@ -1124,10 +1124,7 @@ IHqlExpression * HqlGram::processIndexBuild(attribute & indexAttr, attribute * r
     args.append(*filenameAttr.getExpr());
     if (flags)
         flags->unwindList(args, no_comma);
-    IHqlExpression * sig = getGpgSignature();
-    if (sig)
-        args.append(*sig);
-
+    saveDiskAccessInformation(err, args);
     checkDistributer(flagsAttr.pos, args);
     return createValue(no_buildindex, makeVoidType(), args);
 }
@@ -7042,10 +7039,7 @@ IHqlExpression * HqlGram::createBuildIndexFromIndex(attribute & indexAttr, attri
     if (distribution)
         args.append(*distribution.getClear());
 
-    IHqlExpression * sig = getGpgSignature();
-    if (sig)
-        args.append(*sig);
-
+    saveDiskAccessInformation(indexAttr, args);
     checkDistributer(flagsAttr.pos, args);
     return createValue(no_buildindex, makeVoidType(), args);
 }
@@ -9282,6 +9276,20 @@ bool HqlGram::checkAllowed(const attribute & errpos, const char *category, const
     return true;
 }
 
+void HqlGram::saveDiskAccessInformation(const attribute & errpos, HqlExprArray & options)
+{
+    IHqlExpression * sig = getGpgSignature();
+    if (sig)
+        options.append(*sig);
+}
+
+void HqlGram::saveDiskAccessInformation(const attribute & errpos, OwnedHqlExpr & options)
+{
+    IHqlExpression * sig = getGpgSignature();
+    if (sig)
+        options.setown(createComma(options.getClear(), sig));
+}
+
 bool HqlGram::checkDFSfields(IHqlExpression *dfsRecord, IHqlExpression *defaultRecord, const attribute& errpos)
 {
     if (dfsRecord==defaultRecord)

ecl/hqlcpp/hqlcerrors.hpp

@@ -224,6 +224,7 @@
 #define HQLERR_BodyNotAllowedWithInline         4204
 #define HQLERR_DatasetPassedToRowArg            4205
 #define HQLERR_HashStoredDuplication            4207
+#define HQLERR_DatafileRequiresSigned           4208
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -525,6 +526,7 @@
 #define HQLERR_BodyNotAllowedWithInline_Text    "#body not supported with INLINE attribute"
 #define HQLERR_DatasetPassedToRowArg_Text       "Cannot pass a dataset to row argument %s"
 #define HQLERR_HashStoredDuplication_Text       "Inconsistent #%s(%s, %s) and #%s(%s, %s)"
+#define HQLERR_DatafileRequiresSigned_Text      "Insufficient access rights to use datafiles"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlckey.cpp

@@ -307,8 +307,10 @@ KeyedJoinInfo::KeyedJoinInfo(HqlCppTranslator & _translator, IHqlExpression * _e
         IHqlExpression * rightTable = queryPhysicalRootTable(right);
         if (!rightTable || rightTable->queryNormalizedSelector() != right->queryNormalizedSelector())
             translator.throwError(HQLERR_FullKeyedNeedsFile);
+        translator.ensureDiskAccessAllowed(rightTable);
         expandedFile.setown(convertToPhysicalTable(rightTable, true));
         rawFile.set(queryPhysicalRootTable(expandedFile));
+        translator.ensureDiskAccessAllowed(rawFile);
         keyedMapper.setDataset(key);
     }
     else if (right->getOperator() == no_newkeyindex)
@@ -334,6 +336,7 @@ KeyedJoinInfo::KeyedJoinInfo(HqlCppTranslator & _translator, IHqlExpression * _e
         originalKey.set(key);
     expandedKey.setown(translator.convertToPhysicalIndex(key));
     rawKey.set(queryPhysicalRootTable(expandedKey));
+    translator.ensureDiskAccessAllowed(rawKey);
     canOptimizeTransfer = _canOptimizeTransfer; 
     monitors = NULL;
     counter.set(queryAttributeChild(expr, _countProject_Atom, 0));
@@ -1595,6 +1598,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx
 
 ABoundActivity * HqlCppTranslator::doBuildActivityKeyDiff(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
 {
+    ensureDiskAccessAllowed(expr);
+
     StringBuffer s;
     IHqlExpression * original = expr->queryChild(0);
     IHqlExpression * updated = expr->queryChild(1);
@@ -1642,6 +1647,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyDiff(BuildCtx & ctx, IHqlEx
 
 ABoundActivity * HqlCppTranslator::doBuildActivityKeyPatch(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
 {
+    ensureDiskAccessAllowed(expr);
     StringBuffer s;
     IHqlExpression * original = expr->queryChild(0);
     IHqlExpression * patch = expr->queryChild(1);

ecl/hqlcpp/hqlcpp.cpp

@@ -1476,6 +1476,13 @@ void HqlCppTranslator::setTargetClusterType(ClusterType clusterType)
     targetClusterType = clusterType;
 }
 
+void HqlCppTranslator::ensureDiskAccessAllowed(IHqlExpression * expr)
+{
+    bool isSigned = expr->hasAttribute(_signed_Atom);
+    if (!queryCallback()->allowAccess("datafile", isSigned))
+        reportErrorNoAbort(expr, HQLERR_DatafileRequiresSigned, "You do not have permission to directly access datafiles");
+}
+
 void HqlCppTranslator::checkAbort()
 {
     if (wu() && wu()->aborting())
@@ -2237,6 +2244,17 @@ void HqlCppTranslator::reportError(IHqlExpression * location, int code,const cha
     reportErrorDirect(location, code, errorMsg.str(), true);
 }
 
+void HqlCppTranslator::reportErrorNoAbort(IHqlExpression * location, int code,const char *format, ...)
+{
+    StringBuffer errorMsg;
+    va_list args;
+    va_start(args, format);
+    errorMsg.valist_appendf(format, args);
+    va_end(args);
+
+    reportErrorDirect(location, code, errorMsg.str(), false);
+}
+
 //---------------------------------------------------------------------------
 
 void HqlCppTranslator::doBuildStmtAssign(BuildCtx & ctx, IHqlExpression * target, IHqlExpression * expr)

ecl/hqlcpp/hqlcpp.ipp

@@ -967,6 +967,7 @@ public:
     void reportWarning(WarnErrorCategory category, unsigned id, const char * msg, ...) __attribute__((format(printf, 4, 5)));
     void reportWarning(WarnErrorCategory category, ErrorSeverity explicitSeverity, IHqlExpression * location, unsigned id, const char * msg, ...) __attribute__((format(printf, 6, 7)));
     void reportError(IHqlExpression * location, int code, const char *format, ...) __attribute__((format(printf, 4, 5)));
+    void reportErrorNoAbort(IHqlExpression * location, int code, const char *format, ...) __attribute__((format(printf, 4, 5)));
     void reportErrorDirect(IHqlExpression * location, int code,const char *msg, bool alwaysAbort);
     void addWorkunitException(ErrorSeverity severity, unsigned code, const char * msg, IHqlExpression * location);
     void useFunction(IHqlExpression * funcdef);
@@ -1113,6 +1114,7 @@ public:
     }
 
     void setTargetClusterType(ClusterType clusterType);
+    void ensureDiskAccessAllowed(IHqlExpression * expr);
     void checkAbort();
 
 public:

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10337,6 +10337,8 @@ void HqlCppTranslator::buildSerializedLayoutMember(BuildCtx & ctx, IHqlExpressio
 
 ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
 {
+    ensureDiskAccessAllowed(expr);
+
     IHqlExpression * dataset  = expr->queryChild(0);
     IHqlExpression * filename = queryRealChild(expr, 1);
     IHqlExpression * record = dataset->queryRecord();

ecl/hqlcpp/hqlresource.cpp

@@ -2651,6 +2651,7 @@ IHqlExpression * SpillerInfo::createSpilledRead(IHqlExpression * spillReason)
         args.append(*createValue(no_thor));
         addSpillFlags(args, true);
         args.append(*createUniqueId());
+        args.append(*createExprAttribute(_signed_Atom, createConstant("hpcc")));
         if (options->isChildQuery && options->targetRoxie())
         {
             args.append(*createAttribute(_colocal_Atom));

ecl/hqlcpp/hqlsource.cpp

@@ -2645,7 +2645,7 @@ class DiskReadBuilderBase : public SourceBuilder
 public:
     DiskReadBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
         : SourceBuilder(_translator, _tableExpr, _nameExpr), monitors(_tableExpr, _translator, 0, true)
-    { 
+    {
         fpos.setown(getFilepos(tableExpr, false));
         lfpos.setown(getFilepos(tableExpr, true));
         logicalFilenameMarker.setown(getFileLogicalName(tableExpr));
@@ -2971,6 +2971,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDiskRead(BuildCtx & ctx, IHqlE
 
     if (isPiped)
         return info.buildActivity(ctx, expr, TAKpiperead, "PipeRead", NULL);
+    ensureDiskAccessAllowed(tableExpr);
     if (modeOp == no_csv)
         return info.buildActivity(ctx, expr, TAKcsvread, "CsvRead", NULL);
     return info.buildActivity(ctx, expr, TAKdiskread, "DiskRead", NULL);
@@ -3029,6 +3030,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDiskNormalize(BuildCtx & ctx,
 {
     assertex(!isGroupedActivity(expr));
     IHqlExpression *tableExpr = queryPhysicalRootTable(expr);
+    ensureDiskAccessAllowed(tableExpr);
     HqlExprAttr mode = tableExpr->queryChild(2);
     assertex(mode->getOperator()!=no_pipe);
 
@@ -3186,6 +3188,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDiskAggregate(BuildCtx & ctx,
 {
     assertex(!isGroupedActivity(expr));
     IHqlExpression *tableExpr = queryPhysicalRootTable(expr);
+    ensureDiskAccessAllowed(tableExpr);
+
     HqlExprAttr mode = tableExpr->queryChild(2);
     assertex(mode->getOperator()!=no_pipe);
 
@@ -3263,6 +3267,8 @@ void DiskGroupAggregateBuilder::buildTransform(IHqlExpression * expr)
 ABoundActivity * HqlCppTranslator::doBuildActivityDiskGroupAggregate(BuildCtx & ctx, IHqlExpression * expr)
 {
     IHqlExpression *tableExpr = queryPhysicalRootTable(expr);
+    ensureDiskAccessAllowed(tableExpr);
+
     HqlExprAttr mode = tableExpr->queryChild(2);
     assertex(mode->getOperator()!=no_pipe);
 
@@ -6513,7 +6519,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityIndexRead(BuildCtx & ctx, IHql
     //If the filter is false, then it may get reduced to a NULL operation!
     if (!tableExpr)
         return buildNullIndexActivity(*this, ctx, optimized);
-
+    ensureDiskAccessAllowed(tableExpr);
     if (optimized->getOperator() != no_compound_indexread)
         optimized.setown(createDataset(no_compound_indexread, LINK(optimized)));
 
@@ -6590,6 +6596,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityIndexNormalize(BuildCtx & ctx,
     IHqlExpression *tableExpr = queryPhysicalRootTable(optimized);
     if (!tableExpr)
         return buildNullIndexActivity(*this, ctx, optimized);
+    ensureDiskAccessAllowed(tableExpr);
 
     assertex(tableExpr->getOperator() == no_newkeyindex);
     IndexNormalizeBuilder info(*this, tableExpr, tableExpr->queryChild(3));
@@ -6757,6 +6764,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityIndexAggregate(BuildCtx & ctx,
     IHqlExpression *tableExpr = queryPhysicalRootTable(optimized);
     if (!tableExpr)
         return buildNullIndexActivity(*this, ctx, optimized);
+    ensureDiskAccessAllowed(tableExpr);
 
     assertex(tableExpr->getOperator() == no_newkeyindex);
     node_operator aggOp = querySimpleAggregate(expr, true, false);
@@ -6919,6 +6927,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityIndexGroupAggregate(BuildCtx &
     IHqlExpression *tableExpr = queryPhysicalRootTable(optimized);
     if (!tableExpr)
         return buildNullIndexActivity(*this, ctx, optimized);
+    ensureDiskAccessAllowed(tableExpr);
 
     IHqlExpression * aggregate = expr->queryChild(0);
     assertex(aggregate->getOperator() == no_newaggregate || aggregate->getOperator() == no_aggregate);
@@ -7094,6 +7103,7 @@ void HqlCppTranslator::buildCsvReadTransformer(IHqlExpression * dataset, StringB
 ABoundActivity * HqlCppTranslator::doBuildActivityXmlRead(BuildCtx & ctx, IHqlExpression * expr)
 {
     IHqlExpression * tableExpr = expr;
+    ensureDiskAccessAllowed(tableExpr);
     IHqlExpression * filename = tableExpr->queryChild(0);
     IHqlExpression * mode = tableExpr->queryChild(2);
     node_operator modeType = mode->getOperator();

ecl/hqlcpp/hqlttcpp.cpp

@@ -5480,6 +5480,7 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
             args.append(*LINK(record));
             args.append(*createValue(no_thor));
             args.append(*createAttribute(_noVirtual_Atom));         // don't interpret virtual fields in spilled output
+            args.append(*createExprAttribute(_signed_Atom, createConstant("hpcc")));
 
             if (persistOp == no_persist)
                 args.append(*createAttribute(_workflowPersist_Atom));

ecl/regress/signed1.ecl

@@ -11,11 +11,11 @@ OUTPUT(match1);
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1
 
-iQEcBAEBAgAGBQJXu0KjAAoJEG37B3CNqkF936sH/3vF0RUUzDE6qCPoIYYq69mO
-R+B6gvXuWngYdjZcy1zXuHrHioBY/yt8IYUwdzdLABmq19/kT3W57zG6gWSAqlGM
-XxHnPeSwtwWexUG7rYAuQ7UdtsxXCxfgs8VmMVLgJhkNSWpURYryFp9IWefwlTJD
-P+WUjVM3QnfRLS0Y3xfX0f/uWlmYgY9pDqSLs60+63S+dgxB5fFoJuaUh/iFv/Bk
-SQWPr+A0tyTymUTX5Qi/CZZ2Y7TQ+Qqd2LRvosOUw5j5Lk/nlUTk8C+zVwl1YYw4
-/KjcjM+6zKD0NVO0Y2hSqDXN+OXqXKsoURM7qQwcqTH40CM8rQ7a6tPpNsK3vo0=
-=ML2W
+iQEcBAEBAgAGBQJZJupdAAoJEG37B3CNqkF9V4AH/RLzf+a57pIbYOFnJzFMm9Qi
+I4MW83RFbjnwIy9+QXyCb5+43VtpbZY1/xSjU4ln9H2hrnTMbDVLfGsutiE4rIr/
+ahmLAaLQ2wt64EiF3Bnd6iZUJSzTUGU+BcoEv/jnryryfh/KpCXQHQwcuUGTby4e
+Lc+U675/+kUyDSGkUrK6NDir5bnpZOeWiCaV/lSzWcO9yhbdbE2XWQ5PpGzCvHNg
+VLY4shgMUQAFosrjluMKXU024EdqZVeJRDjyObQYXxGwZ927STcDjwOIpiQtuCad
+QrIskkxZrikOG/yz+/4NOUArhYaQvBtJHuk6GbOhV9JLER6eNOE+PQq+NOLaWlA=
+=imCB
 -----END PGP SIGNATURE-----

ecl/regress/signed2.ecl

@@ -38,11 +38,11 @@ outputraw;
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1
 
-iQEcBAEBAgAGBQJXu0KqAAoJEG37B3CNqkF9McMH/2yv9aM4xsItYWxL90HbeMvR
-Gw1YQVdU6MYU+vCvrDBlfIIWGRr+Mw/1mGyFvUU2FuniXk1yKpdE/aDvGg/eQFzP
-n7lJC+bvWFOYt+0rbF7dC+ixRjltKck+d4m1l/V0BywXPqJfonptLdeV8DedAF6Z
-aLZA1KX0IVvkyDy9JmJA0M444gwGQiHTJJcDdG87NwNwYATOr/Ar4X2eX9vXaNLi
-8u4NAEDfc3XOv58sGKcZ+nj70vbx9LFFZuwRFVgPNxHHaMM2Abl4zKySKCRXCyjM
-2iedVwBhBFCyN5xOcYJzl2uOCgtoj7L9LhJ7XnJkHN698yJL4OqIPbxGi5L4Q2w=
-=nvBc
+iQEcBAEBAgAGBQJZJuqQAAoJEG37B3CNqkF9rYkIALn8qRE0jYvcBd+seNhT+cRL
+MRQGbPGdv4XTRnToJVvmMztglTp9o0jmpnbCiAQ7iSrJLBr2YRHesY5BtFM2RcOD
+uQ2UxxHOOAUk3KcXY/QSnWR8vWj2dCwPzqkIFvf9NWWZiMjwpKeFuklYa7u8xS8R
+UlmLFvFdtaa9SJjzMVjKh6WgQaCjiK7GV4vqDytNXIPczX/in1NoIX0FbxZicKzj
+Jt4jhE1oymvw+KlgmvrVs/Mi1V0wIwGGQE4kNzBxK79zww9ttN1GlBD6knrzSDgv
+2FSo1618z20DdZY+sGlTPZ2XRVs+xL9EzIe2FrvmtjjGvKVngrGsNzulDqhVnoE=
+=ovMK
 -----END PGP SIGNATURE-----

ecl/regress/signedtests/regress.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+###############################################################################
+#  HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+##############################################################################
+# Instructions:
+#   Same as options as regress.sh in parent directory
+#   (calls regress.sh in parent directory)
+../regress.sh $* -x "--allowsigned datafile"

ecl/regress/signedtests/signed1.ecl

@@ -0,0 +1,21 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Rawfile := DATASET('TST::postcodes', { string8 postcode, UNSIGNED8
+                                      __filepos {virtual(fileposition)}}, FLAT);
+INDX_Postcode := INDEX(Rawfile, {postcode, __filepos}, 'TST::postcode.key');
+
+match1 := INDX_Postcode( postcode = 'KT19 1AA' );
+
+OUTPUT(match1);
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1
+
+iQEcBAEBAgAGBQJXu0KjAAoJEG37B3CNqkF936sH/3vF0RUUzDE6qCPoIYYq69mO
+R+B6gvXuWngYdjZcy1zXuHrHioBY/yt8IYUwdzdLABmq19/kT3W57zG6gWSAqlGM
+XxHnPeSwtwWexUG7rYAuQ7UdtsxXCxfgs8VmMVLgJhkNSWpURYryFp9IWefwlTJD
+P+WUjVM3QnfRLS0Y3xfX0f/uWlmYgY9pDqSLs60+63S+dgxB5fFoJuaUh/iFv/Bk
+SQWPr+A0tyTymUTX5Qi/CZZ2Y7TQ+Qqd2LRvosOUw5j5Lk/nlUTk8C+zVwl1YYw4
+/KjcjM+6zKD0NVO0Y2hSqDXN+OXqXKsoURM7qQwcqTH40CM8rQ7a6tPpNsK3vo0=
+=ML2W
+-----END PGP SIGNATURE-----

ecl/regress/signedtests/signed2.ecl

@@ -0,0 +1,48 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and 
+    limitations under the License.
+############################################################################## */
+
+postcodes := DATASET([{'KT19 1AA'}, {'KT19 1AB'}, {'KT19 1AC'}, {'KT19 1AD'},
+                      {'KT20 1AE'}, {'KT20 1AF'}, {'KT20 1AG'}, {'KT20 1AH'}, 
+                      {'KT21 1AI'}, {'KT21 1AJ'}, {'KT21 1AK'}, {'KT21 1AL'}, 
+                      {'KT22 1AM'}, {'KT22 1AN'}, {'KT22 1AO'}, {'KT22 1AA'},
+                      {'KT23 1AB'}, {'KT23 1AC'}, {'KT23 1AD'}, {'KT23 1AE'}, 
+                      {'KT30 1AF'}, {'KT30 1AG'}, {'KT30 1AH'}, {'KT30 1AI'},
+                      {'KT31 1AJ'}, {'KT31 1AK'}, {'KT31 1AL'}, {'KT31 1AM'}, 
+                      {'KT32 1AN'}, {'KT32 1AO'}, {'KT32 1AA'}, {'KT32 1AB'}, 
+                      {'KT40 1AC'}, {'KT40 1AD'}, {'KT40 1AE'}, {'KT40 1AF'}, 
+                      {'KT41 1AG'}, {'KT41 1AH'}, {'KT41 1AI'}, {'KT41 1AJ'}, 
+                      {'KT41 1AK'}, {'KT41 1AL'}, {'KT41 1AM'}, {'KT41 1AN'}, 
+                      {'KT3'}, {'KT4'}], {string8 postcode});
+
+
+outputraw := OUTPUT(postcodes,,'TST::postcodes', OVERWRITE);
+
+outputraw;
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1
+
+iQEcBAEBAgAGBQJXu0KqAAoJEG37B3CNqkF9McMH/2yv9aM4xsItYWxL90HbeMvR
+Gw1YQVdU6MYU+vCvrDBlfIIWGRr+Mw/1mGyFvUU2FuniXk1yKpdE/aDvGg/eQFzP
+n7lJC+bvWFOYt+0rbF7dC+ixRjltKck+d4m1l/V0BywXPqJfonptLdeV8DedAF6Z
+aLZA1KX0IVvkyDy9JmJA0M444gwGQiHTJJcDdG87NwNwYATOr/Ar4X2eX9vXaNLi
+8u4NAEDfc3XOv58sGKcZ+nj70vbx9LFFZuwRFVgPNxHHaMM2Abl4zKySKCRXCyjM
+2iedVwBhBFCyN5xOcYJzl2uOCgtoj7L9LhJ7XnJkHN698yJL4OqIPbxGi5L4Q2w=
+=nvBc
+-----END PGP SIGNATURE-----

ecl/regress/signedtests/signed_error.ecl

@@ -0,0 +1,19 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Rawfile := DATASET('TST::postcodes', { string8 postcode, UNSIGNED8
+                                      __filepos {virtual(fileposition)}}, FLAT);
+
+
+OUTPUT(Rawfile);
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1
+
+iQEcBAEBAgAGBQJXu0KjAAoJEG37B3CNqkF936sH/3vF0RUUzDE6qCPoIYYq69mO
+R+B6gvXuWngYdjZcy1zXuHrHioBY/yt8IYUwdzdLABmq19/kT3W57zG6gWSAqlGM
+XxHnPeSwtwWexUG7rYAuQ7UdtsxXCxfgs8VmMVLgJhkNSWpURYryFp9IWefwlTJD
+P+WUjVM3QnfRLS0Y3xfX0f/uWlmYgY9pDqSLs60+63S+dgxB5fFoJuaUh/iFv/Bk
+SQWPr+A0tyTymUTX5Qi/CZZ2Y7TQ+Qqd2LRvosOUw5j5Lk/nlUTk8C+zVwl1YYw4
+/KjcjM+6zKD0NVO0Y2hSqDXN+OXqXKsoURM7qQwcqTH40CM8rQ7a6tPpNsK3vo0=
+=ML2W
+-----END PGP SIGNATURE-----

ecl/regress/signedtests/unsigned1.ecl

@@ -0,0 +1,5 @@
+Rawfile := DATASET('TST::postcodes', { string8 postcode, UNSIGNED8
+                                      __filepos {virtual(fileposition)}}, FLAT);
+INDX_Postcode := INDEX(Rawfile, {postcode, __filepos}, 'TST::postcode.key');
+
+BUILD(INDX_Postcode);

ecl/regress/signedtests/unsigned2.ecl

@@ -0,0 +1,7 @@
+Rawfile := DATASET('TST::postcodes', { string8 postcode, UNSIGNED8
+                                      __filepos {virtual(fileposition)}}, FLAT);
+INDX_Postcode := INDEX(Rawfile, {postcode, __filepos}, 'TST::postcode.key');
+
+match1 := INDX_Postcode( postcode = 'KT19 1AA' );
+
+OUTPUT(match1);