HPCC-10864: Command option to fail publish if query has foreign files

For roxie, add --no-foreign option to ecl-publish, ecl-queries-copy,
and ecl-packagemap-add to fail if foreign files are being referenced.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/workunit/pkgimpl.hpp

@@ -384,7 +384,7 @@ public:
         Owned<IPropertyTree> query = resolveQueryAlias(querySet, queryname, true);
         if (!query)
             throw MakeStringException(PACKAGE_QUERY_NOT_FOUND, "Query %s not found", queryname);
-        Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, NULL);
+        Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, NULL, true);
         Owned<IWorkUnitFactory> wufactory = getWorkUnitFactory(NULL, NULL);
         Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(query->queryProp("@wuid"), false);
 
@@ -462,7 +462,7 @@ public:
             const char *queryid = queries->query().queryProp("@id");
             if (queryid && *queryid)
             {
-                Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, NULL);
+                Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, NULL, true);
                 Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(queries->query().queryProp("@wuid"), false);
 
                 StringArray libnames, unresolvedLibs;

common/workunit/referencedfilelist.cpp

@@ -36,6 +36,21 @@ bool getIsOpt(const IPropertyTree &graphNode)
         return graphNode.getPropBool("att[@name='_isIndexOpt']/@value", false);
 }
 
+bool checkForeign(const char *lfn)
+{
+    if (*lfn=='~')
+        lfn++;
+    static size_t l = strlen("foreign");
+    if (strncasecmp("foreign", lfn, l)==0)
+    {
+        lfn+=l;
+        while (isspace(*lfn))
+            lfn++;
+        if (lfn[0]==':' && lfn[1]==':')
+            return true;
+    }
+    return false;
+}
 const char *skipForeign(const char *name, StringBuffer *ip)
 {
     if (*name=='~')
@@ -164,7 +179,7 @@ class ReferencedFileList : public CInterface, implements IReferencedFileList
 {
 public:
     IMPLEMENT_IINTERFACE;
-    ReferencedFileList(const char *username, const char *pw)
+    ReferencedFileList(const char *username, const char *pw, bool allowForeignFiles) : allowForeign(allowForeignFiles)
     {
         if (username && pw)
         {
@@ -204,6 +219,7 @@ public:
     StringAttr process;
     StringAttr srcCluster;
     StringAttr remotePrefix;
+    bool allowForeign;
 };
 
 void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstCluster, const char *srcCluster, StringArray *subfiles)
@@ -461,6 +477,9 @@ public:
 
 void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip, const char *srcCluster, const char *prefix)
 {
+    if (!allowForeign && checkForeign(ln))
+        throw MakeStringException(-1, "Foreign file not allowed%s: %s", (flags & RefFileInPackage) ? " (declared in package)" : "", ln);
+
     Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid);
     if (!file->logicalName.length())
         return;
@@ -597,7 +616,11 @@ void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalF
     StringArray childSubFiles;
     ForEachItemIn(i, subfiles)
     {
-        Owned<ReferencedFile> file = new ReferencedFile(subfiles.item(i), NULL, NULL, NULL, true, 0, NULL);
+        const char *lfn = subfiles.item(i);
+        if (!allowForeign && checkForeign(lfn))
+            throw MakeStringException(-1, "Foreign sub file not allowed: %s", lfn);
+
+        Owned<ReferencedFile> file = new ReferencedFile(lfn, NULL, NULL, NULL, true, 0, NULL);
         if (file->logicalName.length() && !map.getValue(file->getLogicalName()))
         {
             file->resolve(process.get(), srcCluster, user, remote, remotePrefix, checkLocalFirst, &childSubFiles, resolveForeign);
@@ -676,7 +699,7 @@ IReferencedFileIterator *ReferencedFileList::getFiles()
     return new ReferencedFileIterator(this);
 }
 
-IReferencedFileList *createReferencedFileList(const char *user, const char *pw)
+IReferencedFileList *createReferencedFileList(const char *user, const char *pw, bool allowForeignFiles)
 {
-    return new ReferencedFileList(user, pw);
+    return new ReferencedFileList(user, pw, allowForeignFiles);
 }

common/workunit/referencedfilelist.hpp

@@ -66,7 +66,7 @@ interface IReferencedFileList : extends IInterface
 
 extern WORKUNIT_API const char *skipForeign(const char *name, StringBuffer *ip=NULL);
 
-extern WORKUNIT_API IReferencedFileList *createReferencedFileList(const char *user, const char *pw);
+extern WORKUNIT_API IReferencedFileList *createReferencedFileList(const char *user, const char *pw, bool allowForeignFiles);
 
 extern WORKUNIT_API void splitDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster);
 extern WORKUNIT_API void splitDerivedDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster, const char *baseCluster, const char *baseIP, const char *basePrefix);

ecl/ecl-package/ecl-package.cpp

@@ -498,7 +498,7 @@ private:
 class EclCmdPackageAdd : public EclCmdCommon
 {
 public:
-    EclCmdPackageAdd() : optActivate(false), optOverWrite(false), optGlobalScope(false)
+    EclCmdPackageAdd() : optActivate(false), optOverWrite(false), optGlobalScope(false), optNoForeign(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -537,6 +537,8 @@ public:
                 continue;
             if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
                 continue;
+            if (iter.matchFlag(optNoForeign, ECLOPT_NO_FOREIGN))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -592,6 +594,7 @@ public:
         request->setOverWrite(optOverWrite);
         request->setGlobalScope(optGlobalScope);
         request->setSourceProcess(optSourceProcess);
+        request->setAllowForeignFiles(!optNoForeign);
 
         Owned<IClientAddPackageResponse> resp = packageProcessClient->AddPackage(request);
         if (resp->getExceptions().ordinality())
@@ -623,6 +626,7 @@ public:
                     "   --pmid                      Identifier of package map - defaults to filename if not specified\n"
                     "   --global-scope              The specified packagemap can be shared across multiple targets\n"
                     "   --source-process            Process cluster to copy files from\n"
+                    "   --no-foreign                Fail if foreign files are used in packagemap\n"
                     "   <target>                    Name of target to use when adding package map information\n"
                     "   <filename>                  Name of file containing package map information\n",
                     stdout);
@@ -640,6 +644,7 @@ private:
     bool optActivate;
     bool optOverWrite;
     bool optGlobalScope;
+    bool optNoForeign;
 };
 
 class EclCmdPackageValidate : public EclCmdCommon

ecl/eclcmd/eclcmd_common.hpp

@@ -65,6 +65,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_OVERWRITE_ENV NULL
 
 #define ECLOPT_DONT_COPY_FILES "--no-files"
+#define ECLOPT_NO_FOREIGN "--no-foreign"
 
 #define ECLOPT_ACTIVE "--active"
 #define ECLOPT_ALL "--all"

ecl/eclcmd/eclcmd_core.cpp

@@ -195,7 +195,7 @@ class EclCmdPublish : public EclCmdWithEclTarget
 {
 public:
     EclCmdPublish() : optNoActivate(false), optSuspendPrevious(false), optDeletePrevious(false),
-        activateSet(false), optNoReload(false), optDontCopyFiles(false), optMsToWait(10000)
+        activateSet(false), optNoReload(false), optDontCopyFiles(false), optMsToWait(10000), optNoForeign(false)
     {
         optObj.accept = eclObjWuid | eclObjArchive | eclObjSharedObject;
         optTimeLimit = (unsigned) -1;
@@ -233,6 +233,8 @@ public:
                 continue;
             if (iter.matchFlag(optDontCopyFiles, ECLOPT_DONT_COPY_FILES))
                 continue;
+            if (iter.matchFlag(optNoForeign, ECLOPT_NO_FOREIGN))
+                continue;
             if (iter.matchFlag(optNoActivate, ECLOPT_NO_ACTIVATE))
             {
                 activateSet=true;
@@ -325,6 +327,7 @@ public:
         req->setWait(optMsToWait);
         req->setNoReload(optNoReload);
         req->setDontCopyFiles(optDontCopyFiles);
+        req->setAllowForeignFiles(!optNoForeign);
 
         if (optTimeLimit != (unsigned) -1)
             req->setTimeLimit(optTimeLimit);
@@ -381,6 +384,7 @@ public:
             "   -A-, --no-activate     Do not activate query when published\n"
             "   --no-reload            Do not request a reload of the (roxie) cluster\n"
             "   --no-files             Do not copy files referenced by query\n"
+            "   --no-foreign           Fail if foreign files are used in query (roxie)\n"
             "   --daliip=<IP>          The IP of the DALI to be used to locate remote files\n"
             "   --source-process       Process cluster to copy files from\n"
             "   --timeLimit=<ms>       Value to set for query timeLimit configuration\n"
@@ -410,6 +414,7 @@ private:
     bool optDontCopyFiles;
     bool optSuspendPrevious;
     bool optDeletePrevious;
+    bool optNoForeign;
 };
 
 class EclCmdRun : public EclCmdWithEclTarget

ecl/eclcmd/queries/ecl-queries.cpp

@@ -292,7 +292,7 @@ private:
 class EclCmdQueriesCopy : public EclCmdCommon
 {
 public:
-    EclCmdQueriesCopy() : optActivate(false), optNoReload(false), optMsToWait(10000), optDontCopyFiles(false), optOverwrite(false)
+    EclCmdQueriesCopy() : optActivate(false), optNoReload(false), optMsToWait(10000), optDontCopyFiles(false), optOverwrite(false), optNoForeign(false)
     {
         optTimeLimit = (unsigned) -1;
         optWarnTimeLimit = (unsigned) -1;
@@ -335,6 +335,8 @@ public:
                 continue;
             if (iter.matchFlag(optDontCopyFiles, ECLOPT_DONT_COPY_FILES))
                 continue;
+            if (iter.matchFlag(optNoForeign, ECLOPT_NO_FOREIGN))
+                continue;
             if (iter.matchOption(optMsToWait, ECLOPT_WAIT))
                 continue;
             if (iter.matchOption(optTimeLimit, ECLOPT_TIME_LIMIT))
@@ -391,6 +393,7 @@ public:
         req->setDontCopyFiles(optDontCopyFiles);
         req->setWait(optMsToWait);
         req->setNoReload(optNoReload);
+        req->setAllowForeignFiles(!optNoForeign);
 
         if (optTimeLimit != (unsigned) -1)
             req->setTimeLimit(optTimeLimit);
@@ -436,6 +439,7 @@ public:
             "   -A, --activate         Activate the new query\n"
             "   --no-reload            Do not request a reload of the (roxie) cluster\n"
             "   -O, --overwrite        Overwrite existing files\n"
+            "   --no-foreign           Fail if foreign files are used in query (roxie)\n"
             "   --wait=<ms>            Max time to wait in milliseconds\n"
             "   --timeLimit=<sec>      Value to set for query timeLimit configuration\n"
             "   --warnTimeLimit=<sec>  Value to set for query warnTimeLimit configuration\n"
@@ -464,6 +468,7 @@ private:
     bool optNoReload;
     bool optOverwrite;
     bool optDontCopyFiles;
+    bool optNoForeign;
 };
 
 class EclCmdQueriesConfig : public EclCmdCommon

esp/scm/ws_packageprocess.ecm

@@ -36,6 +36,7 @@ ESPrequest AddPackageRequest
     string DaliIp;
     bool GlobalScope(0);
     string SourceProcess;
+    bool AllowForeignFiles(true);
 };
 
 

esp/scm/ws_workunits.ecm

@@ -1108,6 +1108,7 @@ ESPrequest [nil_remove] WUPublishWorkunitRequest
     string Comment;
     bool DontCopyFiles(false);
     string SourceProcess;
+    bool AllowForeignFiles(true);
 };
 
 ESPresponse [exceptions_inline] WUPublishWorkunitResponse
@@ -1406,6 +1407,7 @@ ESPrequest [nil_remove] WUQuerySetCopyQueryRequest
     string priority;
     string Comment;
     string SourceProcess;
+    bool AllowForeignFiles(true);
 };
 
 ESPresponse [exceptions_inline] WUQuerySetCopyQueryResponse

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -123,7 +123,7 @@ bool isFileKnownOnCluster(const char *logicalname, const char *target, IUserDesc
     return isFileKnownOnCluster(logicalname, clusterInfo, userdesc);
 }
 
-void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, IConstWUClusterInfo *dstInfo, const char *srcCluster, const char *remotePrefix, bool overWrite, IUserDescriptor* userdesc)
+void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, IConstWUClusterInfo *dstInfo, const char *srcCluster, const char *remotePrefix, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles)
 {
     StringBuffer user;
     StringBuffer password;
@@ -134,7 +134,7 @@ void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const
         userdesc->getPassword(password);
     }
 
-    Owned<IReferencedFileList> wufiles = createReferencedFileList(user, password);
+    Owned<IReferencedFileList> wufiles = createReferencedFileList(user, password, allowForeignFiles);
     wufiles->addFilesFromPackageMap(packageMap);
     SCMStringBuffer processName;
     dstInfo->getRoxieProcess(processName);
@@ -151,13 +151,13 @@ void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const
     }
 }
 
-void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, const char *dstCluster, const char *srcCluster, const char *prefix, bool overWrite, IUserDescriptor* userdesc)
+void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, const char *dstCluster, const char *srcCluster, const char *prefix, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles)
 {
     Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(dstCluster);
     if (!clusterInfo)
         throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", dstCluster);
 
-    cloneFileInfoToDali(notFound, packageMap, lookupDaliIp, clusterInfo, srcCluster, prefix, overWrite, userdesc);
+    cloneFileInfoToDali(notFound, packageMap, lookupDaliIp, clusterInfo, srcCluster, prefix, overWrite, userdesc, allowForeignFiles);
 }
 
 
@@ -175,7 +175,7 @@ void makePackageActive(IPropertyTree *pkgSetRegistry, IPropertyTree *pkgSetTree,
 
 //////////////////////////////////////////////////////////
 
-void addPackageMapInfo(StringArray &filesNotFound, IPropertyTree *pkgSetRegistry, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, const char *prefix, IPropertyTree *packageInfo, bool activate, bool overWrite, IUserDescriptor* userdesc)
+void addPackageMapInfo(StringArray &filesNotFound, IPropertyTree *pkgSetRegistry, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, const char *prefix, IPropertyTree *packageInfo, bool activate, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles)
 {
     if (srcCluster && *srcCluster)
     {
@@ -251,7 +251,7 @@ void addPackageMapInfo(StringArray &filesNotFound, IPropertyTree *pkgSetRegistry
     }
 
     mergePTree(mapTree, baseInfo);
-    cloneFileInfoToDali(filesNotFound, mapTree, lookupDaliIp, clusterInfo, srcCluster, prefix, overWrite, userdesc);
+    cloneFileInfoToDali(filesNotFound, mapTree, lookupDaliIp, clusterInfo, srcCluster, prefix, overWrite, userdesc, allowForeignFiles);
 
     globalLock->commit();
 
@@ -547,7 +547,7 @@ bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageReque
     StringArray filesNotFound;
     StringBuffer pkgSetId;
     buildPkgSetId(pkgSetId, processName.get());
-    addPackageMapInfo(filesNotFound, pkgSetRegistry, target.get(), pmid.str(), pkgSetId.str(), daliip.str(), srcCluster.str(), prefix.str(), LINK(packageTree), activate, overWrite, userdesc);
+    addPackageMapInfo(filesNotFound, pkgSetRegistry, target.get(), pmid.str(), pkgSetId.str(), daliip.str(), srcCluster.str(), prefix.str(), LINK(packageTree), activate, overWrite, userdesc, req.getAllowForeignFiles());
     resp.setFilesNotFound(filesNotFound);
 
     StringBuffer msg;
@@ -784,7 +784,7 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
 
     if (req.getCheckDFS())
     {
-        Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword());
+        Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), true);
         pmfiles->addFilesFromPackageMap(mapTree);
         pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false);
         Owned<IReferencedFileIterator> files = pmfiles->getFiles();

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -434,7 +434,7 @@ static inline void updateQueryPriority(IPropertyTree *queryTree, const char *val
     }
 }
 
-void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const char *remoteIP, const char *remotePrefix, const char *target, const char *srcCluster, const char *queryname, bool overwrite)
+void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const char *remoteIP, const char *remotePrefix, const char *target, const char *srcCluster, const char *queryname, bool overwrite, bool allowForeignFiles)
 {
     if (!target || !*target)
         return;
@@ -446,7 +446,7 @@ void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const cha
         clusterInfo->getRoxieProcess(process);
         if (!process.length())
             return;
-        Owned<IReferencedFileList> wufiles = createReferencedFileList(context.queryUserId(), context.queryPassword());
+        Owned<IReferencedFileList> wufiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), allowForeignFiles);
         Owned<IHpccPackageSet> ps = createPackageSet(process.str());
         StringBuffer queryid;
         if (queryname && *queryname)
@@ -540,7 +540,7 @@ bool CWsWorkunitsEx::onWUPublishWorkunit(IEspContext &context, IEspWUPublishWork
     }
 
     if (!req.getDontCopyFiles())
-        copyQueryFilesToCluster(context, cw, daliIP, srcPrefix, target.str(), srcCluster, queryName.str(), false);
+        copyQueryFilesToCluster(context, cw, daliIP, srcPrefix, target.str(), srcCluster, queryName.str(), false, req.getAllowForeignFiles());
 
     WorkunitUpdate wu(&cw->lock());
     if (req.getUpdateWorkUnitName() && notEmpty(req.getJobName()))
@@ -1587,7 +1587,7 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
         StringBuffer srcCluster;
         StringBuffer srcPrefix;
         splitDerivedDfsLocation(req.getDaliServer(), srcCluster, daliIP, srcPrefix, req.getSourceProcess(), req.getSourceProcess(), remoteIP.str(), NULL);
-        copyQueryFilesToCluster(context, cw, daliIP.str(), srcPrefix, target, srcCluster, queryName.str(), req.getOverwrite());
+        copyQueryFilesToCluster(context, cw, daliIP.str(), srcPrefix, target, srcCluster, queryName.str(), req.getOverwrite(), req.getAllowForeignFiles());
     }
 
     WorkunitUpdate wu(&cw->lock());

initfiles/etc/bash_completion/ecl

@@ -89,7 +89,7 @@ _ecl_opts_queries()
             echo "--help list copy config"
             ;;
         copy)
-            echo -n "--no-reload --wait= --timeLimit= --warnTimeLimit= --memoryLimit= --priority= --daliip= "
+            echo -n "--no-reload --no=foreign --wait= --timeLimit= --warnTimeLimit= --memoryLimit= --priority= --daliip= "
             _ecl_opts_common
             ;;
         config)
@@ -136,7 +136,7 @@ _ecl_opts_packagemap()
             then
                 _ecl_opts_file
             else
-                echo -n "-O --overwrite "
+                echo -n "-O --overwrite --no-foreign"
                 _ecl_opts_common
             fi
             ;;
@@ -166,7 +166,7 @@ _ecl_opts_core_file()
                 _ecl_opts_common
                 ;;
             publish)
-                echo -n "-A --activate --no-reload --timeLimit= --warnTimeLimit= --memoryLimit= --priority= --daliip= "
+                echo -n "-A --activate --no-reload --no-foreign --timeLimit= --warnTimeLimit= --memoryLimit= --priority= --daliip= "
                 _ecl_opts_deploy
                 _ecl_opts_common
                 ;;