HPCC-13243 Indicate Opt and Compulsory in Packagemap validation report

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/workunit/pkgimpl.hpp

@@ -528,7 +528,7 @@ public:
                         unmatchedQueries.append(libname);
                 }
 
-                filelist->addFilesFromQuery(cw, this, queryid);
+                bool isCompulsory = filelist->addFilesFromQuery(cw, this, queryid);
 
                 unsigned unmatchedCount=0;
                 Owned<IReferencedFileIterator> refFiles = filelist->getFiles();
@@ -559,6 +559,10 @@ public:
                         continue;
                     }
                     VStringBuffer fullname("%s/%s", queryid, rf.getLogicalName());
+                    if (!(flags & RefFileNotOptional))
+                        fullname.append("/Optional");
+                    else if (isCompulsory)
+                        fullname.append("/Compulsory");
                     unmatchedFiles.append(fullname);
                     unmatchedCount++;
                 }

common/workunit/referencedfilelist.cpp

@@ -220,8 +220,8 @@ public:
     virtual void addFile(const char *ln, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
     virtual void addFiles(StringArray &files);
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw);
-    virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid);
-    virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg);
+    virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid);
+    virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg);
     virtual void addFilesFromPackageMap(IPropertyTree *pm);
 
     void addFileFromSubFile(IPropertyTree &subFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix);
@@ -612,7 +612,7 @@ void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm)
         addFilesFromPackage(packages->query(), ip, cluster, prefix);
 }
 
-void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)
+bool ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)
 {
     Owned<IConstWUGraphIterator> graphs = &cw->getGraphs(GraphTypeActivities);
     ForEach(*graphs)
@@ -622,11 +622,15 @@ void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackag
         ForEach(*iter)
         {
             IPropertyTree &node = iter->query();
+            bool isOpt = false;
             const char *logicalName = node.queryProp("att[@name='_fileName']/@value");
             if (!logicalName)
-                logicalName = node.queryProp("att[@name='_indexFileName']/@value");
-            if (!logicalName)
                 continue;
+
+            isOpt = node.getPropBool("att[@name='_isIndexOpt']/@value");
+            if (!isOpt)
+                isOpt = node.getPropBool("att[@name='_isOpt']/@value");
+
             ThorActivityKind kind = (ThorActivityKind) node.getPropInt("att[@name='_kind']/@value", TAKnone);
             //not likely to be part of roxie queries, but for forward compatibility:
             if(kind==TAKdiskwrite || kind==TAKindexwrite || kind==TAKcsvwrite || kind==TAKxmlwrite || kind==TAKjsonwrite)
@@ -634,7 +638,7 @@ void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackag
             if (node.getPropBool("att[@name='_isSpill']/@value") ||
                 node.getPropBool("att[@name='_isTransformSpill']/@value"))
                 continue;
-            unsigned flags = 0;
+            unsigned flags = isOpt ? RefFileOptional : RefFileNotOptional;
             if (pkg)
             {
                 const char *pkgid = pkg->locateSuperFile(logicalName);
@@ -659,14 +663,17 @@ void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackag
                 ensureFile(logicalName, flags, NULL, false);
         }
     }
+    return pkg ? pkg->isCompulsory() : false;
 }
 
-void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)
+bool ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)
 {
     const IHpccPackage *pkg = NULL;
     if (pm && queryid && *queryid)
+    {
         pkg = pm->matchPackage(queryid);
-    addFilesFromQuery(cw, pkg);
+    }
+    return addFilesFromQuery(cw, pkg);
 }
 
 void ReferencedFileList::addFilesFromWorkUnit(IConstWorkUnit *cw)

common/workunit/referencedfilelist.hpp

@@ -23,18 +23,21 @@
 #include "package.h"
 #include "dfuutil.hpp"
 
-#define RefFileNone           0x000
-#define RefFileIndex          0x001
-#define RefFileNotOnCluster   0x002
-#define RefFileNotFound       0x004
-#define RefFileRemote         0x008
-#define RefFileForeign        0x010
-#define RefFileSuper          0x020
-#define RefSubFile            0x040
-#define RefFileCopyInfoFailed 0x080
-#define RefFileCloned         0x100
-#define RefFileInPackage      0x200
-#define RefFileNotOnSource    0x400
+#define RefFileNone           0x0000
+#define RefFileIndex          0x0001
+#define RefFileNotOnCluster   0x0002
+#define RefFileNotFound       0x0004
+#define RefFileRemote         0x0008
+#define RefFileForeign        0x0010
+#define RefFileSuper          0x0020
+#define RefSubFile            0x0040
+#define RefFileCopyInfoFailed 0x0080
+#define RefFileCloned         0x0100
+#define RefFileInPackage      0x0200
+#define RefFileNotOnSource    0x0400
+#define RefFileOptional       0x0800 //File referenced in more than one place can be both optional and not optional
+#define RefFileNotOptional    0x1000
+
 
 interface IReferencedFile : extends IInterface
 {
@@ -52,8 +55,8 @@ interface IReferencedFileIterator : extends IIteratorOf<IReferencedFile> { };
 interface IReferencedFileList : extends IInterface
 {
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw)=0;
-    virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)=0;
-    virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)=0;
+    virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)=0;
+    virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)=0;
     virtual void addFilesFromPackageMap(IPropertyTree *pm)=0;
 
     virtual void addFile(const char *ln, const char *daliip=NULL, const char *sourceProcessCluster=NULL, const char *remotePrefix=NULL)=0;

ecl/ecl-package/ecl-package.cpp

@@ -716,6 +716,32 @@ public:
         }
         return true;
     }
+
+    void outputLfnCategoryTree(IPropertyTree &querynode, const char *name)
+    {
+        StringBuffer lcname(name);
+        IPropertyTree *catnode = querynode.queryPropTree(lcname.toLowerCase());
+        if (!catnode)
+            return;
+        fprintf(stderr, "        [%s]\n", name);
+
+        Owned<IPropertyTreeIterator> lfnnodes = catnode->getElements("*");
+        ForEach(*lfnnodes)
+            fprintf(stderr, "          %s\n", lfnnodes->query().queryName());
+    }
+    void outputLfnTree(IPropertyTree *lfntree)
+    {
+        Owned<IPropertyTreeIterator> querynodes = lfntree->getElements("*");
+        ForEach(*querynodes)
+        {
+            IPropertyTree &querynode = querynodes->query();
+            fprintf(stderr, "     --%s\n", querynode.queryName());
+            outputLfnCategoryTree(querynode, "Compulsory");
+            outputLfnCategoryTree(querynode, "Required");
+            outputLfnCategoryTree(querynode, "Optional");
+        }
+    }
+
     virtual int processCMD()
     {
         Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
@@ -778,9 +804,25 @@ public:
         if (unusedFiles.ordinality()>0)
         {
             validateMessages = true;
-            fputs("\n   Files without matching package definitions:\n", stderr);
+            fputs("\n   Query files without matching package definitions:\n", stderr);
+            Owned<IPropertyTree> filetree = createPTree();
             ForEachItemIn(i, unusedFiles)
-                fprintf(stderr, "      %s\n", unusedFiles.item(i));
+            {
+                StringArray info;
+                info.appendList(unusedFiles.item(i), "/");
+                if (info.length()>=2)
+                {
+                    IPropertyTree *querynode = ensurePTree(filetree, info.item(0));
+                    if (querynode)
+                    {
+                        StringBuffer category = (info.length()>=3) ? info.item(2) : "required";
+                        IPropertyTree *cat = ensurePTree(querynode, category.toLowerCase());
+                        if (cat)
+                            ensurePTree(cat, info.item(1));
+                    }
+                }
+            }
+            outputLfnTree(filetree);
         }
 
         StringArray &notInDFS = resp->getFiles().getNotInDFS();