Merge pull request #13267 from ghalliday/issue23106

HPCC-23106 Introduce an experimental generic disk read activity

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: James McMullan <james.mcmullan@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/CMakeLists.txt

@@ -31,6 +31,7 @@ set (    SRCS
          csvsplitter.cpp 
          thorcommon.cpp 
          thorfile.cpp 
+         thormeta.cpp
          thorparse.cpp 
          thorpipe.cpp 
          thorread.cpp
@@ -53,6 +54,7 @@ set (    SRCS
          csvsplitter.hpp 
          thorcommon.hpp 
          thorfile.hpp 
+         thormeta.hpp
          thorparse.hpp 
          thorpipe.hpp 
          thorread.hpp
@@ -70,6 +72,8 @@ set (    SRCS
          thorsort.hpp
          persistent.cpp
          persistent.hpp
+         thorstore.cpp
+         thorstore.hpp
          
          roxiedebug.hpp
          roxiedebug.ipp

common/thorhelper/roxiehelper.cpp

@@ -2669,12 +2669,16 @@ StringBuffer & mangleHelperFileName(StringBuffer & out, const char * in, const c
 {
     out = in;
     if (flags & (TDXtemporary | TDXjobtemp))
-        out.append("__").append(wuid);
+        out.append("__").appendLower(wuid);
     return out;
 }
 
-StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in)
+
+//Replace any occurrences of :: in the logical name with __scope__, and optionally append with the wuid
+StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in, const char * wuid)
 {
+    if (*in == '~')
+        in++;
     char const * start = in;
     while(true)
     {
@@ -2690,6 +2694,8 @@ StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in)
             break;
         }
     }
+    if (wuid)
+        out.append("__").appendLower(wuid);
     return out;
 }
 
@@ -2735,6 +2741,10 @@ StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fn
         sb.replaceString("::",PATHSEPSTR);
         makeAbsolutePath(sb.str(), logicalName.clear());
     }
+    else if (strchr(fname, PATHSEPCHAR))
+    {
+        logicalName.append(fname);
+    }
     else
     {
         SCMStringBuffer lfn;

common/thorhelper/roxiehelper.hpp

@@ -564,7 +564,7 @@ private:
 //==============================================================================================================
 
 THORHELPER_API StringBuffer & mangleHelperFileName(StringBuffer & out, const char * in, const char * wuid, unsigned int flags);
-THORHELPER_API StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in);
+THORHELPER_API StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in, const char * optWuid);
 THORHELPER_API StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally, bool ignoreForeignPrefix);
 
 THORHELPER_API ISectionTimer * queryNullSectionTimer();

common/thorhelper/thorcommon.cpp

@@ -2023,14 +2023,28 @@ static IOutputMetaData *_getDaliLayoutInfo(MemoryBuffer &layoutBin, IPropertyTre
                 error.setown(E); // Save to throw later if we can't recover via ECL
             }
         }
-        if (props.hasProp("ECL"))
+        if (props.hasProp("meta"))
+        {
+            props.getPropBin("meta", layoutBin);
+            try
+            {
+                return createTypeInfoOutputMetaData(layoutBin, isGrouped);
+            }
+            catch (IException *E)
+            {
+                EXCLOG(E);
+                error.setown(E); // Save to throw later if we can't recover via ECL
+            }
+        }
+        const char * layoutECL = props.queryProp("ECL");
+        if (!layoutECL)
+            layoutECL = props.queryProp("@ecl");
+        if (layoutECL)
         {
             const char *kind = props.queryProp("@kind");
             bool isIndex = (kind && streq(kind, "key"));
-            StringBuffer layoutECL;
-            props.getProp("ECL", layoutECL);
             MultiErrorReceiver errs;
-            Owned<IHqlExpression> expr = parseQuery(layoutECL.str(), &errs);
+            Owned<IHqlExpression> expr = parseQuery(layoutECL, &errs);
             if (expr && (errs.errCount() == 0))
             {
                 if (props.hasProp("_record_layout"))  // Some old indexes need the payload count patched in from here

common/thorhelper/thormeta.cpp

@@ -0,0 +1,531 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "jsocket.hpp"
+#include "thorfile.hpp"
+
+#include "eclhelper.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
+
+#include "dautils.hpp"
+#include "dadfs.hpp"
+#include "dameta.hpp"
+
+#include "thormeta.hpp"
+#include "rtlcommon.hpp"
+#include "thorcommon.hpp"
+
+//Should be common with agentctx.hpp
+#define WRN_SkipMissingOptIndex             5400
+#define WRN_SkipMissingOptFile              5401
+#define WRN_UseLayoutTranslation            5402
+#define WRN_UnsupportedAlgorithm            5403
+#define WRN_MismatchGroupInfo               5404
+#define WRN_MismatchCompressInfo            5405
+#define WRN_RemoteReadFailure               5406
+
+//Default format - if not specified in ecl and not known to dali etc.
+static constexpr const char * defaultFileFormat = "flat";
+
+IPropertyTree * resolveLogicalFilename(const char * filename, IUserDescriptor * user, ResolveOptions options)
+{
+    //This may go via esp instead at some point....
+    return resolveLogicalFilenameFromDali(filename, user, options);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+CLogicalFile::CLogicalFile(const CStorageSystems & storage, const IPropertyTree * _metaXml, IOutputMetaData * _expectedMeta)
+: metaTree(_metaXml), expectedMeta(_expectedMeta)
+{
+    name = metaTree->queryProp("@name");
+    numParts = metaTree->getPropInt("@numParts", 1);
+    fileSize = metaTree->getPropInt64("@rawSize"); // logical file size - used for file positions
+    mergedMeta.set(metaTree);
+
+    parts.reserve(numParts);
+    Owned<IPropertyTreeIterator> partIter = metaTree->getElements("part");
+    if (partIter->first())
+    {
+        offset_t baseOffset = 0;
+        do
+        {
+            offset_t partSize = partIter->query().getPropInt("@rawSize");
+            offset_t numRows = partIter->query().getPropInt("@numRows");
+            parts.emplace_back(parts.size(), numRows, partSize, baseOffset);
+            baseOffset += partSize;
+        } while (partIter->next());
+        assertex(parts.size() == numParts);
+
+        if (baseOffset)
+        {
+            assertex(fileSize == 0 || fileSize == baseOffset);
+            fileSize = baseOffset;
+        }
+    }
+    else
+    {
+        for (unsigned part=0; part < numParts; part++)
+            parts.emplace_back(part, 0, 0, 0);
+    }
+
+    Owned<IPropertyTreeIterator> planeIter = metaTree->getElements("planes");
+    ForEach(*planeIter)
+    {
+        const char * planeName = planeIter->query().queryProp(nullptr);
+        planes.append(storage.queryPlane(planeName));
+    }
+    if (planes.ordinality() == 0)
+    {
+        throwUnexpectedX("No plane associated with file");
+    }
+
+    actualCrc = metaTree->getPropInt("@metaCrc");
+}
+
+
+void CLogicalFile::applyHelperOptions(const IPropertyTree * helperOptions)
+{
+    if (!helperOptions || isEmptyPTree(helperOptions))
+    {
+        mergedMeta.set(metaTree);
+    }
+    else
+    {
+        //Use mergeConfiguration() instead of synchronizePTree because it merges attributes, combines single elements, appends lists
+        Owned<IPropertyTree> merged = createPTreeFromIPT(metaTree);
+        mergeConfiguration(*merged, *helperOptions, nullptr, true);
+        mergedMeta.setown(merged.getClear());
+    }
+}
+
+
+void CLogicalFile::noteLocation(unsigned part, unsigned superPartNum, offset_t baseOffset)
+{
+    auto & cur = parts[part];
+    cur.superPartNum = superPartNum;
+    cur.baseOffset = baseOffset;
+}
+
+const IPropertyTree * CLogicalFile::queryFileMeta() const
+{
+    return mergedMeta;
+}
+
+offset_t CLogicalFile::queryOffsetOfPart(unsigned part) const
+{
+    return queryPart(part).baseOffset;
+}
+
+offset_t CLogicalFile::getPartSize(unsigned part) const
+{
+    if (part < parts.size())
+        return parts[part].fileSize;
+    return (offset_t)-1;
+}
+
+bool CLogicalFile::isLocal(unsigned part, unsigned copy) const
+{
+    return queryPlane(copy)->isLocal(part);
+}
+
+bool CLogicalFile::onAttachedStorage(unsigned copy) const
+{
+    return queryPlane(copy)->isAttachedStorage();
+}
+
+
+//expand name as path, e.g. copy and translate :: into /
+StringBuffer & CLogicalFile::expandLogicalAsPhysical(StringBuffer & target, unsigned copy) const
+{
+    const char * const separator = queryScopeSeparator(copy);
+    const char * cur = name;
+    for (;;)
+    {
+        const char * colon = strstr(cur, "::");
+        if (!colon)
+            break;
+
+        //MORE: Process special characters?  Revisit commoning up these functions as HPCC-25337
+        target.append(colon - cur, cur);
+        target.append(separator);
+        cur = colon + 2;
+    }
+
+    return target.append(cur);
+}
+
+StringBuffer & CLogicalFile::expandPath(StringBuffer & target, unsigned part, unsigned copy) const
+{
+    const char * separator = queryScopeSeparator(copy);
+    if (isExternal())
+    {
+        //expandExternalPath always adds a separator character at the start
+        if (endsWith(target.str(), separator))
+            target.setLength(target.length()-strlen(separator));
+
+        //skip file::
+        const char * coloncolon = strstr(name, "::");
+        assertex(coloncolon);
+        //skip ip::
+        const char * next = coloncolon+2;
+        const char * s = strstr(next, "::");
+        assertex(s);
+        IException * e = nullptr;
+        //Slightly strangely expandExternalPath() expects s to point to the leading ::
+        expandExternalPath(target, target, name, s, false, &e);
+        if (e)
+            throw e;
+    }
+    else
+    {
+        if (!endsWith(target.str(), separator))
+            target.append(separator);
+        expandLogicalAsPhysical(target, copy);
+    }
+
+    //Add part number suffix
+    if (includePartSuffix())
+    {
+        target.append("._").append(part+1).append("_of_").append(numParts);
+    }
+
+    return target;
+}
+
+StringBuffer & CLogicalFile::getURL(StringBuffer & target, unsigned part, unsigned copy) const
+{
+    if (planes.ordinality())
+    {
+        planes.item(copy)->getURL(target, part);
+    }
+    return expandPath(target, part, copy);
+}
+
+
+IOutputMetaData * CLogicalFile::queryActualMeta() const
+{
+    if (!actualMeta)
+    {
+        actualMeta.setown(getDaliLayoutInfo(*metaTree));
+        if (!actualMeta)
+        {
+            //MORE: Old files (pre 7.0) do not have the serialized file format, some new files cannot create them
+            //we should possibly have a way of distinguishing between the two
+            actualMeta.set(expectedMeta);
+        }
+    }
+    return actualMeta;
+}
+
+const char * CLogicalFile::queryFormat() const
+{
+    return metaTree->queryProp("@format");
+}
+
+unsigned CLogicalFile::getNumCopies() const
+{
+    return planes.ordinality();
+}
+
+const char * CLogicalFile::queryScopeSeparator(unsigned copy) const
+{
+    return planes.item(copy)->queryScopeSeparator();
+}
+
+bool CLogicalFile::includePartSuffix() const
+{
+    return !metaTree->getPropBool("@singlePartNoSuffix");
+}
+
+StringBuffer & CLogicalFile::getTracingFilename(StringBuffer & out, unsigned part) const
+{
+    return out.append(name).append(":").append(part);
+}
+
+const char * CLogicalFile::queryLogicalFilename() const
+{
+    return name ? name : "";
+}
+
+
+//---------------------------------------------------------------------------------------------------------------------
+
+CLogicalFileSlice::CLogicalFileSlice(CLogicalFile * _file, unsigned _part, offset_t _startOffset, offset_t _length)
+: file(_file), part(_part), startOffset(_startOffset), length(_length)
+{
+}
+
+bool CLogicalFileSlice::isWholeFile() const
+{
+    if ((startOffset == 0) && file)
+    {
+        if ((length == unknownFileSize) || (length == file->getPartSize(part)))
+            return true;
+    }
+    return false;
+}
+
+StringBuffer & CLogicalFileSlice::getTracingFilename(StringBuffer & out) const
+{
+    file->getTracingFilename(out, part);
+    out.append('{').append(startOffset).append("..");
+    if (length != unknownFileSize)
+        out.append(startOffset + length);
+    return out.append('}');
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+void CLogicalFileCollection::appendFile(CLogicalFile & file)
+{
+    files.append(file);
+    totalSize += file.getFileSize();
+    unsigned numParts = file.getNumParts();
+    if (numParts > maxParts)
+        maxParts = numParts;
+}
+
+
+void CLogicalFileCollection::calcLocations()
+{
+    //If there is only a single file, then the global part numbers and base offsets match the values within the file.
+    if (files.ordinality() == 1)
+        return;
+
+    //If there are multiple parts then they need to be calculated within the interleaved files in the superfile.
+    unsigned superPartNum = 0;
+    offset_t baseOffset = 0;
+    for (unsigned part=0; part < maxParts; part++)
+    {
+        ForEachItemIn(i, files)
+        {
+            CLogicalFile & cur = files.item(i);
+            unsigned numParts = cur.getNumParts();
+            if (numParts > part)
+            {
+                cur.noteLocation(part, superPartNum, baseOffset);
+                superPartNum++;
+                baseOffset += cur.getPartSize(part);
+            }
+        }
+    }
+    //We may need a function to map from super part numbers to a file/part - in which case we'll need to create a reverse
+    //mapping {fileIndex,part} if it needs to be done efficiently
+}
+
+/*
+Calculate which parts of which files will be included in this channel
+
+There is an implicit ordering of the file parts - file1, part1, file2, part1, file<n> part1, file1, part2 etc.
+
+if distribution and ordering are preservered:
+    files are processed in order, multiple parts can be processed on a single node as long as global ordering is retained.
+
+if ordering is preserved and distribution does not need to be:
+    files must be processed in order, parts can be processed on any node as long as global ordering is retained.
+    e.g. if number of reading nodes is > number of file parts, then part1s can be split between nodes1 and 2, part2 between nodes 3 and 4
+         or if multiple files, node1 can process file1 part1, node2 file2 part1, node3 file1 part2, node4 file1 part2
+
+if distribution is preserved, but ordering does not need to be:
+    on bare metal, it might be better to have part1,part<n+1> on node1, part2,part<n+2> on node2 since first parts more likely to be local
+
+if neither distribution or ordering are preserved
+    if non local storage, could try and ensure that workload is even across the nodes.  Could split parts between nodes (but beware
+    of making the read parts too small e.g. for blob read granularity)
+
+on bare metal, if channels is a multiple of the maximum number of parts it is likely to be worth reading the same as inorder
+*/
+
+void CLogicalFileCollection::calcPartition(SliceVector & slices, unsigned numChannels, unsigned channel, bool preserveDistribution, bool preserveOrder)
+{
+    calcLocations();
+
+    //MORE: Revisit and improve the following code to optimize cases detailed above, once they are likely to occur.
+    //Likely to require code generator/language improvements first
+    unsigned partsPerNode = (maxParts < numChannels) ? 1 : (maxParts + numChannels - 1) / numChannels;
+    unsigned startPart = channel * partsPerNode;
+    unsigned endPart = startPart + partsPerNode;
+    if (endPart > maxParts)
+        endPart = maxParts;
+
+    for (unsigned part=startPart; part < endPart; part++)
+    {
+        collectPartSlices(slices, part);
+    }
+}
+
+
+void CLogicalFileCollection::collectPartSlices(SliceVector & slices, unsigned part)
+{
+    unsigned numFiles = files.ordinality();
+    for (unsigned from = 0; from < numFiles; from++)
+    {
+        CLogicalFile & cur = files.item(from);
+        unsigned numParts = cur.getNumParts();
+        if (part < numParts)
+            slices.emplace_back(&cur, part, 0, unknownFileSize);
+    }
+}
+
+
+void CLogicalFileCollection::reset()
+{
+    totalSize = 0;
+    files.kill();
+    maxParts = 0;
+}
+
+void CLogicalFileCollection::init(IFileCollectionContext * _context, const char * _wuid,  bool _isTemporary, bool _resolveLocally, bool _isCodeSigned, IUserDescriptor * _user, IOutputMetaData * _expectedMeta)
+{
+    context = _context;
+    wuid.set(_wuid);
+    isTemporary = _isTemporary;
+    resolveLocally = _resolveLocally;
+    isCodeSigned = _isCodeSigned;
+    user = _user;
+    expectedMeta = _expectedMeta;
+}
+
+
+//The following function is call each time the activity is started - more than once if in a child query
+void CLogicalFileCollection::setEclFilename(const char * _filename, IPropertyTree * _helperOptions)
+{
+    assertex(!isTemporary);
+    //Check if the same parameters have been passed, and if so avoid rebuilding the information
+    if (strisame(filename, _filename))
+    {
+        if (areMatchingPTrees(helperOptions, _helperOptions))
+            return;
+
+        //The file list can stay the same, but the options will need to be recalculated.
+        helperOptions.set(_helperOptions);
+    }
+    else
+    {
+        reset();
+        filename.set(_filename);
+        helperOptions.set(_helperOptions);
+
+        processLogicalFilename();
+    }
+
+    applyHelperOptions();
+}
+
+void CLogicalFileCollection::processLogicalFilename()
+{
+    Owned<IPropertyTree> resolvedMeta;
+    if (resolveLocally)
+    {
+        resolvedMeta.setown(createPTree("meta"));
+        IPropertyTree * storage = resolvedMeta->addPropTree("storage");
+        IPropertyTree * plane = storage->addPropTree("planes");
+        plane->setProp("@prefix", ".");
+        plane->setProp("@name", "local");
+
+        IPropertyTree * file = resolvedMeta->addPropTree("file");
+        file->setProp("@name", filename);
+        file->setProp("@prefix", ".");
+        file->setPropBool("@singlePartNoSuffix", true);
+        file->addProp("planes", "local");
+    }
+    else
+    {
+        //MORE: These options could be restricted e.g., ROpartinfo/ROsizes only if a count operation, or if virtual(fileposition) used
+        ResolveOptions options = ROincludeLocation|ROpartinfo|ROsizes;
+        resolvedMeta.setown(resolveLogicalFilename(filename, user, options));
+    }
+    processResolvedMeta(resolvedMeta);
+}
+
+//Walk the information that was generated by resolving the filename and generate a set of file objects
+void CLogicalFileCollection::processResolvedMeta(IPropertyTree * _resolved)
+{
+    resolved.set(_resolved);
+    storageSystems.setFromMeta(resolved);
+
+    bool expectedGrouped = helperOptions->getPropBool("@grouped");
+    Owned<IPropertyTreeIterator> fileIter = resolved->getElements("file");
+    ForEach(*fileIter)
+    {
+        IPropertyTree & cur = fileIter->query();
+        if (cur.getPropBool("@missing"))
+        {
+            const char * filename = cur.queryProp("@name");
+            if (!helperOptions->getPropBool("@optional", false))
+            {
+                StringBuffer errorMsg("");
+                throw makeStringException(0, errorMsg.append(": Logical file name '").append(filename).append("' could not be resolved").str());
+            }
+            else
+            {
+                StringBuffer buff;
+                buff.appendf("Input file '%s' was missing but declared optional", filename);
+                context->noteException(SeverityInformation, WRN_SkipMissingOptFile, buff.str());
+            }
+        }
+        else
+        {
+            CLogicalFile * file = new CLogicalFile(storageSystems, &cur, expectedMeta);
+            appendFile(*file);
+
+            bool isGrouped = file->isGrouped();
+            if (isGrouped != expectedGrouped)
+            {
+                StringBuffer msg;
+                msg.append("DFS and code generated group info for file ").append(filename).append(" differs: DFS(").append(isGrouped ? "grouped" : "ungrouped").append("), CodeGen(").append(expectedGrouped ? "ungrouped" : "grouped").append("), using DFS info");
+                throw makeStringException(WRN_MismatchGroupInfo, msg.str());
+            }
+        }
+    }
+}
+
+void CLogicalFileCollection::setTempFilename(const char * _filename, IPropertyTree * _helperOptions, const IPropertyTree * spillPlane)
+{
+    assertex(isTemporary);
+    //Temp file parameters should never change if they are called again in a child query
+    if (filename)
+    {
+        if (!strisame(filename, _filename) || !areMatchingPTrees(helperOptions, _helperOptions))
+            throwUnexpected();
+        return;
+    }
+
+    filename.set(_filename);
+    helperOptions.setown(createPTreeFromIPT(_helperOptions));
+    helperOptions->setProp("@name", filename);
+    //Partinfo does not need to be supplied for temporary files.  Deos the number of parts?
+
+    IPropertyTree * plane = helperOptions->addPropTreeArrayItem("planes", createPTree("planes"));
+    plane->setProp("", spillPlane->queryProp("@name"));
+
+    storageSystems.registerPlane(spillPlane);
+
+    CLogicalFile * file = new CLogicalFile(storageSystems, helperOptions, expectedMeta);
+    appendFile(*file);
+    file->applyHelperOptions(nullptr);
+}
+
+
+void CLogicalFileCollection::applyHelperOptions()
+{
+    ForEachItemIn(i, files)
+        files.item(i).applyHelperOptions(helperOptions);
+}

common/thorhelper/thormeta.hpp

@@ -0,0 +1,211 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef __THORMETA_HPP_
+#define __THORMETA_HPP_
+
+#ifdef THORHELPER_EXPORTS
+ #define THORHELPER_API DECL_EXPORT
+#else
+ #define THORHELPER_API DECL_IMPORT
+#endif
+
+#include "jrowstream.hpp"
+#include "rtlkey.hpp"
+#include <vector>
+#include "thorstore.hpp"
+
+
+interface IDistributedFile;
+
+//--------------------------------------------------------------------------------------------------------------------
+
+
+//This class represents a logical file part.
+//The superPartNum/baseOffset may not match the values for the file if this file is part of a superfile.
+class CLogicalFilePart
+{
+public:
+    CLogicalFilePart() = default;
+    CLogicalFilePart(unsigned _superPartNum, offset_t _numRows, offset_t _fileSize, offset_t _baseOffset)
+    : superPartNum(_superPartNum), numRows(_numRows), fileSize(_fileSize), baseOffset(_baseOffset)
+    {
+    }
+
+public://should be private
+    unsigned superPartNum;
+    offset_t numRows = 0;
+    offset_t fileSize = 0;
+    offset_t baseOffset = 0; // sum of previous file sizes
+//  Owned<CSplitPointTable> splits;   // This is where split points would be saved and accessed
+};
+
+class THORHELPER_API CLogicalFile : public CInterface
+{
+public:
+    CLogicalFile(const CStorageSystems & storage, const IPropertyTree * xml, IOutputMetaData * _expectedMeta);
+
+    StringBuffer & getURL(StringBuffer & target, unsigned part, unsigned copy) const;
+    offset_t getFileSize() const { return fileSize; }
+    unsigned getNumCopies() const;
+    unsigned getNumParts() const { return numParts; }
+    offset_t getPartSize(unsigned part) const;
+    bool isDistributed() const { return numParts > 1; }  // MORE: Only if originally a logical file...
+    bool isExternal() const { return metaTree->getPropBool("@external"); }
+    bool isGrouped() const { return metaTree->getPropBool("@grouped"); }
+    bool isLogicalFile() const { return name != nullptr; }
+    bool isLocal(unsigned part, unsigned copy) const;
+    bool isMissing() const { return metaTree->getPropBool("@missing"); }
+    bool onAttachedStorage(unsigned copy) const;
+    const CStoragePlane * queryPlane(unsigned idx) const { return planes.item(idx); }
+
+    unsigned queryActualCrc() const { return actualCrc; }
+    IOutputMetaData * queryActualMeta() const;
+    const char * queryFormat() const;
+    const IPropertyTree * queryFileMeta() const;
+    const char * queryLogicalFilename() const;
+    offset_t queryOffsetOfPart(unsigned part) const;
+    const CLogicalFilePart & queryPart(unsigned part) const
+    {
+        assertex(part < parts.size());
+        return parts[part];
+    }
+    StringBuffer & getTracingFilename(StringBuffer & out, unsigned part) const;
+
+    const char * queryPhysicalPath() const { UNIMPLEMENTED; }    // MORE!!!
+    bool includePartSuffix() const;
+
+    void applyHelperOptions(const IPropertyTree * helperOptions);
+    void noteLocation(unsigned part, unsigned superPartNum, offset_t baseOffset);
+
+protected:
+    StringBuffer & expandLogicalAsPhysical(StringBuffer & target, unsigned copy) const;
+    StringBuffer & expandPath(StringBuffer & target, unsigned part, unsigned copy) const;
+    const char * queryScopeSeparator(unsigned copy) const;
+
+private:
+    const IPropertyTree * metaTree = nullptr;
+    Owned<const IPropertyTree> mergedMeta;
+    IOutputMetaData * expectedMeta = nullptr;  // same as CLogicalFileCollection::expectedMeta
+    //All of the following are derived from the xml
+    const char * name = nullptr;
+    unsigned numParts = 0;
+    offset_t fileSize = 0;
+    std::vector<CLogicalFilePart> parts;
+    ConstPointerArrayOf<CStoragePlane> planes; // An array of locations the file is stored.  replicas are expanded out.
+    mutable Owned<IOutputMetaData> actualMeta;
+    mutable unsigned actualCrc = 0;
+};
+
+//This class is the unit that is passed to the disk reading classes to represent a section from a filepart.
+class THORHELPER_API CLogicalFileSlice
+{
+    friend class CLogicalFileCollection;
+public:
+    CLogicalFileSlice(CLogicalFile * _file, unsigned _part, offset_t _startOffset = 0, offset_t _length = unknownFileSize);
+
+    StringBuffer & getURL(StringBuffer & url, unsigned copy) const { return file->getURL(url, part, copy); }
+
+    unsigned getNumCopies() const { return file->getNumCopies(); }
+    bool isEmpty() const { return !file || length == 0; }
+    bool isLogicalFile() const { return file->isLogicalFile(); }
+    bool isRemoteReadCandidate(unsigned copy) const;
+    bool isWholeFile() const;
+    bool isLocal(unsigned copy) const { return file->isLocal(part, copy); }
+    bool onAttachedStorage(unsigned copy) const { return file->onAttachedStorage(copy); }
+
+    CLogicalFile * queryFile() const { return file; }
+    const char * queryFormat() const { return file->queryFormat(); }
+    const IPropertyTree * queryFileMeta() const { return file->queryFileMeta(); }
+    offset_t queryLength() const { return length; }
+    unsigned queryPartNumber() const { return file->queryPart(part).superPartNum; }
+    offset_t queryOffsetOfPart() const { return file->queryOffsetOfPart(part); }
+    offset_t queryStartOffset() const { return startOffset; }
+    const char * queryLogicalFilename() const { return file->queryLogicalFilename(); }
+    StringBuffer & getTracingFilename(StringBuffer & out) const;
+
+    void setAccessed() {}  // MORE:
+
+private:
+    CLogicalFileSlice() = default;
+
+private:
+    CLogicalFile * file = nullptr;
+    unsigned part = 0;
+    offset_t startOffset = 0;
+    offset_t length = unknownFileSize;
+    //MORE: What about HDFS records that are split over multiple files??
+};
+
+//MORE: Should this be a vector of owned pointers instead?
+using SliceVector = std::vector<CLogicalFileSlice>;
+
+class IFileCollectionContext
+{
+public:
+    virtual void noteException(unsigned severity, unsigned code, const char * text) = 0;
+};
+
+//The following class is always used to access a collection of files - even if it is only a single physical file.
+class CDfsLogicalFileName;
+class THORHELPER_API CLogicalFileCollection
+{
+public:
+    CLogicalFileCollection() = default;
+    CLogicalFileCollection(MemoryBuffer & in);
+
+    void init(IFileCollectionContext * _context, const char * _wuid,  bool _isTemporary,  bool _resolveLocally, bool _isCodeSigned, IUserDescriptor * _user, IOutputMetaData * _expectedMeta); // called once
+    void calcPartition(SliceVector & slices, unsigned numChannels, unsigned channel, bool preserveDistribution, bool preserveOrder);
+    void serialize(MemoryBuffer & out) const;
+    void setEclFilename(const char * filename, IPropertyTree * helperOptions);
+    void setTempFilename(const char * filename, IPropertyTree * helperOptions, const IPropertyTree * spillPlane);
+
+protected:
+    void appendFile(CLogicalFile & file);
+    void applyHelperOptions();
+    void calcLocations();
+    void collectPartSlices(SliceVector & slices, unsigned part);
+    void processFile(IDistributedFile * file, IOutputMetaData * expectedMeta, IPropertyTree * inputOptions, IPropertyTree * formatOptions);
+    void processFilename(CDfsLogicalFileName & logicalFilename, IUserDescriptor *user, bool isTemporary, IOutputMetaData * expectedMeta, IPropertyTree * inputOptions, IPropertyTree * formatOptions);
+    void processMissing(const char * filename, IPropertyTree * inputOptions);
+    void processPhysicalFilename(const char * path, IOutputMetaData * expectedMeta, IPropertyTree * inputOptions, IPropertyTree * formatOptions);
+    void processProtocolFilename(const char * name, const char * colon, const char * slash, IOutputMetaData * expectedMeta, IPropertyTree * inputOptions, IPropertyTree * formatOptions);
+    void processLogicalFilename();
+    void processResolvedMeta(IPropertyTree * _resolved);
+    void reset();
+
+private:
+    //Options that are constant for the lifetime of the class - not linked because they are owned by something else.
+    StringAttr wuid;
+    IFileCollectionContext * context = nullptr;
+    IUserDescriptor * user = nullptr;
+    IOutputMetaData * expectedMeta = nullptr;
+    bool isTemporary = false;
+    bool isCodeSigned = false;
+    bool resolveLocally = false;
+    //The following may be reset e.g. if used within a child query
+    StringAttr filename;
+    Owned<IPropertyTree> helperOptions;    // defined by the helper functions
+    //derived information
+    Owned<IPropertyTree> resolved;
+    CStorageSystems storageSystems;
+    CIArrayOf<CLogicalFile> files;
+    offset_t totalSize = 0;
+    unsigned maxParts = 0;
+};
+
+#endif

common/thorhelper/thormeta.txt

@@ -0,0 +1,8 @@
+Questions arising out of walking through the sketched out implementation:
+
+How do split points fit in?   They can be used when distribution does not need to be preserved.
+When are locations resolved?  Likely to be late on the agent nodes which allows better fallback, and potentially hot swapping to a backup copy.
+What information is signed?  The entire blob of xml that comes back from the esp service is signed.
+How do we reuse buffering classes cleanly?
+
+Implementing FETCH and partition reading required basic interface to request reading for a range of the file.  Fits in well with the range support of S3/azure

common/thorhelper/thorread.cpp

@@ -34,6 +34,7 @@
 #include "rtlcommon.hpp"
 #include "thorcommon.hpp"
 #include "csvsplitter.hpp"
+#include "thormeta.hpp"
 
 //---------------------------------------------------------------------------------------------------------------------
 
@@ -44,8 +45,8 @@
 class DiskReadMapping : public CInterfaceOf<IDiskReadMapping>
 {
 public:
-    DiskReadMapping(RecordTranslationMode _mode, const char * _format, unsigned _actualCrc, IOutputMetaData & _actual, unsigned _expectedCrc, IOutputMetaData & _expected, unsigned _projectedCrc, IOutputMetaData & _output, const IPropertyTree * _options)
-    : mode(_mode), format(_format), actualCrc(_actualCrc), actualMeta(&_actual), expectedCrc(_expectedCrc), expectedMeta(&_expected), projectedCrc(_projectedCrc), projectedMeta(&_output), options(_options)
+    DiskReadMapping(RecordTranslationMode _mode, const char * _format, unsigned _actualCrc, IOutputMetaData & _actual, unsigned _expectedCrc, IOutputMetaData & _expected, unsigned _projectedCrc, IOutputMetaData & _output, const IPropertyTree * _fileOptions)
+    : mode(_mode), format(_format), actualCrc(_actualCrc), actualMeta(&_actual), expectedCrc(_expectedCrc), expectedMeta(&_expected), projectedCrc(_projectedCrc), projectedMeta(&_output), fileOptions(_fileOptions)
     {}
 
     virtual const char * queryFormat() const override { return format; }
@@ -55,7 +56,7 @@ public:
     virtual IOutputMetaData * queryActualMeta() const override { return actualMeta; }
     virtual IOutputMetaData * queryExpectedMeta() const override{ return expectedMeta; }
     virtual IOutputMetaData * queryProjectedMeta() const override{ return projectedMeta; }
-    virtual const IPropertyTree * queryOptions() const override { return options; }
+    virtual const IPropertyTree * queryFileOptions() const override { return fileOptions; }
     virtual RecordTranslationMode queryTranslationMode() const override { return mode; }
 
     virtual const IDynamicTransform * queryTranslator() const override
@@ -71,11 +72,18 @@ public:
 
     virtual bool matches(const IDiskReadMapping * other) const
     {
-        return mode == other->queryTranslationMode() && streq(format, other->queryFormat()) &&
-                ((actualCrc && actualCrc == other->getActualCrc()) || (actualMeta == other->queryActualMeta())) &&
-                ((expectedCrc && expectedCrc == other->getExpectedCrc()) || (expectedMeta == other->queryExpectedMeta())) &&
-                ((projectedCrc && projectedCrc == other->getProjectedCrc()) || (projectedMeta == other->queryProjectedMeta())) &&
-                areMatchingPTrees(options, other->queryOptions());
+        if ((mode != other->queryTranslationMode()) || !streq(format, other->queryFormat()))
+            return false;
+        //if crc is set, then a matching crc counts as a match, otherwise meta must be identical
+        if (((actualCrc && actualCrc == other->getActualCrc()) || (actualMeta == other->queryActualMeta())) &&
+            ((expectedCrc && expectedCrc == other->getExpectedCrc()) || (expectedMeta == other->queryExpectedMeta())) &&
+            ((projectedCrc && projectedCrc == other->getProjectedCrc()) || (projectedMeta == other->queryProjectedMeta())))
+        {
+            if (!areMatchingPTrees(fileOptions->queryPropTree("formatOptions"), other->queryFileOptions()->queryPropTree("formatOptions")))
+                return false;
+            return true;
+        }
+        return false;
     }
 
     virtual bool expectedMatchesProjected() const
@@ -96,7 +104,7 @@ protected:
     Linked<IOutputMetaData> actualMeta;
     Linked<IOutputMetaData> expectedMeta;
     Linked<IOutputMetaData> projectedMeta;
-    Linked<const IPropertyTree> options;
+    Linked<const IPropertyTree> fileOptions;
     mutable Owned<const IDynamicTransform> translator;
     mutable Owned<const IKeyTranslator> keyedTranslator;
     mutable SpinLock translatorLock; // use a spin lock since almost certainly not going to contend
@@ -135,7 +143,8 @@ void DiskReadMapping::ensureTranslators() const
     const RtlRecord & sourceRecord = sourceMeta->queryRecordAccessor(true);
     if (strsame(format, "csv"))
     {
-        type_vals format = options->hasProp("ascii") ? type_string : type_utf8;
+        const IPropertyTree * formatOptions = fileOptions->queryPropTree("formatOptions");
+        type_vals format = formatOptions->hasProp("ascii") ? type_string : type_utf8;
         translator.setown(createRecordTranslatorViaCallback(projectedRecord, sourceRecord, format));
     }
     else if (strsame(format, "xml"))
@@ -145,7 +154,14 @@ void DiskReadMapping::ensureTranslators() const
     else
     {
         if ((projectedMeta != sourceMeta) && (projectedCrc != sourceCrc))
-            translator.setown(createRecordTranslator(projectedRecord, sourceRecord));
+        {
+            //Special case the situation where the output record matches the input record with some virtual fields
+            //appended.  This allows alien datatypes or ifblocks in records to also hav virtual file positions/
+            if (fileOptions->getPropBool("@cloneAppendVirtuals"))
+                translator.setown(createCloneVirtualRecordTranslator(projectedRecord, *sourceMeta));
+            else
+                translator.setown(createRecordTranslator(projectedRecord, sourceRecord));
+        }
     }
 
     if (translator)
@@ -173,16 +189,16 @@ void DiskReadMapping::ensureTranslators() const
     checkedTranslators = true;
 }
 
-THORHELPER_API IDiskReadMapping * createDiskReadMapping(RecordTranslationMode mode, const char * format, unsigned actualCrc, IOutputMetaData & actual, unsigned expectedCrc, IOutputMetaData & expected, unsigned projectedCrc, IOutputMetaData & projected, const IPropertyTree * options)
+THORHELPER_API IDiskReadMapping * createDiskReadMapping(RecordTranslationMode mode, const char * format, unsigned actualCrc, IOutputMetaData & actual, unsigned expectedCrc, IOutputMetaData & expected, unsigned projectedCrc, IOutputMetaData & projected, const IPropertyTree * fileOptions)
 {
     assertex(expectedCrc);
-    assertex(options);
-    return new DiskReadMapping(mode, format, actualCrc, actual, expectedCrc, expected, projectedCrc, projected, options);
+    assertex(fileOptions);
+    return new DiskReadMapping(mode, format, actualCrc, actual, expectedCrc, expected, projectedCrc, projected, fileOptions);
 }
 
 THORHELPER_API IDiskReadMapping * createUnprojectedMapping(IDiskReadMapping * mapping)
 {
-    return createDiskReadMapping(mapping->queryTranslationMode(), mapping->queryFormat(), mapping->getActualCrc(), *mapping->queryActualMeta(), mapping->getExpectedCrc(), *mapping->queryExpectedMeta(), mapping->getExpectedCrc(), *mapping->queryExpectedMeta(), mapping->queryOptions());
+    return createDiskReadMapping(mapping->queryTranslationMode(), mapping->queryFormat(), mapping->getActualCrc(), *mapping->queryActualMeta(), mapping->getExpectedCrc(), *mapping->queryExpectedMeta(), mapping->getExpectedCrc(), *mapping->queryExpectedMeta(), mapping->queryFileOptions());
 }
 
 
@@ -250,7 +266,7 @@ DiskRowReader::DiskRowReader(IDiskReadMapping * _mapping)
     //Options contain information that is the same for each file that is being read, and potentially expensive to reconfigure.
     translator = mapping->queryTranslator();
     keyedTranslator = mapping->queryKeyedTranslator();
-    const IPropertyTree * options = mapping->queryOptions();
+    const IPropertyTree * options = mapping->queryFileOptions();
     if (options->hasProp("encryptionKey"))
     {
         encryptionKey.resetBuffer();
@@ -283,7 +299,7 @@ bool DiskRowReader::matches(const char * format, bool streamRemote, IDiskReadMap
     //if ((expectedDiskMeta != &_expected) || (projectedDiskMeta != &_projected) || (actualDiskMeta != &_actual))
     //    return false;
 
-    const IPropertyTree * options = otherMapping->queryOptions();
+    const IPropertyTree * options = otherMapping->queryFileOptions();
     if (options->hasProp("encryptionKey"))
     {
         MemoryBuffer tempEncryptionKey;
@@ -343,11 +359,12 @@ public:
     LocalDiskRowReader(IDiskReadMapping * _mapping);
 
     virtual bool matches(const char * format, bool streamRemote, IDiskReadMapping * otherMapping) override;
-    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) override;
-    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(const CLogicalFileSlice & slice, const FieldFilterArray & expectedFilter, unsigned copy) override;
 
 protected:
-    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter);
+    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter);
     virtual bool isBinary() const = 0;
 
 protected:
@@ -370,13 +387,13 @@ bool LocalDiskRowReader::matches(const char * format, bool streamRemote, IDiskRe
 }
 
 
-bool LocalDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & _expectedFilter)
+bool LocalDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputMeta, const FieldFilterArray & _expectedFilter)
 {
-    assertex(meta);
-    grouped = meta->getPropBool("grouped");
-    compressed = meta->getPropBool("compressed", false);
-    blockcompressed = meta->getPropBool("blockCompressed", false);
-    bool forceCompressed = meta->getPropBool("forceCompressed", false);
+    assertex(inputMeta);
+    grouped = inputMeta->getPropBool("@grouped");
+    compressed = inputMeta->getPropBool("@compressed", false);
+    blockcompressed = inputMeta->getPropBool("@blockCompressed", false);
+    bool forceCompressed = inputMeta->getPropBool("@forceCompressed", false);
 
     logicalFilename.set(_logicalFilename);
     filePart = _partNumber;
@@ -396,7 +413,7 @@ bool LocalDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFi
 
     if (isBinary())
     {
-        size32_t dfsRecordSize = meta->getPropInt("dfsRecordSize");
+        size32_t dfsRecordSize = inputMeta->getPropInt("@recordSize");
         size32_t fixedDiskRecordSize = actualDiskMeta->getFixedSize();
         if (dfsRecordSize)
         {
@@ -447,9 +464,15 @@ bool LocalDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFi
     if (!inputfileio)
         return false;
 
-    unsigned __int64 filesize = inputfileio->size();
+    if (length == unknownFileSize)
+    {
+        offset_t filesize = inputfileio->size();
+        assertex(startOffset <= filesize);
+        length = filesize - startOffset;
+    }
+
     //MORE: Allow a previously created input stream to be reused to avoid reallocating the buffer
-    inputStream.setown(createFileSerialStream(inputfileio, 0, filesize, readBufferSize));
+    inputStream.setown(createFileSerialStream(inputfileio, startOffset, length, readBufferSize));
 
     expectedFilter.clear();
     ForEachItemIn(i, _expectedFilter)
@@ -457,18 +480,32 @@ bool LocalDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFi
     return true;
 }
 
-bool LocalDiskRowReader::setInputFile(const char * localFilename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+bool LocalDiskRowReader::setInputFile(const char * localFilename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter)
 {
     Owned<IFile> inputFile = createIFile(localFilename);
-    return setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, meta, expectedFilter);
+    return setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, 0, unknownFileSize, inputOptions, expectedFilter);
 }
 
-bool LocalDiskRowReader::setInputFile(const RemoteFilename & filename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+bool LocalDiskRowReader::setInputFile(const RemoteFilename & filename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter)
 {
     Owned<IFile> inputFile = createIFile(filename);
-    return setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, meta, expectedFilter);
+    return setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, 0, unknownFileSize, inputOptions, expectedFilter);
 }
 
+bool LocalDiskRowReader::setInputFile(const CLogicalFileSlice & slice, const FieldFilterArray & expectedFilter, unsigned copy)
+{
+    const char * logicalFilename = slice.queryLogicalFilename();
+    offset_t baseOffset = slice.queryOffsetOfPart();
+
+    StringBuffer url;
+    slice.getURL(url, copy);
+    Owned<IFile> inputFile = createIFile(url);
+
+    //MORE: These need to be passed on to the input reader
+    offset_t startOffset = slice.queryStartOffset();
+    offset_t length = slice.queryLength();
+    return setInputFile(inputFile, logicalFilename, slice.queryPartNumber(), baseOffset, startOffset, length, slice.queryFileMeta(), expectedFilter);
+}
 
 
 //---------------------------------------------------------------------------------------------------------------------
@@ -492,7 +529,7 @@ public:
     virtual bool matches(const char * format, bool streamRemote, IDiskReadMapping * otherMapping) override;
 
 protected:
-    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override;
     virtual bool isBinary() const { return true; }
 
     inline bool fieldFilterMatch(const void * buffer)
@@ -516,7 +553,7 @@ private:
     inline const void * inlineNextRow(PROCESS processor) __attribute__((always_inline));
 
 protected:
-    ISourceRowPrefetcher * actualRowPrefetcher = nullptr;
+    Owned<ISourceRowPrefetcher> actualRowPrefetcher;
     const RtlRecord  * actualRecord = nullptr;
     RowFilter actualFilter;               // This refers to the actual disk layout
     bool eogPending = false;
@@ -527,7 +564,7 @@ protected:
 BinaryDiskRowReader::BinaryDiskRowReader(IDiskReadMapping * _mapping)
 : LocalDiskRowReader(_mapping)
 {
-    actualRowPrefetcher = actualDiskMeta->createDiskPrefetcher();
+    actualRowPrefetcher.setown(actualDiskMeta->createDiskPrefetcher());
     actualRecord = &actualDiskMeta->queryRecordAccessor(true);
     needToTranslate = (translator && translator->needsTranslate());
 }
@@ -546,9 +583,9 @@ bool BinaryDiskRowReader::matches(const char * format, bool streamRemote, IDiskR
     return LocalDiskRowReader::matches(format, streamRemote, otherMapping);
 }
 
-bool BinaryDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+bool BinaryDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter)
 {
-    if (!LocalDiskRowReader::setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, meta, expectedFilter))
+    if (!LocalDiskRowReader::setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, startOffset, length, inputOptions, expectedFilter))
         return false;
 
     actualFilter.clear().appendFilters(expectedFilter);
@@ -737,9 +774,9 @@ public:
         projectedRecord = &mapping->queryProjectedMeta()->queryRecordAccessor(true);
     }
 
-    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & _expectedFilter) override
+    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & _expectedFilter) override
     {
-        if (!LocalDiskRowReader::setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, meta, _expectedFilter))
+        if (!LocalDiskRowReader::setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, startOffset, length, inputOptions, _expectedFilter))
             return false;
 
         projectedFilter.clear().appendFilters(_expectedFilter);
@@ -836,9 +873,9 @@ public:
     virtual bool matches(const char * format, bool streamRemote, IDiskReadMapping * otherMapping) override;
 
 protected:
-    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override;
 
-    void processOption(CSVSplitter::MatchItem element, const IPropertyTree & config, const char * option, const char * dft, const char * dft2 = nullptr);
+    void processOption(CSVSplitter::MatchItem element, const IPropertyTree & csvOptions, const char * option, const char * dft, const char * dft2 = nullptr);
 
     inline bool fieldFilterMatchProjected(const void * buffer)
     {
@@ -869,24 +906,25 @@ protected:
 CsvDiskRowReader::CsvDiskRowReader(IDiskReadMapping * _mapping)
 : ExternalFormatDiskRowReader(_mapping)
 {
-    const IPropertyTree & config = *mapping->queryOptions();
+    const IPropertyTree & fileOptions = *mapping->queryFileOptions();
+    const IPropertyTree & csvOptions = *fileOptions.queryPropTree("formatOptions");
 
-    maxRowSize = config.getPropInt64("maxRowSize", defaultMaxCsvRowSizeMB) * 1024 * 1024;
-    preserveWhitespace = config.getPropBool("preserveWhitespace", false);
-    preserveWhitespace = config.getPropBool("notrim", preserveWhitespace);
+    maxRowSize = csvOptions.getPropInt64("maxRowSize", defaultMaxCsvRowSizeMB) * 1024 * 1024;
+    preserveWhitespace = csvOptions.getPropBool("preserveWhitespace", false);
+    preserveWhitespace = csvOptions.getPropBool("notrim", preserveWhitespace);
 
     const RtlRecord * inputRecord = &mapping->queryActualMeta()->queryRecordAccessor(true);
     unsigned numInputFields = inputRecord->getNumFields();
     csvSplitter.init(numInputFields, maxRowSize, csvQuote, csvSeparate, csvTerminate, csvEscape, preserveWhitespace);
 
     //MORE: How about options from the file? - test writing with some options and then reading without specifying them
-    processOption(CSVSplitter::QUOTE, config, "quote", "\"");
-    processOption(CSVSplitter::SEPARATOR, config, "separator", ",");
-    processOption(CSVSplitter::TERMINATOR, config, "terminator", "\n", "\r\n");
-    if (config.getProp("escape", csvEscape))
+    processOption(CSVSplitter::QUOTE, csvOptions, "quote", "\"");
+    processOption(CSVSplitter::SEPARATOR, csvOptions, "separator", ",");
+    processOption(CSVSplitter::TERMINATOR, csvOptions, "terminator", "\n", "\r\n");
+    if (csvOptions.getProp("escape", csvEscape))
         csvSplitter.addEscape(csvEscape);
 
-    headerLines = config.getPropInt64("heading");
+    headerLines = csvOptions.getPropInt64("heading");
     fieldFetcher.setown(new CFieldFetcher(csvSplitter, numInputFields));
 }
 
@@ -898,12 +936,12 @@ bool CsvDiskRowReader::matches(const char * format, bool streamRemote, IDiskRead
     return ExternalFormatDiskRowReader::matches(format, streamRemote, otherMapping);
 }
 
-void CsvDiskRowReader::processOption(CSVSplitter::MatchItem element, const IPropertyTree & config, const char * option, const char * dft, const char * dft2)
+void CsvDiskRowReader::processOption(CSVSplitter::MatchItem element, const IPropertyTree & csvOptions, const char * option, const char * dft, const char * dft2)
 {
-    if (config.hasProp(option))
+    if (csvOptions.hasProp(option))
     {
-        bool useAscii = mapping->queryOptions()->hasProp("ascii");
-        Owned<IPropertyTreeIterator> iter = config.getElements(option);
+        bool useAscii = csvOptions.hasProp("ascii");
+        Owned<IPropertyTreeIterator> iter = csvOptions.getElements(option);
         ForEach(*iter)
         {
             const char * value = iter->query().queryProp("");
@@ -926,9 +964,9 @@ void CsvDiskRowReader::processOption(CSVSplitter::MatchItem element, const IProp
     }
 }
 
-bool CsvDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & _expectedFilter)
+bool CsvDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & _expectedFilter)
 {
-    if (!ExternalFormatDiskRowReader::setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, meta, _expectedFilter))
+    if (!ExternalFormatDiskRowReader::setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, startOffset, length, inputOptions, _expectedFilter))
         return false;
 
     //Skip any header lines..
@@ -1048,7 +1086,7 @@ class CompoundProjectRowReader : extends CInterfaceOf<IDiskRowStream>, implement
     MemoryBufferBuilder bufferBuilder;
     RtlDynamicRowBuilder allocatedBuilder;
     Linked<IEngineRowAllocator> outputAllocator;
-    IDiskRowStream * rawInputStream;
+    IDiskRowStream * rawInputStream = nullptr;
 public:
     CompoundProjectRowReader(IDiskRowReader * _input, IDiskReadMapping * _mapping)
     : inputReader(_input), mapping(_mapping), bufferBuilder(tempOutputBuffer, 0), allocatedBuilder(nullptr)
@@ -1059,27 +1097,27 @@ public:
     }
     IMPLEMENT_IINTERFACE_USING(CInterfaceOf<IDiskRowStream>)
 
-    virtual IDiskRowStream * queryAllocatedRowStream(IEngineRowAllocator * _outputAllocator)
+    virtual IDiskRowStream * queryAllocatedRowStream(IEngineRowAllocator * _outputAllocator) override
     {
         allocatedBuilder.setAllocator(_outputAllocator);
         outputAllocator.set(_outputAllocator);
         return this;
     }
 
-    virtual bool matches(const char * _format, bool _streamRemote, IDiskReadMapping * _mapping)
+    virtual bool matches(const char * _format, bool _streamRemote, IDiskReadMapping * _mapping) override
     {
         return false;
     }
 
-    virtual void clearInput()
+    virtual void clearInput() override
     {
         inputReader->clearInput();
         rawInputStream = nullptr;
     }
 
-    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override
     {
-        if (inputReader->setInputFile(localFilename, logicalFilename, partNumber, baseOffset, meta, expectedFilter))
+        if (inputReader->setInputFile(localFilename, logicalFilename, partNumber, baseOffset, inputOptions, expectedFilter))
         {
             rawInputStream = inputReader->queryAllocatedRowStream(nullptr);
             return true;
@@ -1087,9 +1125,19 @@ public:
         return false;
     }
 
-    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override
     {
-        if (inputReader->setInputFile(filename, logicalFilename, partNumber, baseOffset, meta, expectedFilter))
+        if (inputReader->setInputFile(filename, logicalFilename, partNumber, baseOffset, inputOptions, expectedFilter))
+        {
+            rawInputStream = inputReader->queryAllocatedRowStream(nullptr);
+            return true;
+        }
+        return false;
+    }
+
+    virtual bool setInputFile(const CLogicalFileSlice & slice, const FieldFilterArray & expectedFilter, unsigned copy) override
+    {
+        if (inputReader->setInputFile(slice, expectedFilter, copy))
         {
             rawInputStream = inputReader->queryAllocatedRowStream(nullptr);
             return true;
@@ -1098,9 +1146,9 @@ public:
     }
 
 //interface IRowReader
-    virtual bool getCursor(MemoryBuffer & cursor) { return rawInputStream->getCursor(cursor); }
-    virtual void setCursor(MemoryBuffer & cursor) { rawInputStream->setCursor(cursor); }
-    virtual void stop() { rawInputStream->stop(); }
+    virtual bool getCursor(MemoryBuffer & cursor) override { return rawInputStream->getCursor(cursor); }
+    virtual void setCursor(MemoryBuffer & cursor) override { rawInputStream->setCursor(cursor); }
+    virtual void stop() override { rawInputStream->stop(); }
 
     virtual const void *nextRow(size32_t & resultSize) override
     {
@@ -1158,13 +1206,13 @@ public:
         compoundReader.setown(new CompoundProjectRowReader(expectedReader, mapping));
     }
 
-    virtual IDiskRowStream * queryAllocatedRowStream(IEngineRowAllocator * _outputAllocator)
+    virtual IDiskRowStream * queryAllocatedRowStream(IEngineRowAllocator * _outputAllocator) override
     {
         assertex(activeReader);
         return activeReader->queryAllocatedRowStream(_outputAllocator);
     }
 
-    virtual bool matches(const char * _format, bool _streamRemote, IDiskReadMapping * _mapping)
+    virtual bool matches(const char * _format, bool _streamRemote, IDiskReadMapping * _mapping) override
     {
         return directReader->matches(_format, _streamRemote, _mapping);
     }
@@ -1172,31 +1220,41 @@ public:
     //Specify where the raw binary input for a particular file is coming from, together with its actual format.
     //Does this make sense, or should it be passed a filename?  an actual format?
     //Needs to specify a filename rather than a ISerialStream so that the interface is consistent for local and remote
-    virtual void clearInput()
+    virtual void clearInput() override
     {
         directReader->clearInput();
         compoundReader->clearInput();
         activeReader = nullptr;
     }
 
-    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override
     {
         bool useProjected = canFilterDirectly(expectedFilter);
         if (useProjected)
             activeReader = directReader;
         else
             activeReader = compoundReader;
-        return activeReader->setInputFile(localFilename, logicalFilename, partNumber, baseOffset, meta, expectedFilter);
+        return activeReader->setInputFile(localFilename, logicalFilename, partNumber, baseOffset, inputOptions, expectedFilter);
     }
 
-    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override
     {
         bool useProjected = canFilterDirectly(expectedFilter);
         if (useProjected)
             activeReader = directReader;
         else
             activeReader = compoundReader;
-        return activeReader->setInputFile(filename, logicalFilename, partNumber, baseOffset, meta, expectedFilter);
+        return activeReader->setInputFile(filename, logicalFilename, partNumber, baseOffset, inputOptions, expectedFilter);
+    }
+
+    virtual bool setInputFile(const CLogicalFileSlice & slice, const FieldFilterArray & expectedFilter, unsigned copy) override
+    {
+        bool useProjected = canFilterDirectly(expectedFilter);
+        if (useProjected)
+            activeReader = directReader;
+        else
+            activeReader = compoundReader;
+        return activeReader->setInputFile(slice, expectedFilter, copy);
     }
 
 protected:
@@ -1238,8 +1296,9 @@ public:
     virtual bool matches(const char * _format, bool _streamRemote, IDiskReadMapping * _mapping) override;
 
 // IDiskRowReader
-    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) override;
-    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override;
+    virtual bool setInputFile(const CLogicalFileSlice & slice, const FieldFilterArray & expectedFilter, unsigned copy) override;
 
 private:
     template <class PROCESS>
@@ -1275,7 +1334,7 @@ bool RemoteDiskRowReader::matches(const char * _format, bool _streamRemote, IDis
     return DiskRowReader::matches(_format, _streamRemote, _mapping);
 }
 
-bool RemoteDiskRowReader::setInputFile(const RemoteFilename & rfilename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilters)
+bool RemoteDiskRowReader::setInputFile(const RemoteFilename & rfilename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilters)
 {
     // NB: only binary handles can be remotely processed by dafilesrv at the moment
 
@@ -1332,11 +1391,15 @@ bool RemoteDiskRowReader::setInputFile(const RemoteFilename & rfilename, const c
     return true;
 }
 
-bool RemoteDiskRowReader::setInputFile(const char * localFilename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter)
+bool RemoteDiskRowReader::setInputFile(const char * localFilename, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter)
 {
     throwUnexpected();
 }
 
+bool RemoteDiskRowReader::setInputFile(const CLogicalFileSlice & slice, const FieldFilterArray & expectedFilter, unsigned copy)
+{
+    UNIMPLEMENTED;
+}
 
 template <class PROCESS>
 const void *RemoteDiskRowReader::inlineNextRow(PROCESS processor)

common/thorhelper/thorread.hpp

@@ -27,6 +27,7 @@
 #include "jrowstream.hpp"
 #include "rtlkey.hpp"
 
+//--- Classes and interfaces for reading instances of files
 //The following is constant for the life of a disk read activity
 interface IDiskReadOutputMapping : public IInterface
 {
@@ -51,7 +52,7 @@ public:
     virtual IOutputMetaData * queryActualMeta() const = 0;
     virtual IOutputMetaData * queryExpectedMeta() const = 0;
     virtual IOutputMetaData * queryProjectedMeta() const = 0;
-    virtual const IPropertyTree * queryOptions() const = 0;
+    virtual const IPropertyTree * queryFileOptions() const = 0;
     virtual RecordTranslationMode queryTranslationMode() const = 0;
 
     virtual bool matches(const IDiskReadMapping * other) const = 0;
@@ -61,7 +62,7 @@ public:
     virtual const IKeyTranslator *queryKeyedTranslator() const = 0; // translates from expected to actual
 };
 
-THORHELPER_API IDiskReadMapping * createDiskReadMapping(RecordTranslationMode mode, const char * format, unsigned actualCrc, IOutputMetaData & actual, unsigned expectedCrc, IOutputMetaData & expected, unsigned projectedCrc, IOutputMetaData & projected, const IPropertyTree * options);
+THORHELPER_API IDiskReadMapping * createDiskReadMapping(RecordTranslationMode mode, const char * format, unsigned actualCrc, IOutputMetaData & actual, unsigned expectedCrc, IOutputMetaData & expected, unsigned projectedCrc, IOutputMetaData & projected, const IPropertyTree * fileOptions);
 
 
 typedef IConstArrayOf<IFieldFilter> FieldFilterArray;
@@ -73,6 +74,7 @@ public:
 };
 
 interface ITranslator;
+class CLogicalFileSlice;
 interface IDiskRowReader : extends IRowReader
 {
 public:
@@ -82,8 +84,9 @@ public:
     //Does this make sense, or should it be passed a filename?  an actual format?
     //Needs to specify a filename rather than a ISerialStream so that the interface is consistent for local and remote
     virtual void clearInput() = 0;
-    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) = 0;
-    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * meta, const FieldFilterArray & expectedFilter) = 0;
+    virtual bool setInputFile(const char * localFilename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) = 0;
+    virtual bool setInputFile(const RemoteFilename & filename, const char * logicalFilename, unsigned partNumber, offset_t baseOffset, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) = 0;
+    virtual bool setInputFile(const CLogicalFileSlice & slice, const FieldFilterArray & expectedFilter, unsigned copy) = 0;
 };
 
 //Create a row reader for a thor binary file.  The expected, projected, actual and options never change.  The file providing the data can change.

common/thorhelper/thorstore.cpp

@@ -0,0 +1,286 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "jsocket.hpp"
+#include "thorfile.hpp"
+
+#include "eclhelper.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
+
+#include "dautils.hpp"
+#include "dadfs.hpp"
+
+#include "thorstore.hpp"
+
+
+//If the path associated with a plane contains one or more hashes in a row those hashes
+//are replaced with the device number, padded with 0s to the width of the number of hashes
+StringBuffer & expandPlanePath(StringBuffer & target, const char * path, unsigned device)
+{
+    for (;;)
+    {
+        const char * hash = strchr(path, '#');
+        if (!hash)
+            break;
+
+        target.append(hash-path, path);
+        unsigned width = 1;
+        while (hash[1] == '#')
+        {
+            hash++;
+            width++;
+        }
+
+        target.appendf("%0*u", width, device);
+        path = hash+1;
+    }
+    target.append(path);
+    return target;
+}
+
+CStorageHostGroup::CStorageHostGroup(const IPropertyTree * _xml)
+: xml(_xml)
+{
+}
+
+const char * CStorageHostGroup::queryName() const
+{
+    return xml->queryProp("@name");
+}
+
+const char * CStorageHostGroup::queryHost(unsigned idx) const
+{
+    VStringBuffer xpath("hosts[%u]", idx+1);
+    return xml->queryProp(xpath);
+}
+
+bool CStorageHostGroup::isLocal(unsigned device) const
+{
+    const char * hostname = queryHost(device);
+    if (hostname)
+    {
+        //MORE: Likely to be inefficient - should search differently?
+        IpAddress ip(hostname);
+        return ip.isLocal();
+    }
+    return false;
+}
+
+/*
+ * A StoragePlane represents a set of storage devices that are used together to store a distributed file.  Each
+ * storage plane has a fixed number of logical devices.  If there are > 1 then the storage plane index can be included
+ * within the file path - which allows data to be striped across multiple devices.
+ *
+ * For S3 etc. it specifies the region, bucket and root path.
+ * For local NFS mounted NAS it specifies the mount point.
+ * For other NAS solutions it could indicate a list of ips, and root paths.
+ * For locally attached storage it specifies the root path, and a set of hostnames.  Also whether dafilesrv is allowed/required.
+ *
+ * For NAS/cloud situation, each cluster should define the default location the data is stored (although it can be overridden in the OUTPUT)..
+ *
+ * For an attached storage scheme, each cluster defined in the environment implicitly defines a locally attached storage plane.  It would
+ * be useful for any cluster to be able to specify a separate plane for the spill files e.g., ramdisk/faster disks.
+ * It could alternatively rely on an implicit plane (for the current node),
+ *
+ * There should be a special local plane for spill files and other temporary files.  How do these fit in?  Should
+ * there be a special ip group ('!') that refers to the cluster for the current component?
+ *
+ * What problems are there resolving ips - e.g. in the globally shared storage plane?  Is it done in the plane or in the logical file?
+ */
+
+
+CStoragePlane::CStoragePlane(const IPropertyTree * _xml, const CStorageHostGroup * _host)
+ : xml(_xml), hostGroup(_host)
+{
+    name = xml->queryProp("@name");
+    numDevices = xml->getPropInt("@numDevices", 1);
+    size = xml->getPropInt("@size", numDevices);
+    offset = xml->getPropInt("@offset", 0);
+    startDelta = xml->getPropInt("@start", 0);
+    startDrive = 0; // Not sure if we really want to support multiple drives..
+}
+
+bool CStoragePlane::containsHost(const char * host) const
+{
+    Owned<IPropertyTreeIterator> iter = xml->getElements("hosts");
+    ForEach(*iter)
+    {
+        if (streq(iter->query().queryProp(""), host))
+            return true;
+    }
+    return false;
+}
+
+bool CStoragePlane::containsPath(const char * path)
+{
+    return startsWith(path, queryPath());
+}
+
+bool CStoragePlane::matchesHost(const char * host)
+{
+    return (xml->getCount("hosts") == 1) && containsHost(host);
+}
+
+const char * CStoragePlane::queryPath() const
+{
+    const char * path = xml->queryProp("@prefix");
+    return path ? path : "/";
+}
+
+const char * CStoragePlane::queryScopeSeparator() const
+{
+    //MORE: Could support it set via a property
+    return "/";
+}
+
+StringBuffer & CStoragePlane::getURL(StringBuffer & target, unsigned part) const
+{
+    unsigned device = getDevice(part);
+    if (hostGroup)
+    {
+        target.append("//");
+        target.append(hostGroup->queryHost(device));
+    }
+
+    //MORE: Should this allow drive to modify and use $D<n>$ syntax instead?  I think that functionality  can be lost.
+    //unsigned drive = getDrive(part);
+    expandPlanePath(target, queryPath(), device);
+    return target;
+}
+
+
+unsigned CStoragePlane::getWidth() const
+{
+    return numDevices;  // Could subdivide even if not done by ip
+}
+
+#if 0
+//What is the cost of accessing part "part" from host "accessIp"
+unsigned CStoragePlane::getCost(unsigned part, const const char * accessIp) const
+{
+    if (!hostGroup)
+        return xml->getPropInt("@accessCost", remoteCost);
+    unsigned device = getDevice(part);
+    if (hosts->isLocal(device, accessIp))
+        return directCost;
+    if (hostGroup->isSameNetwork(device, accessIp))
+        return localDCost;
+    return remoteCost;
+}
+#endif
+
+unsigned CStoragePlane::getDevice(unsigned part) const
+{
+    unsigned nodeDelta = (part + startDelta) % size;
+    unsigned device = (nodeDelta + offset);
+    assertex(device < getWidth());
+    return device;
+}
+
+unsigned CStoragePlane::getDrive(unsigned part) const
+{
+    unsigned driveDelta = (part + startDelta) / size;
+    unsigned drive = (startDrive + driveDelta) % getNumDrives();
+    return drive;
+}
+
+bool CStoragePlane::isLocal(unsigned part) const
+{
+    if (hostGroup)
+        return hostGroup->isLocal(getDevice(part));
+    return false;
+}
+
+bool CStoragePlane::isAttachedStorage() const
+{
+    return (hostGroup != nullptr);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+const CStorageHostGroup * CStorageSystems::queryHostGroup(const char * search) const
+{
+    if (!search)
+        return nullptr;
+
+    ForEachItemIn(i, hostGroups)
+    {
+        CStorageHostGroup & cur = hostGroups.item(i);
+        if (strsame(search, cur.queryName()))
+            return &cur;
+    }
+    return nullptr;
+}
+
+const CStoragePlane * CStorageSystems::queryPlane(const char * search) const
+{
+    if (!search)
+        return nullptr;
+
+    ForEachItemIn(i, planes)
+    {
+        CStoragePlane & cur = planes.item(i);
+        if (strsame(search, cur.queryName()))
+            return &cur;
+    }
+    return nullptr;
+}
+
+const CStoragePlane * CStorageSystems::queryNullPlane() const
+{
+    if (!nullPlaneXml)
+    {
+        nullPlaneXml.setown(createPTree("plane"));
+        nullPlaneXml->setProp("@name", "implicitExternalPlane");
+        nullPlane.setown(new CStoragePlane(nullPlaneXml, nullptr));
+    }
+
+    return nullPlane;
+}
+
+void CStorageSystems::setFromMeta(const IPropertyTree * xml)
+{
+    const IPropertyTree * storage = xml->queryPropTree("storage");
+
+    //MORE: Is it worth checking if the hostGroups and storageplanes are the same as last time?
+    //if areMatchingPTrees(storage, savedStorage) return;
+    hostGroups.kill();
+    planes.kill();
+
+    if (!storage)
+        return;
+
+    Owned<IPropertyTreeIterator> hostIter = storage->getElements("hostGroups");
+    ForEach(*hostIter)
+        hostGroups.append(*new CStorageHostGroup(&hostIter->query()));
+
+    Owned<IPropertyTreeIterator> planeIter = storage->getElements("planes");
+    ForEach(*planeIter)
+    {
+        IPropertyTree * cur = &planeIter->query();
+        const CStorageHostGroup * hosts = queryHostGroup(cur->queryProp("@hosts"));
+        planes.append(*new CStoragePlane(cur, hosts));
+    }
+}
+
+void CStorageSystems::registerPlane(const IPropertyTree * plane)
+{
+    assertex(planes.empty());
+    planes.append(*new CStoragePlane(plane, nullptr));
+}

common/thorhelper/thorstore.hpp

@@ -0,0 +1,170 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef __THORSTORE_HPP_
+#define __THORSTORE_HPP_
+
+#ifdef THORHELPER_EXPORTS
+ #define THORHELPER_API DECL_EXPORT
+#else
+ #define THORHELPER_API DECL_IMPORT
+#endif
+
+#include <vector>
+
+//All of the following will move to a different location (?dali) once the proof of concept is completed.
+
+//=====================================================================================================================
+
+/*
+ * File processing sequence:
+ *
+ * a) create a CLogicalFileCollection
+ *   A) with an implicit superfile
+ *      process each of the implicit subfiles in turn
+ *      Do we allow logical files and
+ *   B) with a logical super file
+ *      process each of the subfiles in turn
+ *   C) With a logical file
+ *      - Resolve the file
+ *      - extract the file information from dali
+ *      - extract the split points (from dali/disk).
+ *        potentially quite large, and expensive?  May want a hint to indicate how many ways the file will be read
+ *        to avoid retrieving if there will be no benefit.
+ *   D) With a absolute filename  [<host>|group?][:port]//path   (or local path if contains a /)
+ *      - NOTE: ./abc can be used to refer to a physical file in the current directory
+ *      - Add as a single (or multiple) part physical file, no logical distribution.
+ *      - not sure if we should allow the port or not.  Could be used to force use of dafilesrv??
+ *      - Need to check access rights - otherwise could extract security keys etc.
+ *   E) As a URL transport-protocol://path
+ *      - Expand wild cards (or other magic) to produce a full list of parts and nodes.  No logical distribution.
+ *      - Retrieve meta information for the file
+ *      - Retrieve partition information for the parts.
+ *      - possibly allow thor:array//path as another representation for (D).  What would make sense?
+ *   F) FILE::...
+ *      Check access rights, then translate to same as (D)
+ *   G) REMOTE::...
+ *      Call into the remote esp to extract the file information and merge it with the location information
+ *
+ * b) perform protocol dependent processing
+ *    - possibly part of stage (a) or a separate phase
+ *    - passing #parts that it will be read from to allow split information to be optimized.
+ *    A) thor
+         - Translate logical nodes to physical ips.
+         - gather any missing file sizes
+      B) s3/azure blobs
+         - Expand any wildcards in the filenames and create entries for each expanded file.
+         - gather file sizes
+      C) HDFS
+         - gather files sizes
+         - gather split points
+ * c) serialize to the slaves
+ * d) deserialize from the master
+ * e) call fileCollection::partition(numChannels, myChannel) to get a list of partitions
+ * f) iterate though the partitions for the current channel
+ *    a) need a reader for the format - how determine?
+ *    b) Where we determine if it should be read directly or via dafilesrv?
+ *    c) request a row reader for the
+ *
+ * Questions:
+ *    Where are the translators created?
+ *    What is signed?  It needs to be a self contained block of data that can easily be serialized and deserialized.
+ *      I don't think it needs to contain information about the storage array - only the logical file.
+ */
+
+class CStorageSystems;
+
+//What is a good term for a collection of drives
+//storage array/system/
+
+//This describes a set of disks that can be used to store a logical file.
+//  "device" is used to represent the index within the storage plane
+class THORHELPER_API CStorageHostGroup : public CInterface
+{
+public:
+    CStorageHostGroup(const IPropertyTree * _xml);
+
+    const char * queryName() const;
+    const char * queryHost(unsigned idx) const;
+    bool isLocal(unsigned device) const;
+
+private:
+    const IPropertyTree * xml = nullptr;
+};
+
+
+//This describes the a set of disks that can be used to store a logical file.
+//  "device" is used to represent the index within the storage plane
+class THORHELPER_API CStoragePlane : public CInterface
+{
+    friend class CStorageSystems;
+
+public:
+    CStoragePlane() = default;
+    CStoragePlane(const IPropertyTree * _xml, const CStorageHostGroup * _host);
+
+    bool containsHost(const char * host) const;
+    bool containsPath(const char * path);
+    bool matches(const char * search) const { return strsame(name, search); }
+    bool matchesHost(const char * host);
+
+//    unsigned getCost(unsigned device, const IpAddress & accessIp, PhysicalGroup & peerIPs) const;
+    StringBuffer & getURL(StringBuffer & target, unsigned part) const;
+    unsigned getWidth() const;
+    bool isLocal(unsigned part) const;
+    bool isAttachedStorage() const;
+    const char * queryName() const { return name; }
+    const char * queryPath() const;
+    const char * queryScopeSeparator() const;
+
+protected:
+    unsigned getNumDrives() const { return 1; }  // MORE: Should we allow support for drives as well as devices?
+    unsigned getDevice(unsigned part) const;
+    unsigned getDrive(unsigned part) const;
+
+private:
+    const IPropertyTree * xml = nullptr;
+    const char * name = nullptr;
+    const CStorageHostGroup * hostGroup = nullptr;
+    unsigned numDevices = 0;
+    unsigned offset = 0;  // offset + size <= plane->getWidth();
+    unsigned size = 0;  // size <= plane->getWidth();
+    unsigned startDelta = 0;  // allows different replication to be represented 0|1|width|....  Can be > size if plane has multiple drives
+    unsigned startDrive = 0;
+};
+
+class CStorageSystems
+{
+public:
+    void registerPlane(const IPropertyTree * _meta);
+    void setFromMeta(const IPropertyTree * _meta);
+
+    const CStorageHostGroup * queryHostGroup(const char * search) const;
+    const CStoragePlane * queryPlane(const char * search) const;
+    const CStoragePlane * queryNullPlane() const;
+
+protected:
+    CIArrayOf<CStorageHostGroup> hostGroups;
+    CIArrayOf<CStoragePlane> planes;
+    mutable Owned<IPropertyTree> nullPlaneXml;
+    mutable Owned<CStoragePlane> nullPlane;
+};
+
+//Exapand the a string, substituting hashes for the device number.  Default number of digits matches the number of hashes
+extern THORHELPER_API StringBuffer & expandPlanePath(StringBuffer & target, const char * path, unsigned device);
+
+#endif

dali/base/dafdesc.cpp

@@ -3434,10 +3434,10 @@ const char * queryDefaultStoragePlane()
 
 //---------------------------------------------------------------------------------------------------------------------
 
-class CStoragePlane : public CInterfaceOf<IStoragePlane>
+class CStoragePlaneInfo : public CInterfaceOf<IStoragePlane>
 {
 public:
-    CStoragePlane(IPropertyTree * _xml) : xml(_xml) {}
+    CStoragePlaneInfo(IPropertyTree * _xml) : xml(_xml) {}
 
     virtual const char * queryPrefix() const override { return xml->queryProp("@prefix"); }
     virtual unsigned numDevices() const override { return xml->getPropInt("@numDevices", 1); }
@@ -3464,5 +3464,5 @@ IStoragePlane * getStoragePlane(const char * name, bool required)
         return nullptr;
     }
 
-    return new CStoragePlane(match);
+    return new CStoragePlaneInfo(match);
 }

dali/base/dameta.cpp

@@ -40,21 +40,6 @@ IPropertyTree * queryStoragePlane(const char * name)
 
 //Cloned for now - export and use from elsewhere
 
-static void copyPropIfMissing(IPropertyTree & target, const char * targetName, IPropertyTree & source, const char * sourceName)
-{
-    if (source.hasProp(sourceName) && !target.hasProp(targetName))
-    {
-        if (source.isBinary(sourceName))
-        {
-            MemoryBuffer value;
-            source.getPropBin(sourceName, value);
-            target.setPropBin(targetName, value.length(), value.toByteArray());
-        }
-        else
-            target.setProp(targetName, source.queryProp(sourceName));
-    }
-}
-
 static void copySeparatorPropIfMissing(IPropertyTree & target, const char * targetName, IPropertyTree & source, const char * sourceName)
 {
     //Legacy - commas are quoted if they occur in a separator list, so need to remove the leading backslashes
@@ -92,7 +77,9 @@ public:
 protected:
     void ensureHostGroup(const char * name);
     void ensurePlane(const char * plane);
-    IPropertyTree * processExternalFile(CDfsLogicalFileName & logicalFilename);
+    void ensureExternalPlane(const char * name, const char * host);
+    IPropertyTree * processExternal(CDfsLogicalFileName & logicalFilename);
+    void processExternalFile(CDfsLogicalFileName & logicalFilename);
     void processExternalPlane(CDfsLogicalFileName & logicalFilename);
     void processFile(IDistributedFile & file);
     void processFilename(CDfsLogicalFileName & logicalFilename);
@@ -139,20 +126,62 @@ void LogicalFileResolver::ensurePlane(const char * name)
 }
 
 
-IPropertyTree * LogicalFileResolver::processExternalFile(CDfsLogicalFileName & logicalFilename)
+void LogicalFileResolver::ensureExternalPlane(const char * name, const char * host)
+{
+    VStringBuffer xpath("planes[@name='%s']", name);
+    IPropertyTree * storage = ensurePTree(meta, "storage");
+    if (storage->hasProp(xpath))
+        return;
+
+    Owned<IPropertyTree> plane = createPTree("planes");
+    plane->setProp("@name", name);
+    plane->setProp("@hosts", name);
+    Owned<IPropertyTree> hostGroup = createPTree("hostGroups");
+    hostGroup->setProp("@name", name);
+    hostGroup->setProp("@hosts", name);
+    IPropertyTree * hosts = createPTree("hosts");
+    hosts->setProp("", host);
+    hostGroup->addPropTreeArrayItem("hosts", hosts);
+
+    storage->addPropTreeArrayItem("planes", plane.getClear());
+    storage->addPropTreeArrayItem("hostGroups", hostGroup.getClear());
+}
+
+
+IPropertyTree * LogicalFileResolver::processExternal(CDfsLogicalFileName & logicalFilename)
 {
     IPropertyTree * fileMeta = meta->addPropTree("file");
     fileMeta->setProp("@name", logicalFilename.get(false));
     fileMeta->setPropInt("@numParts", 1);
     fileMeta->setProp("@format", "unknown");
     fileMeta->setPropBool("@external", true);
+    fileMeta->setPropBool("@singlePartNoSuffix", true);
 
     return fileMeta;
 }
 
+void LogicalFileResolver::processExternalFile(CDfsLogicalFileName & logicalFilename)
+{
+    IPropertyTree * fileMeta = processExternal(logicalFilename);
+
+    //MORE: In the future we could go and grab the meta information from disk for a file and determine the number of parts etc.
+    //to provide an implicit multi part file import
+    if (options & ROincludeLocation)
+    {
+        StringBuffer hostName;
+        logicalFilename.getExternalHost(hostName);
+        StringBuffer planeName;
+        planeName.append("external_").append(hostName);
+        IPropertyTree * plane = createPTree("planes");
+        plane = fileMeta->addPropTreeArrayItem("planes", plane);
+        plane->setProp("", planeName);
+        ensureExternalPlane(planeName, hostName);
+    }
+}
+
 void LogicalFileResolver::processExternalPlane(CDfsLogicalFileName & logicalFilename)
 {
-    IPropertyTree * fileMeta = processExternalFile(logicalFilename);
+    IPropertyTree * fileMeta = processExternal(logicalFilename);
 
     //MORE: In the future we could go and grab the meta information from disk for a file and determine the number of parts etc.
     //to provide an implicit multi part file import

dali/base/dautils.cpp

@@ -323,6 +323,24 @@ bool CDfsLogicalFileName::getExternalPlane(StringBuffer & plane) const
 }
 
 
+bool CDfsLogicalFileName::isExternalFile() const
+{
+    return external && startsWithIgnoreCase(lfn, EXTERNAL_SCOPE "::");
+}
+
+bool CDfsLogicalFileName::getExternalHost(StringBuffer & host) const
+{
+    if (!isExternalFile())
+        return false;
+
+    const char * start = lfn.str() + strlen(EXTERNAL_SCOPE "::");
+    const char * end = strstr(start,"::");
+    assertex(end);
+    host.append(end-start, start);
+    return true;
+}
+
+
 void CDfsLogicalFileName::set(const CDfsLogicalFileName &other)
 {
     lfn.set(other.lfn);
@@ -435,6 +453,94 @@ void normalizeNodeName(const char *node, unsigned len, SocketEndpoint &ep, bool
     ep.set(nodename.str());
 }
 
+
+//s points to the second "::" in the external filename (file::ip or plane::<plane>::)
+bool expandExternalPath(StringBuffer &dir, StringBuffer &tail, const char * filename, const char * s, bool iswin, IException **e)
+{
+    if (e)
+        *e = NULL;
+    if (!s) {
+        if (e)
+            *e = MakeStringException(-1,"Invalid format for external file (%s)",filename);
+        return false;
+    }
+    if (s[2]=='>') {
+        dir.append('/');
+        tail.append(s+2);
+        return true;
+    }
+
+    // check for ::c$/
+    if (iswin&&(s[3]=='$'))
+        s += 2;                 // no leading '\'
+    const char *s1=s;
+    const char *t1=NULL;
+    for (;;) {
+        s1 = strstr(s1,"::");
+        if (!s1)
+            break;
+        t1 = s1;
+        s1 = s1+2;
+    }
+    //The following code is never actually executed, since s always points at the leading '::'
+    if (!t1||!*t1) {
+        if (e)
+            *e = MakeStringException(-1,"No directory specified in external file name (%s)",filename);
+        return false;
+    }
+    size32_t odl = dir.length();
+    bool start=true;
+    while (s!=t1) {
+        char c=*(s++);
+        if (isPathSepChar(c)) {
+            if (e)
+                *e = MakeStringException(-1,"Path cannot contain separators, use '::' to separate directories: (%s)",filename);
+            return false;
+        }
+        if ((c==':')&&(s!=t1)&&(*s==':')) {
+            dir.append(iswin?'\\':'/');
+            s++;
+            //Disallow ::..:: to gain access to parent subdirectories
+            if (strncmp(s, "..::", 4) == 0)
+            {
+                if (e)
+                    *e = MakeStringException(-1,"External filename cannot contain relative path '..' (%s)", filename);
+                return false;
+            }
+        }
+        else if (c==':') {
+            if (e)
+                *e = MakeStringException(-1,"Path cannot contain single ':', use 'c$' to indicate 'c:' (%s)",filename);
+            return false;
+        }
+        else if (iswin&&start&&(s!=t1)&&(*s=='$')) {
+            dir.append(c).append(':');
+            s++;
+        }
+        else {
+            if ((c=='^')&&(s!=t1)) {
+                c = toupper(*s);
+                s++;
+            }
+            dir.append(c);
+        }
+        start = false;
+    }
+    t1+=2; // skip ::
+    //Always ensure there is a // - if not directory is provided it will be in the root
+    if ((dir.length()==0)||(!isPathSepChar(dir.charAt(dir.length()-1))))
+        dir.append(iswin?'\\':'/');
+    while (*t1) {
+        char c = *(t1++);
+        if ((c=='^')&&*t1) {
+            c = toupper(*t1);
+            t1++;
+        }
+        tail.append(c);
+    }
+    return true;
+}
+
 void CDfsLogicalFileName::normalizeName(const char *name, StringAttr &res, bool strict)
 {
     // NB: If !strict(default) allows spaces to exist either side of scopes (no idea why would want to permit that, but preserving for bwrd compat.)
@@ -1197,7 +1303,6 @@ bool CDfsLogicalFileName::getExternalPath(StringBuffer &dir, StringBuffer &tail,
     if (multi)
         DBGLOG("CDfsLogicalFileName::makeFullnameQuery called on multi-lfn %s",get());
 
-    size32_t odl = dir.length();
     const char *s = skipScope(lfn,EXTERNAL_SCOPE);
     if (s)
     {
@@ -1231,83 +1336,7 @@ bool CDfsLogicalFileName::getExternalPath(StringBuffer &dir, StringBuffer &tail,
             }
         }
     }
-    if (!s) {
-        if (e)
-            *e = MakeStringException(-1,"Invalid format for external file (%s)",get());
-        return false;
-    }
-    if (s[2]=='>') {
-        dir.append('/');
-        tail.append(s+2);
-        return true;
-    }
-
-    // check for ::c$/
-    if (iswin&&(s[3]=='$'))
-        s += 2;                 // no leading '\'
-    const char *s1=s;
-    const char *t1=NULL;
-    for (;;) {
-        s1 = strstr(s1,"::");
-        if (!s1)
-            break;
-        t1 = s1;
-        s1 = s1+2;
-    }
-    if (!t1||!*t1) {
-        if (e)
-            *e = MakeStringException(-1,"No directory specified in external file name (%s)",get());
-        return false;
-    }
-    bool start=true;
-    while (s!=t1) {
-        char c=*(s++);
-        if (isPathSepChar(c)) {
-            if (e)
-                *e = MakeStringException(-1,"Path cannot contain separators, use '::' to separate directories: (%s)",get());
-            return false;
-        }
-        if ((c==':')&&(s!=t1)&&(*s==':')) {
-            dir.append(iswin?'\\':'/');
-            s++;
-            //Disallow ::..:: to gain access to parent subdirectories
-            if (strncmp(s, "..::", 4) == 0)
-            {
-                if (e)
-                    *e = MakeStringException(-1,"External filename cannot contain relative path '..' (%s)", get());
-                return false;
-            }
-        }
-        else if (c==':') {
-            if (e)
-                *e = MakeStringException(-1,"Path cannot contain single ':', use 'c$' to indicate 'c:' (%s)",get());
-            return false;
-        }
-        else if (iswin&&start&&(s!=t1)&&(*s=='$')) {
-            dir.append(c).append(':');
-            s++;
-        }
-        else {
-            if ((c=='^')&&(s!=t1)) {
-                c = toupper(*s);
-                s++;
-            }
-            dir.append(c);
-        }
-        start = false;
-    }
-    t1+=2; // skip ::
-    if ((dir.length()!=odl)&&(!isPathSepChar(dir.charAt(dir.length()-1))))
-        dir.append(iswin?'\\':'/');
-    while (*t1) {
-        char c = *(t1++);
-        if ((c=='^')&&*t1) {
-            c = toupper(*t1);
-            t1++;
-        }
-        tail.append(c);
-    }
-    return true;
+    return expandExternalPath(dir, tail, get(), s, iswin, e);
 }
 
 bool CDfsLogicalFileName::getExternalFilename(RemoteFilename &rfn) const

dali/base/dautils.hpp

@@ -95,6 +95,8 @@ public:
     bool isExternal() const { return external; }
     bool isExternalPlane() const;
     bool getExternalPlane(StringBuffer & plane) const;
+    bool isExternalFile() const;
+    bool getExternalHost(StringBuffer & host) const;
     /*
      * Multi files are temporary SuperFiles only. SuperFiles created
      * by the user do not fit into this category and are created
@@ -539,5 +541,6 @@ extern da_decl IPropertyTree * getDropZonePlane(const char * name);
 extern da_decl void setPageCacheTimeoutMilliSeconds(unsigned timeoutSeconds);
 extern da_decl void setMaxPageCacheItems(unsigned _maxPageCacheItems);
 extern da_decl IRemoteConnection* connectXPathOrFile(const char* path, bool safe, StringBuffer& xpath);
+extern da_decl bool expandExternalPath(StringBuffer &dir, StringBuffer &tail, const char * filename, const char * s, bool iswin, IException **e);
 
 #endif

ecl/eclagent/eclagent.cpp

@@ -619,7 +619,7 @@ const char *EclAgent::queryTempfilePath()
 
 StringBuffer & EclAgent::getTempfileBase(StringBuffer & buff)
 {
-    return buff.append(queryTempfilePath()).append(PATHSEPCHAR).append(wuid);
+    return buff.append(queryTempfilePath()).append(PATHSEPCHAR).appendLower(wuid);
 }
 
 const char *EclAgent::queryTemporaryFile(const char *fname)

ecl/eclagent/eclgraph.cpp

@@ -240,7 +240,12 @@ static IHThorActivity * createActivity(IAgentContext & agent, unsigned activityI
     case TAKspillread:
         return createDiskReadActivity(agent, activityId, subgraphId, (IHThorDiskReadArg &)arg, kind, graph, node);
     case TAKnewdiskread:
-        return createNewDiskReadActivity(agent, activityId, subgraphId, (IHThorNewDiskReadArg &)arg, kind, graph, node);
+        {
+            bool isGeneric = (((IHThorNewDiskReadArg &)arg).getFlags() & TDXgeneric) != 0;
+            if (isGeneric)
+                return createGenericDiskReadActivity(agent, activityId, subgraphId, (IHThorNewDiskReadArg &)arg, kind, graph, node);
+            return createNewDiskReadActivity(agent, activityId, subgraphId, (IHThorNewDiskReadArg &)arg, kind, graph, node);
+        }
     case TAKdisknormalize:
         return createDiskNormalizeActivity(agent, activityId, subgraphId, (IHThorDiskNormalizeArg &)arg, kind, graph, node);
     case TAKdiskaggregate:

ecl/hql/hqlutil.cpp

@@ -10273,6 +10273,10 @@ void getFieldTypeInfo(FieldTypeInfoStruct &out, ITypeInfo *type)
                 out.fieldType |= RFTMlinkcounted;
                 out.fieldType &= ~RFTMunknownsize;
             }
+
+            // DATASET(record, COUNT()/SIZEOF) cannot currently be serialized/deserialized
+            if (queryAttributeModifier(type, _childAttr_Atom))
+                out.fieldType |= RFTMnoserialize;
             break;
         }
     case type_dictionary:
@@ -10586,3 +10590,24 @@ const RtlTypeInfo *buildRtlType(IRtlFieldTypeDeserializer &deserializer, ITypeIn
 
     return deserializer.addType(info, type);
 }
+
+
+IHqlExpression * queryAttributeModifier(ITypeInfo * type, IAtom * name)
+{
+    ITypeInfo * cur = type;
+    for(;;)
+    {
+        typemod_t mod = cur->queryModifier();
+        if (mod == typemod_none)
+            break;
+        if (mod == typemod_attr)
+        {
+            IHqlExpression * attr = (IHqlExpression *)cur->queryModifierExtra();
+            if (!name || (name == attr->queryName()))
+                return attr;
+        }
+
+        cur = cur->queryTypeBase();
+    }
+    return nullptr;
+}

ecl/hql/hqlutil.hpp

@@ -874,5 +874,6 @@ protected:
 
 extern HQL_API bool joinHasRightOnlyHardMatch(IHqlExpression * expr, bool allowSlidingMatch);
 extern HQL_API void gatherParseWarnings(IErrorReceiver * errs, IHqlExpression * expr, IErrorArray & warnings);
+extern HQL_API IHqlExpression * queryAttributeModifier(ITypeInfo * type, IAtom * name);
 
 #endif

ecl/hqlcpp/hqlcpp.cpp

@@ -1863,7 +1863,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.checkDuplicateMinActivities, "checkDuplicateMinActivities", 100),
         DebugOption(options.diskReadsAreSimple, "diskReadsAreSimple", false), // Not yet enabled - needs filters to default to generating keyed info first
         DebugOption(options.allKeyedFiltersOptional, "allKeyedFiltersOptional", false),
-        DebugOption(options.genericDiskReads, "genericDiskReads", false),
+        DebugOption(options.genericDiskReads, "genericDiskReads", false), // Can be enabled for hthor, but locking not currently supported
         DebugOption(options.generateActivityFormats, "generateActivityFormats", false),
         DebugOption(options.generateDiskFormats, "generateDiskFormats", false),
         DebugOption(options.maxOptimizeSize, "maxOptimizeSize", 5),             // Remove the overhead from very small functions e.g. function prolog

ecl/hqlcpp/hqlsource.cpp

@@ -305,8 +305,6 @@ void VirtualFieldsInfo::gatherVirtualFields(IHqlExpression * _record, bool ignor
         if (virtualAttr)
         {
             selects.append(*LINK(cur));
-            if (isUnknownSize(cur->queryType()))
-                simpleVirtualsAtEnd = false;
             if (virtuals.find(*virtualAttr) == NotFound)
                 virtuals.append(*LINK(virtualAttr));
         }
@@ -634,7 +632,7 @@ static bool forceLegacyMapping(IHqlExpression * expr)
 class SourceBuilder
 {
 public:
-    SourceBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
+    SourceBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr, bool canReadGenerically)
         : tableExpr(_tableExpr), newInputMapping(false), translator(_translator)
     { 
         nameExpr.setown(foldHqlExpression(_nameExpr));
@@ -663,6 +661,7 @@ public:
         isUnfilteredCount = false;
         requiresOrderedMerge = false;
         genericDiskReads = translator.queryOptions().genericDiskReads;
+        genericDiskRead = genericDiskReads && canReadGenerically;
         rootSelfRow = NULL;
         activityKind = TAKnone;
 
@@ -673,7 +672,7 @@ public:
             else
                 newInputMapping = translator.queryOptions().newDiskReadMapping;
 
-            if (forceLegacyMapping(tableExpr))
+            if (!genericDiskRead && forceLegacyMapping(tableExpr))
                 newInputMapping = false;
 
             //If this index has been translated using the legacy method then ensure we continue to use that method
@@ -798,12 +797,13 @@ public:
     bool            useImplementationClass;
     bool            isUnfilteredCount;
     bool            isVirtualLogicalFilenameUsed = false;
+    bool            isVirtualLogicalFileposUsed = false;
     bool            transformUsesVirtualLogicalFilename = false;
     bool            transformUsesVirtualFilePosition = false;
     bool            requiresOrderedMerge;
     bool            newInputMapping;
     bool            extractCanMatch = false;
-    bool            genericDiskReads;
+    bool            genericDiskReads = false;
     bool            genericDiskRead = false;
     bool            hasDynamicOptions = false;
 
@@ -926,6 +926,8 @@ void SourceBuilder::analyse(IHqlExpression * expr)
             {
                 if (containsVirtualField(tableExpr->queryRecord(), logicalFilenameAtom))
                     isVirtualLogicalFilenameUsed = true;
+                if (containsVirtualField(tableExpr->queryRecord(), filepositionAtom) || containsVirtualField(tableExpr->queryRecord(), localFilePositionAtom))
+                    isVirtualLogicalFileposUsed = true;
             }
         }
         break;
@@ -2703,9 +2705,10 @@ void SourceBuilder::deduceDiskRecords()
     {
         projectedRecord.set(tableExpr->queryRecord());
         expectedRecord.setown(getSerializedForm(physicalRecord, diskAtom));
-        //MORE: HPCC-18469 Reduce projected to the fields that are actually required by the dataset, and will need to remap field references.
 
-        if (fieldInfo.hasVirtuals())
+        //If the record translator can proccess the record then virtual fields can be anywhere, otherwise they need
+        //to be at the end so they can be appended - and the projected format will match the serialized disk format
+        if (!canDefinitelyProcessWithTranslator(projectedRecord) && !fieldInfo.canAppendVirtuals())
         {
             StringBuffer typeName;
             unsigned recordTypeFlags = translator.buildRtlType(typeName, projectedRecord->queryType());
@@ -2845,14 +2848,13 @@ class DiskReadBuilderBase : public SourceBuilder
 {
 public:
     DiskReadBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr, bool canReadGenerically)
-        : SourceBuilder(_translator, _tableExpr, _nameExpr), monitors(_tableExpr, _translator, 0, true, true)
+        : SourceBuilder(_translator, _tableExpr, _nameExpr, canReadGenerically), monitors(_tableExpr, _translator, 0, true, true)
     {
         fpos.setown(getFilepos(tableExpr, false));
         lfpos.setown(getFilepos(tableExpr, true));
         logicalFilenameMarker.setown(getFileLogicalName(tableExpr));
         mode = tableExpr->queryChild(2);
         modeOp = mode->getOperator();
-        genericDiskRead = genericDiskReads && canReadGenerically;
         includeFormatCrc = ((modeOp != no_csv) || genericDiskRead) && (modeOp != no_pipe);
     }
 
@@ -2963,6 +2965,8 @@ void DiskReadBuilderBase::buildFlagsMember(IHqlExpression * expr)
     if (tableExpr->hasAttribute(unsortedAtom)) flags.append("|TDRunsorted");
     if (tableExpr->hasAttribute(optAtom)) flags.append("|TDRoptional");
     if (tableExpr->hasAttribute(_workflowPersist_Atom)) flags.append("|TDXupdateaccessed");
+    if (genericDiskRead) flags.append("|TDXgeneric");
+
     if (isPreloaded) flags.append("|TDRpreload");
     if (monitors.isKeyed()) flags.append("|TDRkeyed");
     if (limitExpr)
@@ -2989,19 +2993,20 @@ void DiskReadBuilderBase::buildFlagsMember(IHqlExpression * expr)
     if (isUnfilteredCount) flags.append("|TDRunfilteredcount");
     if (isVirtualLogicalFilenameUsed || transformUsesVirtualLogicalFilename)
         flags.append("|TDRfilenamecallback");
+    if (isVirtualLogicalFileposUsed || transformUsesVirtualFilePosition)
+        flags.append("|TDRfileposcallback");
     if (transformUsesVirtualFilePosition || transformUsesVirtualLogicalFilename)
         flags.append("|TDRtransformvirtual");
     if (requiresOrderedMerge) flags.append("|TDRorderedmerge");
     if (hasDynamicOptions) flags.append("|TDRdynformatoptions");
+    if (fieldInfo.hasVirtuals() && fieldInfo.canAppendVirtuals())
+    {
+        if (!canDefinitelyProcessWithTranslator(projectedRecord))
+            flags.append("|TDRcloneappendvirtual");
+    }
 
     if (flags.length())
         translator.doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
-
-    //New activity doesn't currently support virtual callbacks from the transform.
-    //At a later date this error will be removed, and a new variant of the activity will be created
-    //that does not imposing the overhead of tracking filepositions on the general cases.
-    if (genericDiskRead && (transformUsesVirtualFilePosition || transformUsesVirtualLogicalFilename))
-        throwError(HQLERR_NoVirtualAndAlien);
 }
 
 
@@ -3113,6 +3118,11 @@ void DiskReadBuilder::analyseGraph(IHqlExpression * expr)
     DiskReadBuilderBase::analyseGraph(expr);
     if (newInputMapping && extractCanMatch && firstTransformer)
     {
+        //If the record cannot be read using the serialized meta information, do not reduce the fields because the translator
+        //cannot perform the mapping.
+        if (!canDefinitelyProcessWithTranslator(projectedRecord))
+            return;
+
         //Calculate the minimum set of fields required by any post-filters and projects.
         projectedRecord.setown(getMinimumInputRecord(translator, firstTransformer));
         if (projectedRecord != firstTransformer->queryChild(0)->queryRecord())
@@ -3329,11 +3339,12 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDiskRead(BuildCtx & ctx, IHqlE
         const bool forceAllProjectedSerialized = options.forceAllProjectedDiskSerialized;
         //Reading from a spill file uses the in-memory format to optimize on-demand spilling.
         bool optimizeInMemorySpill = targetThor();
-        bool useInMemoryFormat = optimizeInMemorySpill && isSimpleProjectingDiskRead(expr);
+        IHqlExpression * record = tableExpr->queryRecord();
+        bool useInMemoryFormat = optimizeInMemorySpill && isSimpleProjectingDiskRead(expr) && !canDefinitelyProcessWithTranslator(record);
         if (forceAllProjectedSerialized || !useInMemoryFormat)
         {
             //else if the the table isn't serialized, then map to a serialized table, and then project to the real format
-            if (recordRequiresSerialization(tableExpr->queryRecord(), diskAtom))
+            if (recordRequiresSerialization(record, diskAtom))
             {
                 OwnedHqlExpr transformed = buildTableFromSerialized(expr);
                 //Need to wrap a possible no_usertable, otherwise the localisation can go wrong.
@@ -3697,7 +3708,7 @@ class ChildBuilderBase : public SourceBuilder
 {
 public:
     ChildBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
-        : SourceBuilder(_translator, _tableExpr, _nameExpr)
+        : SourceBuilder(_translator, _tableExpr, _nameExpr, false)
     { 
     }
 
@@ -3972,7 +3983,7 @@ class IndexReadBuilderBase : public SourceBuilder
     friend class MonitorRemovalTransformer;
 public:
     IndexReadBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
-        : SourceBuilder(_translator, _tableExpr, _nameExpr),
+        : SourceBuilder(_translator, _tableExpr, _nameExpr, false),
           monitors(_tableExpr, _translator, -(int)numPayloadFields(_tableExpr), false, getHintBool(_tableExpr, createValueSetsAtom, _translator.queryOptions().createValueSets))
     {
     }
@@ -4878,7 +4889,7 @@ class FetchBuilder : public SourceBuilder
 {
 public:
     FetchBuilder(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr, IHqlExpression * _fetchExpr)
-        : SourceBuilder(_translator, _tableExpr, _nameExpr)
+        : SourceBuilder(_translator, _tableExpr, _nameExpr, false)
     {
         compoundExpr.set(_fetchExpr);
         fetchExpr.set(queryFetch(_fetchExpr));

ecl/hqlcpp/hqlsource.ipp

@@ -36,6 +36,7 @@ public:
     void gatherVirtualFields(IHqlExpression * record, bool ignoreVirtuals, bool ensureSerialized);
     bool hasVirtuals()      { return virtuals.ordinality() != 0; }
     bool hasVirtualsOrDeserialize() { return requiresDeserialize || virtuals.ordinality() != 0; }
+    bool canAppendVirtuals() { return simpleVirtualsAtEnd; }
 
 public:
     HqlExprArray    physicalFields;

ecl/hthor/hthor.cpp

@@ -54,6 +54,8 @@
 #include "ftbase.ipp"
 #include "rtldynfield.hpp"
 #include "rtlnewkey.hpp"
+
+#include "thormeta.hpp"
 #include "thorread.hpp"
 
 #define EMPTY_LOOP_LIMIT 1000
@@ -532,7 +534,7 @@ void CHThorDiskWriteActivity::resolve()
     else
     {
         StringBuffer mangledName;
-        mangleLocalTempFilename(mangledName, mangledHelperFileName.str());
+        mangleLocalTempFilename(mangledName, mangledHelperFileName.str(), nullptr);
         filename.set(agent.noteTemporaryFile(mangledName.str()));
         PROGLOG("DISKWRITE: using temporary filename %s", filename.get());
     }
@@ -8267,7 +8269,7 @@ void CHThorDiskReadBaseActivity::resolve()
     if (helper.getFlags() & (TDXtemporary | TDXjobtemp))
     {
         StringBuffer mangledFilename;
-        mangleLocalTempFilename(mangledFilename, mangledHelperFileName.str());
+        mangleLocalTempFilename(mangledFilename, mangledHelperFileName.str(), nullptr);
         tempFileName.set(agent.queryTemporaryFile(mangledFilename.str()));
         logicalFileName.set(tempFileName);
         gatherInfo(NULL);
@@ -10650,7 +10652,7 @@ void CHThorNewDiskReadBaseActivity::resolveFile()
     if (helper.getFlags() & (TDXtemporary | TDXjobtemp))
     {
         StringBuffer mangledFilename;
-        mangleLocalTempFilename(mangledFilename, mangledHelperFileName.str());
+        mangleLocalTempFilename(mangledFilename, mangledHelperFileName.str(), nullptr);
         tempFileName.set(agent.queryTemporaryFile(mangledFilename.str()));
         logicalFileName = tempFileName.str();
         gatherInfo(NULL);
@@ -10760,7 +10762,7 @@ CHThorNewDiskReadBaseActivity::InputFileInfo * CHThorNewDiskReadBaseActivity::ex
     Owned<IPropertyTree> meta = createPTree();
     unsigned actualCrc = helper.getDiskFormatCrc();
     Linked<IOutputMetaData> actualDiskMeta = expectedDiskMeta;
-    Linked<const IPropertyTree> fileFormatOptions = curFormatOptions;
+    Linked<IPropertyTree> fileFormatOptions = createPTreeFromIPT(curFormatOptions);
     bool compressed = false;
     bool blockcompressed = false;
     const char * readFormat = helper.queryFormat();
@@ -10785,7 +10787,7 @@ CHThorNewDiskReadBaseActivity::InputFileInfo * CHThorNewDiskReadBaseActivity::ex
 
             size32_t dfsSize = props.getPropInt("@recordSize");
             if (dfsSize != 0)
-                meta->setPropInt("dfsRecordSize", dfsSize);
+                meta->setPropInt("@recordSize", dfsSize);
         }
         compressed = distributedFile->isCompressed(&blockcompressed); //try new decompression, fall back to old unless marked as block
 
@@ -10802,25 +10804,22 @@ CHThorNewDiskReadBaseActivity::InputFileInfo * CHThorNewDiskReadBaseActivity::ex
 
         //MORE: There should probably be a generic way of storing and extracting format options for a file
         IPropertyTree & options = distributedFile->queryAttributes();
-        Linked<IPropertyTree> tempOptions = createPTreeFromIPT(fileFormatOptions);
-        queryInheritProp(*tempOptions, "quote", options, "@csvQuote");
-        queryInheritSeparatorProp(*tempOptions, "separator", options, "@csvSeparate");
-        queryInheritProp(*tempOptions, "terminator", options, "@csvTerminate");
-        queryInheritProp(*tempOptions, "escape", options, "@csvEscape");
+        queryInheritProp(*fileFormatOptions, "quote", options, "@csvQuote");
+        queryInheritSeparatorProp(*fileFormatOptions, "separator", options, "@csvSeparate");
+        queryInheritProp(*fileFormatOptions, "terminator", options, "@csvTerminate");
+        queryInheritProp(*fileFormatOptions, "escape", options, "@csvEscape");
+        dbglogXML(fileFormatOptions);
         dbglogXML(fileFormatOptions);
-        dbglogXML(tempOptions);
-        if (!areMatchingPTrees(fileFormatOptions, tempOptions))
-            fileFormatOptions.setown(tempOptions.getClear());
     }
 
-    meta->setPropBool("grouped", grouped);
-    meta->setPropBool("compressed", compressed);
-    meta->setPropBool("blockCompressed", blockcompressed);
-    meta->setPropBool("forceCompressed", (helper.getFlags() & TDXcompress) != 0);
+    meta->setPropBool("@grouped", grouped);
+    meta->setPropBool("@compressed", compressed);
+    meta->setPropBool("@blockCompressed", blockcompressed);
+    meta->setPropBool("@forceCompressed", (helper.getFlags() & TDXcompress) != 0);
+    meta->setPropTree("formatOptions", fileFormatOptions.getClear());
 
     InputFileInfo & target = * new InputFileInfo;
     target.file = distributedFile;
-    target.formatOptions.swap(fileFormatOptions);
     target.meta.setown(meta.getClear());
     target.actualCrc = actualCrc;
     target.actualMeta.swap(actualDiskMeta);
@@ -10980,7 +10979,7 @@ bool CHThorNewDiskReadBaseActivity::openFilePart(const char * filename)
 
     unsigned expectedCrc = helper.getDiskFormatCrc();
     unsigned projectedCrc = helper.getProjectedFormatCrc();
-    IDiskRowReader * reader = ensureRowReader(format, false, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, expectedCrc, *expectedDiskMeta, fileInfo->formatOptions);
+    IDiskRowReader * reader = ensureRowReader(format, false, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, expectedCrc, *expectedDiskMeta, fileInfo->meta);
     if (reader->setInputFile(filename, logicalFileName, 0, offsetOfPart, fileInfo->meta, fieldFilters))
     {
         initStream(reader, filename);
@@ -11035,7 +11034,7 @@ bool CHThorNewDiskReadBaseActivity::openFilePart(ILocalOrDistributedFile * local
         {
             StringBuffer path;
             rfn.getPath(path);
-            IDiskRowReader * reader = ensureRowReader(format, false, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, actualCrc, *actualDiskMeta, fileInfo->formatOptions);
+            IDiskRowReader * reader = ensureRowReader(format, false, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, actualCrc, *actualDiskMeta, fileInfo->meta);
             if (reader->setInputFile(path.str(), logicalFileName, whichPart, offsetOfPart, fileInfo->meta, fieldFilters))
             {
                 initStream(reader, path.str());
@@ -11058,7 +11057,7 @@ bool CHThorNewDiskReadBaseActivity::openFilePart(ILocalOrDistributedFile * local
             filenamelist.append('\n').append(filename);
             try
             {
-                IDiskRowReader * reader = ensureRowReader(format, tryRemoteStream, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, actualCrc, *actualDiskMeta, fileInfo->formatOptions);
+                IDiskRowReader * reader = ensureRowReader(format, tryRemoteStream, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, actualCrc, *actualDiskMeta, fileInfo->meta);
                 if (reader->setInputFile(rfilename, logicalFileName, whichPart, offsetOfPart, fileInfo->meta, fieldFilters))
                 {
                     initStream(reader, filename);
@@ -11298,6 +11297,566 @@ const void *CHThorNewDiskReadActivity::nextRow()
 
 //=====================================================================================================
 
+bool RemoteReadChecker::onlyReadLocally(const CLogicalFileSlice & slice, unsigned copy)
+{
+    //Allow all operations to be forced to be executed locally.
+    if (forceRemoteDisabled.getValue(false))
+        return true;
+
+    //If not locally attached then there is no benefit in reading remotely
+    if (!slice.onAttachedStorage(copy))
+        return true;
+
+    //If the file is not local then execute it remotely
+    if (!slice.isLocal(copy))
+        return false;
+
+    StringBuffer localPath;
+    slice.getURL(localPath, copy);
+    if (forceRemoteRead.getValue(testForceRemote(localPath)))
+        return false;
+    return true;
+}
+
+
+CHThorGenericDiskReadBaseActivity::CHThorGenericDiskReadBaseActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNewDiskReadBaseArg &_arg, IHThorCompoundBaseArg & _segHelper, ThorActivityKind _kind, EclGraph & _graph, IPropertyTree *_node)
+: CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind, _graph), helper(_arg), segHelper(_segHelper), remoteReadChecker(_agent.queryWorkUnit())
+{
+    helper.setCallback(this);
+    expectedDiskMeta = helper.queryDiskRecordSize();
+    projectedDiskMeta = helper.queryProjectedDiskRecordSize();
+    isCodeSigned = false;
+    if (_node)
+    {
+        const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layouttranslation']/@value");
+        if (recordTranslationModeHintText)
+            recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText, true);
+        isCodeSigned = isActivityCodeSigned(*_node);
+    }
+
+    grouped = ((helper.getFlags() & TDXgrouped) != 0);
+    inputOptions.setown(createPTree());
+    inputOptions->setPropBool("@grouped", grouped);
+    inputOptions->setPropBool("@forceCompressed", (helper.getFlags() & TDXcompress) != 0);
+    if (helper.getFlags() & TDRoptional)
+        inputOptions->setPropBool("@optional", true);
+    if ((helper.getFlags() & TDRcloneappendvirtual) != 0)
+        inputOptions->setPropBool("@cloneAppendVirtuals", true);
+
+    CPropertyTreeWriter writer(ensurePTree(inputOptions, "formatOptions"));
+    helper.getFormatOptions(writer);
+
+    outputGrouped = helper.queryOutputMeta()->isGrouped();  // It is possible for input to be incorrectly marked as grouped, and input not or vice-versa
+    bool isTemporary = (helper.getFlags() & (TDXtemporary | TDXjobtemp)) != 0;
+    files.init(this, agent.queryWuid(), isTemporary, agent.queryResolveFilesLocally(), isCodeSigned, agent.queryCodeContext()->queryUserDescriptor(), expectedDiskMeta);
+    if (isTemporary)
+    {
+        StringBuffer spillPath;
+        agent.getTempfileBase(spillPath);
+
+        //Should probably be in eclagent
+        spillPlane.setown(createPTree("planes"));
+        spillPlane->setProp("@name", "localspill");
+        spillPlane->setProp("@prefix", spillPath);
+    }
+}
+
+CHThorGenericDiskReadBaseActivity::~CHThorGenericDiskReadBaseActivity()
+{
+    close();
+}
+
+void CHThorGenericDiskReadBaseActivity::ready()
+{
+    CHThorActivityBase::ready();
+
+    opened = false;
+    curSlice = NotFound;
+
+    resolveFile();
+
+    fieldFilters.kill();
+    segHelper.createSegmentMonitors(this);
+}
+
+void CHThorGenericDiskReadBaseActivity::stop()
+{
+    close();
+    CHThorActivityBase::stop();
+}
+
+unsigned __int64 CHThorGenericDiskReadBaseActivity::getFilePosition(const void * row)
+{
+    //These functions do not need to be implemented - they will be implemented by the translation layer
+    throwUnexpected();
+}
+
+unsigned __int64 CHThorGenericDiskReadBaseActivity::getLocalFilePosition(const void * row)
+{
+    throwUnexpected();
+}
+
+void CHThorGenericDiskReadBaseActivity::noteException(unsigned severity, unsigned code, const char * text)
+{
+    //MORE: This should really supply the activity and the scope - a general issue for hthor errors...
+    agent.addWuExceptionEx(text, code, severity, MSGAUD_user, "hthor");
+}
+
+
+const char * CHThorGenericDiskReadBaseActivity::queryLogicalFilename(const void * row)
+{
+    throwUnexpected();
+}
+
+void CHThorGenericDiskReadBaseActivity::resolveFile()
+{
+    //If in a child query, and the filenames haven't changed, the information about the resolved filenames will also not have changed
+    //Assume that is also true for format properties - require dynamic if they are to be recalculated.
+    if (resolved && !(helper.getFlags() & (TDXvarfilename|TDRdynformatoptions)))
+        return;
+    resolved = true;
+
+    //Update the inputOptions and formatOptions if they depend on the current context
+    curInputOptions.set(inputOptions);
+    //Check for encryption key
+    void *k;
+    size32_t kl;
+    helper.getEncryptKey(kl,k);
+    if (kl || (helper.getFlags() & TDRdynformatoptions))
+    {
+        curInputOptions.setown(createPTreeFromIPT(inputOptions));
+        if (kl)
+        {
+            curInputOptions->setPropBin("encryptionKey", kl, k);
+            curInputOptions->setPropBool("blockcompressed", true);
+            curInputOptions->setPropBool("compressed", true);
+        }
+
+        if (helper.getFlags() & TDRdynformatoptions)
+        {
+            Owned<IPropertyTree> helperFormatOptions = createPTree("formatOptions");
+            CPropertyTreeWriter writer(helperFormatOptions);
+            helper.getFormatDynOptions(writer);
+
+            IPropertyTree * curFormatOptions = ensurePTree(curInputOptions, "formatOptions");
+            mergeConfiguration(*curFormatOptions, *helperFormatOptions, nullptr, true);
+        }
+    }
+
+    //Extract meta information from the helper.  Another (possibly more efficient) alternative to an IPropertyTree would be a class.
+    bool isTemporary = (helper.getFlags() & (TDXtemporary | TDXjobtemp)) != 0;
+    OwnedRoxieString fileName(helper.getFileName());
+    if (isTemporary)
+    {
+        StringBuffer mangledFilename;
+        mangleLocalTempFilename(mangledFilename, fileName, agent.queryWuid());    // should this occur inside setEclFilename?
+        curInputOptions->setPropBool("@singlePartNoSuffix", true);
+        files.setTempFilename(mangledFilename, curInputOptions, spillPlane);
+    }
+    else
+    {
+        StringBuffer lfn;
+        expandLogicalFilename(lfn, fileName, agent.queryWorkUnit(), false, false);
+        files.setEclFilename(lfn, curInputOptions);
+    }
+    slices.clear();
+    files.calcPartition(slices, 1, 0, false, true);
+    curSlice = 0;
+}
+
+void CHThorGenericDiskReadBaseActivity::close()
+{
+    closepart();
+    if (activeSlice)
+        activeSlice->setAccessed();
+}
+
+void CHThorGenericDiskReadBaseActivity::closepart()
+{
+    if (activeReader)
+    {
+        activeReader->clearInput();
+        activeReader = nullptr;
+        activeSlice = nullptr;
+    }
+}
+
+bool CHThorGenericDiskReadBaseActivity::openFirstPart()
+{
+    if (openFilePart(0U))
+        return true;
+    setEmptyStream();
+    return false;
+}
+
+bool CHThorGenericDiskReadBaseActivity::openNextPart()
+{
+    if (curSlice == NotFound)
+        return false;
+
+    if (activeSlice)
+        closepart();
+
+    if (openFilePart(curSlice+1))
+        return true;
+    setEmptyStream();
+    return false;
+}
+
+void CHThorGenericDiskReadBaseActivity::initStream(CLogicalFileSlice * slice, IDiskRowReader * reader)
+{
+    activeSlice = slice;
+    activeReader = reader;
+    inputRowStream = reader->queryAllocatedRowStream(rowAllocator);
+
+    StringBuffer report("Reading file ");
+    activeSlice->getTracingFilename(report);
+    agent.reportProgress(report.str());
+}
+
+void CHThorGenericDiskReadBaseActivity::setEmptyStream()
+{
+    inputRowStream = queryNullDiskRowStream();
+    finishedParts = true;
+}
+
+IDiskRowReader * CHThorGenericDiskReadBaseActivity::ensureRowReader(const char * format, bool streamRemote, unsigned expectedCrc, IOutputMetaData & expected, unsigned projectedCrc, IOutputMetaData & projected, unsigned actualCrc, IOutputMetaData & actual, CLogicalFileSlice * slice)
+{
+    bool translateFromActual = strsame(format, slice->queryFormat());
+    //Backwards compatibility - there should be an option to override this
+    if (strsame(format, "csv") || strsame(format, "xml"))
+        translateFromActual = false;
+
+    //If the actual and expected file formats do not translate from the actual file format - use the expected format instead
+    Owned<IDiskReadMapping> mapping;
+    if (translateFromActual)
+        mapping.setown(createDiskReadMapping(getLayoutTranslationMode(), format, actualCrc, actual, expectedCrc, expected, projectedCrc, projected, slice->queryFileMeta()));
+    else
+        mapping.setown(createDiskReadMapping(getLayoutTranslationMode(), format, expectedCrc, expected, expectedCrc, expected, projectedCrc, projected, slice->queryFileMeta()));
+
+    ForEachItemIn(i, readers)
+    {
+        IDiskRowReader & cur = readers.item(i);
+        if (cur.matches(format, streamRemote, mapping))
+            return &cur;
+    }
+    IDiskRowReader * reader = createDiskReader(format, streamRemote, mapping);
+    readers.append(*reader);
+    return reader;
+}
+
+bool CHThorGenericDiskReadBaseActivity::openFilePart(unsigned whichSlice)
+{
+    for (;;)
+    {
+        if (whichSlice >= slices.size())
+        {
+            curSlice = NotFound;
+            return false;
+        }
+
+        if (openFilePart(&slices[whichSlice]))
+        {
+            curSlice = whichSlice;
+            activeSlice = &slices[whichSlice];
+            return true;
+        }
+
+        whichSlice++;
+    }
+}
+
+bool CHThorGenericDiskReadBaseActivity::openFilePart(CLogicalFileSlice * nextSlice)
+{
+    unsigned expectedCrc = helper.getDiskFormatCrc();
+    unsigned projectedCrc = helper.getProjectedFormatCrc();
+    unsigned actualCrc = nextSlice->queryFile()->queryActualCrc();
+    IOutputMetaData * actualDiskMeta = nextSlice->queryFile()->queryActualMeta();
+
+    bool tryRemoteStream = actualDiskMeta->queryTypeInfo()->canInterpret() && actualDiskMeta->queryTypeInfo()->canSerialize() &&
+                           projectedDiskMeta->queryTypeInfo()->canInterpret() && projectedDiskMeta->queryTypeInfo()->canSerialize();
+
+
+    /*
+     * If a file part can be accessed local, then read it locally
+     * If a file part supports a remote stream, then use that
+     * Otherwise failover to the legacy remote access.
+     */
+    const char * format = helper.queryFormat();
+    // If format is not specified in the ECL then it is deduced from the file.  It must be the same for all copies of a file part
+    if (!format)
+        format = nextSlice->queryFormat();
+
+    Owned<IException> saveOpenExc;
+    StringBuffer filenamelist;
+    std::vector<unsigned> remoteCandidates;
+
+    // scan for local part 1st
+    //MORE: Order of copies should be optimized at this point....
+    unsigned numCopies = nextSlice->getNumCopies();
+    for (unsigned copy=0; copy<numCopies; copy++)
+    {
+        if (remoteReadChecker.onlyReadLocally(*nextSlice, copy))
+        {
+            IDiskRowReader * reader = ensureRowReader(format, false, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, actualCrc, *actualDiskMeta, nextSlice);
+            if (reader->setInputFile(*nextSlice, fieldFilters, copy))
+            {
+                initStream(nextSlice, reader);
+                return true;
+            }
+        }
+        else
+            remoteCandidates.push_back(copy);
+    }
+
+    //First try remote streaming, and if that does not succeed, fall back to remote reading.
+    bool allowFallbackToNonStreaming = true;
+    for (;;)
+    {
+        for (unsigned copy: remoteCandidates)
+        {
+            StringBuffer filename;
+            nextSlice->getURL(filename, copy);
+            filenamelist.append('\n').append(filename);
+            try
+            {
+                IDiskRowReader * reader = ensureRowReader(format, tryRemoteStream, expectedCrc, *expectedDiskMeta, projectedCrc, *projectedDiskMeta, actualCrc, *actualDiskMeta, nextSlice);
+                if (reader->setInputFile(*nextSlice, fieldFilters, copy))
+                {
+                    initStream(nextSlice, reader);
+                    return true;
+                }
+            }
+            catch (IException *E)
+            {
+                saveOrRelease(saveOpenExc, E);
+            }
+        }
+
+        if (!tryRemoteStream || !allowFallbackToNonStreaming)
+            break;
+        tryRemoteStream = false;
+    }
+
+    if (!(helper.getFlags() & TDRoptional))
+    {
+        //Should this be unconditional?  If the logical file exists, but the file can't be opened, it isn't really what OPT means.
+        StringBuffer s;
+        StringBuffer tracingName;
+        nextSlice->getTracingFilename(tracingName);
+
+        if (filenamelist)
+        {
+            if (saveOpenExc.get())
+            {
+                if (!nextSlice->isLogicalFile())
+                    saveOpenExc->errorMessage(s);
+                else
+                {
+                    s.append("Could not open logical file ").append(tracingName).append(" in any of these locations:").append(filenamelist).append(" (");
+                    saveOpenExc->errorMessage(s).append(")");
+                }
+            }
+            else
+                s.append("Could not open logical file ").append(tracingName).append(" in any of these locations:").append(filenamelist).append(" (").append((unsigned)GetLastError()).append(")");
+        }
+        else
+        {
+            const char * filename = nextSlice->queryFile()->queryLogicalFilename();
+            s.append("Could not open local physical file ").append(filename).append(" (").append((unsigned)GetLastError()).append(")");
+        }
+        agent.fail(1, s.str());
+    }
+    return false;
+}
+
+
+bool CHThorGenericDiskReadBaseActivity::openNext()
+{
+    return openNextPart();
+}
+
+void CHThorGenericDiskReadBaseActivity::open()
+{
+    assertex(!opened);
+    opened = true;
+    if (!segHelper.canMatchAny())
+    {
+        setEmptyStream();
+    }
+    else
+    {
+        if (!openFirstPart())
+            setEmptyStream();
+    }
+}
+
+void CHThorGenericDiskReadBaseActivity::append(FFoption option, const IFieldFilter * filter)
+{
+    if (filter->isWild())
+        filter->Release();
+    else
+        fieldFilters.append(*filter);
+}
+
+//=====================================================================================================
+
+CHThorGenericDiskReadActivity::CHThorGenericDiskReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNewDiskReadArg &_arg, ThorActivityKind _kind, EclGraph & _graph, IPropertyTree *_node)
+: CHThorGenericDiskReadBaseActivity(_agent, _activityId, _subgraphId, _arg, _arg, _kind, _graph, _node), helper(_arg), outBuilder(NULL)
+{
+    hasMatchFilter = helper.hasMatchFilter();
+    useRawStream = hasMatchFilter || helper.needTransform();
+}
+
+void CHThorGenericDiskReadActivity::ready()
+{
+    PARENT::ready();
+    outBuilder.setAllocator(rowAllocator);
+    lastGroupProcessed = processed;
+    needTransform = helper.needTransform() || fieldFilters.length();
+    limit = helper.getRowLimit();
+    if (helper.getFlags() & TDRlimitskips)
+        limit = (unsigned __int64) -1;
+    stopAfter = helper.getChooseNLimit();
+    if (!helper.transformMayFilter() && !helper.hasMatchFilter())
+        remoteLimit = stopAfter;
+    finishedParts = false;
+}
+
+
+void CHThorGenericDiskReadActivity::stop()
+{
+    outBuilder.clear();
+    PARENT::stop();
+}
+
+
+void CHThorGenericDiskReadActivity::onLimitExceeded()
+{
+    if ( agent.queryCodeContext()->queryDebugContext())
+        agent.queryCodeContext()->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
+    helper.onLimitExceeded();
+}
+
+const void *CHThorGenericDiskReadActivity::nextRow()
+{
+    //Avoid this check on each row- e.g., initialising streams with a null stream, which returns eof, and falls through to eof processing
+    if (!opened) open();
+
+    // Only check once per row returned.  Potentially means that heavily filtered datasets may wait a long time to check for abort
+    queryUpdateProgress();
+
+    //Avoid this test...  Combine the limit checking with choosen, and have choosen/limit triggering set the
+    //stream to a special no more rows stream so that subsequent calls do not read records.
+    if ((processed - initialProcessed) >= stopAfter)
+        return nullptr;
+
+    try
+    {
+        if (useRawStream)
+        {
+            for (;;)
+            {
+                //Returns a row in the serialized form of the projected format
+                size32_t nextSize;
+                const byte * next = (const byte *)inputRowStream->nextRow(nextSize);
+                if (!isSpecialRow(next))
+                {
+                    if (likely(!hasMatchFilter || helper.canMatch(next)))
+                    {
+                        size32_t thisSize = helper.transform(outBuilder.ensureRow(), next);
+                        if (thisSize != 0)
+                        {
+                            if (unlikely((processed - initialProcessed) >= limit))
+                            {
+                                outBuilder.clear();
+                                onLimitExceeded();
+                                return nullptr;
+                            }
+                            processed++;
+                            return outBuilder.finalizeRowClear(thisSize);
+                        }
+                    }
+                }
+                else
+                {
+                    switch (getSpecialRowType(next))
+                    {
+                    case SpecialRow::eof:
+                        if (!openNext())
+                            return next; // i.e. eof
+                        //rawStream will have changed, but it cannot change into a rowStream
+                        break;
+                    case SpecialRow::eos:
+                        return next;
+                    case SpecialRow::eog:
+                        if (outputGrouped && (processed != lastGroupProcessed))
+                        {
+                            lastGroupProcessed = processed;
+                            //MORE: Change to return next - i.e. an eog marker
+                            return nullptr;
+                        }
+                        break;
+                    default:
+                        throwUnexpected();
+                    }
+                }
+            }
+        }
+        else
+        {
+            //This branch avoids a memcpy from actual to projected followed by a deserialize - since it can map directly
+            //May be more efficient to use this branch if serialized==deserialized and there is a filter, but no transform.
+            //It would be possibel to have two (or more) different implementations, which were created based on
+            //whether there was a limit, a transform etc., but unlikely to save more than a couple of boolean tests.
+            for (;;)
+            {
+                const byte * next = (const byte *)inputRowStream->nextRow();
+                if (!isSpecialRow(next))
+                {
+                    if (unlikely((processed - initialProcessed) >= limit))
+                    {
+                        ReleaseRoxieRow(next);
+                        onLimitExceeded();
+                        return nullptr;
+                    }
+                    processed++;
+                    return next;
+                }
+                else
+                {
+                    switch (getSpecialRowType(next))
+                    {
+                    case SpecialRow::eof:
+                        if (!openNext())
+                            return next;
+                        //rowStream will have changed
+                        break;
+                    case SpecialRow::eos:
+                        return next;
+                    case SpecialRow::eog:
+                        if (processed != lastGroupProcessed)
+                        {
+                            lastGroupProcessed = processed;
+                            return nullptr;
+                        }
+                        break;
+                    default:
+                        throwUnexpected();
+                    }
+                }
+            }
+        }
+    }
+    catch(IException * e)
+    {
+        throw makeWrappedException(e);
+    }
+    return NULL;
+}
+
+//=====================================================================================================
+
 MAKEFACTORY(DiskWrite);
 MAKEFACTORY(Iterate);
 MAKEFACTORY(Filter);
@@ -11358,6 +11917,11 @@ extern HTHOR_API IHThorActivity *createHashAggregateActivity(IAgentContext &_age
     return new CHThorHashAggregateActivity(_agent, _activityId, _subgraphId, arg, kind, _graph, _isGroupedAggregate);
 }
 
+extern HTHOR_API IHThorActivity *createGenericDiskReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNewDiskReadArg &arg, ThorActivityKind kind, EclGraph & _graph, IPropertyTree * node)
+{
+    return new CHThorGenericDiskReadActivity(_agent, _activityId, _subgraphId, arg, kind, _graph, node);
+}
+
 MAKEFACTORY(Null);
 MAKEFACTORY(SideEffect);
 MAKEFACTORY(Action);

ecl/hthor/hthor.hpp

@@ -173,6 +173,7 @@ extern HTHOR_API IHThorActivity *createDiskCountActivity(IAgentContext &_agent,
 extern HTHOR_API IHThorActivity *createDiskGroupAggregateActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskGroupAggregateArg &arg, ThorActivityKind kind, EclGraph & _graph, IPropertyTree *node);
 
 extern HTHOR_API IHThorActivity *createNewDiskReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNewDiskReadArg &arg, ThorActivityKind kind, EclGraph & _graph, IPropertyTree *node);
+extern HTHOR_API IHThorActivity *createGenericDiskReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorNewDiskReadArg &arg, ThorActivityKind kind, EclGraph & _graph, IPropertyTree * node);
 
 extern HTHOR_API IHThorActivity *createIndexReadActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorIndexReadArg &arg, ThorActivityKind kind, EclGraph & _graph, IPropertyTree *_node);
 extern HTHOR_API IHThorActivity *createIndexNormalizeActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorIndexNormalizeArg &arg, ThorActivityKind kind, EclGraph & _graph, IPropertyTree *_node);

ecl/hthor/hthor.ipp

@@ -43,6 +43,8 @@
 #include "rtlrecord.hpp"
 #include "roxiemem.hpp"
 #include "roxierowbuff.hpp"
+
+#include "thormeta.hpp"
 #include "thorread.hpp"
 
 roxiemem::IRowManager * queryRowManager();
@@ -2927,7 +2929,6 @@ protected:
     {
         IDistributedFile * file;
         Owned<IOutputMetaData> actualMeta;
-        Owned<const IPropertyTree> formatOptions;
         Owned<const IPropertyTree> meta;
         unsigned actualCrc;
     };
@@ -3043,6 +3044,169 @@ protected:
 };
 
 
+//---------------------------------------------------------------------------------------------------------------------
+
+//Could be a useful general class for caching workunit debug and tree values
+template <class T>
+class CachedValue
+{
+public:
+    T getValue(const T dft = false) { return hasValue ? value : dft; }
+
+protected:
+    bool hasValue = false;
+    T value = false;
+};
+
+class CachedBoolValue : public CachedValue<bool>
+{
+public:
+    CachedBoolValue(IConstWorkUnit * wu, const char * name)
+    {
+        hasValue = wu->hasDebugValue(name);
+        if (hasValue)
+            value = wu->getDebugValueBool(name, false);
+    }
+
+    CachedBoolValue(IPropertyTree * tree, const char * name)
+    {
+        hasValue = tree->queryProp(name) != nullptr;
+        if (hasValue)
+            value = tree->getPropBool(name);
+    }
+};
+
+class RemoteReadChecker
+{
+public:
+    RemoteReadChecker(IConstWorkUnit * wu)
+    : forceRemoteDisabled(wu, "forceRemoteDisabled"), forceRemoteRead(wu, "forceRemoteRead")
+    {
+    }
+
+    bool onlyReadLocally(const CLogicalFileSlice & nextSlice, unsigned copy);
+
+protected:
+    CachedBoolValue forceRemoteDisabled;
+    CachedBoolValue forceRemoteRead;
+};
+
+//---------------------------------------------------------------------------------------------------------------------
+
+class CHThorGenericDiskReadBaseActivity : public CHThorActivityBase, implements IThorDiskCallback, implements IIndexReadContext, public IFileCollectionContext
+{
+protected:
+    IHThorNewDiskReadBaseArg &helper;
+    IHThorCompoundBaseArg & segHelper;
+    IDiskRowReader * activeReader = nullptr;
+    CLogicalFileCollection files;
+    Owned<IPropertyTree> spillPlane;
+    std::vector<CLogicalFileSlice> slices;
+    IArrayOf<IDiskRowReader> readers;
+    IDiskRowStream * inputRowStream = nullptr;
+    RemoteReadChecker remoteReadChecker;
+    IOutputMetaData *expectedDiskMeta = nullptr;
+    IOutputMetaData *projectedDiskMeta = nullptr;
+    IConstArrayOf<IFieldFilter> fieldFilters;  // These refer to the expected layout
+    Owned<IPropertyTree> inputOptions;
+    Owned<IPropertyTree> curInputOptions;
+    CLogicalFileSlice * activeSlice = nullptr;
+    unsigned curSlice = 0;
+    RecordTranslationMode recordTranslationModeHint = RecordTranslationMode::Unspecified;
+    bool useRawStream = false; // Constant for the lifetime of the activity
+    bool grouped = false;
+    bool outputGrouped = false;
+    bool opened = false;
+    bool finishedParts = false;
+    bool isCodeSigned = false;
+    bool resolved = false;
+    unsigned __int64 stopAfter = 0;
+
+protected:
+    void close();
+    void resolveFile();
+    StringBuffer &translateLFNtoLocal(const char *filename, StringBuffer &localName);
+
+    inline void queryUpdateProgress()
+    {
+        agent.reportProgress(NULL);
+    }
+
+    RecordTranslationMode getLayoutTranslationMode()
+    {
+        if (recordTranslationModeHint != RecordTranslationMode::Unspecified)
+            return recordTranslationModeHint;
+        return agent.getLayoutTranslationMode();
+    }
+
+public:
+    CHThorGenericDiskReadBaseActivity(IAgentContext &agent, unsigned _activityId, unsigned _subgraphId, IHThorNewDiskReadBaseArg &_arg, IHThorCompoundBaseArg & _segHelper, ThorActivityKind _kind, EclGraph & _graph, IPropertyTree *node);
+    ~CHThorGenericDiskReadBaseActivity();
+    IMPLEMENT_IINTERFACE_USING(CHThorActivityBase)
+
+    virtual void ready();
+    virtual void stop();
+
+    IHThorInput *queryOutput(unsigned index)                { return this; }
+
+//interface IHThorInput
+    virtual bool isGrouped()                                { return outputGrouped; }
+    virtual IOutputMetaData * queryOutputMeta() const       { return outputMeta; }
+
+//interface IFilePositionProvider
+    virtual unsigned __int64 getFilePosition(const void * row);
+    virtual unsigned __int64 getLocalFilePosition(const void * row);
+    virtual const char * queryLogicalFilename(const void * row);
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
+
+//interface IIndexReadContext
+    virtual void append(IKeySegmentMonitor *segment) override { throwUnexpected(); }
+    virtual void append(FFoption option, const IFieldFilter * filter) override;
+
+//interface IFileCollectionContext
+    virtual void noteException(unsigned severity, unsigned code, const char * text) override;
+
+protected:
+    bool openFirstPart();
+    void initStream(CLogicalFileSlice * slice, IDiskRowReader * reader);
+    bool openFilePart(unsigned whichSlice);
+    bool openFilePart(CLogicalFileSlice * nextSlice);
+    void setEmptyStream();
+
+    virtual void open();
+    virtual bool openNext();
+    virtual void closepart();
+
+    bool openNextPart();
+    IDiskRowReader * ensureRowReader(const char * format, bool streamRemote, unsigned expectedCrc, IOutputMetaData & expected, unsigned projectedCrc, IOutputMetaData & projected, unsigned actualCrc, IOutputMetaData & actual, CLogicalFileSlice * slice);
+};
+
+
+class CHThorGenericDiskReadActivity : public CHThorGenericDiskReadBaseActivity
+{
+    typedef CHThorGenericDiskReadBaseActivity PARENT;
+protected:
+    IHThorNewDiskReadArg &helper;
+    bool needTransform = false;
+    bool hasMatchFilter = false;
+    unsigned __int64 lastGroupProcessed = 0;
+    RtlDynamicRowBuilder outBuilder;
+    unsigned __int64 limit = 0;
+    unsigned __int64 remoteLimit = 0;
+
+public:
+    CHThorGenericDiskReadActivity(IAgentContext &agent, unsigned _activityId, unsigned _subgraphId, IHThorNewDiskReadArg &_arg, ThorActivityKind _kind, EclGraph & _graph, IPropertyTree *node);
+
+    virtual void ready();
+    virtual void stop();
+    virtual bool needsAllocator() const { return true; }
+
+    //interface IHThorInput
+    virtual const void *nextRow();
+
+protected:
+    void onLimitExceeded();
+};
 
 
 #define MAKEFACTORY(NAME) \

ecl/regress/alienread_bad.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//check that virtual fields in the middle of a record with alein datatypes reports an error.
+#option ('targetClusterType', 'hthor');
+#option ('genericDiskReads', true);
+
+import lib_stringlib;
+prefix := 'regress::'+ __TARGET_PLATFORM__ + '::';
+
+extractXStringLength(data x, unsigned len) := transfer(((data4)(x[1..len])), unsigned4);
+
+//A pretty weird type example - a length prefixed string, where the number of bytes used for the length is configurable...
+xstring(unsigned len) := type
+    export integer physicallength(data x) := extractXStringLength(x, len)+len;
+    export string load(data x) := (string)x[(len+1)..extractXStringLength(x, len)+len];
+    export data store(string x) := transfer(length(x), data4)[1..len]+(data)x;
+end;
+
+dstring(string del) := TYPE
+    export integer physicallength(string s) := StringLib.StringUnboundedUnsafeFind(s,del)+length(del)-1;
+    export string load(string s) := s[1..StringLib.StringUnboundedUnsafeFind(s,del)-1];
+    export string store(string s) := s+del; // Untested (vlength output generally broken)
+END;
+
+
+alienString := xstring(4);
+alienVarString := dstring('\000');
+
+alienRecordEx := RECORD
+    alienString         surname;
+    alienString         forename;
+    alienVarString      addr;
+    string4             extra;
+    unsigned        fpos{virtual(fileposition)};
+    alienVarString      extra2;
+    string          filename{virtual(logicalfilename)};
+    unsigned        lfpos{virtual(localfileposition)};
+END;
+
+filenameRaw := prefix+'alientest_raw';
+filenameAlien := prefix+'alientest_alien';
+
+output(DATASET(filenameRaw, alienRecordEx, THOR, HINT(layoutTranslation('alwaysEcl'))),,NAMED('RawAlien'));

ecl/regress/standaloneread.ecl

@@ -0,0 +1,5 @@
+ds := DATASET('localout1', { unsigned c }, XML('Dataset/Row') );
+ds2 := DATASET('subdir::localout1', { unsigned c }, XML('Dataset/Row') );
+
+output(ds, { c+1 },'localout2',csv,overwrite);
+output(ds, { c*2 },'subdir::localout2',csv,overwrite);

ecl/regress/standalonewrite.ecl

@@ -0,0 +1,5 @@
+ds := DATASET(10, transform({ unsigned c }, SELF.c := COUNTER));
+
+output(ds,,'localout1',xml,overwrite);
+
+output(ds,,'subdir::localout1',xml,overwrite);

rtl/eclrtl/rtldynfield.cpp

@@ -1902,6 +1902,146 @@ extern ECLRTL_API const IDynamicTransform *createRecordTranslatorViaCallback(con
     return new GeneralRecordTranslator(destRecInfo, srcRecInfo, false, rawType);
 }
 
+//---------------------------------------------------------------------------------------------------------------------
+
+class CloneVirtualRecordTranslator : public CInterfaceOf<IDynamicTransform>
+{
+public:
+    CloneVirtualRecordTranslator(const RtlRecord &_destRecInfo, IOutputMetaData & _sourceMeta)
+        : destRecInfo(_destRecInfo), sourceMeta(_sourceMeta)
+    {
+        init();
+    }
+// IDynamicTransform impl.
+    virtual void describe() const override
+    {
+        doDescribe(0);
+    }
+    virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const byte *sourceRec) const override
+    {
+        size32_t sourceSize = sourceMeta.getRecordSize(sourceRec);
+        return doAppendVirtuals(builder, callback, 0, sourceSize, sourceRec);
+    }
+    virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const RtlRow &sourceRow) const override
+    {
+        const byte * source = sourceRow.queryRow();
+        size32_t sourceSize = sourceMeta.getRecordSize(source);
+        return doAppendVirtuals(builder, callback, 0, sourceSize, source);
+    }
+    virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const IDynamicFieldValueFetcher & fetcher) const override
+    {
+        throwUnexpected();
+    }
+    virtual bool canTranslate() const override
+    {
+        return true;
+    }
+    virtual bool needsTranslate() const override
+    {
+        return true;
+    }
+    virtual bool needsNonVirtualTranslate() const override
+    {
+        return false;
+    }
+    virtual bool keyedTranslated() const override
+    {
+        return false;
+    }
+private:
+    void doDescribe(unsigned indent) const
+    {
+        for (unsigned idx = firstVirtual; idx < destRecInfo.getNumFields(); idx++)
+        {
+            const RtlFieldInfo *field = destRecInfo.queryField(idx);
+            const char * dest = destRecInfo.queryName(idx);
+            const char * result = "";
+            switch (getVirtualInitializer(field->initializer))
+            {
+            case FVirtualFilePosition:
+                result = "FILEPOSITION";
+                break;
+            case FVirtualLocalFilePosition:
+                result = "LOCALFILEPOSITION";
+                break;
+            case FVirtualFilename:
+                result = "LOGICALFILENAME";
+                break;
+            }
+            DBGLOG("Use virtual(%s) for field %s", result, dest);
+        }
+    }
+    size32_t doAppendVirtuals(ARowBuilder &builder, IVirtualFieldCallback & callback, size32_t offset, size32_t sourceSize, const void *sourceRow) const
+    {
+        size32_t estimate = sourceSize + fixedVirtualSize;
+        builder.ensureCapacity(offset+estimate, "record");
+        memcpy(builder.getSelf() + offset, sourceRow, sourceSize);
+
+        unsigned destOffset = offset + sourceSize;
+        for (unsigned idx = firstVirtual; idx < destRecInfo.getNumFields(); idx++)
+        {
+            const RtlFieldInfo *field = destRecInfo.queryField(idx);
+            const RtlTypeInfo *type = field->type;
+            switch (getVirtualInitializer(field->initializer))
+            {
+            case FVirtualFilePosition:
+                destOffset = type->buildInt(builder, destOffset, field, callback.getFilePosition(sourceRow));
+                break;
+            case FVirtualLocalFilePosition:
+                destOffset = type->buildInt(builder, destOffset, field, callback.getLocalFilePosition(sourceRow));
+                break;
+            case FVirtualFilename:
+                {
+                    const char * filename = callback.queryLogicalFilename(sourceRow);
+                    destOffset = type->buildString(builder, destOffset, field, strlen(filename), filename);
+                    break;
+                }
+            default:
+                throwUnexpected();
+            }
+        }
+        return destOffset;
+    }
+
+    void init()
+    {
+        unsigned idx = 0;
+        for (; idx < destRecInfo.getNumFields(); idx++)
+        {
+            const RtlFieldInfo *field = destRecInfo.queryField(idx);
+            const byte * initializer = (const byte *) field->initializer;
+            if (isVirtualInitializer(initializer))
+                break;
+        }
+        firstVirtual = idx;
+
+        size32_t size = 0;
+        for (; idx < destRecInfo.getNumFields(); idx++)
+        {
+            const RtlFieldInfo *field = destRecInfo.queryField(idx);
+            const RtlTypeInfo *type = field->type;
+            const byte * initializer = (const byte *) field->initializer;
+            assertex(isVirtualInitializer(initializer));
+            size += type->getMinSize();
+        }
+
+        fixedVirtualSize = size;
+    }
+
+protected:
+    const RtlRecord &destRecInfo;
+    IOutputMetaData & sourceMeta;
+    unsigned firstVirtual = 0;
+    size32_t fixedVirtualSize = 0;
+};
+
+
+extern ECLRTL_API const IDynamicTransform *createCloneVirtualRecordTranslator(const RtlRecord &_destRecInfo, IOutputMetaData & _source)
+{
+    return new CloneVirtualRecordTranslator(_destRecInfo, _source);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
 extern ECLRTL_API void throwTranslationError(const RtlRecord & destRecInfo, const RtlRecord & srcRecInfo, const char * filename)
 {
     Owned<const IDynamicTransform> translator = createRecordTranslator(destRecInfo, srcRecInfo);

rtl/eclrtl/rtldynfield.hpp

@@ -133,8 +133,8 @@ interface IDynamicTransform : public IInterface
 {
     virtual void describe() const = 0;
     virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const byte *sourceRec) const = 0;
-    virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const RtlRow &sourceRow) const = 0;
-    virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const IDynamicFieldValueFetcher & fetcher) const = 0;
+    virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const RtlRow &sourceRow) const = 0;  // allows offsets to be reused if already calculated
+    virtual size32_t translate(ARowBuilder &builder, IVirtualFieldCallback & callback, const IDynamicFieldValueFetcher & fetcher) const = 0; // called when reading from non binary e.g. xml/csv
     virtual bool canTranslate() const = 0;
     virtual bool needsTranslate() const = 0;
     virtual bool keyedTranslated() const = 0;
@@ -181,6 +181,7 @@ interface IDynamicTransformViaCallback : public IInterface
 };
 
 extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo);
+extern ECLRTL_API const IDynamicTransform *createCloneVirtualRecordTranslator(const RtlRecord &_destRecInfo, IOutputMetaData & _source);
 extern ECLRTL_API const IDynamicTransform *createRecordTranslatorViaCallback(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo, type_vals rawType);
 extern ECLRTL_API void throwTranslationError(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo, const char * filename);
 

rtl/eclrtl/rtlformat.cpp

@@ -832,8 +832,9 @@ void CPropertyTreeWriter::outputString(unsigned len, const char *field, const ch
         len = rtlTrimStrLen(len, field);
     if ((flags & XWFopt) && (rtlTrimStrLen(len, field) == 0))
         return;
-    StringBuffer tmp(len, field);
-    target->setProp(fieldname, tmp.str());
+    StringBuffer tmp;
+    appendStringAsUtf8(tmp, len, field);
+    target->addProp(fieldname, tmp.str());
 }
 
 
@@ -897,8 +898,14 @@ void CPropertyTreeWriter::outputUnicode(unsigned len, const UChar *field, const
         len = rtlTrimUnicodeStrLen(len, field);
     if ((flags & XWFopt) && (rtlTrimUnicodeStrLen(len, field) == 0))
         return;
+
     StringBuffer fieldStr;
-    outputXmlUnicode(len, field, fieldname, fieldStr);
+    char * buff = 0;
+    unsigned bufflen = 0;
+    rtlUnicodeToCodepageX(bufflen, buff, len, field, "utf-8");
+    fieldStr.append(bufflen, buff);
+    rtlFree(buff);
+
     target->setProp(fieldname, fieldStr);
 }
 
@@ -909,8 +916,8 @@ void CPropertyTreeWriter::outputUtf8(unsigned len, const char *field, const char
     if ((flags & XWFopt) && (rtlTrimUtf8StrLen(len, field) == 0))
         return;
     StringBuffer fieldStr;
-    outputXmlUtf8(len, field, nullptr, fieldStr);
-    target->setProp(fieldname, fieldStr);
+    fieldStr.append(rtlUtf8Size(len, field), field);
+    target->setProp(fieldname, fieldStr); //MORE: addProp - discuss in code review!
 }
 
 void CPropertyTreeWriter::outputXmlns(const char *name, const char *uri)

rtl/include/eclhelper.hpp

@@ -1110,6 +1110,7 @@ enum
     TDXupdateaccessed   = 0x0010,
     TDXdynamicfilename  = 0x0020,
     TDXjobtemp          = 0x0040,       // stay around while a wu is being executed.
+    TDXgeneric          = 0x0080,       // generic form of disk read/write
 
 //disk read flags
     TDRoptional         = 0x00000100,
@@ -1122,7 +1123,7 @@ enum
     TDRcountkeyedlimit  = 0x00008000,
     TDRkeyedlimitskips  = 0x00010000,
     TDRlimitskips       = 0x00020000,
-    //unused              0x00040000,
+    TDRfileposcallback  = 0x00040000,
     TDRaggregateexists  = 0x00080000,       // only aggregate is exists()
     TDRgroupmonitors    = 0x00100000,       // are segement monitors created for all group by conditions.
     TDRlimitcreates     = 0x00200000,
@@ -1132,6 +1133,8 @@ enum
     TDRtransformvirtual = 0x02000000,       // transform uses a virtual field.
     TDRdynformatoptions = 0x04000000,
     TDRinvariantfilename= 0x08000000,       // filename is non constant but has the same value for the whole query
+    TDRprojectleading   = 0x10000000,       // The projeted format matches leading fields of the source row (so no need to project if disappearing quickly)
+    TDRcloneappendvirtual = 0x20000000,     // Can clone the disk record and then append the virtual fields.
 
 //disk write flags
     TDWextend           = 0x0100,
@@ -2404,6 +2407,28 @@ struct IHThorIndexReadBaseArg : extends IHThorCompoundBaseArg
     virtual IHThorSteppedSourceExtra *querySteppingExtra() = 0;
 };
 
+
+/*
+ queryDiskRecordSize()
+    This is the expected record on disk.  It is always the serialialized form.  With the new disk reading implementation it may contain
+    the virtual fields as well as real fields.
+
+ queryProjectedDiskRecordSize()
+    This is the format that the transsform/match function expect as input.  It is generally the deserialized vesion of
+    the expected record with any fields that are not required removed.  If not transform is required then the row is
+    will match the queryOutputMetaData()
+
+There are some record formats that are not supported by the translation code (see canDefinitelyProcessWithTranslator() in hqlattr):
+    alien data types
+    dataset(record, count(x))
+    dataset(record, sizeof(x))
+    ifblock(complex-expression)
+
+    If these occur then the compound disk operations are disabled - field projection will not happen.  The projectedDiskRecordSize
+    will match the queryDiskRecordSize() and the transform will perform the deserialization.  The generated queryDiskRecordSize()
+    function will need to be used for calculating the size of the input record.
+
+*/
 struct IHThorDiskReadBaseArg : extends IHThorCompoundBaseArg
 {
     virtual const char * getFileName() = 0;

system/jlib/jptree.cpp

@@ -9244,3 +9244,18 @@ jlib_decl IPropertyTree * queryCostsConfiguration()
 {
     return queryComponentConfig().queryPropTree("costs");
 }
+
+void copyPropIfMissing(IPropertyTree & target, const char * targetName, IPropertyTree & source, const char * sourceName)
+{
+    if (source.hasProp(sourceName) && !target.hasProp(targetName))
+    {
+        if (source.isBinary(sourceName))
+        {
+            MemoryBuffer value;
+            source.getPropBin(sourceName, value);
+            target.setPropBin(targetName, value.length(), value.toByteArray());
+        }
+        else
+            target.setProp(targetName, source.queryProp(sourceName));
+    }
+}

system/jlib/jptree.hpp

@@ -362,4 +362,6 @@ jlib_decl void dbglogYAML(const IPropertyTree *tree, unsigned indent = 0, unsign
 // Defines the threshold where attribute value maps are created for sibling ptrees for fast lookups
 jlib_decl void setPTreeMappingThreshold(unsigned threshold);
 
+jlib_decl void copyPropIfMissing(IPropertyTree & target, const char * targetName, IPropertyTree & source, const char * sourceName);
+
 #endif

system/jlib/jregexp.cpp

@@ -1557,6 +1557,8 @@ bool StringMatcher::queryAddEntry(unsigned len, const char * text, unsigned acti
         entry & curElement = curTable[c];
         if (--len == 0)
         {
+            if (curElement.value == action)
+                return true;
             if (curElement.value != 0)
                 return false;
             curElement.value = action;

system/jlib/jscm.hpp

@@ -60,6 +60,7 @@ public:
     inline Shared(const Shared & other)          { ptr = other.getLink(); }
 #if defined(__cplusplus) && __cplusplus >= 201100
     inline Shared(Shared && other)               { ptr = other.getClear(); }
+    explicit operator bool() const               { return ptr != nullptr; }
 #endif
     inline ~Shared()                             { ::Release(ptr); }
     inline Shared<CLASS> & operator = (const Shared<CLASS> & other) { this->set(other.get()); return *this;  }

testing/regress/ecl/alien2.ecl

@@ -1,6 +1,6 @@
 /*##############################################################################
 
-    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

testing/regress/ecl/fileposition.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//version multiPart=false
+//xversion multiPart=true           // cannot use this because results depend on the number of thor slaves
+//noroxie      - see HPCC-22629
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, true);
+optRemoteRead := #IFDEFINED(root.optRemoteRead, false);
+
+import $.setup;
+import Std.System.ThorLib;
+Files := setup.Files(multiPart, false);
+
+// Roxie needs this to resolve files at run time
+#option ('allowVariableRoxieFilenames', 1);
+#option('forceRemoteRead', optRemoteRead);
+
+inDs := Files.seqDupFile;
+
+sequential(
+    output(inDs, { seq, filepos, unsigned part := (localfilepos - 0x8000000000000000) >> 48,  unsigned offset := (localfilepos & 0xFFFFFFFFFFFF) })
+);

testing/regress/ecl/key/fileposition.xml

@@ -0,0 +1,34 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><filepos>0</filepos><part>0</part><offset>0</offset></Row>
+ <Row><seq>2</seq><filepos>8</filepos><part>0</part><offset>8</offset></Row>
+ <Row><seq>3</seq><filepos>16</filepos><part>0</part><offset>16</offset></Row>
+ <Row><seq>4</seq><filepos>24</filepos><part>0</part><offset>24</offset></Row>
+ <Row><seq>5</seq><filepos>32</filepos><part>0</part><offset>32</offset></Row>
+ <Row><seq>6</seq><filepos>40</filepos><part>0</part><offset>40</offset></Row>
+ <Row><seq>7</seq><filepos>48</filepos><part>0</part><offset>48</offset></Row>
+ <Row><seq>8</seq><filepos>56</filepos><part>0</part><offset>56</offset></Row>
+ <Row><seq>9</seq><filepos>64</filepos><part>0</part><offset>64</offset></Row>
+ <Row><seq>10</seq><filepos>72</filepos><part>0</part><offset>72</offset></Row>
+ <Row><seq>11</seq><filepos>80</filepos><part>0</part><offset>80</offset></Row>
+ <Row><seq>12</seq><filepos>88</filepos><part>0</part><offset>88</offset></Row>
+ <Row><seq>13</seq><filepos>96</filepos><part>0</part><offset>96</offset></Row>
+ <Row><seq>14</seq><filepos>104</filepos><part>0</part><offset>104</offset></Row>
+ <Row><seq>15</seq><filepos>112</filepos><part>0</part><offset>112</offset></Row>
+ <Row><seq>16</seq><filepos>120</filepos><part>0</part><offset>120</offset></Row>
+ <Row><seq>1</seq><filepos>128</filepos><part>1</part><offset>0</offset></Row>
+ <Row><seq>2</seq><filepos>136</filepos><part>1</part><offset>8</offset></Row>
+ <Row><seq>3</seq><filepos>144</filepos><part>1</part><offset>16</offset></Row>
+ <Row><seq>4</seq><filepos>152</filepos><part>1</part><offset>24</offset></Row>
+ <Row><seq>5</seq><filepos>160</filepos><part>1</part><offset>32</offset></Row>
+ <Row><seq>6</seq><filepos>168</filepos><part>1</part><offset>40</offset></Row>
+ <Row><seq>7</seq><filepos>176</filepos><part>1</part><offset>48</offset></Row>
+ <Row><seq>8</seq><filepos>184</filepos><part>1</part><offset>56</offset></Row>
+ <Row><seq>9</seq><filepos>192</filepos><part>1</part><offset>64</offset></Row>
+ <Row><seq>10</seq><filepos>200</filepos><part>1</part><offset>72</offset></Row>
+ <Row><seq>11</seq><filepos>208</filepos><part>1</part><offset>80</offset></Row>
+ <Row><seq>12</seq><filepos>216</filepos><part>1</part><offset>88</offset></Row>
+ <Row><seq>13</seq><filepos>224</filepos><part>1</part><offset>96</offset></Row>
+ <Row><seq>14</seq><filepos>232</filepos><part>1</part><offset>104</offset></Row>
+ <Row><seq>15</seq><filepos>240</filepos><part>1</part><offset>112</offset></Row>
+ <Row><seq>16</seq><filepos>248</filepos><part>1</part><offset>120</offset></Row>
+</Dataset>

testing/regress/ecl/key/setup.xml

@@ -34,3 +34,5 @@
 </Dataset>
 <Dataset name='Result 18'>
 </Dataset>
+<Dataset name='Result 26'>
+</Dataset>

testing/regress/ecl/setup/files.ecl

@@ -80,6 +80,8 @@ EXPORT DG_DictFilename      := filePrefix + 'SerialLibraryDict';
 EXPORT DG_DictKeyFilename   := indexPrefix + 'SerialLibraryKeyDict';
 EXPORT DG_BookKeyFilename   := indexPrefix + 'SerialBookKey';
 
+EXPORT SEQ_Filename              := filePrefix + 'Sequence';
+
 //record structures
 EXPORT DG_FetchRecord := RECORD
   INTEGER8 sequence;
@@ -207,4 +209,14 @@ EXPORT getSearchIndex() := INDEX(TS.textSearchIndex, NameSearchIndex);
 EXPORT getSearchSuperIndex() := INDEX(TS.textSearchIndex, '{' + NameSearchIndex + ',' + NameWordIndex() + '}');
 EXPORT getSearchSource() := DATASET(NameSearchSource, TS.textSourceRecord, THOR);
 
+EXPORT SeqRecord := { unsigned seq; };
+EXPORT SeqReadRecord :=
+  RECORD(SeqRecord)
+    unsigned8 filepos{virtual(fileposition)};
+    unsigned8 localfilepos{virtual(localfileposition)};
+  END;
+
+EXPORT SeqFile := DATASET(SEQ_Filename, SeqReadRecord, THOR);
+EXPORT SeqDupFile := DATASET('{' + SEQ_Filename + ',' + SEQ_Filename + '}', SeqReadRecord, THOR);
+
 END;

testing/regress/ecl/setup/setup.ecl

@@ -141,3 +141,6 @@ IF (createMultiPart,
         buildindex(LocalFiles.DG_IntIndex, overwrite,NOROOT,BLOOM(DG_parentId), PARTITION(DG_parentId));
    )
 );
+
+seqDs := DATASET(16, transform(Files.SeqRecord, self.seq := counter), DISTRIBUTED);
+OUTPUT(seqDs, , Files.Seq_Filename, OVERWRITE);

testing/regress/ecl/setup/thor/setup.xml

@@ -48,3 +48,5 @@
 </Dataset>
 <Dataset name='Result 25'>
 </Dataset>
+<Dataset name='Result 26'>
+</Dataset>