Merge pull request #4672 from ghalliday/issue9743

HPCC-9743 Speed up #APPEND and #LOOP processing

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlexpr.cpp

@@ -7254,9 +7254,7 @@ bool CFileContents::preloadFromFile()
         mb.setLength(sizeRead);
     } while (rd);
     ensureUtf8(mb);
-    mb.append((byte)0);
-    mb.truncate();
-    fileContents.setown(static_cast<char *>(mb.detach()));
+    setContentsOwn(mb);
     return true;
 }
 
@@ -7296,25 +7294,44 @@ void CFileContents::ensureLoaded()
     byte * contents = static_cast<byte *>(buffer.reserve(sizeToRead));
     size32_t sizeRead = io->read(0, sizeToRead, contents);
     ensureUtf8(buffer);
-    buffer.append((byte)0);
+    setContentsOwn(buffer);
 
-    fileContents.setown(static_cast<char *>(buffer.detach()));
     if (sizeRead != sizeToRead)
         throw MakeStringException(1, "File %s only read %u of %u bytes", file->queryFilename(), sizeRead, sizeToRead);
 }
 
 CFileContents::CFileContents(const char *query, ISourcePath * _sourcePath) 
-: fileContents(query), sourcePath(_sourcePath)
+: sourcePath(_sourcePath)
 {
+    if (query)
+        setContents(strlen(query), query);
+
     delayedRead = false;
 }
 
 CFileContents::CFileContents(unsigned len, const char *query, ISourcePath * _sourcePath) 
-: fileContents(query, len), sourcePath(_sourcePath)
+: sourcePath(_sourcePath)
 {
+    setContents(len, query);
     delayedRead = false;
 }
 
+
+void CFileContents::setContents(unsigned len, const char * query)
+{
+    void * buffer = fileContents.allocate(len+1);
+    memcpy(buffer, query, len);
+    ((byte *)buffer)[len] = '\0';
+}
+
+void CFileContents::setContentsOwn(MemoryBuffer & buffer)
+{
+    buffer.append((byte)0);
+    buffer.truncate();
+    size32_t len = buffer.length();
+    fileContents.setOwn(len, buffer.detach());
+}
+
 IFileContents * createFileContentsFromText(unsigned len, const char * text, ISourcePath * sourcePath)
 {
     return new CFileContents(len, text, sourcePath);

ecl/hql/hqlexpr.ipp

@@ -397,7 +397,7 @@ class CFileContents : public CInterfaceOf<IFileContents>
 {
 private:
     Linked<IFile> file;
-    StringAttr fileContents;
+    MemoryAttr fileContents;
     Linked<ISourcePath> sourcePath;
     bool delayedRead;
 
@@ -408,21 +408,24 @@ public:
 
     virtual IFile * queryFile() { return file; }
     virtual ISourcePath * querySourcePath() { return sourcePath; }
-    virtual const char *getText()
+    virtual const char * getText()
     {
         ensureLoaded();
-        return fileContents.sget();
+        return (const char *)fileContents.get();
     }
+    //NB: This is the string length, so subtract one to remove the null terminator
     virtual size32_t length() 
     { 
         ensureLoaded();
-        return fileContents.length();
+        return fileContents.length()-1;
     }
 
 private:
     bool preloadFromFile();
     void ensureLoaded();
     void ensureUtf8(MemoryBuffer & contents);
+    void setContents(size32_t len, const char * query);
+    void setContentsOwn(MemoryBuffer & contents);
 };
 
 class HQL_API CHqlAnnotation: public CHqlExpression

ecl/hql/hqlgram.hpp

@@ -1115,9 +1115,13 @@ class HqlLex
         bool isMacroActive(IHqlExpression *expr);
         bool isAborting();
         void pushMacro(IHqlExpression *expr);
+        void pushText(IFileContents * text, int startLineNo, int startColumn);
         void pushText(const char *s, int startLineNo, int startColumn);
         bool getParameter(StringBuffer &curParam, const char* for_what, int* startLine=NULL, int* startCol=NULL);
+        IValue *foldConstExpression(const YYSTYPE & errpos, IHqlExpression * expr, IXmlScope *xmlScope, int startLine, int startCol);
         IValue *parseConstExpression(const YYSTYPE & errpos, StringBuffer &curParam, IXmlScope *xmlScope, int line, int col);
+        IValue *parseConstExpression(const YYSTYPE & errpos, IFileContents * contents, IXmlScope *xmlScope, int line, int col);
+        IHqlExpression * parseECL(IFileContents * contents, IXmlScope *xmlScope, int startLine, int startCol);
         IHqlExpression * parseECL(const char * curParam, IXmlScope *xmlScope, int startLine, int startCol);
         void setMacroParam(const YYSTYPE & errpos, IHqlExpression* funcdef, StringBuffer& curParam, IIdAtom * argumentName, unsigned& parmno,IProperties *macroParms);
         unsigned getTypeSize(unsigned lengthTypeName);
@@ -1166,8 +1170,8 @@ private:
         Owned<IProperties> macroParms;
         IIterator *forLoop;
         IHqlExpression *macroExpr;
-        StringBuffer forBody;
-        StringBuffer forFilter;
+        Owned<IFileContents> forBody;
+        Owned<IFileContents> forFilter;
 
         IXmlScope *xmlScope;
 

ecl/hql/hqlparse.cpp

@@ -302,18 +302,24 @@ IHqlExpression * HqlLex::createIntegerConstant(__int64 value, bool isSigned)
     return createConstant(createIntValue(value, makeIntType(8, isSigned)));
 }
 
-void HqlLex::pushText(const char *s, int startLineNo, int startColumn)
+void HqlLex::pushText(IFileContents * text, int startLineNo, int startColumn)
 {
 #ifdef TIMING_DEBUG
     MTIME_SECTION(timer, "HqlLex::pushText");
 #endif
-    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath);
-    inmacro = new HqlLex(yyParser, macroContents, NULL, NULL);
+    inmacro = new HqlLex(yyParser, text, NULL, NULL);
     inmacro->set_yyLineNo(startLineNo);
     inmacro->set_yyColumn(startColumn);
 }
 
 
+void HqlLex::pushText(const char *s, int startLineNo, int startColumn)
+{
+    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath);
+    pushText(macroContents, startLineNo, startColumn);
+}
+
+
 void HqlLex::pushText(const char *s)
 {
 #ifdef TIMING_DEBUG
@@ -1089,16 +1095,17 @@ void HqlLex::doFor(YYSTYPE & returnToken, bool doAll)
         return;
     }
     name = returnToken.getId();
-    forFilter.clear();
+
+    StringBuffer forFilterText;
     // Note - we gather the for filter and body in skip mode (deferring evaluation of #if etc) since the context will be different each time...
     skipping = 1;
     int tok = yyLex(returnToken, false,0);
     if (tok == '(')
     {
-        forFilter.append('(');
-        while (getParameter(forFilter, forwhat.str()))
-            forFilter.append(") AND (");
-        forFilter.append(')');
+        forFilterText.append('(');
+        while (getParameter(forFilterText, forwhat.str()))
+            forFilterText.append(") AND (");
+        forFilterText.append(')');
         tok = yyLex(returnToken, false,0);
     }
     if (tok != ')')
@@ -1109,8 +1116,9 @@ void HqlLex::doFor(YYSTYPE & returnToken, bool doAll)
         pushText(get_yyText());
         returnToken.release();
     }
+
     // Now gather the tokens we are going to repeat...
-    forBody.clear();
+    StringBuffer forBodyText;
     for (;;)
     {
         int tok = yyLex(returnToken, false,0);
@@ -1128,12 +1136,17 @@ void HqlLex::doFor(YYSTYPE & returnToken, bool doAll)
         }
         if (tok == HASHEND && !skipping)
             break;
-        forBody.append(' ');
-        getTokenText(forBody);
+        forBodyText.append(' ');
+        getTokenText(forBodyText);
         returnToken.release();
     } 
     ::Release(forLoop);
+
     forLoop = getSubScopes(returnToken, name->getAtomNamePtr(), doAll);
+    if (forFilterText.length())
+        forFilter.setown(createFileContentsFromText(forFilterText, sourcePath));
+    forBody.setown(createFileContentsFromText(forBodyText, sourcePath));
+
     loopTimes = 0;
     if (forLoop && forLoop->first()) // more - check filter
         checkNextLoop(returnToken, true, startLine, startCol);
@@ -1147,7 +1160,7 @@ void HqlLex::doLoop(YYSTYPE & returnToken)
     
 
     // Now gather the tokens we are going to repeat...
-    forBody.clear();
+    StringBuffer forBodyText;
     // Note - we gather the for filter and body in skip mode (deferring evaluation of #if etc) since the context will be different each time...
     skipping = 1;
     hasHashbreak = false;
@@ -1168,8 +1181,8 @@ void HqlLex::doLoop(YYSTYPE & returnToken)
         }
         if (tok == HASHEND && !skipping)
             break;
-        forBody.append(' ');
-        getTokenText(forBody);
+        forBodyText.append(' ');
+        getTokenText(forBodyText);
         returnToken.release();
     } 
     if (!hasHashbreak)
@@ -1177,9 +1190,11 @@ void HqlLex::doLoop(YYSTYPE & returnToken)
         reportError(returnToken, ERR_TMPLT_NOBREAKINLOOP,"No #BREAK inside %s: infinite loop will occur", forwhat.str());
         return;
     }
+
     ::Release(forLoop);
     forLoop = new CDummyScopeIterator(ensureTopXmlScope(returnToken));
     forFilter.clear();
+    forBody.setown(createFileContentsFromText(forBodyText, sourcePath));
     loopTimes = 0;
     if (forLoop->first()) // more - check filter
         checkNextLoop(returnToken, true,startLine,startCol);
@@ -1458,7 +1473,7 @@ void HqlLex::checkNextLoop(const YYSTYPE & errpos, bool first, int startLine, in
     {
         bool filtered;
         IXmlScope *subscope = (IXmlScope *) &forLoop->query();
-        if (forFilter.length())
+        if (forFilter)
         {
 #ifdef TIMING_DEBUG
             MTIME_SECTION(timer, "HqlLex::checkNextLoopcond");
@@ -1471,7 +1486,7 @@ void HqlLex::checkNextLoop(const YYSTYPE & errpos, bool first, int startLine, in
             filtered = false;
         if (!filtered)
         {
-            pushText(forBody.str(),startLine,startCol);
+            pushText(forBody,startLine,startCol);
             inmacro->xmlScope = LINK(subscope);
             return;
         }
@@ -1674,7 +1689,7 @@ void HqlLex::doDefined(YYSTYPE & returnToken)
         pushText(param2Text);
 }
 
-IHqlExpression *HqlLex::parseECL(const char * text, IXmlScope *xmlScope, int startLine, int startCol)
+IHqlExpression *HqlLex::parseECL(IFileContents * contents, IXmlScope *xmlScope, int startLine, int startCol)
 {
 #ifdef TIMING_DEBUG
     MTIME_SECTION(timer, "HqlLex::parseConstExpression");
@@ -1684,19 +1699,22 @@ IHqlExpression *HqlLex::parseECL(const char * text, IXmlScope *xmlScope, int sta
 
     HqlGramCtx parentContext(yyParser->lookupCtx);
     yyParser->saveContext(parentContext, false);
-    Owned<IFileContents> contents = createFileContentsFromText(text, querySourcePath());
     HqlGram parser(parentContext, scope, contents, xmlScope, true);
     parser.getLexer()->set_yyLineNo(startLine);
     parser.getLexer()->set_yyColumn(startCol);
     return parser.yyParse(false, false);
 }
 
-IValue *HqlLex::parseConstExpression(const YYSTYPE & errpos, StringBuffer &curParam, IXmlScope *xmlScope, int startLine, int startCol)
+
+IHqlExpression *HqlLex::parseECL(const char * text, IXmlScope *xmlScope, int startLine, int startCol)
+{
+    Owned<IFileContents> contents = createFileContentsFromText(text, querySourcePath());
+    return parseECL(contents, xmlScope, startLine, startCol);
+}
+
+
+IValue *HqlLex::foldConstExpression(const YYSTYPE & errpos, IHqlExpression * expr, IXmlScope *xmlScope, int startLine, int startCol)
 {
-#ifdef TIMING_DEBUG
-    MTIME_SECTION(timer, "HqlLex::parseConstExpression");
-#endif
-    OwnedHqlExpr expr = parseECL(curParam, xmlScope, startLine, startCol);
     OwnedIValue value;
     if (expr)
     {
@@ -1724,6 +1742,24 @@ IValue *HqlLex::parseConstExpression(const YYSTYPE & errpos, StringBuffer &curPa
     return value.getClear();
 }
 
+IValue *HqlLex::parseConstExpression(const YYSTYPE & errpos, StringBuffer &curParam, IXmlScope *xmlScope, int startLine, int startCol)
+{
+#ifdef TIMING_DEBUG
+    MTIME_SECTION(timer, "HqlLex::parseConstExpression");
+#endif
+    OwnedHqlExpr expr = parseECL(curParam, xmlScope, startLine, startCol);
+    return foldConstExpression(errpos, expr, xmlScope, startLine, startCol);
+}
+
+IValue *HqlLex::parseConstExpression(const YYSTYPE & errpos, IFileContents * text, IXmlScope *xmlScope, int startLine, int startCol)
+{
+#ifdef TIMING_DEBUG
+    MTIME_SECTION(timer, "HqlLex::parseConstExpression");
+#endif
+    OwnedHqlExpr expr = parseECL(text, xmlScope, startLine, startCol);
+    return foldConstExpression(errpos, expr, xmlScope, startLine, startCol);
+}
+
 int hexchar(char c)
 {
     if (c >= 'A' && c <= 'F')

ecl/hql/hqlxmldb.cpp

@@ -203,9 +203,7 @@ bool CXmlScope::appendValue(const char *name, const char *value)
     CXmlScope *scope = _findValue(name);
     if (scope)
     {
-        StringBuffer s;
-        scope->locals->getProp(name, s);
-        scope->locals->setProp(name, s.append(value).str());
+        scope->locals->appendProp(name, value);
         return true;
     }
     else

system/jlib/jprop.cpp

@@ -291,6 +291,29 @@ public:
                 properties.remove(propname);
         }
     }
+    virtual void appendProp(PTYPE propname, const char *val)
+    {
+        if (propname && val)
+        {
+            StringAttr * mapping = properties.getValue(propname);
+            if (mapping)
+            {
+                if (*val)
+                {
+                    char * str = mapping->detach();
+                    unsigned len1 = strlen(str);
+                    unsigned len2 = strlen(val);
+                    char * newstr = (char *)realloc(str, len1 + len2+1);
+                    assertex(newstr);
+                    memcpy(newstr+len1, val, len2);
+                    newstr[len1+len2] = '\0';
+                    mapping->setown(newstr);
+                }
+            }
+            else
+                properties.setValue(propname, val);
+        }
+    }
     virtual bool removeProp(PTYPE propname)
     {
         if (propname)

system/jlib/jprop.hpp

@@ -43,6 +43,7 @@ interface jlib_decl IPropertiesOf : extends serializable
     virtual const char *queryProp(PTYPE propname) = 0;
     virtual void setProp(PTYPE propname, int val) = 0;
     virtual void setProp(PTYPE propname, const char *val) = 0;
+    virtual void appendProp(PTYPE propname, const char *val) = 0;
     virtual bool hasProp(PTYPE propname) = 0;
     virtual PITER *getIterator() = 0;
     virtual void loadFile(const char *filename) = 0;