浏览代码

Merge branch 'candidate-6.0.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
Richard Chapman 9 年之前
父节点
当前提交
eabf030ca1

+ 42 - 0
common/thorhelper/roxiehelper.cpp

@@ -693,6 +693,23 @@ public:
     }
     }
 };
 };
 
 
+class CParallelQuickSortAlgorithm : public CInplaceSortAlgorithm
+{
+public:
+    CParallelQuickSortAlgorithm(ICompare *_compare) : CInplaceSortAlgorithm(_compare) {}
+
+    virtual void prepare(IInputBase *input)
+    {
+        curIndex = 0;
+        if (input->nextGroup(sorted))
+        {
+            cycle_t startCycles = get_cycles_now();
+            parqsortvec(const_cast<void * *>(sorted.getArray()), sorted.ordinality(), *compare);
+            elapsedCycles += (get_cycles_now() - startCycles);
+        }
+    }
+};
+
 class CTbbQuickSortAlgorithm : public CInplaceSortAlgorithm
 class CTbbQuickSortAlgorithm : public CInplaceSortAlgorithm
 {
 {
 public:
 public:
@@ -744,6 +761,17 @@ public:
     }
     }
 };
 };
 
 
+class CParallelStableQuickSortAlgorithm : public CStableInplaceSortAlgorithm
+{
+public:
+    CParallelStableQuickSortAlgorithm(ICompare *_compare) : CStableInplaceSortAlgorithm(_compare) {}
+
+    virtual void sortRows(void * * rows, size_t numRows, void * * temp)
+    {
+        parqsortvecstableinplace(rows, numRows, *compare, temp);
+    }
+};
+
 class CMergeSortAlgorithm : public CStableInplaceSortAlgorithm
 class CMergeSortAlgorithm : public CStableInplaceSortAlgorithm
 {
 {
 public:
 public:
@@ -1415,11 +1443,21 @@ extern ISortAlgorithm *createQuickSortAlgorithm(ICompare *_compare)
     return new CQuickSortAlgorithm(_compare);
     return new CQuickSortAlgorithm(_compare);
 }
 }
 
 
+extern ISortAlgorithm *createParallelQuickSortAlgorithm(ICompare *_compare)
+{
+    return new CParallelQuickSortAlgorithm(_compare);
+}
+
 extern ISortAlgorithm *createStableQuickSortAlgorithm(ICompare *_compare)
 extern ISortAlgorithm *createStableQuickSortAlgorithm(ICompare *_compare)
 {
 {
     return new CStableQuickSortAlgorithm(_compare);
     return new CStableQuickSortAlgorithm(_compare);
 }
 }
 
 
+extern ISortAlgorithm *createParallelStableQuickSortAlgorithm(ICompare *_compare)
+{
+    return new CParallelStableQuickSortAlgorithm(_compare);
+}
+
 extern ISortAlgorithm *createTbbQuickSortAlgorithm(ICompare *_compare)
 extern ISortAlgorithm *createTbbQuickSortAlgorithm(ICompare *_compare)
 {
 {
     return new CTbbQuickSortAlgorithm(_compare);
     return new CTbbQuickSortAlgorithm(_compare);
@@ -1467,6 +1505,10 @@ extern ISortAlgorithm *createSortAlgorithm(RoxieSortAlgorithm _algorithm, ICompa
         return createQuickSortAlgorithm(_compare);
         return createQuickSortAlgorithm(_compare);
     case stableQuickSortAlgorithm:
     case stableQuickSortAlgorithm:
         return createStableQuickSortAlgorithm(_compare);
         return createStableQuickSortAlgorithm(_compare);
+    case parallelQuickSortAlgorithm:
+        return createParallelQuickSortAlgorithm(_compare);
+    case parallelStableQuickSortAlgorithm:
+        return createParallelStableQuickSortAlgorithm(_compare);
     case spillingQuickSortAlgorithm:
     case spillingQuickSortAlgorithm:
     case stableSpillingQuickSortAlgorithm:
     case stableSpillingQuickSortAlgorithm:
         return createSpillingQuickSortAlgorithm(_compare, _rowManager, _rowMeta, _ctx, _tempDirectory, _activityId, _algorithm==stableSpillingQuickSortAlgorithm);
         return createSpillingQuickSortAlgorithm(_compare, _rowManager, _rowMeta, _ctx, _tempDirectory, _activityId, _algorithm==stableSpillingQuickSortAlgorithm);

+ 21 - 6
common/thorhelper/roxiehelper.hpp

@@ -99,12 +99,25 @@ public:
 
 
 //==============================================================================================================
 //==============================================================================================================
 
 
-typedef enum { heapSortAlgorithm, insertionSortAlgorithm,
-              quickSortAlgorithm, stableQuickSortAlgorithm, spillingQuickSortAlgorithm, stableSpillingQuickSortAlgorithm,
-              mergeSortAlgorithm, spillingMergeSortAlgorithm,
-              parallelMergeSortAlgorithm, spillingParallelMergeSortAlgorithm,
-              tbbQuickSortAlgorithm, tbbStableQuickSortAlgorithm,
-              unknownSortAlgorithm } RoxieSortAlgorithm;
+//MORE: This should just contain the algorithm, and use a separate field for stable|spilling|parallel
+//Should be implemented in a subsequent pull request, which also uses ALGORITHM('x') instead of requiring STABLE/UNSTABLE
+typedef enum {
+    heapSortAlgorithm,                  // heap sort
+    insertionSortAlgorithm,             // insertion sort - purely for comparison
+    quickSortAlgorithm,                 // jlib implementation of quicksort
+    stableQuickSortAlgorithm,           // jlib version of quick sort that uses an extra array indirect to ensure it is stable
+    spillingQuickSortAlgorithm,         // quickSortAlgorithm with the ability to spill
+    stableSpillingQuickSortAlgorithm,   // stableQuickSortAlgorithm with the ability to spill
+    mergeSortAlgorithm,                 // stable merge sort
+    spillingMergeSortAlgorithm,         // stable merge sort that can spill to disk
+    parallelMergeSortAlgorithm,         // parallel version of stable merge sort
+    spillingParallelMergeSortAlgorithm, // parallel version of stable merge sort that can spill to disk
+    tbbQuickSortAlgorithm,              // (parallel) quick sort implemented by the TBB libraries
+    tbbStableQuickSortAlgorithm,        // stable version of tbbQuickSortAlgorithm
+    parallelQuickSortAlgorithm,         // parallel version of the internal quicksort implementation (for comparison)
+    parallelStableQuickSortAlgorithm,   // stable version of parallelQuickSortAlgorithm
+    unknownSortAlgorithm
+} RoxieSortAlgorithm;
 
 
 interface ISortAlgorithm : extends IInterface
 interface ISortAlgorithm : extends IInterface
 {
 {
@@ -117,6 +130,8 @@ interface ISortAlgorithm : extends IInterface
 
 
 extern THORHELPER_API ISortAlgorithm *createQuickSortAlgorithm(ICompare *_compare);
 extern THORHELPER_API ISortAlgorithm *createQuickSortAlgorithm(ICompare *_compare);
 extern THORHELPER_API ISortAlgorithm *createStableQuickSortAlgorithm(ICompare *_compare);
 extern THORHELPER_API ISortAlgorithm *createStableQuickSortAlgorithm(ICompare *_compare);
+extern THORHELPER_API ISortAlgorithm *createParallelQuickSortAlgorithm(ICompare *_compare);
+extern THORHELPER_API ISortAlgorithm *createParallelStableQuickSortAlgorithm(ICompare *_compare);
 extern THORHELPER_API ISortAlgorithm *createInsertionSortAlgorithm(ICompare *_compare, roxiemem::IRowManager *_rowManager, unsigned _activityId);
 extern THORHELPER_API ISortAlgorithm *createInsertionSortAlgorithm(ICompare *_compare, roxiemem::IRowManager *_rowManager, unsigned _activityId);
 extern THORHELPER_API ISortAlgorithm *createHeapSortAlgorithm(ICompare *_compare);
 extern THORHELPER_API ISortAlgorithm *createHeapSortAlgorithm(ICompare *_compare);
 extern THORHELPER_API ISortAlgorithm *createSpillingQuickSortAlgorithm(ICompare *_compare, roxiemem::IRowManager &_rowManager, IOutputMetaData * _rowMeta, ICodeContext *_ctx, const char *_tempDirectory, unsigned _activityId, bool _stable);
 extern THORHELPER_API ISortAlgorithm *createSpillingQuickSortAlgorithm(ICompare *_compare, roxiemem::IRowManager &_rowManager, IOutputMetaData * _rowMeta, ICodeContext *_ctx, const char *_tempDirectory, unsigned _activityId, bool _stable);

+ 14 - 1
docs/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -189,7 +189,20 @@
               <?dbfo keep-together="always"?>
               <?dbfo keep-together="always"?>
 
 
               <para>Select the <emphasis role="bold">Compiler</emphasis> tab.
               <para>Select the <emphasis role="bold">Compiler</emphasis> tab.
-              The compiler details are automatically specified:</para>
+              </para>
+
+              <para>The compiler details are automatically specified. If you
+              want to override the defaults, check the <emphasis
+              role="bold">Override Automatic Compiler Selection</emphasis>
+              checkbox.</para>
+
+              <para><emphasis role="bold">Note:</emphasis> You can have
+              multiple versions of the compiler and client tools installed.
+              This allows you to have multiple configurations where the
+              compiler matches each server you need to access. Check the
+              <emphasis role="bold">Override Automatic Compiler
+              Selection</emphasis> checkbox and specify the Complier that
+              matches your HPCC instance.</para>
 
 
               <para><graphic fileref="../../images/CT03a.jpg" /></para>
               <para><graphic fileref="../../images/CT03a.jpg" /></para>
             </listitem>
             </listitem>

+ 19 - 1
ecl/hthor/hthor.cpp

@@ -3813,7 +3813,12 @@ void CHThorGroupSortActivity::createSorter()
             sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
             sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
     }
     }
     else if(stricmp(algoname, "parquicksort") == 0)
     else if(stricmp(algoname, "parquicksort") == 0)
-        sorter.setown(new CParallelStableQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
+    {
+        if((flags & TAFstable) != 0)
+            sorter.setown(new CParallelStableQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
+        else
+            sorter.setown(new CParallelQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
+    }
     else if(stricmp(algoname, "mergesort") == 0)
     else if(stricmp(algoname, "mergesort") == 0)
     {
     {
         if((flags & TAFparallel) != 0)
         if((flags & TAFparallel) != 0)
@@ -3969,6 +3974,19 @@ void CQuickSorter::performSort()
     }
     }
 }
 }
 
 
+// Quick sort
+
+void CParallelQuickSorter::performSort()
+{
+    size32_t numRows = rowsToSort.numCommitted();
+    if (numRows)
+    {
+        const void * * rows = rowsToSort.getBlock(numRows);
+        parqsortvec((void * *)rows, numRows, *compare);
+        finger = 0;
+    }
+}
+
 // StableQuick sort
 // StableQuick sort
 
 
 bool CStableSorter::addRow(const void * next)
 bool CStableSorter::addRow(const void * next)

+ 7 - 0
ecl/hthor/hthor.ipp

@@ -1126,6 +1126,13 @@ public:
     virtual void performSort();
     virtual void performSort();
 };
 };
 
 
+class CParallelQuickSorter : public CSimpleSorterBase
+{
+public:
+    CParallelQuickSorter(ICompare * _compare, roxiemem::IRowManager * _rowManager, size32_t _initialSize, size32_t _commitDelta) : CSimpleSorterBase(_compare, _rowManager, _initialSize, _commitDelta) {}
+    virtual void performSort();
+};
+
 class CStableSorter : public CSimpleSorterBase
 class CStableSorter : public CSimpleSorterBase
 {
 {
 public:
 public:

+ 48 - 35
esp/xslt/esdl2ecl.xslt

@@ -17,26 +17,57 @@
 ##############################################################################
 ##############################################################################
 -->
 -->
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
-	<xsl:output method="text" version="1.0" encoding="UTF-8" indent="yes"/>
-	<xsl:param name="sourceFileName" select="'UNKNOWN'"/>
-	<xsl:variable name="docname" select="/expesdl/esxdl/@name"/>
-	<xsl:template match="/">
-		<xsl:apply-templates select="expesdl"/>
-	</xsl:template>
-	<xsl:template name="doNotChangeManuallyComment">
-		<xsl:text>/*** Not to be hand edited (changes will be lost on re-generation) ***/
+    <xsl:output method="text" version="1.0" encoding="UTF-8" indent="yes"/>
+    <xsl:param name="sourceFileName" select="'UNKNOWN'"/>
+    <xsl:param name="importsList" select="''"/>
+    <xsl:variable name="docname" select="/expesdl/esxdl/@name"/>
+    <xsl:template match="/">
+        <xsl:apply-templates select="expesdl"/>
+    </xsl:template>
+    <xsl:template match="text()" name="outputImports">
+    <xsl:param name="import" select="."/>
+        <xsl:if test="string-length($import) > 0">
+            <xsl:text>import </xsl:text><xsl:value-of select="$import"/><xsl:text>;
+</xsl:text>
+        </xsl:if>
+    </xsl:template>
+    <xsl:template match="text()" name="handleImports">
+        <xsl:param name="imports" select="."/>
+
+        <xsl:choose>
+            <xsl:when test="contains($imports, ',')">
+                <xsl:call-template name="outputImports">
+                    <xsl:with-param name="import" select="substring-before($imports, ',')"/>
+                </xsl:call-template>
+                <xsl:call-template name="handleImports">
+                    <xsl:with-param name="imports" select="substring-after($imports, ',')"/>
+                </xsl:call-template>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:call-template name="outputImports">
+                    <xsl:with-param name="import" select="$imports"/>
+                </xsl:call-template>
+                <xsl:text>
+</xsl:text>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:template>
+
+    <xsl:template name="doNotChangeManuallyComment">
+        <xsl:text>/*** Not to be hand edited (changes will be lost on re-generation) ***/
 /*** ECL Interface generated by esdl2ecl version 1.0 from </xsl:text><xsl:copy-of select="$sourceFileName"/> <xsl:text>.xml. ***/
 /*** ECL Interface generated by esdl2ecl version 1.0 from </xsl:text><xsl:copy-of select="$sourceFileName"/> <xsl:text>.xml. ***/
 /*===================================================*/
 /*===================================================*/
 
 
-import $.share;
-
 </xsl:text>
 </xsl:text>
-	</xsl:template>
-	<xsl:template match="expesdl">
-			<xsl:apply-templates select="esxdl"/>
-	</xsl:template>
-	<xsl:template match="esxdl">
-	             <xsl:call-template name="doNotChangeManuallyComment"/>
+    </xsl:template>
+    <xsl:template match="expesdl">
+            <xsl:apply-templates select="esxdl"/>
+    </xsl:template>
+    <xsl:template match="esxdl">
+                 <xsl:call-template name="doNotChangeManuallyComment"/>
+                  <xsl:call-template name="handleImports">
+                    <xsl:with-param name="imports" select="$importsList"/>
+                  </xsl:call-template>
 			<xsl:text>export </xsl:text>
 			<xsl:text>export </xsl:text>
 			<xsl:choose>
 			<xsl:choose>
 				<xsl:when test="starts-with(@name, 'wsm_')"><xsl:value-of select="substring(@name, 5)"/></xsl:when>
 				<xsl:when test="starts-with(@name, 'wsm_')"><xsl:value-of select="substring(@name, 5)"/></xsl:when>
@@ -45,17 +76,6 @@ import $.share;
 			<xsl:text> := MODULE
 			<xsl:text> := MODULE
 
 
 </xsl:text>
 </xsl:text>
-		<xsl:if test="$docname='wsm_share'">
-		<xsl:text>export t_IntegerArrayItem := record
-	integer value { xpath('')};
-end;
-
-export t_StringArrayItem := record
-	string value { xpath(''), MAXLENGTH(8192) };
-end;
-
-</xsl:text>
-		</xsl:if>
 		<xsl:apply-templates select="EsdlStruct"/>
 		<xsl:apply-templates select="EsdlStruct"/>
 		<xsl:apply-templates select="EsdlRequest"/>
 		<xsl:apply-templates select="EsdlRequest"/>
 		<xsl:apply-templates select="EsdlResponse"/>
 		<xsl:apply-templates select="EsdlResponse"/>
@@ -89,7 +109,7 @@ end;
 	</xsl:template>
 	</xsl:template>
 	<xsl:template match="EsdlArray[@type='string']">
 	<xsl:template match="EsdlArray[@type='string']">
 		<xsl:if test="not(@ecl_hide) and (@ecl_keep or not(@get_data_from))">
 		<xsl:if test="not(@ecl_hide) and (@ecl_keep or not(@get_data_from))">
-		<xsl:text>	dataset(</xsl:text><xsl:call-template name="output_arrayitem_ref"/><xsl:text>) </xsl:text><xsl:call-template name="output_ecl_name"/>
+		<xsl:text>	set of string </xsl:text><xsl:call-template name="output_ecl_name"/>
 		<xsl:text> {xpath('</xsl:text>
 		<xsl:text> {xpath('</xsl:text>
 		<xsl:if test="not(@flat_array)"><xsl:value-of select="@name"/></xsl:if><xsl:text>/</xsl:text><xsl:call-template name="output_item_tag"/><xsl:text>')</xsl:text>
 		<xsl:if test="not(@flat_array)"><xsl:value-of select="@name"/></xsl:if><xsl:text>/</xsl:text><xsl:call-template name="output_item_tag"/><xsl:text>')</xsl:text>
 		<xsl:choose>
 		<xsl:choose>
@@ -312,13 +332,6 @@ end;
 	<xsl:text>')</xsl:text>
 	<xsl:text>')</xsl:text>
 </xsl:template>
 </xsl:template>
 
 
-<xsl:template name="output_arrayitem_ref">
-	<xsl:choose>
-		<xsl:when test="@ecl_item"><xsl:text>t_</xsl:text><xsl:value-of select="@ecl_item"/></xsl:when>
-		<xsl:otherwise><xsl:if test="$sourceFileName!='share'"><xsl:text>share.</xsl:text></xsl:if><xsl:text>t_StringArrayItem</xsl:text></xsl:otherwise>
-	</xsl:choose>
-</xsl:template>
-
 <xsl:template name="output_item_tag">
 <xsl:template name="output_item_tag">
     <xsl:choose>
     <xsl:choose>
          <xsl:when test="@item_tag"><xsl:value-of select="@item_tag"/></xsl:when>
          <xsl:when test="@item_tag"><xsl:value-of select="@item_tag"/></xsl:when>

+ 1 - 0
plugins/kafka/CMakeLists.txt

@@ -90,6 +90,7 @@ if (USE_KAFKA)
                 ./../../system/jlib
                 ./../../system/jlib
                 ${PROJECT_BINARY_DIR}/include
                 ${PROJECT_BINARY_DIR}/include
                 ${CMAKE_BINARY_DIR}
                 ${CMAKE_BINARY_DIR}
+                ${CMAKE_BINARY_DIR}/oss
             )
             )
 
 
         ADD_DEFINITIONS( -D_USRDLL -DECL_KAFKA_EXPORTS )
         ADD_DEFINITIONS( -D_USRDLL -DECL_KAFKA_EXPORTS )

+ 10 - 1
roxie/ccd/ccdserver.cpp

@@ -7373,6 +7373,15 @@ public:
             case TAFstable|TAFspill: sortAlgorithm = stableSpillingQuickSortAlgorithm; break;
             case TAFstable|TAFspill: sortAlgorithm = stableSpillingQuickSortAlgorithm; break;
             }
             }
         }
         }
+        else if (stricmp(algorithmName, "parquicksort")==0)
+        {
+            switch (sortFlags & TAFstable)
+            {
+            case 0: sortAlgorithm = parallelQuickSortAlgorithm; break;
+            case TAFstable: sortAlgorithm = parallelStableQuickSortAlgorithm; break;
+            default: throwUnexpected();
+            }
+        }
         else if (stricmp(algorithmName, "heapsort")==0)
         else if (stricmp(algorithmName, "heapsort")==0)
             sortAlgorithm = heapSortAlgorithm; // NOTE - we do allow UNSTABLE('heapsort') in order to facilitate runtime selection. Also explicit selection of heapsort overrides request to spill
             sortAlgorithm = heapSortAlgorithm; // NOTE - we do allow UNSTABLE('heapsort') in order to facilitate runtime selection. Also explicit selection of heapsort overrides request to spill
         else if (stricmp(algorithmName, "insertionsort")==0)
         else if (stricmp(algorithmName, "insertionsort")==0)
@@ -11861,7 +11870,7 @@ public:
     CRoxieServerJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
     CRoxieServerJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
         : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind)
         : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind)
     {
     {
-        forceSpill = _queryFactory.queryOptions().allSortsMaySpill || _graphNode.getPropBool("hint[@name='spill']/@value", false);
+        forceSpill = _graphNode.getPropBool("hint[@name='spill']/@value", _queryFactory.queryOptions().allSortsMaySpill);
         input2 = 0;
         input2 = 0;
         input2idx = 0;
         input2idx = 0;
     }
     }

+ 12 - 1
system/mp/mpcomm.cpp

@@ -826,11 +826,22 @@ protected: friend class CMPPacketReader;
 #ifdef _TRACE
 #ifdef _TRACE
                                 EXCLOG(e, "MP: Failed to connect");
                                 EXCLOG(e, "MP: Failed to connect");
 #endif
 #endif
-                                e->Release();
                                 if ((retrycount--==0)||(tm.timeout==MP_ASYNC_SEND))
                                 if ((retrycount--==0)||(tm.timeout==MP_ASYNC_SEND))
                                 {   // don't bother retrying on async send
                                 {   // don't bother retrying on async send
+                                    e->Release();
                                     throw new CMPException(MPERR_connection_failed,remoteep);
                                     throw new CMPException(MPERR_connection_failed,remoteep);
                                 }
                                 }
+
+                                // if other side closes, connect again
+                                if (e->errorCode() == JSOCKERR_graceful_close)
+                                {
+                                    LOG(MCdebugInfo(100), unknownJob, "MP: Retrying (other side closed connection, probably due to clash)");
+                                    e->Release();
+                                    break;
+                                }
+
+                                e->Release();
+
 #ifdef _TRACE
 #ifdef _TRACE
                                 LOG(MCdebugInfo(100), unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getUrlStr(str).toCharArray(),retrycount+1);
                                 LOG(MCdebugInfo(100), unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getUrlStr(str).toCharArray(),retrycount+1);
 #endif
 #endif

+ 2 - 0
tools/esdlcmd-xml/esdl2xml.hpp

@@ -75,6 +75,8 @@ public:
                    transform(subfile, outdir, out, outputIncludes, true);
                    transform(subfile, outdir, out, outputIncludes, true);
                 }
                 }
             }
             }
+            if (optVerbose)
+                fprintf(stdout, "Finished processing ESDL definition\n");
         }
         }
         else if (optVerbose)
         else if (optVerbose)
             fprintf(stdout, "ESDL definition: %s has already been loaded!\n", source);
             fprintf(stdout, "ESDL definition: %s has already been loaded!\n", source);

+ 23 - 8
tools/esdlcmd/esdl2ecl.cpp

@@ -238,11 +238,14 @@ public:
                 continue;
                 continue;
             if (iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML) || iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML_x))
             if (iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML) || iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML_x))
                 continue;
                 continue;
-            if (iter.matchFlag(optHPCCCompFilesDir, HPCC_COMPONENT_FILES_DIR) || iter.matchFlag(optHPCCCompFilesDir, HPCC_COMPONENT_FILES_DIR_CDE))
+            if (iter.matchFlag(optHPCCCompFilesDir, HPCC_COMPONENT_FILES_DIR_CDE))
                 continue;
                 continue;
             if (iter.matchFlag(optRollUpEclToSingleFile, ESDL_OPTION_ROLLUP))
             if (iter.matchFlag(optRollUpEclToSingleFile, ESDL_OPTION_ROLLUP))
                 continue;
                 continue;
-
+            if (iter.matchOption(optECLIncludesList, ESDL_OPTION_ECL_INCLUDE_LIST))
+                continue;
+            if (iter.matchOption(optECLHeaderBlock, ESDL_OPTION_ECL_HEADER_BLOCK))
+                continue;
             if (matchCommandLineOption(iter, true)!=EsdlCmdOptionMatch)
             if (matchCommandLineOption(iter, true)!=EsdlCmdOptionMatch)
                 return false;
                 return false;
         }
         }
@@ -258,7 +261,11 @@ public:
             return EsdlCmdOptionMatch;
             return EsdlCmdOptionMatch;
         if (iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML) || iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML_x))
         if (iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML) || iter.matchFlag(optOutputExpandedXML, ESDL_CONVERT_EXPANDEDXML_x))
             return EsdlCmdOptionMatch;
             return EsdlCmdOptionMatch;
-        if (iter.matchFlag(optHPCCCompFilesDir, HPCC_COMPONENT_FILES_DIR) || iter.matchFlag(optHPCCCompFilesDir, HPCC_COMPONENT_FILES_DIR_CDE))
+        if (iter.matchFlag(optHPCCCompFilesDir, HPCC_COMPONENT_FILES_DIR_CDE))
+            return EsdlCmdOptionMatch;
+        if (iter.matchOption(optHPCCCompFilesDir, ESDL_OPTION_ECL_INCLUDE_LIST))
+            return EsdlCmdOptionMatch;
+        if (iter.matchOption(optECLHeaderBlock, ESDL_OPTION_ECL_HEADER_BLOCK))
             return EsdlCmdOptionMatch;
             return EsdlCmdOptionMatch;
 
 
         return EsdlCmdCommon::matchCommandLineOption(iter, true);
         return EsdlCmdCommon::matchCommandLineOption(iter, true);
@@ -338,7 +345,7 @@ public:
                 StringBuffer xmlfile;
                 StringBuffer xmlfile;
                 toXML(&file, xmlfile, 0,0);
                 toXML(&file, xmlfile, 0,0);
 
 
-                outputEcl(srcPath.str(), filename, optOutDirPath.get(), idxxml.str(), xmlfile);
+                outputEcl(srcPath.str(), filename, optOutDirPath.get(), idxxml.str(), xmlfile, optECLIncludesList.str(), optECLHeaderBlock.str());
             }
             }
         }
         }
         else
         else
@@ -352,7 +359,7 @@ public:
                     StringBuffer xmlfile;
                     StringBuffer xmlfile;
                     toXML(file, xmlfile, 0,0);
                     toXML(file, xmlfile, 0,0);
 
 
-                    outputEcl(srcPath.str(), srcName.str(), optOutDirPath.get(), idxxml.str(), xmlfile);
+                    outputEcl(srcPath.str(), srcName.str(), optOutDirPath.get(), idxxml.str(), xmlfile, optECLIncludesList.str(), optECLHeaderBlock.str());
                 }
                 }
             }
             }
         }
         }
@@ -367,15 +374,19 @@ public:
                 "esdl ecl sourcePath outputPath [options]\n"
                 "esdl ecl sourcePath outputPath [options]\n"
                 "\nsourcePath must be absolute path to the ESDL Definition file containing the"
                 "\nsourcePath must be absolute path to the ESDL Definition file containing the"
                 "EsdlService definition for the service you want to work with.\n"
                 "EsdlService definition for the service you want to work with.\n"
-                "outputPath must be the absolute path where the ECL output with be created."
+                "outputPath must be the absolute path where the ECL output with be created.\n"
                 "   Options:\n"
                 "   Options:\n"
                 "      -x, --expandedxml     Output expanded XML files\n"
                 "      -x, --expandedxml     Output expanded XML files\n"
                 "      --includes            Process all included files\n"
                 "      --includes            Process all included files\n"
-                "      --rollup              Roll-up all processed includes to single ecl output file"
+                "      --rollup              Roll-up all processed includes to single ecl output file\n"
+                "      -cde                  HPCC Component files directory (xslt files)\n"
+                "      --ecl-imports         Coma-delimited import list to be attached to output ECL\n"
+                "                            each entry generates a corresponding import *.<entry>\n"
+                "      --ecl-header          Text included in target header (must be valid ECL) \n"
                 ,stdout);
                 ,stdout);
     }
     }
 
 
-    void outputEcl(const char *srcpath, const char *file, const char *path, const char *types, const char * xml)
+    void outputEcl(const char *srcpath, const char *file, const char *path, const char *types, const char * xml, const char * eclimports, const char * eclheader)
     {
     {
         DBGLOG("Generating ECL file for %s", file);
         DBGLOG("Generating ECL file for %s", file);
 
 
@@ -443,6 +454,8 @@ public:
 
 
         Owned<IProperties> params = createProperties();
         Owned<IProperties> params = createProperties();
         params->setProp("sourceFileName", finger);
         params->setProp("sourceFileName", finger);
+        params->setProp("importsList", eclimports);
+        params->setProp("eclHeader", eclheader);
         StringBuffer esdl2eclxslt (optHPCCCompFilesDir.get());
         StringBuffer esdl2eclxslt (optHPCCCompFilesDir.get());
         esdl2eclxslt.append("/xslt/esdl2ecl.xslt");
         esdl2eclxslt.append("/xslt/esdl2ecl.xslt");
         esdl2eclxsltTransform(expstr.str(), esdl2eclxslt.str(), params, outfile.str());
         esdl2eclxsltTransform(expstr.str(), esdl2eclxslt.str(), params, outfile.str());
@@ -488,4 +501,6 @@ public:
     bool optProcessIncludes;
     bool optProcessIncludes;
     bool optOutputExpandedXML;
     bool optOutputExpandedXML;
     StringAttr optHPCCCompFilesDir;
     StringAttr optHPCCCompFilesDir;
+    StringAttr optECLIncludesList;
+    StringAttr optECLHeaderBlock;
 };
 };

+ 3 - 2
tools/esdlcmd/esdlcmd_common.hpp

@@ -57,8 +57,7 @@ typedef IEsdlCommand *(*EsdlCommandFactory)(const char *cmdname);
 #define ESDL_CONVERT_EXPANDEDXML        "--expandedxml"
 #define ESDL_CONVERT_EXPANDEDXML        "--expandedxml"
 #define ESDL_CONVERT_EXPANDEDXML_x      "-x"
 #define ESDL_CONVERT_EXPANDEDXML_x      "-x"
 
 
-#define HPCC_COMPONENT_FILES_DIR        "--compfilesdir"
-#define HPCC_COMPONENT_FILES_DIR_CDE    "--CDE"
+#define HPCC_COMPONENT_FILES_DIR_CDE    "-cde"
 
 
 #define ESDLOPT_XSLT_PATH               "--xslt"
 #define ESDLOPT_XSLT_PATH               "--xslt"
 
 
@@ -98,6 +97,8 @@ typedef IEsdlCommand *(*EsdlCommandFactory)(const char *cmdname);
 #define ESDL_OPTION_CONFIG              "--config"
 #define ESDL_OPTION_CONFIG              "--config"
 #define ESDL_OPTION_OVERWRITE           "--overwrite"
 #define ESDL_OPTION_OVERWRITE           "--overwrite"
 #define ESDL_OPTION_ROLLUP              "--rollup"
 #define ESDL_OPTION_ROLLUP              "--rollup"
+#define ESDL_OPTION_ECL_INCLUDE_LIST    "--ecl-imports"
+#define ESDL_OPTION_ECL_HEADER_BLOCK    "--ecl-header"
 
 
 
 
 bool matchVariableOption(ArgvIterator &iter, const char prefix, IArrayOf<IEspNamedValue> &values);
 bool matchVariableOption(ArgvIterator &iter, const char prefix, IArrayOf<IEspNamedValue> &values);