HPCC-2877 Initial commit of first version of new regress

New Python based regression suite.

Signed-off-by: Philip Schwartz <philip.schwartz@lexisnexis.com>

testing/regress/ecl/key/setup.xml

@@ -0,0 +1,86 @@
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_Parent.d00&apos;,&apos;REGRESS::hthor::DG_Child.d00&apos;,&apos;DG_ParentID&apos;,&apos;DG_ParentID&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_Child.d00&apos;,&apos;REGRESS::hthor::DG_GrandChild.d00&apos;,&apos;DG_ParentID&apos;,&apos;DG_ParentID&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_Parent.d00&apos;,&apos;REGRESS::hthor::DG_CSV&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_Parent.d00&apos;,&apos;REGRESS::hthor::DG_XML&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_FLAT&apos;,&apos;REGRESS::hthor::DG_INDEX&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_FLAT&apos;,&apos;REGRESS::hthor::DG_INDEX&apos;,&apos;__fileposition__&apos;,&apos;filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_FLAT_EVENS&apos;,&apos;REGRESS::hthor::DG_INDEX_EVENS&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_FLAT_EVENS&apos;,&apos;REGRESS::hthor::DG_INDEX_EVENS&apos;,&apos;__fileposition__&apos;,&apos;filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_VAR&apos;,&apos;REGRESS::hthor::DG_VARINDEX&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_VAR&apos;,&apos;REGRESS::hthor::DG_VARINDEX&apos;,&apos;__fileposition__&apos;,&apos;__filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_VAR&apos;,&apos;REGRESS::hthor::DG_VARVARINDEX&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::DG_VAR&apos;,&apos;REGRESS::hthor::DG_VARVARINDEX&apos;,&apos;__fileposition__&apos;,&apos;__filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::SimplePersonBook&apos;,&apos;REGRESS::hthor::SimplePersonBookIndex&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::SimplePersonBook&apos;,&apos;REGRESS::hthor::SimplePersonBookIndex&apos;,&apos;__fileposition__&apos;,&apos;filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::House&apos;,&apos;REGRESS::hthor::Person&apos;,&apos;id&apos;,&apos;houseid&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::Person&apos;,&apos;REGRESS::hthor::Book&apos;,&apos;id&apos;,&apos;personid&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::House&apos;,&apos;REGRESS::hthor::HousePersonBook&apos;,&apos;id&apos;,&apos;id&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::House&apos;,&apos;REGRESS::hthor::PersonBook&apos;,&apos;id&apos;,&apos;houseid&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::House&apos;,&apos;REGRESS::hthor::HouseIndexID&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::House&apos;,&apos;REGRESS::hthor::HouseIndexID&apos;,&apos;__fileposition__&apos;,&apos;filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::Person&apos;,&apos;REGRESS::hthor::PersonIndexID&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::Person&apos;,&apos;REGRESS::hthor::PersonIndexID&apos;,&apos;__fileposition__&apos;,&apos;filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::Book&apos;,&apos;REGRESS::hthor::BookIndexID&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::Book&apos;,&apos;REGRESS::hthor::BookIndexID&apos;,&apos;__fileposition__&apos;,&apos;filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::House&apos;,&apos;REGRESS::hthor::HouseIndex&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::House&apos;,&apos;REGRESS::hthor::HouseIndex&apos;,&apos;__fileposition__&apos;,&apos;filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::Person&apos;,&apos;REGRESS::hthor::PersonIndex&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::PersonIndexID&apos;,&apos;REGRESS::hthor::PersonIndex&apos;,&apos;id&apos;,&apos;id&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::Book&apos;,&apos;REGRESS::hthor::BookIndex&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::BookIndexID&apos;,&apos;REGRESS::hthor::BookIndex&apos;,&apos;id&apos;,&apos;id&apos;,&apos;link&apos;) done</Message></Info>
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>
+<Dataset name='Result 6'>
+</Dataset>
+<Dataset name='Result 7'>
+</Dataset>
+<Dataset name='Result 8'>
+</Dataset>
+<Dataset name='Result 9'>
+</Dataset>
+<Dataset name='Result 10'>
+</Dataset>
+<Dataset name='Result 11'>
+</Dataset>
+<Dataset name='Result 12'>
+</Dataset>
+<Dataset name='Result 13'>
+</Dataset>
+<Dataset name='Result 14'>
+</Dataset>
+<Dataset name='Result 15'>
+</Dataset>
+<Dataset name='Result 16'>
+</Dataset>
+<Dataset name='Result 17'>
+</Dataset>
+<Dataset name='Result 18'>
+</Dataset>
+<Dataset name='Result 19'>
+</Dataset>
+<Dataset name='Result 20'>
+</Dataset>
+<Dataset name='Result 21'>
+</Dataset>
+<Dataset name='Result 22'>
+</Dataset>
+<Dataset name='Result 23'>
+</Dataset>
+<Dataset name='Result 24'>
+</Dataset>
+<Dataset name='Result 25'>
+</Dataset>
+<Dataset name='Result 26'>
+</Dataset>
+<Dataset name='Result 27'>
+</Dataset>
+<Dataset name='Result 28'>
+</Dataset>

testing/regress/ecl/key/setup_fetch.xml

@@ -0,0 +1,16 @@
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::C.DG_FetchFile&apos;,&apos;REGRESS::hthor::C.DG_FetchFilePreload&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::C.DG_FetchFile&apos;,&apos;REGRESS::hthor::C.DG_FetchFilePreloadIndexed&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::C.DG_FetchFile&apos;,&apos;REGRESS::hthor::DG_FetchIndex1&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::C.DG_FetchFile&apos;,&apos;REGRESS::hthor::DG_FetchIndex1&apos;,&apos;__fileposition__&apos;,&apos;__filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::C.DG_FetchFile&apos;,&apos;REGRESS::hthor::DG_FetchIndex2&apos;,&apos;&apos;,&apos;&apos;,&apos;view&apos;) done</Message></Info>
+<Info><Source>fileservices</Source><Message>AddFileRelationship(&apos;REGRESS::hthor::C.DG_FetchFile&apos;,&apos;REGRESS::hthor::DG_FetchIndex2&apos;,&apos;__fileposition__&apos;,&apos;__filepos&apos;,&apos;link&apos;) done</Message></Info>
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>

testing/regress/ecl/key/setupxml.xml

@@ -0,0 +1,2 @@
+<Dataset name='Result 1'>
+</Dataset>

testing/regress/ecl/setup/files.ecl

@@ -0,0 +1,514 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+//skip type==setup TBD
+
+//define constants
+EXPORT files(string p) := module
+SHARED DG_GenFlat           := true;   //TRUE gens FlatFile
+SHARED DG_GenChild          := true;   //TRUE gens ChildFile
+SHARED DG_GenGrandChild     := true;   //TRUE gens GrandChildFile
+SHARED DG_GenIndex          := true;   //TRUE gens FlatFile AND the index
+SHARED DG_GenCSV            := true;   //TRUE gens CSVFile
+SHARED DG_GenXML            := true;   //TRUE gens XMLFile
+SHARED DG_GenVar            := true;   //TRUE gens VarFile only IF MaxField >= 3
+
+EXPORT DG_MaxField          := 3;    // maximum number of fields to use building the data records
+EXPORT DG_MaxChildren       := 3;    //maximum (1 to n) number of child recs
+
+                    // generates (#parents * DG_MaxChildren) records
+EXPORT DG_MaxGrandChildren  := 3;    //maximum (1 to n) number of grandchild recs
+                    // generates (#children * DG_MaxGrandChildren) records
+
+SHARED useDynamic := false;
+SHARED useLayoutTrans := false;
+SHARED useVarIndex := false;
+SHARED prefix := 'hthor';
+
+#if (useDynamic=true)
+SHARED  VarString EmptyString := '' : STORED('dummy');
+SHARED  filePrefix := prefix + EmptyString;
+ #option ('allowVariableRoxieFilenames', 1)
+#else
+SHARED  filePrefix := prefix;
+#end
+
+EXPORT DG_FileOut           := '~REGRESS::' + filePrefix + '::DG_';
+EXPORT DG_ParentFileOut     := '~REGRESS::' + filePrefix + '::DG_Parent.d00';
+EXPORT DG_ChildFileOut      := '~REGRESS::' + filePrefix + '::DG_Child.d00';
+EXPORT DG_GrandChildFileOut := '~REGRESS::' + filePrefix + '::DG_GrandChild.d00';
+EXPORT DG_FetchFileName     := '~REGRESS::' + filePrefix + '::C.DG_FetchFile';
+EXPORT DG_FetchFilePreloadName := '~REGRESS::' + filePrefix + '::C.DG_FetchFilePreload';
+EXPORT DG_FetchFilePreloadIndexedName := '~REGRESS::' + filePrefix + '::C.DG_FetchFilePreloadIndexed';
+EXPORT DG_FetchIndex1Name   := '~REGRESS::' + filePrefix + '::DG_FetchIndex1';
+EXPORT DG_FetchIndex2Name   := '~REGRESS::' + filePrefix + '::DG_FetchIndex2';
+EXPORT DG_FetchIndexDiffName:= '~REGRESS::' + filePrefix + '::DG_FetchIndexDiff';
+EXPORT DG_MemFileName       := '~REGRESS::' + filePrefix + '::DG_MemFile';
+EXPORT DG_IntegerDatasetName:= '~REGRESS::' + filePrefix + '::DG_IntegerFile';
+EXPORT DG_IntegerIndexName  := '~REGRESS::' + filePrefix + '::DG_IntegerIndex';
+
+//record structures
+EXPORT DG_FetchRecord := RECORD
+  INTEGER8 sequence;
+  STRING2  State;
+  STRING20 City;
+  STRING25 Lname;
+  STRING15 Fname;
+END;
+
+EXPORT DG_FetchFile   := DATASET(DG_FetchFileName,{DG_FetchRecord,UNSIGNED8 __filepos {virtual(fileposition)}},FLAT);
+EXPORT DG_FetchFilePreload := PRELOAD(DATASET(DG_FetchFilePreloadName,{DG_FetchRecord,UNSIGNED8 __filepos {virtual(fileposition)}},FLAT));
+EXPORT DG_FetchFilePreloadIndexed := PRELOAD(DATASET(DG_FetchFilePreloadIndexedName,{DG_FetchRecord,UNSIGNED8 __filepos {virtual(fileposition)}},FLAT),1);
+
+#IF (useLayoutTrans=false)
+  #IF (useVarIndex=true)
+    EXPORT DG_FetchIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{STRING fn := TRIM(Fname), state, STRING100 x {blob}:= fname, __filepos},DG_FetchIndex1Name);
+    EXPORT DG_FetchIndex2 := INDEX(DG_FetchFile,{Lname,Fname},{STRING fn := TRIM(Fname), state, STRING100 x {blob}:= fname, __filepos},DG_FetchIndex2Name);
+  #ELSE
+    EXPORT DG_FetchIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{state ,__filepos},DG_FetchIndex1Name);
+    EXPORT DG_FetchIndex2 := INDEX(DG_FetchFile,{Lname,Fname},{state, __filepos}, DG_FetchIndex2Name);
+  #END
+#ELSE
+ // Declare all indexes such that layout translation is required... Used at run-time only, not at setup time...
+  #IF (useVarIndex=true)
+    EXPORT DG_FetchIndex1 := INDEX(DG_FetchFile,{Fname,Lname},{STRING fn := TRIM(Fname), state, STRING100 x {blob}:= fname, __filepos},DG_FetchIndex1Name);
+    EXPORT DG_FetchIndex2 := INDEX(DG_FetchFile,{Fname,Lname},{STRING fn := TRIM(Fname), state, STRING100 x {blob}:= fname, __filepos},DG_FetchIndex2Name);
+  #ELSE
+    EXPORT DG_FetchIndex1 := INDEX(DG_FetchFile,{Fname,Lname},{state ,__filepos},DG_FetchIndex1Name);
+    EXPORT DG_FetchIndex2 := INDEX(DG_FetchFile,{Fname,Lname},{state, __filepos}, DG_FetchIndex2Name);
+  #END
+#END
+EXPORT DG_OutRec := RECORD
+    unsigned4  DG_ParentID;
+    string10  DG_firstname;
+    string10  DG_lastname;
+    unsigned1 DG_Prange;
+END;
+
+EXPORT DG_OutRecChild := RECORD
+    unsigned4  DG_ParentID;
+    unsigned4  DG_ChildID;
+    string10  DG_firstname;
+    string10  DG_lastname;
+    unsigned1 DG_Prange;
+END;
+
+EXPORT DG_VarOutRec := RECORD
+  DG_OutRec;
+  IFBLOCK(self.DG_Prange%2=0)
+    string20 ExtraField;
+  END;
+END;
+
+//DATASET declarations
+EXPORT DG_BlankSet := dataset([{0,'','',0}],DG_OutRec);
+
+EXPORT DG_FlatFile      := DATASET(DG_FileOut+'FLAT',{DG_OutRec,UNSIGNED8 filepos{virtual(fileposition)}},FLAT);
+EXPORT DG_FlatFileEvens := DATASET(DG_FileOut+'FLAT_EVENS',{DG_OutRec,UNSIGNED8 filepos{virtual(fileposition)}},FLAT);
+
+EXPORT DG_indexFile      := INDEX(DG_FlatFile,
+    RECORD
+#if(useLayoutTrans=false)
+      DG_firstname;
+      DG_lastname;
+#else
+      DG_lastname;
+      DG_firstname;
+#end
+    END,
+     RECORD
+      DG_Prange;
+      filepos
+    END,DG_FileOut+'INDEX');
+
+EXPORT DG_indexFileEvens := INDEX(DG_FlatFileEvens,
+    RECORD
+#if(useLayoutTrans=false)
+      DG_firstname;
+      DG_lastname;
+#else
+      DG_lastname;
+      DG_firstname;
+#end
+    END,
+    RECORD
+      DG_Prange;
+      filepos
+    END,DG_FileOut+'INDEX_EVENS');
+
+EXPORT DG_CSVFile   := DATASET(DG_FileOut+'CSV',DG_OutRec,CSV);
+EXPORT DG_XMLFile   := DATASET(DG_FileOut+'XML',DG_OutRec,XML);
+
+EXPORT DG_VarOutRecPlus := RECORD
+  DG_VarOutRec,
+  unsigned8 __filepos { virtual(fileposition)};
+END;
+
+EXPORT DG_VarFile   := DATASET(DG_FileOut+'VAR',DG_VarOutRecPlus,FLAT);
+EXPORT DG_VarIndex  := INDEX(DG_VarFile,{
+#if(useLayoutTrans=false)
+      DG_firstname;
+      DG_lastname;
+#else
+      DG_lastname;
+      DG_firstname;
+#end
+__filepos},DG_FileOut+'VARINDEX');
+EXPORT DG_VarVarIndex  := INDEX(DG_VarFile,{
+#if(useLayoutTrans=false)
+      DG_firstname;
+      DG_lastname;
+#else
+      DG_lastname;
+      DG_firstname;
+#end
+__filepos},{ string temp_blob1 := TRIM(ExtraField); string10000 temp_blob2 {blob} := ExtraField },DG_FileOut+'VARVARINDEX');
+
+EXPORT DG_ParentFile  := DATASET(DG_ParentFileOut,{DG_OutRec,UNSIGNED8 filepos{virtual(fileposition)}},FLAT);
+EXPORT DG_ChildFile   := DATASET(DG_ChildFileOut,{DG_OutRecChild,UNSIGNED8 filepos{virtual(fileposition)}},FLAT);
+EXPORT DG_GrandChildFile := DATASET(DG_GrandChildFileOut,{DG_OutRecChild,UNSIGNED8 filepos{virtual(fileposition)}},FLAT);
+
+//define data atoms - each set has 16 elements
+EXPORT SET OF STRING10 DG_Fnames := ['DAVID','CLAIRE','KELLY','KIMBERLY','PAMELA','JEFFREY','MATTHEW','LUKE',
+                              'JOHN' ,'EDWARD','CHAD' ,'KEVIN'   ,'KOBE'  ,'RICHARD','GEORGE' ,'DIRK'];
+EXPORT SET OF STRING10 DG_Lnames := ['BAYLISS','DOLSON','BILLINGTON','SMITH'   ,'JONES'   ,'ARMSTRONG','LINDHORFF','SIMMONS',
+                              'WYMAN'  ,'MORTON','MIDDLETON' ,'NOWITZKI','WILLIAMS','TAYLOR'   ,'CHAPMAN'  ,'BRYANT'];
+EXPORT SET OF UNSIGNED1 DG_PrangeS := [1, 2, 3, 4, 5, 6, 7, 8,
+                                9,10,11,12,13,14,15,16];
+EXPORT SET OF STRING10 DG_Streets := ['HIGH'  ,'CITATION'  ,'MILL','25TH' ,'ELGIN'    ,'VICARAGE','YAMATO' ,'HILLSBORO',
+                               'SILVER','KENSINGTON','MAIN','EATON','PARK LANE','HIGH'    ,'POTOMAC','GLADES'];
+EXPORT SET OF UNSIGNED1 DG_ZIPS := [101,102,103,104,105,106,107,108,
+                             109,110,111,112,113,114,115,116];
+EXPORT SET OF UNSIGNED1 DG_AGES := [31,32,33,34,35,36,37,38,
+                             39,40,41,42,43,44,45,56];
+EXPORT SET OF STRING2 DG_STATES := ['FL','GA','SC','NC','TX','AL','MS','TN',
+                             'KY','CA','MI','OH','IN','IL','WI','MN'];
+EXPORT SET OF STRING3 DG_MONTHS := ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG',
+                             'SEP','OCT','NOV','DEC','ABC','DEF','GHI','JKL'];
+
+EXPORT t_personfile := DATASET('t_personfile', RECORD
+  unsigned integer4 hhid;
+  unsigned integer4 personid;
+  string20 firstname;
+  string20 lastname;
+  string20 middlename;
+  unsigned integer1 age;
+  unsigned integer8 ssn;
+END, THOR);
+
+EXPORT t_tradesfile := DATASET('t_tradesfile', RECORD
+  unsigned integer4 personid;
+  string20 tradeid;
+  real4 amount;
+  string8 date;
+END, THOR);
+
+EXPORT t_hhfile := DATASET('t_hhfile', RECORD
+  unsigned integer4 hhid;
+  string2 State;
+  string5 zip;
+  string20 City;
+  string40 street;
+  unsigned integer4 houseNumber;
+END, THOR);
+
+
+//----------------------------- Child query related definitions ----------------------------------
+
+// Raw record definitions:
+
+EXPORT sqHouseRec :=
+            record
+string          addr;
+string10        postcode;
+unsigned2       yearBuilt := 0;
+            end;
+
+
+EXPORT sqPersonRec :=
+            record
+string          forename;
+string          surname;
+udecimal8       dob;
+udecimal8       booklimit := 0;
+unsigned2       aage := 0;
+            end;
+
+EXPORT sqBookRec :=
+            record
+string          name;
+string          author;
+unsigned1       rating100;
+udecimal8_2     price := 0;
+            end;
+
+
+// Nested record definitions
+EXPORT sqPersonBookRec :=
+            record
+sqPersonRec;
+dataset(sqBookRec)      books;
+            end;
+
+EXPORT sqHousePersonBookRec :=
+            record
+sqHouseRec;
+dataset(sqPersonBookRec) persons;
+            end;
+
+
+// Record definitions with additional ids
+
+EXPORT sqHouseIdRec :=
+            record
+unsigned4       id;
+sqHouseRec;
+            end;
+
+
+EXPORT sqPersonIdRec :=
+            record
+unsigned4       id;
+sqPersonRec;
+            end;
+
+
+EXPORT sqBookIdRec :=
+            record
+unsigned4       id;
+sqBookRec;
+            end;
+
+
+// Same with parent linking field.
+
+EXPORT sqPersonRelatedIdRec :=
+            record
+sqPersonIdRec;
+unsigned4       houseid;
+            end;
+
+
+EXPORT sqBookRelatedIdRec :=
+            record
+sqBookIdRec;
+unsigned4       personid;
+            end;
+
+
+// Nested definitions with additional ids...
+
+EXPORT sqPersonBookIdRec :=
+            record
+sqPersonIdRec;
+dataset(sqBookIdRec)        books;
+            end;
+
+EXPORT sqHousePersonBookIdRec :=
+            record
+sqHouseIdRec;
+dataset(sqPersonBookIdRec) persons;
+            end;
+
+
+EXPORT sqPersonBookRelatedIdRec :=
+            RECORD
+                sqPersonBookIdRec;
+unsigned4       houseid;
+            END;
+
+EXPORT sqNestedBlob :=
+            RECORD
+udecimal8       booklimit := 0;
+            END;
+
+EXPORT sqSimplePersonBookRec :=
+            RECORD
+string20        surname;
+string10        forename;
+udecimal8       dob;
+//udecimal8     booklimit := 0;
+sqNestedBlob    limit{blob};
+unsigned1       aage := 0;
+dataset(sqBookIdRec)        books{blob};
+            END;
+EXPORT sqNamePrefix := '~REGRESS::' + filePrefix + '::';
+EXPORT sqHousePersonBookName := sqNamePrefix + 'HousePersonBook';
+EXPORT sqPersonBookName := sqNamePrefix + 'PersonBook';
+EXPORT sqHouseName := sqNamePrefix + 'House';
+EXPORT sqPersonName := sqNamePrefix + 'Person';
+EXPORT sqBookName := sqNamePrefix + 'Book';
+EXPORT sqSimplePersonBookName := sqNamePrefix + 'SimplePersonBook';
+
+EXPORT sqHousePersonBookIndexName := sqNamePrefix + 'HousePersonBookIndex';
+EXPORT sqPersonBookIndexName := sqNamePrefix + 'PersonBookIndex';
+EXPORT sqHouseIndexName := sqNamePrefix + 'HouseIndex';
+EXPORT sqPersonIndexName := sqNamePrefix + 'PersonIndex';
+EXPORT sqBookIndexName := sqNamePrefix + 'BookIndex';
+EXPORT sqSimplePersonBookIndexName := sqNamePrefix + 'SimplePersonBookIndex';
+EXPORT sqHousePersonBookIdExRec := record
+sqHousePersonBookIdRec;
+unsigned8           filepos{virtual(fileposition)};
+                end;
+
+EXPORT sqPersonBookRelatedIdExRec := record
+sqPersonBookRelatedIdRec;
+unsigned8           filepos{virtual(fileposition)};
+                end;
+
+EXPORT sqHouseIdExRec := record
+sqHouseIdRec;
+unsigned8           filepos{virtual(fileposition)};
+                end;
+
+EXPORT sqPersonRelatedIdExRec := record
+sqPersonRelatedIdRec;
+unsigned8           filepos{virtual(fileposition)};
+                end;
+
+EXPORT sqBookRelatedIdExRec := record
+sqBookRelatedIdRec;
+unsigned8           filepos{virtual(fileposition)};
+                end;
+
+EXPORT sqSimplePersonBookExRec := record
+sqSimplePersonBookRec;
+unsigned8           filepos{virtual(fileposition)};
+                end;
+
+// Dataset definitions:
+
+
+EXPORT sqHousePersonBookDs := dataset(sqHousePersonBookName, sqHousePersonBookIdExRec, thor);
+EXPORT sqPersonBookDs := dataset(sqPersonBookName, sqPersonBookRelatedIdRec, thor);
+EXPORT sqHouseDs := dataset(sqHouseName, sqHouseIdExRec, thor);
+EXPORT sqPersonDs := dataset(sqPersonName, sqPersonRelatedIdRec, thor);
+EXPORT sqBookDs := dataset(sqBookName, sqBookRelatedIdRec, thor);
+
+EXPORT sqHousePersonBookExDs := dataset(sqHousePersonBookName, sqHousePersonBookIdExRec, thor);
+EXPORT sqPersonBookExDs := dataset(sqPersonBookName, sqPersonBookRelatedIdExRec, thor);
+EXPORT sqHouseExDs := dataset(sqHouseName, sqHouseIdExRec, thor);
+EXPORT sqPersonExDs := dataset(sqPersonName, sqPersonRelatedIdExRec, thor);
+EXPORT sqBookExDs := dataset(sqBookName, sqBookRelatedIdExRec, thor);
+
+EXPORT sqSimplePersonBookDs := dataset(sqSimplePersonBookName, sqSimplePersonBookExRec, thor);
+EXPORT sqSimplePersonBookIndex := index(sqSimplePersonBookDs, { surname, forename, aage  }, { sqSimplePersonBookDs }, sqSimplePersonBookIndexName);
+
+//related datasets:
+//Don't really work because inheritance structure isn't preserved.
+
+EXPORT relatedBooks(sqPersonIdRec parentPerson) := sqBookDs(personid = parentPerson.id);
+EXPORT relatedPersons(sqHouseIdRec parentHouse) := sqPersonDs(houseid = parentHouse.id);
+
+EXPORT sqNamesTable1 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable2 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable3 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable4 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable5 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable6 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable7 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable8 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+EXPORT sqNamesTable9 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
+
+EXPORT sqNamesIndex1 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex2 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex3 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex4 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex5 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex6 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex7 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex8 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+EXPORT sqNamesIndex9 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
+
+
+//----------------------------- Text search definitions ----------------------------------
+EXPORT TS_MaxTerms             := 50;
+EXPORT TS_MaxStages            := 50;
+EXPORT TS_MaxProximity         := 10;
+EXPORT TS_MaxWildcard          := 1000;
+EXPORT TS_MaxMatchPerDocument  := 1000;
+EXPORT TS_MaxFilenameLength        := 255;
+EXPORT TS_MaxActions           := 255;
+EXPORT TS_MaxTagNesting        := 40;
+EXPORT TS_MaxColumnsPerLine := 10000;          // used to create a pseudo document position
+
+EXPORT TS_kindType         := enum(unsigned1, UnknownEntry=0, TextEntry, OpenTagEntry, CloseTagEntry, OpenCloseTagEntry, CloseOpenTagEntry);
+EXPORT TS_sourceType       := unsigned2;
+EXPORT TS_wordCountType    := unsigned8;
+EXPORT TS_segmentType      := unsigned1;
+EXPORT TS_wordPosType      := unsigned8;
+EXPORT TS_docPosType       := unsigned8;
+EXPORT TS_documentId       := unsigned8;
+EXPORT TS_termType         := unsigned1;
+EXPORT TS_distanceType     := integer8;
+EXPORT TS_indexWipType     := unsigned1;
+EXPORT TS_wipType          := unsigned8;
+EXPORT TS_stageType        := unsigned1;
+EXPORT TS_dateType         := unsigned8;
+
+EXPORT TS_sourceType TS_docid2source(TS_documentId x) := (x >> 48);
+EXPORT TS_documentId TS_docid2doc(TS_documentId x) := (x & 0xFFFFFFFFFFFF);
+EXPORT TS_documentId TS_createDocId(TS_sourceType source, TS_documentId doc) := (TS_documentId)(((unsigned8)source << 48) | doc);
+EXPORT boolean      TS_docMatchesSource(TS_documentId docid, TS_sourceType source) := (docid between TS_createDocId(source,0) and (TS_documentId)(TS_createDocId(source+1,0)-1));
+
+EXPORT TS_wordType := string20;
+EXPORT TS_wordFlags    := enum(unsigned1, HasLower=1, HasUpper=2);
+
+EXPORT TS_wordIdType       := unsigned4;
+
+EXPORT TS_NameWordIndex        := '~REGRESS::' + filePrefix + '::TS_wordIndex';
+EXPORT TS_NameSearchIndex      := '~REGRESS::' + filePrefix + '::TS_searchIndex';
+
+EXPORT TS_wordIndex        := index({ TS_kindType kind, TS_wordType word, TS_documentId doc, TS_segmentType segment, TS_wordPosType wpos, TS_indexWipType wip } , { TS_wordFlags flags, TS_wordType original, TS_docPosType dpos}, TS_NameWordIndex);
+EXPORT TS_searchIndex      := index(TS_wordIndex, TS_NameSearchIndex);
+
+EXPORT TS_wordIndexRecord := recordof(TS_wordIndex);
+
+//----------------------------- End of text search definitions --------------------------
+
+
+
+EXPORT DG_MemFileRec := RECORD
+    unsigned2 u2;
+    unsigned3 u3;
+    big_endian unsigned2 bu2;
+    big_endian unsigned3 bu3;
+    integer2 i2;
+    integer3 i3;
+    big_endian integer2 bi2;
+    big_endian integer3 bi3;
+END;
+
+EXPORT DG_MemFile := DATASET(DG_MemFileName,DG_MemFileRec,FLAT);
+
+
+//record structures
+EXPORT DG_NestedIntegerRecord := RECORD
+  big_endian UNSIGNED4 i4;
+  big_endian UNSIGNED3 u3;
+END;
+
+EXPORT DG_IntegerRecord := RECORD
+    INTEGER6    i6;
+    DG_NestedIntegerRecord nested;
+    integer5    i5;
+    integer3    i3;
+END;
+
+EXPORT DG_IntegerDataset := DATASET(DG_IntegerDatasetName, DG_IntegerRecord, thor);
+EXPORT DG_IntegerIndex := INDEX(DG_IntegerDataset, { i6, nested }, { DG_IntegerDataset }, DG_IntegerIndexName);
+
+END;

testing/regress/ecl/setup/setup.ecl

@@ -0,0 +1,430 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import Std.File AS FileServices;
+import $; C := $.files('');
+
+C.DG_OutRec norm1(C.DG_OutRec l, integer cc) := transform
+  self.DG_firstname := C.DG_Fnames[cc];
+  self := l;
+  end;
+DG_Norm1Recs := normalize( C.DG_BlankSet, 4, norm1(left, counter));
+
+C.DG_OutRec norm2(C.DG_OutRec l, integer cc) := transform
+  self.DG_lastname := C.DG_Lnames[cc];
+  self := l;
+  end;
+DG_Norm2Recs := normalize( DG_Norm1Recs, 4, norm2(left, counter));
+
+C.DG_OutRec norm3(C.DG_OutRec l, integer cc) := transform
+  self.DG_Prange := C.DG_Pranges[cc];
+  self := l;
+  end;
+DG_Norm3Recs := normalize( DG_Norm2Recs, 4, norm3(left, counter));
+
+//output data files
+DG_OutputRecs := DG_Norm3Recs;
+
+//***************************************************************************
+
+DG_OutputRecs SeqParent(DG_OutputRecs l, integer c) := transform
+  self.DG_ParentID := c-1;  //use -1 so max records (16^8) fit in unsigned4
+  self := l;
+  end;
+DG_ParentRecs := project( DG_OutputRecs, SeqParent(left, counter));
+
+C.DG_OutRecChild GenChildren(DG_OutputRecs l) := transform
+  self.DG_ChildID := 0;
+  self := l;
+  end;
+DG_ChildRecs1 := normalize(DG_ParentRecs, C.DG_MaxChildren, GenChildren(left));
+
+C.DG_OutRecChild SeqChildren(C.DG_OutRecChild l, integer cc) := transform
+  self.DG_ChildID := cc-1;
+  self := l;
+  end;
+DG_ChildRecs := project( DG_ChildRecs1, SeqChildren(left, counter));
+output(DG_ParentRecs,,C.DG_ParentFileOut,overwrite);
+output(DG_ChildRecs,,C.DG_ChildFileOut,overwrite);
+fileServices.AddFileRelationship( C.DG_ParentFileOut, C.DG_ChildFileOut, 'DG_ParentID', 'DG_ParentID', 'link', '1:M', false);
+C.DG_OutRecChild GenGrandChildren(C.DG_OutRecChild l) := transform
+  self := l;
+  end;
+DG_GrandChildRecs := normalize( DG_ChildRecs, C.DG_MaxGrandChildren, GenGrandChildren(left));
+output(DG_GrandChildRecs,,C.DG_GrandChildFileOut,overwrite);
+fileServices.AddFileRelationship( C.DG_ChildFileOut, C.DG_GrandChildFileOut, 'DG_ParentID', 'DG_ParentID', 'link', '1:M', false);
+
+//output data files
+
+//***************************************************************************
+
+output(DG_ParentRecs,,C.DG_FileOut+'CSV',CSV,overwrite);
+fileServices.AddFileRelationship( C.DG_ParentFileOut, C.DG_FileOut+'CSV', '', '', 'view', '1:1', false);
+output(DG_ParentRecs,,C.DG_FileOut+'XML',XML,overwrite);
+fileServices.AddFileRelationship( C.DG_ParentFileOut, C.DG_FileOut+'XML', '', '', 'view', '1:1', false);
+EvensFilter := DG_ParentRecs.DG_firstname in [C.DG_Fnames[2],C.DG_Fnames[4],C.DG_Fnames[6],C.DG_Fnames[8],
+                                              C.DG_Fnames[10],C.DG_Fnames[12],C.DG_Fnames[14],C.DG_Fnames[16]];
+
+SEQUENTIAL(
+    PARALLEL(output(DG_ParentRecs,,C.DG_FileOut+'FLAT',overwrite),
+             output(DG_ParentRecs(EvensFilter),,C.DG_FileOut+'FLAT_EVENS',overwrite)),
+    PARALLEL(buildindex(C.DG_IndexFile,overwrite),
+             buildindex(C.DG_IndexFileEvens,overwrite))
+    );
+
+    fileServices.AddFileRelationship( __nameof__(C.DG_FlatFile), __nameof__(C.DG_IndexFile), '', '', 'view', '1:1', false);
+    fileServices.AddFileRelationship( __nameof__(C.DG_FlatFile), __nameof__(C.DG_IndexFile), '__fileposition__', 'filepos', 'link', '1:1', true);
+    fileServices.AddFileRelationship( __nameof__(C.DG_FlatFileEvens), __nameof__(C.DG_IndexFileEvens), '', '', 'view', '1:1', false);
+    fileServices.AddFileRelationship( __nameof__(C.DG_FlatFileEvens), __nameof__(C.DG_IndexFileEvens), '__fileposition__', 'filepos', 'link', '1:1', true);
+
+C.DG_VarOutRec Proj1(C.DG_OutRec L) := TRANSFORM
+  SELF := L;
+  SELF.ExtraField := IF(self.DG_Prange<=10,
+                        trim(self.DG_lastname[1..self.DG_Prange]+self.DG_firstname[1..self.DG_Prange],all),
+                        trim(self.DG_lastname[1..self.DG_Prange-10]+self.DG_firstname[1..self.DG_Prange-10],all));
+END;
+DG_VarOutRecs := PROJECT(DG_ParentRecs,Proj1(LEFT));
+
+sequential(
+  output(DG_VarOutRecs,,C.DG_FileOut+'VAR',overwrite),
+  buildindex(C.DG_VarIndex, overwrite),
+  buildindex(C.DG_VarVarIndex, overwrite),
+  fileServices.AddFileRelationship( __nameof__(C.DG_VarFile), __nameof__(C.DG_VarIndex), '', '', 'view', '1:1', false),
+  fileServices.AddFileRelationship( __nameof__(C.DG_VarFile), __nameof__(C.DG_VarIndex), '__fileposition__', '__filepos', 'link', '1:1', true),
+  fileServices.AddFileRelationship( __nameof__(C.DG_VarFile), __nameof__(C.DG_VarVarIndex), '', '', 'view', '1:1', false),
+  fileServices.AddFileRelationship( __nameof__(C.DG_VarFile), __nameof__(C.DG_VarVarIndex), '__fileposition__', '__filepos', 'link', '1:1', true)
+);
+
+//******************************** Child query setup code ***********************
+
+udecimal8 baseDate := 20050101;
+
+rawHouse := dataset([
+    { '99 Maltings Road', 'SW1A0AA', 1720,
+        [{ 'Gavin', 'Halliday', 19700101, 1000, 0,
+            [
+            { 'To kill a mocking bird', 'Harper Lee', 95},
+            { 'Clarion Language Manual', 'Richard Taylor', 1, 399.99 },
+            { 'The bible', 'Various', 98 },
+            { 'Compilers', 'Aho, Sethi, Ullman', 80 },
+            { 'Lord of the Rings', 'JRR Tolkien', 95 }
+            ]
+        },
+        { 'Abigail', 'Halliday', 20000101, 40, 0,
+            [
+            { 'The thinks you can think', 'Dr. Seuss', 90, 5.99 },
+            { 'Where is flop?', '', 85, 4.99 },
+            { 'David and Goliath', '', 20, 2.99 },
+            { 'The story of Jonah', '', 80, 6.99 }
+            ]
+        },
+        { 'Liz', 'Halliday', 19700909, 0, 0,
+            [
+            { 'The Life of Pi', 'Yan Martel', 90 },
+            { 'BNF', 'Various', 60 },
+            { 'The Third Policeman', 'Flann O\'Brien', 85 }
+            ]
+        }]
+    },
+    { 'Buckingham Palace', 'WC1', 1702,
+        [{'Elizabeth', 'Windsor', 19260421, 0, 0,
+            [
+            { 'The bible', 'Various', 93 },
+            { 'The Prince', 'Machiavelli', 40 },
+            { 'Atlas of the world', 'Times', 70 },
+            { 'Girl guide annual', 'Various', 50 },
+            { 'Rwandan war dances', 'Unknown', 30 }
+            ]
+        },
+        {'Philip', 'Windsor', 19210610, 0, 0,
+            [
+            { 'Greek tragedies', 'Various', 30 },
+            { 'Social etiquette', 'RU Correct', 10},
+            { 'The Rituals of Dinner: The Origins, Evolution, Eccentricities and the Meaning of Table Manners', 'Margaret Visser', 60 },
+            { 'The cat in the hat', 'Dr. Seuss', 85 }
+            ]
+        }]
+    },
+    { 'Bedrock', '', 0,
+        [{'Fred', 'Flintstone', 00000101, 0, 0, [] },
+        {'Wilma', 'Flintstone', 00020202, 0, 0,
+            [
+            { 'Dinosaur stews', 'Trog', 55 }
+            ]
+        }]
+    },
+    { 'Wimpole Hall', 'AG1 6QT', 1203,
+        [{'John', 'Grabdale', 19361008, 0, 0,
+            [
+            { 'Pride and prejudice, 1st edition', 'Jane Austen', 95, 12000 },
+            { 'Mein Kampf, 1st edition', 'Adolph Hitler', 80, 10000 }
+            ]
+        }]
+    },
+    { '56 New Road', 'SG8 1S2', 2003,
+        [{'Brian', 'Jones', 19540206, 0, 0,
+            [
+            { 'All quiet on the western front', 'Erich Maria Remarque', 85, 4.99 },
+            { 'Fox in Socks', 'Dr. Seuss', 99, 4.99 }
+            ]
+        },
+        {'Julia', 'Jones', 19550312, 0, 0,
+            [
+            { 'Zen and the art of motorcyle maintenance', 'Robert Pirsig', 90, 7.99 },
+            { 'Where the wild things are', 'Maurice Sendak', 70, 4.79 },
+            { 'The bible', 'Various', 10 , 5.99 },
+            { 'The cat in the hat', 'Dr. Seuss', 80 }
+            ]
+        }]
+    }
+    ], C.sqHousePersonBookRec);
+
+
+//First reproject the datasets to
+
+C.sqBookIdRec addIdToBook(C.sqBookRec l) :=
+            transform
+                self.id := 0;
+                self := l;
+            end;
+
+C.sqPersonBookIdRec addIdToPerson(C.sqPersonBookRec l) :=
+            transform
+                unsigned2 aage := if (l.dob < baseDate, (unsigned2)((baseDate - l.dob) / 10000), 0);
+                self.id := 0;
+                self.books := project(l.books, addIdToBook(LEFT));
+                self.aage := if (aage > 200, 99, aage);
+                self := l;
+            end;
+
+C.sqHousePersonBookIdRec addIdToHouse(C.sqHousePersonBookRec l) :=
+            transform
+                self.id := 0;
+                self.persons := project(l.persons, addIdToPerson(LEFT));
+                self := l;
+            end;
+
+
+projected := project(rawHouse, addIdToHouse(LEFT));
+
+//version 1 assign unique ids a really inefficient way...
+//doesn't actually work....
+
+C.sqBookIdRec setBookId(C.sqHousePersonBookIdRec lh, C.sqBookIdRec l, unsigned4 basebookid) :=
+            transform
+                unsigned maxbookid := max(lh.persons, max(lh.persons.books, id));
+                self.id := if(maxbookid=0, basebookid, maxbookid)+1;
+                self := l;
+            end;
+
+C.sqPersonBookIdRec setPersonId(C.sqHousePersonBookIdRec lh, C.sqPersonBookIdRec l, unsigned4 basepersonid, unsigned4 basebookid) :=
+            transform
+                unsigned4 maxpersonid := max(lh.persons, id);
+                self.id := if(maxpersonid=0, basepersonid, maxpersonid)+1;
+                self.books := project(l.books, setBookId(lh, LEFT, basebookid));
+                self := l;
+            end;
+
+C.sqHousePersonBookIdRec setHouseId(C.sqHousePersonBookIdRec l, C.sqHousePersonBookIdRec r, unsigned4 id) :=
+            transform
+                unsigned prevmaxpersonid := max(l.persons, id);
+                unsigned prevmaxbookid := max(l.persons, max(l.persons.books, id));
+                self.id := id;
+                self.persons := project(r.persons, setPersonId(r, LEFT, prevmaxpersonid, prevmaxbookid));
+                self := r;
+            end;
+
+
+final1 := iterate(projected, setHouseId(LEFT, RIGHT, counter));
+
+
+//------------------ Common extraction functions... ---------------
+
+C.sqHouseIdRec extractHouse(C.sqHousePersonBookIdRec l) :=
+            TRANSFORM
+                SELF := l;
+            END;
+
+C.sqPersonBookRelatedIdRec extractPersonBook(C.sqHousePersonBookIdRec l, C.sqPersonBookIdRec r) :=
+            TRANSFORM
+                SELF.houseid := l.id;
+                SELF := r;
+            END;
+
+C.sqPersonRelatedIdRec extractPerson(C.sqPersonBookRelatedIdRec l) :=
+            TRANSFORM
+                SELF := l;
+            END;
+
+C.sqBookRelatedIdRec extractBook(C.sqBookIdRec l, unsigned4 personid) :=
+            TRANSFORM
+                SELF.personid := personid;
+                SELF := l;
+            END;
+
+
+
+//------------------- Add Sequence numbers by normalized/project/denormalize
+
+//normalize, adding parent ids as we do it.  Once all normalized and sequenced then combine them back together
+
+DoAssignSeq(ds, o) := macro
+#uniquename (trans)
+typeof(ds) %trans%(ds l, unsigned c) :=
+        transform
+            self.id := c;
+            self := l;
+        end;
+o := sorted(project(ds, %trans%(LEFT, COUNTER)), id);
+endmacro;
+
+DoAssignSeq(projected, projectedSeq);
+normSeqHouse := project(projectedSeq, extractHouse(LEFT));
+normPersonBooks := normalize(projectedSeq, left.persons, extractPersonBook(LEFT, RIGHT));
+DoAssignSeq(normPersonBooks, normSeqPersonBooks);
+normSeqPerson := project(normSeqPersonBooks, extractPerson(LEFT));
+normBook := normalize(normSeqPersonBooks, count(left.books), extractBook(LEFT.books[COUNTER], LEFT.id));
+DoAssignSeq(normBook, normSeqBook);
+
+// finally denormalize by joining back together.
+
+C.sqPersonBookRelatedIdRec expandPerson(C.sqPersonRelatedIdRec l) :=
+        TRANSFORM
+            SELF := l;
+            SELF.books := [];
+        END;
+
+C.sqHousePersonBookIdRec expandHouse(C.sqHouseIdRec l) :=
+        TRANSFORM
+            SELF := l;
+            SELF.persons := [];
+        END;
+
+C.sqPersonBookRelatedIdRec combinePersonBook(C.sqPersonBookRelatedIdRec l, C.sqBookRelatedIdRec r) :=
+        TRANSFORM
+            SELF.books := l.books + row({r.id, r.name, r.author, r.rating100, r.price}, C.sqBookIdRec);
+            SELF := l;
+        END;
+
+C.sqHousePersonBookIdRec combineHousePerson(C.sqHousePersonBookIdRec l, C.sqPersonBookRelatedIdRec r) :=
+        TRANSFORM
+            SELF.persons := l.persons + row(r, C.sqPersonBookIdRec);
+            SELF := l;
+        END;
+
+normSeqHouseEx := project(normSeqHouse, expandHouse(LEFT));
+normSeqPersonEx := project(normSeqPerson, expandPerson(LEFT));
+normSeqPersonBook := denormalize(normSeqPersonEx, sorted(normSeqBook, personid), left.id = right.personid, combinePersonBook(left, right), local);
+final3 := denormalize(normSeqHouseEx, sorted(normSeqPersonBook, houseid), left.id = right.houseid, combineHousePerson(left, right), local);
+
+
+
+//------------ Now generate the different output files.... -----------------
+// Try and do everything as many different ways as possible...!
+
+final := final3;
+
+houseOut := project(final, extractHouse(LEFT));
+personBooks := normalize(final, left.persons, extractPersonBook(LEFT, RIGHT));
+personOut := project(personBooks, extractPerson(LEFT));
+bookOut := normalize(personBooks, count(left.books), extractBook(LEFT.books[COUNTER], LEFT.id));
+
+simplePersonBooks := project(personBooks, transform(C.sqSimplePersonBookRec, SELF := LEFT, SELF.limit.booklimit := LEFT.booklimit));
+
+output(final,, C.sqHousePersonBookName,overwrite);
+output(personBooks,, C.sqPersonBookName,overwrite);
+output(houseOut,,C.sqHouseName,overwrite);
+output(personOut,,C.sqPersonName,overwrite);
+output(bookOut,,C.sqBookName,overwrite);
+
+output(simplePersonBooks,, C.sqSimplePersonBookName,overwrite);
+buildindex(
+  C.sqSimplePersonBookDs,
+  { surname, forename, aage  }, { C.sqSimplePersonBookDs }, C.sqSimplePersonBookIndexName, overwrite
+);
+fileServices.AddFileRelationship( __nameof__(C.sqSimplePersonBookDs), C.sqSimplePersonBookIndexName, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( __nameof__(C.sqSimplePersonBookDs), C.sqSimplePersonBookIndexName, '__fileposition__', 'filepos', 'link', '1:1', true);
+
+fileServices.AddFileRelationship( C.sqHouseName, C.sqPersonName, 'id', 'houseid', 'link', '1:M', false);
+fileServices.AddFileRelationship( C.sqPersonName, C.sqBookName, 'id', 'personid', 'link', '1:M', false);
+
+fileServices.AddFileRelationship( C.sqHouseName, C.sqHousePersonBookName, 'id', 'id', 'link', '1:1', false);
+fileServices.AddFileRelationship( C.sqHouseName, C.sqPersonBookName, 'id', 'houseid', 'link', '1:M', false);
+
+//Now build some indexes - with numeric fields in the key
+buildindex(C.sqHouseExDs, { id }, { addr, filepos }, C.sqHouseIndexName+'ID', overwrite);
+buildindex(C.sqPersonExDs, { id }, { filepos }, C.sqPersonIndexName+'ID', overwrite);
+buildindex(C.sqBookExDs, { id }, { filepos }, C.sqBookIndexName+'ID', overwrite);
+
+fileServices.AddFileRelationship( __nameof__(C.sqHouseExDs), C.sqHouseIndexName+'ID', '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( __nameof__(C.sqHouseExDs), C.sqHouseIndexName+'ID', '__fileposition__', 'filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( __nameof__(C.sqPersonExDs), C.sqPersonIndexName+'ID', '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( __nameof__(C.sqPersonExDs), C.sqPersonIndexName+'ID', '__fileposition__', 'filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( __nameof__(C.sqBookExDs), C.sqBookIndexName+'ID', '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( __nameof__(C.sqBookExDs), C.sqBookIndexName+'ID', '__fileposition__', 'filepos', 'link', '1:1', true);
+
+//Some more conventional indexes - some requiring a double lookup to resolve the payload
+buildindex(C.sqHouseExDs, { string40 addr := C.sqHouseExDs.addr, postcode }, { filepos }, C.sqHouseIndexName, overwrite);
+buildindex(C.sqPersonExDs, { string40 forename := C.sqPersonExDs.forename, string40 surname := C.sqPersonExDs.surname }, { id }, C.sqPersonIndexName, overwrite);
+buildindex(C.sqBookExDs, { string40 name := C.sqBookExDs.name, string40 author := C.sqBookExDs.author }, { id }, C.sqBookIndexName, overwrite);
+
+fileServices.AddFileRelationship( __nameof__(C.sqHouseExDs), C.sqHouseIndexName, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( __nameof__(C.sqHouseExDs), C.sqHouseIndexName, '__fileposition__', 'filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( __nameof__(C.sqPersonExDs), C.sqPersonIndexName, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( C.sqPersonIndexName+'ID', C.sqPersonIndexName, 'id', 'id', 'link', '1:1', true);
+fileServices.AddFileRelationship( __nameof__(C.sqBookExDs), C.sqBookIndexName, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( C.sqBookIndexName+'ID', C.sqBookIndexName, 'id', 'id', 'link', '1:1', true);
+
+//Should try creating a dataset with a set of ids which are used as a link...  (e.g., bookids->bookfile)
+
+C.DG_MemFileRec t_u2(C.DG_MemFileRec l, integer c) := transform self.u2 := c-2; self := l; END;
+C.DG_MemFileRec t_u3(C.DG_MemFileRec l, integer c) := transform self.u3 := c-2; self := l; END;
+C.DG_MemFileRec t_bu2(C.DG_MemFileRec l, integer c) := transform self.bu2 := c-2; self := l; END;
+C.DG_MemFileRec t_bu3(C.DG_MemFileRec l, integer c) := transform self.bu3 := c-2; self := l; END;
+C.DG_MemFileRec t_i2(C.DG_MemFileRec l, integer c) := transform self.i2 := c-2; self := l; END;
+C.DG_MemFileRec t_i3(C.DG_MemFileRec l, integer c) := transform self.i3 := c-2; self := l; END;
+C.DG_MemFileRec t_bi2(C.DG_MemFileRec l, integer c) := transform self.bi2 := c-2; self := l; END;
+C.DG_MemFileRec t_bi3(C.DG_MemFileRec l, integer c) := transform self.bi3 := c-2; self := l; END;
+
+n_blank := dataset([{0,0,0,0, 0,0,0,0}],C.DG_MemFileRec);
+
+n_u2 := NORMALIZE(n_blank, 4, t_u2(left, counter));
+n_u3 := NORMALIZE(n_u2, 4, t_u3(left, counter));
+
+n_bu2 := NORMALIZE(n_u3, 4, t_bu2(left, counter));
+n_bu3 := NORMALIZE(n_bu2, 4, t_bu3(left, counter));
+
+n_i2 := NORMALIZE(n_bu3, 4, t_i2(left, counter));
+n_i3 := NORMALIZE(n_i2, 4, t_i3(left, counter));
+
+n_bi2 := NORMALIZE(n_i3, 4, t_bi2(left, counter));
+n_bi3 := NORMALIZE(n_bi2, 4, t_bi3(left, counter));
+
+output(n_bi3,,C.DG_MemFileName,overwrite);
+
+
+C.DG_IntegerRecord createIntegerRecord(unsigned8 c) := transform
+    SELF.i6 := c;
+    SELF.nested.i4 := c;
+    SELF.nested.u3 := c;
+    SELF.i5 := c;
+    SELF.i3 := c;
+END;
+
+singleNullRowDs := dataset([transform({unsigned1 i}, self.i := 0;)]);
+output(normalize(singleNullRowDs, 100, createIntegerRecord(counter)),,C.DG_IntegerDatasetName,overwrite);
+build(C.DG_IntegerIndex,overwrite);

testing/regress/ecl/setup/setup_fetch.ecl

@@ -0,0 +1,63 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import Std.File AS FileServices;
+import $; C := $.files('');
+
+FetchData   := DATASET([
+    {3000,   'FL', 'Boca Raton',  'London',   'Bridge'},
+    {3500,   'FL', 'Boca Raton',  'Anderson', 'Sue'},
+    {3500,   'FL', 'Boca Raton',  'Anderson', 'John'},
+    {35,     'FL', 'Boca Raton',  'Smith',    'Frank'},
+    {3500,   'FL', 'Boca Raton',  'Johnson',  'Joe'},
+    {50,     'FL', 'Boca Raton',  'Smith',    'Sue'},
+    {135,    'FL', 'Boca Raton',  'Smith',    'Nancy'},
+    {3500,   'FL', 'Boca Raton',  'Johnson',  'Sue'},
+    {235,    'FL', 'Boca Raton',  'Smith',    'Fred'},
+    {335,    'FL', 'Boca Raton',  'Taylor',   'Frank'},
+    {3500,   'FL', 'Boca Raton',  'Johnson',  'Jane'},
+    {3500,   'FL', 'Boca Raton',  'Jones',    'Frank'},
+    {3500,   'FL', 'Boca Raton',  'Jones',    'Tommy'},
+    {3500,   'FL', 'Boca Raton',  'Doe',        'John'},
+    {3500,   'FL', 'Boca Raton',  'Anderson', 'Joe'},
+    {3500,   'FL', 'Boca Raton',  'Doe',        'Jane'},
+    {3500,   'FL', 'Boca Raton',  'Doe',        'Joe'},
+    {3500,   'FL', 'Boca Raton',  'Johnson',  'Larry'},
+    {3500,   'FL', 'Boca Raton',  'Johnson',  'John'},
+    {3500,   'FL', 'Boca Raton',  'Anderson', 'Larry'},
+    {3500,   'FL', 'Boca Raton',  'Anderson', 'Jane'},
+    {30,     'FL', 'Boca Raton',  'Smith',      'Zeek'}], C.DG_FetchRecord);
+
+// Try to make sure that there are at least 2 parts with data on... This gets a little hairy in places!
+// Note that in order to facilitate testing of preload / preloadIndexed versions of data, we generate the same file under 3 names
+
+twoways := distribute(FetchData, IF(lname < 'Jom', 0, 1));
+output(sort(twoways,record,local),,C.DG_FetchFileName,OVERWRITE);
+output(sort(twoways,record,local),,C.DG_FetchFilePreloadName,OVERWRITE);
+output(sort(twoways,record,local),,C.DG_FetchFilePreloadIndexedName,OVERWRITE);
+
+sortedFile := SORT(C.DG_FETCHFILE, Lname,Fname,state ,__filepos, LOCAL);
+BUILDINDEX(sortedFile,{Lname,Fname},{STRING fn := TRIM(Fname), state, STRING100 x {blob}:= fname, __filepos},C.DG_FetchIndex1Name, OVERWRITE, SORTED);
+BUILDINDEX(sortedFile,{Lname,Fname},{STRING fn := TRIM(Fname), state, STRING100 x {blob}:= fname, __filepos},C.DG_FetchIndex2Name, OVERWRITE, SORTED);
+
+fileServices.AddFileRelationship( C.DG_FetchFileName, C.DG_FetchFilePreloadName, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( C.DG_FetchFileName, C.DG_FetchFilePreloadIndexedName, '', '', 'view', '1:1', false);
+
+fileServices.AddFileRelationship( C.DG_FetchFileName, C.DG_FetchIndex1Name, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( C.DG_FetchFileName, C.DG_FetchIndex1Name, '__fileposition__', '__filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( C.DG_FetchFileName, C.DG_FetchIndex2Name, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( C.DG_FetchFileName, C.DG_FetchIndex2Name, '__fileposition__', '__filepos', 'link', '1:1', true);

testing/regress/ecl/setup/sourcedoc.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+##############################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>testing/ecl/setup</title>
+
+    <para>
+        The testing/ecl/setup directory contains the sources for the testing/ecl/setup library.
+    </para>
+</section>

testing/regress/hpcc/__init__.py

@@ -0,0 +1,18 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''

testing/regress/hpcc/common/__init__.py

@@ -0,0 +1,18 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''

testing/regress/hpcc/common/config.py

@@ -0,0 +1,67 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import ConfigParser
+import json
+
+try:
+    from cStringIO import StringIO
+except:
+    from StringIO import StringIO
+
+from collections import deque, namedtuple
+from ..common.dict import _dict
+
+
+class ConfigGenerator:
+    def __init__(self):
+        pass
+
+    def parseConfig(self, config, section='Default'):
+        self.section = section
+        self.conf = ConfigParser.ConfigParser()
+        s = StringIO(config)
+        self.conf.readfp(s)
+
+    def sections(self):
+        return self.conf.sections()
+
+    def get(self, item, section=None):
+        if not section:
+            section = self.section
+        return self.conf.get(section, item)
+
+
+class Config:
+    def __init__(self, file):
+        self.fileName = file
+        self.configObj = self.loadConfig(self.fileName)
+
+    def loadConfig(self, file):
+        try:
+            fp = open(file)
+            js = json.load(fp)
+            rC = namedtuple("Regress", js.keys())
+            return _dict(getattr(rC(**js), "Regress"))
+        except IOError as e:
+            print(e)
+
+    #implement writing out of a config.
+    def writeConfig(self, file):
+        pass

testing/regress/hpcc/common/dict.py

@@ -0,0 +1,29 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+
+class _dict(object):
+    def __init__(self, d):
+        self.__d = d
+
+    def __getattr__(self, attr):
+        try:
+            return self.__d[attr]
+        except KeyError:
+            raise AttributeError

testing/regress/hpcc/common/error.py

@@ -0,0 +1,39 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+ERROR = {
+    "1000": "Command not found.",
+    "1001": "Command return code non-zero.",
+    "1002": "Command timed out.",
+    "2000": "Value must be of Type Suite or ECLFile.",
+    "3000": "Return is null",
+    "3001": "Return diff does not match."
+}
+
+
+class Error(Exception):
+    def __init__(self, code, **kwargs):
+        self.code = code
+        self.err = kwargs.pop('err', False)
+
+    def __str__(self):
+        if self.err:
+            return "Error (%s): %s \n %s" % (self.code,
+                                             ERROR[self.code], self.err)
+        return "Error (%s): %s " % (self.code, ERROR[self.code])

testing/regress/hpcc/common/logger.py

@@ -0,0 +1,114 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import logging
+import sys
+import time
+
+try:
+    import curses
+except:
+    curses = None
+
+
+class Logger(object):
+    class _LogFormatter(logging.Formatter):
+        def __init__(self, color, *args, **kwargs):
+            logging.Formatter.__init__(self, *args, **kwargs)
+            self._color = color
+            if color:
+                fg_color = (curses.tigetstr("setaf")
+                            or curses.tigetstr("setf") or "")
+                self._colors = {
+                    logging.ERROR: curses.tparm(fg_color, 1),
+                    logging.INFO: curses.tparm(fg_color, 2),
+                    logging.CRITICAL: curses.tparm(fg_color, 3),
+                    logging.WARNING: curses.tparm(fg_color, 5),
+                    logging.DEBUG: curses.tparm(fg_color, 4)
+                }
+                self._normal = curses.tigetstr("sgr0")
+
+        def format(self, record):
+            try:
+                record.message = record.getMessage()
+            except Exception, e:
+                record.message = "Bad message (%r): %r" % (e, record.__dict__)
+            record.asctime = time.strftime(
+                "%y-%m-%d %H:%M:%S", self.converter(record.created))
+            if record.__dict__['levelname'] == "ERROR":
+                prefix = '[Failure]' % \
+                    record.__dict__
+            elif record.__dict__['levelname'] == "CRITICAL":
+                prefix = '[Error]' % \
+                    record.__dict__
+            elif record.__dict__['levelname'] == "DEBUG":
+                prefix = '[TEST]' % \
+                    record.__dict__
+            elif record.__dict__['levelname'] == "INFO":
+                prefix = '[Pass]' % \
+                    record.__dict__
+            elif record.__dict__['levelname'] == "WARNING":
+                prefix = '[Running]' % \
+                    record.__dict__
+            if self._color:
+                prefix = (self._colors.get(record.levelno, self._normal) +
+                          prefix + self._normal)
+            formatted = prefix + " " + record.message
+            if record.exc_info:
+                if not record.exc_text:
+                    record.exc_text = self.formatException(record.exc_info)
+            if record.exc_text:
+                formatted = formatted.rstrip() + "\n" + record.exc_text
+            return formatted.replace("\n", "\n    ")
+
+    def addHandler(self, fd, level='debug'):
+        root_logger = logging.getLogger()
+        channel = logging.FileHandler(fd)
+        channel.setLevel(getattr(logging, level.upper()))
+        root_logger.addHandler(channel)
+
+    def enable_pretty_logging(self):
+        root_logger = logging.getLogger()
+        color = False
+        if curses and sys.stderr.isatty():
+            try:
+                curses.setupterm()
+                if curses.tigetnum("colors") > 0:
+                    color = True
+            except:
+                pass
+        channel = logging.StreamHandler()
+        channel.setFormatter(Logger._LogFormatter(color=color))
+        root_logger.addHandler(channel)
+
+    def __init__(self, level='debug'):
+        logging.getLogger().setLevel(getattr(logging, level.upper()))
+        self.enable_pretty_logging()
+
+
+def main():
+    logging.info("INFO")
+    logging.debug("sort.ecl")
+    logging.warn("WARN")
+    logging.error("ERROR")
+    logging.critical("CRITICAL")
+
+if __name__ == '__main__':
+    regLog = Logger('debug')
+    main()

testing/regress/hpcc/common/report.py

@@ -0,0 +1,96 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+from ..regression.suite import Suite
+from ..util.ecl.file import ECLFile
+from ..common.dict import _dict
+from ..common.error import Error
+
+import logging
+import sys
+
+
+class Tee(object):
+    def __init__(self, name, mode):
+        self.file = open(name, mode)
+        self.stdout = sys.stdout
+        sys.stdout = self
+
+    def __del__(self):
+        sys.stdout = self.stdout
+        self.file.close()
+
+    def write(self, data):
+        self.file.write(data)
+        self.stdout.write(data)
+
+
+class Report:
+    def __init__(self, name, suite=None):
+        report = {'_pass': [], '_fail': []}
+        self.report = _dict(report)
+        self.name = name
+
+    def display(self):
+        reportStr = "\n"
+        reportStr += "Results\n"
+        reportStr += "-------------------------------------------------\n"
+        reportStr += "Passing: %i\n" % len(self.report._pass)
+        reportStr += "Failure: %i\n" % len(self.report._fail)
+        reportStr += "-------------------------------------------------\n"
+        if self.report._fail:
+            for result in self.report._fail:
+                reportStr += result.Diff
+                reportStr += "\n"
+            reportStr += "-------------------------------------------------\n"
+        logging.warn(reportStr)
+
+    def getResult(self, eclfile):
+        pass
+
+    def addResult(self, result):
+        if isinstance(result, Suite):
+            self.__addSuite(result)
+        elif isinstance(result, ECLFile):
+            self.__addEclFile(result)
+        else:
+            raise Error('2000')
+
+    def __addSuite(self, suite):
+        for eclfile in suite:
+            result = {}
+            result['File'] = eclfile.ecl
+            if eclfile.diff:
+                result['Result'] = 'Fail'
+                result['Diff'] = eclfile.diff
+                self.report._fail.append(_dict(result))
+            else:
+                result['Result'] = 'Pass'
+                self.report._pass.append(_dict(result))
+
+    def __addEclFile(self, eclfile):
+        result = {}
+        result['File'] = eclfile.ecl
+        if eclfile.diff:
+            result['Result'] = 'Fail'
+            result['Diff'] = eclfile.diff
+            self.report._fail.append(_dict(result))
+        else:
+            result['Result'] = 'Pass'
+            self.report._pass.append(_dict(result))

testing/regress/hpcc/common/shell.py

@@ -0,0 +1,63 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import sys
+from subprocess import (
+    PIPE,
+    Popen,
+    CalledProcessError
+)
+from ..common.error import Error
+
+#TODO: Find a better way since which is tempramental.
+CMD = {
+    "eclcc": "/usr/bin/eclcc",
+    "ecl": "/usr/bin/ecl",
+    "configgen": "/opt/HPCCSystems/sbin/configgen"
+}
+
+
+class Shell:
+    def command(self, *command_args):
+        def __command(*args):
+            all_args = command_args + args
+            return self.__run(*all_args)
+        return __command
+
+    def __run(self, *args, **kwargs):
+        args = [i for i in args if i is not None]
+        process = Popen(
+            args, stdout=kwargs.pop('stdout', PIPE),
+            stderr=kwargs.pop('stderr', PIPE),
+            close_fds=kwargs.pop('close_fds', True), **kwargs)
+        stdout, stderr = process.communicate()
+        if process.returncode:
+            exception = CalledProcessError(
+                process.returncode, repr(args))
+            exception.output = ''.join(filter(None, [stdout, stderr]))
+            raise Error('1001', err=exception.output)
+        return stdout
+
+    # Currently hacked to use the CMD dict as which can be tempramental.
+    # - What other methods can be used?
+    # - Support multiple versions of eclcc?
+    def which(self, command):
+        if command in CMD:
+            return CMD[command]
+        return self.__run("which", command).rstrip('\n')

testing/regress/hpcc/regression/__init__.py

@@ -0,0 +1,18 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''

testing/regress/hpcc/regression/regress.py

@@ -0,0 +1,100 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import logging
+import os
+import sys
+
+from ..common.config import Config
+from ..common.logger import Logger
+from ..common.report import Report, Tee
+from ..regression.suite import Suite
+from ..util.ecl.cc import ECLCC
+from ..util.ecl.command import ECLcmd
+
+
+class Regression:
+    def __init__(self, config="regress.json"):
+        self.config = Config(config).configObj
+        self.suites = {}
+        self.log = Logger("DEBUG")
+
+    def bootstrap(self):
+        archives = os.path.join(self.config.baseDir, self.config.archiveDir)
+        results = os.path.join(self.config.baseDir, self.config.resultDir)
+        self.createDirectory(archives)
+        self.createDirectory(results)
+        self.setup = self.Setup()
+        for cluster in self.config.Clusters:
+            self.createSuite(cluster)
+
+    def createDirectory(self, dir_n):
+        if not os.path.isdir(dir_n):
+            os.makedirs(dir_n)
+
+    def createSuite(self, cluster):
+        dir_ec = os.path.join(self.config.baseDir, self.config.eclDir)
+        dir_a = os.path.join(self.config.baseDir, self.config.archiveDir)
+        dir_ex = os.path.join(self.config.baseDir, self.config.keyDir)
+        dir_r = os.path.join(self.config.baseDir, self.config.resultDir)
+        self.suites[cluster] = Suite(cluster, dir_ec, dir_a, dir_ex, dir_r)
+
+    def Setup(self):
+        setup = os.path.join(self.config.baseDir, self.config.setupDir)
+        dir_a = os.path.join(self.config.baseDir, self.config.archiveDir)
+        dir_ex = os.path.join(self.config.baseDir, self.config.keyDir)
+        dir_r = os.path.join(self.config.baseDir, self.config.resultDir)
+        return Suite('setup', setup, dir_a, dir_ex, dir_r)
+
+    def runSuite(self, name, suite):
+        logDir = os.path.join(self.config.baseDir, self.config.logDir)
+        server = self.config.ip
+        report = Report(name)
+        logName = name + ".log"
+        if name == "setup":
+            cluster = 'hthor'
+        else:
+            cluster = name
+        log = os.path.join(logDir, logName)
+        self.log.addHandler(log, 'DEBUG')
+        logging.debug("Suite: %s" % name)
+        logging.debug("Queries: %s" % repr(len(suite.getSuite())))
+        cnt = 1
+        for query in suite.getSuite():
+            logging.warn("%s. Test: %s" % (repr(cnt), query.ecl))
+            ECLCC().makeArchive(query)
+            res = ECLcmd().runCmd("run", cluster, query, report,
+                                  server=server, username=self.config.username,
+                                  password=self.config.password)
+            wuid = query.getWuid()
+            if wuid:
+                url = "http://" + self.config.server
+                url += "/WsWorkunits/WUInfo?Wuid="
+                url += wuid
+            if res:
+                logging.info("Pass %s" % wuid)
+                logging.info("URL %s" % url)
+            else:
+                if not wuid:
+                    logging.error("Fail No WUID")
+                else:
+                    logging.error("Fail %s" % wuid)
+                    logging.error("URL %s" % url)
+            cnt += 1
+        report.display()

testing/regress/hpcc/regression/suite.py

@@ -0,0 +1,47 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import os
+from ..util.ecl.file import ECLFile
+
+
+class Suite:
+    def __init__(self, name, dir_ec, dir_a, dir_ex, dir_r):
+        self.name = name
+        self.suite = []
+        self.dir_ec = dir_ec
+        self.dir_a = dir_a
+        self.dir_ex = dir_ex
+        self.dir_r = dir_r
+        self.buildSuite()
+
+    def buildSuite(self):
+        if not os.path.isdir(self.dir_ec):
+            raise Exception("ECL Directory does not exist.")
+        for files in os.listdir(self.dir_ec):
+            if files.endswith(".ecl"):
+                ecl = os.path.join(self.dir_ec, files)
+                eclfile = ECLFile(ecl, self.dir_a, self.dir_ex,
+                                  self.dir_r)
+                if not eclfile.testSkip(self.name)['skip']:
+                    self.suite.append(eclfile)
+        self.suite.reverse()
+
+    def getSuite(self):
+        return self.suite

testing/regress/hpcc/util/__init__.py

@@ -0,0 +1,18 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''

testing/regress/hpcc/util/ecl/__init__.py

@@ -0,0 +1,18 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''

testing/regress/hpcc/util/ecl/cc.py

@@ -0,0 +1,81 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import os
+from ...common.error import Error
+from ...common.shell import Shell
+from ...util.ecl.file import ECLFile
+
+
+class ECLCC(Shell):
+    def __init__(self):
+        self.defaults = []
+        self.cmd = self.which('eclcc')
+
+    def __ECLCC(self):
+        return self.command(self.cmd, *self.defaults)
+
+    def getArchive(self, file):
+        try:
+            return self.__ECLCC()('-E', file)
+        except Error as err:
+            return repr(err)
+
+    def makeArchive(self, ecl):
+        self.addIncludePath(ecl.dir_ec)
+        dirname = ecl.dir_a
+        filename = ecl.getArchive()
+        if not os.path.isdir(dirname):
+            os.mkdir(dirname)
+        if os.path.isfile(filename):
+            os.unlink(filename)
+        FILE = open(filename, "w")
+        FILE.write(self.getArchive(ecl.getEcl()))
+        FILE.close()
+
+    def setVerbose(self):
+        self.defaults.append("--verbose")
+
+    def addIncludePath(self, path):
+        self.defaults.append('-I')
+        self.defaults.append(path)
+
+    def addLibraryPath(self, path):
+        self.defaults.append('-L')
+        self.defaults.append(path)
+
+    def compile(self, eclfile, **kwargs):
+        args = []
+
+        if kwargs.pop('shared', False):
+            args.append('-shared')
+
+        out = kwargs.pop('out', False)
+        if out:
+            args.append('-o' + out)
+
+        args.append(eclfile.getEcl())
+
+        if not kwargs.pop('noCompile', False):
+            try:
+                self.__ECLCC()(*args)
+            except Error as err:
+                print(repr(err))
+        else:
+            print self.defaults, args

testing/regress/hpcc/util/ecl/command.py

@@ -0,0 +1,84 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import logging
+
+from ...common.shell import Shell
+from ...util.ecl.file import ECLFile
+from ...common.error import Error
+
+
+class ECLcmd(Shell):
+    def __init__(self):
+        self.defaults = []
+        self.cmd = self.which('ecl')
+
+    def __ECLcmd(self):
+        return self.command(self.cmd, *self.defaults)
+
+    def runCmd(self, cmd, cluster, eclfile, report, **kwargs):
+        args = []
+        args.append(cmd)
+        args.append('-v')
+        args.append('--noroot')
+        args.append('--cluster=' + cluster)
+        name = kwargs.pop('name', False)
+        username = kwargs.pop('username', False)
+        password = kwargs.pop('password', False)
+        server = kwargs.pop('server', False)
+        if server:
+            args.append('--server=' + server)
+        if not name:
+            name = eclfile.ecl
+        if username:
+            args.append("--username=" + username)
+        if password:
+            args.append("--password=" + password)
+        args.append("--name=" + name)
+        args.append(eclfile.getArchive())
+        data = ""
+        wuid = ""
+        try:
+            results = self.__ECLcmd()(*args)
+            data = '\n'.join(line for line in
+                             results.split('\n') if line) + "\n"
+            ret = data.split('\n')
+            result = ""
+            cnt = 0
+            for i in ret:
+                if "wuid:" in i:
+                    wuid = i.split()[1]
+                if cnt > 4:
+                    result += i + "\n"
+                cnt += 1
+            data = '\n'.join(line for line in
+                             result.split('\n') if line) + "\n"
+
+        except Error as err:
+            data = repr(err)
+            logging.error("------" + err + "------")
+            return err + "\n"
+        finally:
+            eclfile.addResults(data, wuid)
+            test = eclfile.testResults()
+            report.addResult(eclfile)
+            if not test:
+                return False
+            else:
+                return True

testing/regress/hpcc/util/ecl/file.py

@@ -0,0 +1,118 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import difflib
+import logging
+import os
+
+
+class ECLFile:
+    ecl = None
+    xml_e = None
+    xml_r = None
+    xml_a = None
+    ecl_c = None
+    dir_ec = None
+    dir_ex = None
+    dir_r = None
+    dir_a = None
+    diff = ''
+    wuid = None
+
+    def __init__(self, ecl, dir_a, dir_ex, dir_r):
+        self.dir_ec = os.path.dirname(ecl)
+        self.dir_ex = dir_ex
+        self.dir_r = dir_r
+        self.dir_a = dir_a
+        baseEcl = os.path.basename(ecl)
+        baseXml = os.path.splitext(baseEcl)[0] + '.xml'
+        self.ecl = baseEcl
+        self.xml_e = baseXml
+        self.xml_r = baseXml
+        self.xml_a = 'archive_' + baseXml
+
+    def getExpected(self):
+        return os.path.join(self.dir_ex, self.xml_e)
+
+    def getResults(self):
+        return os.path.join(self.dir_r, self.xml_r)
+
+    def getArchive(self):
+        return os.path.join(self.dir_a, self.xml_a)
+
+    def getEcl(self):
+        return os.path.join(self.dir_ec, self.ecl)
+
+    def getWuid(self):
+        return self.wuid
+
+    def addResults(self, results, wuid):
+        filename = self.getResults()
+        self.wuid = wuid
+        if not os.path.isdir(self.dir_r):
+            os.mkdir(self.dir_r)
+        if os.path.isfile(filename):
+            os.unlink(filename)
+        FILE = open(filename, "w")
+        FILE.write(results)
+        FILE.close()
+
+    def __checkSkip(self, skipText, skip):
+        eclText = open(self.getEcl(), 'r')
+        skipLines = []
+        for line in eclText:
+            if skipText in line:
+                skipLines.append(line.rstrip('\n'))
+        if len(skipLines) > 0:
+            for skipLine in skipLines:
+                skipParts = skipLine.split()
+                skipType = skipParts[1]
+                skipReason = None
+                if len(skipParts) == 3:
+                    skipReason = skipParts[2]
+                if "==" in skipType:
+                    skipType = skipType.split("==")[1]
+                if not skip:
+                    return {'reason': skipReason, 'type': skipType}
+                if skipType == skip:
+                    return {'skip': True, 'reason': skipReason}
+        return {'skip': False}
+
+    def testSkip(self, skip=None):
+        return self.__checkSkip("//skip", skip)
+
+    def testVarSkip(self, skip=None):
+        return self.__checkSkip("//varskip", skip)
+
+    def testResults(self):
+        d = difflib.Differ()
+        try:
+            expected = open(self.getExpected(), 'r').readlines()
+            recieved = open(self.getResults(), 'r').readlines()
+        except IOError as e:
+            logging.critical(e)
+
+        for line in difflib.unified_diff(recieved,
+                                         expected,
+                                         fromfile=self.xml_r,
+                                         tofile=self.xml_e):
+            self.diff += line
+        if not self.diff:
+            return True
+        return False

testing/regress/regress

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import argparse
+
+from hpcc.regression.regress import Regression
+
+if __name__ == "__main__":
+    prog = "regress"
+    description = 'HPCC Platform Regression suite'
+    parser = argparse.ArgumentParser(prog=prog, description=description)
+    parser.add_argument('--version', '-v', action='version',
+                        version='%(prog)s 0.0.1')
+    parser.add_argument('--config', help="Config file to use.",
+                        nargs='?', default="regress.json")
+    subparsers = parser.add_subparsers(help='sub-command help')
+    parser_list = subparsers.add_parser('list', help='list help')
+    parser_list.add_argument('clusters', help="Print clusters from config.",
+                             action='store_true')
+    parser_run = subparsers.add_parser('run', help='run help')
+    parser_run.add_argument('cluster', help="Run the cluster suite.",
+                            nargs='?', default='setup')
+    args = parser.parse_args()
+    regress = Regression(args.config)
+    if 'clusters' in args:
+        Clusters = ['setup']
+        for cluster in regress.config.Clusters:
+            Clusters.append(str(cluster))
+        print "Avaliable Clusters: "
+        for i in Clusters:
+            print i
+    if 'cluster' in args:
+        regress.bootstrap()
+        if 'setup' in args.cluster:
+            regress.runSuite('setup', regress.setup)
+        else:
+            regress.runSuite(args.cluster, regress.suites[args.cluster])

testing/regress/regress.json

@@ -0,0 +1,21 @@
+{
+    "Regress":{
+        "ip": "127.0.0.1",
+        "username": "regress",
+        "password": "regress",
+        "roxie": "127.0.0.1:9876",
+        "server": "127.0.0.1:8010",
+        "eclDir": "ecl",
+        "setupDir": "ecl/setup",
+        "keyDir": "ecl/key",
+        "archiveDir": "regression/archives",
+        "resultDir": "regression/results",
+        "logDir": "regression",
+        "baseDir": ".",
+        "Clusters": [
+            "hthor",
+            "thor",
+            "roxie"
+        ]
+    }
+}