Merge branch 'candidate-3.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlgram2.cpp

@@ -8738,7 +8738,7 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
     IHqlExpression * failure = NULL;
     DefineIdSt* defineid = nameattr.getDefineId();
     _ATOM name = defineid->id;
-    ITypeInfo *type = defineid->getType();
+    Owned<ITypeInfo> type = defineid->getType();
     assertex(name);
 
     ActiveScopeInfo & activeScope = defineScopes.tos();
@@ -8748,7 +8748,6 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
         {
             activeScope.resetParameters();
             delete defineid;
-            ::Release(type);
             return;
         }
 
@@ -8787,8 +8786,7 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
         if (type)
         {
             reportError(ERR_SVC_NOYPENEEDED, nameattr, "Service can not have a type");
-            type->Release();
-            type = NULL;
+            type.clear();
         }
         if (activeScope.isParametered)
         {
@@ -8825,7 +8823,13 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
             }
             break;
         }
+    }
 
+    if (type && expr->isTransform())
+    {
+        ITypeInfo * recordType = queryRecordType(type);
+        assertex(recordType);
+        type.setown(makeTransformType(LINK(recordType)));
     }
 
     IHqlScope * localScope = activeScope.localScope;
@@ -8863,15 +8867,14 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
                 //check the parameters and return type (if specified) are compatible, promote expression return type to same
                 if (type)
                 {
-                    if (!isSameUnqualifiedType(type, matchType))
+                    if (!isSameFullyUnqualifiedType(type, matchType))
                     {
                         //allow dataset with no record to match, as long as base type is the same
                         if (queryRecord(type) != queryNullRecord() || (type->getTypeCode() != matchType->getTypeCode()))
                             reportError(ERR_SAME_TYPE_REQUIRED, nameattr, "Explicit type for %s doesn't match definition in base module", name->str());
                         else
                         {
-                            type->Release();
-                            type = LINK(matchType);
+                            type.set(matchType);
                         }
                     }
                 }
@@ -8886,7 +8889,7 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
                         }
                     }
                     else
-                        type = LINK(matchType);
+                        type.set(matchType);
                 }
             }
         }
@@ -8895,52 +8898,42 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
     // type cast if necessary
     if (type && etype)
     {
-        if (op == no_transform)
-        {
-            if (!recordTypesMatch(type, etype))
-                canNotAssignTypeError(type,etype,paramattr);
-        }
-        else
+        if (type != etype)
         {
-            if (type != etype)
+            if (!type->assignableFrom(etype))
             {
-                if (!type->assignableFrom(etype))
-                {
-                    if (queryRecord(type) != queryNullRecord())
-                    {
-                        canNotAssignTypeError(type,etype,paramattr);
-                        switch (type->getTypeCode())
-                        {
-                        case type_record:
-                            expr.set(queryNullRecord());
-                            break;
-                        default:
-                            expr.setown(createNullExpr(type));
-                            break;
-                        }
-                    }
-                }
-                else
+                if (queryRecord(type) != queryNullRecord())
                 {
-                    switch (etype->getTypeCode())
+                    canNotAssignTypeError(type,etype,paramattr);
+                    switch (type->getTypeCode())
                     {
-                    case type_table:
-                    case type_groupedtable:
                     case type_record:
-                    case type_row:
-                    case type_transform:
+                        expr.set(queryNullRecord());
                         break;
                     default:
-                        expr.setown(forceEnsureExprType(expr, type));
+                        expr.setown(createNullExpr(type));
                         break;
                     }
                 }
             }
+            else
+            {
+                switch (etype->getTypeCode())
+                {
+                case type_table:
+                case type_groupedtable:
+                case type_record:
+                case type_row:
+                case type_transform:
+                    break;
+                default:
+                    expr.setown(forceEnsureExprType(expr, type));
+                    break;
+                }
+            }
         }
     }
 
-    ::Release(type);
-
     // env symbol
     doDefineSymbol(defineid, expr.getClear(), failure, nameattr, assignattr.pos.position, semiattr.pos.position, activeScope.isParametered);
 }

ecl/regress/badtransform1.ecl

@@ -0,0 +1,48 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+testModule := MODULE
+
+  EXPORT Original := MODULE
+    EXPORT Layout := RECORD
+      STRING v;
+    END;
+
+    EXPORT File := DATASET([{'a'}], Layout);
+  END;
+
+
+  EXPORT parent := MODULE, VIRTUAL
+    SHARED integer Trans(Original.Layout L) := TRANSFORM
+      SELF := L;
+    END;
+
+    EXPORT File := PROJECT(Original.File, Trans(LEFT));
+  END;
+
+
+  EXPORT child := MODULE(parent)
+    SHARED Original.Layout Trans(Original.Layout L) := TRANSFORM
+      SELF.v := L.v + '2';
+    END;
+  END;
+
+END;
+
+testmodule.child.file;

ecl/regress/badtransform2.ecl

@@ -0,0 +1,20 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+{ unsigned i } x := TRANSFORM({ unsigned j}, SELF.j := 10; );

ecl/regress/derivedtransform.ecl

@@ -0,0 +1,48 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+testModule := MODULE
+
+  EXPORT Original := MODULE
+    EXPORT Layout := RECORD
+      STRING v;
+    END;
+
+    EXPORT File := DATASET([{'a'}], Layout);
+  END;
+
+
+  EXPORT parent := MODULE, VIRTUAL
+    SHARED Original.Layout Trans(Original.Layout L) := TRANSFORM
+      SELF := L;
+    END;
+
+    EXPORT File := PROJECT(Original.File, Trans(LEFT));
+  END;
+
+
+  EXPORT child := MODULE(parent)
+    SHARED Original.Layout Trans(Original.Layout L) := TRANSFORM
+      SELF.v := L.v + '2';
+    END;
+  END;
+
+END;
+
+testmodule.child.file;

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/ECLEngine.java

@@ -292,7 +292,38 @@ public class ECLEngine
 							}
 							selectstruct.append(" );");
 						}
+						else if (col.getColumnName().equalsIgnoreCase("SUM"))
+						{
+							eclEnteties.put("SUMFN", "TRUE");
+							selectstruct.append(col.getAlias() + " := ");
+
+							selectstruct.append("SUM( ");
+							if (parser.hasGroupByColumns())
+							{
+								selectstruct.append(" GROUP ");
+							}
+							else
+							{
+								selectstruct.append(datasource);
+								if (eclEnteties.size() > 0)
+									addFilterClause(selectstruct, eclEnteties);
+							}
 
+							List<HPCCColumnMetaData> funccols = col.getFunccols();
+							if (funccols.size() > 0)
+							{
+								String paramname = funccols.get(0).getColumnName();
+								eclEnteties.put("FNCOLS", paramname);
+								if (!paramname.equals("*") && funccols.get(0).getColumnType() != HPCCColumnMetaData.COLUMN_TYPE_CONSTANT)
+								{
+									selectstruct.append(", ");
+									selectstruct.append(datasource);
+									selectstruct.append(".");
+									selectstruct.append(paramname);
+								}
+							}
+							selectstruct.append(" );");
+						}
 					}
 					else
 						selectstruct.append(col.getEclType()).append(" ").append(col.getColumnName()).append(" := ").append(datasource).append(".").append(col.getColumnName()).append("; ");
@@ -449,8 +480,17 @@ public class ECLEngine
 							addFilterClause(sb, parameters);
 						sb.append(");");
 					}
+					else if (parameters.containsKey("SUMFN"))
+					{
+						sb.append("scalarout := SUM(fileds");
+						if (parameters.size() > 0)
+							addFilterClause(sb, parameters);
 
-					if (parameters.containsKey("MAXFN"))
+						sb.append(" , fileds.");
+						sb.append(parameters.get("FNCOLS"));
+						sb.append(");");
+					}
+					else if (parameters.containsKey("MAXFN"))
 					{
 						sb.append("scalarout := MAX(fileds");
 						if (parameters.size() > 0)
@@ -470,6 +510,7 @@ public class ECLEngine
 						sb.append(parameters.get("FNCOLS"));
 						sb.append(");");
 					}
+					sb.append("\n");
 				}
 
 				if (parameters.containsKey("SCALAROUTNAME"))
@@ -532,6 +573,12 @@ public class ECLEngine
 							sb.append("scalarout := COUNT(idxds");
 							sb.append(", KEYED);\n");
 						}
+						if (parameters.containsKey("SUMFN"))
+						{
+							sb.append("scalarout := SUM(idxds");
+							sb.append(parameters.get("FNCOLS"));
+							sb.append(", KEYED);\n");
+						}
 						if (parameters.containsKey("MAXFN"))
 						{
 							sb.append("scalarout := MAX(idxds, fileds.");
@@ -593,6 +640,12 @@ public class ECLEngine
 					{
 						if (parameters.containsKey("COUNTFN"))
 							sb.append("scalarout := COUNT(idxds);");
+						if (parameters.containsKey("SUMFN"))
+						{
+							sb.append("scalarout := SUM(idxds, fileds.");
+							sb.append(parameters.get("FNCOLS"));
+							sb.append(");");
+						}
 						if (parameters.containsKey("MAXFN"))
 						{
 							sb.append("scalarout := MAX(idxds, fileds.");
@@ -717,23 +770,15 @@ public class ECLEngine
 			ArrayList<HPCCColumnMetaData> storeProcInParams = hpccquery.getAllInFields();
 			String[] procInParamValues = parser.getStoredProcInParamVals();
 
+			if (procInParamValues.length != storeProcInParams.size())
+				throw new Exception("Invalid number of parameter passed in.");
+
 			for (int i = 0; i < procInParamValues.length; i++)
 			{
 				String key = storeProcInParams.get(i).getColumnName();
-				{
-					sb.append("&").append(key).append("=").append(procInParamValues[i]);
-				}
+				sb.append("&").append(key).append("=").append(procInParamValues[i]);
 			}
 
-			//if (parameters.size() > 0)
-			//{
-			//	for (Object key : parameters.keySet())
-			//	{
-			//		Object value = parameters.get(key);
-			//		sb.append("&").append(key).append("=").append(value);
-			//	}
-			//}
-
 			long startTime = System.currentTimeMillis();
 			// Send data
 			URL url = new URL(urlString);

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/ECLFunctions.java

@@ -14,6 +14,7 @@ public class ECLFunctions
 		functions.put("COUNT", new ECLFunction("COUNT", true, new HPCCColumnMetaData("countreturn", 0, java.sql.Types.NUMERIC), true));
 		functions.put("MAX", new ECLFunction("MAX", true, new HPCCColumnMetaData("maxreturn", 0, java.sql.Types.NUMERIC), false));
 		functions.put("MIN", new ECLFunction("MIN", true, new HPCCColumnMetaData("minreturn", 0, java.sql.Types.NUMERIC), false));
+		functions.put("SUM", new ECLFunction("SUM", true, new HPCCColumnMetaData("sumreturn", 0, java.sql.Types.NUMERIC), false));
 	}
 
 	static ECLFunction getEclFunction( String funcname)

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCDriver.java

@@ -27,10 +27,7 @@ public class HPCCDriver implements Driver
 		}
 	}
 
-	public HPCCDriver()
-	{
-	}
-
+	public HPCCDriver()	{}
 
 	public Connection connect(String url, Properties info) throws SQLException
 	{
@@ -66,12 +63,10 @@ public class HPCCDriver implements Driver
 		return new HPCCConnection(info);
 	}
 
-
 	public boolean acceptsURL(String url) throws SQLException {
 		return true;
 	}
 
-
 	public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException
 	{
 		DriverPropertyInfo[] infoArray = new DriverPropertyInfo[1];
@@ -79,18 +74,16 @@ public class HPCCDriver implements Driver
 		return infoArray;
 	}
 
-
 	public int getMajorVersion()
 	{
 		return HPCCVersionTracker.HPCCMajor;
 	}
 
-
-	public int getMinorVersion() {
+	public int getMinorVersion()
+	{
 		return HPCCVersionTracker.HPCCMinor;
 	}
 
-
 	public boolean jdbcCompliant() {
 		return true;
 	}
@@ -140,9 +133,9 @@ public class HPCCDriver implements Driver
 			//"select count(persons.zip) as zipcount, persons.city as mycity from tutorial::rp::tutorialperson as persons where persons.city = 'ABBEVILLE' "
 			//"select min(zip) as maxzip from tutorial::rp::tutorialperson as persons where persons.city = 'ABBEVILLE' "
 			//"select 1 as ONE"
-			"call myroxie::fetchpeoplebyzipservice(33445)"
+			//"call myroxie::fetchpeoplebyzipservice(33445)"
 			//"call fetchpeoplebyzipservice(33445)"
-
+			"call fetchpeoplebyzipservice()"
 			//"select MIN(zip), city from tutorial::rp::tutorialperson where zip  > '33445'"
 
 			//"select tbl.* from progguide::exampledata::peopleaccts tbl"
@@ -287,7 +280,7 @@ public class HPCCDriver implements Driver
 			//System.out.println("Tables found: ");
 			//while (tables.next()) {
 			//	System.out.println("   " + tables.getString("TABLE_NAME") + " Remarks: \'" + tables.getString("REMARKS")+"\'");
-		//	}
+			//}
 
 			ResultSet procs = conn.getMetaData().getProcedures(null, null, null);
 
@@ -296,12 +289,6 @@ public class HPCCDriver implements Driver
 				System.out.println("   " + procs.getString("PROCEDURE_NAME"));
 			}
 
-			ResultSet procs2 = conn.getMetaData().getProcedures(null, null, null);
-
-			System.out.println("procs found: ");
-			while (procs2.next()) {
-				System.out.println("   " + procs2.getString("PROCEDURE_NAME"));
-			}
 			/*
 			ResultSet proccols = conn.getMetaData().getProcedureColumns(null, null, null, null);
 
@@ -311,10 +298,10 @@ public class HPCCDriver implements Driver
 			}
 			*/
 			//getProcedureColumns catalog: null, schemaPattern: null, procedureNamePattern: fetchpeoplebyzipservice columnanmepat: null
-
 		}
-		catch (SQLException e) {
-			e.printStackTrace();
+		catch (SQLException e)
+		{
+			System.err.println(e.getMessage());
 		}
 	}
 }

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCJDBCUtils.java

@@ -122,7 +122,7 @@ public class HPCCJDBCUtils
 
 	public static String removeAllNewLines(String str)
 	{
-		return str.replaceAll("\\r\\n|\\r|\\n", " ");
+		return str.trim().replaceAll("\\r\\n|\\r|\\n", " ");
 	}
 
 	public static boolean isLiteralString(String str)

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCResultSet.java

@@ -70,7 +70,7 @@ public class HPCCResultSet implements ResultSet
 			rows = new ArrayList();
 			String indextousename = null;
 			SQLParser parser = new SQLParser();
-			parser.parse(query);
+			parser.process(query);
 			int sqlreqtype = parser.getSqlType();
 			//not sure this is actually needed...
 			parser.populateParametrizedExpressions(inParameters);

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/SQLParser.java

@@ -3,16 +3,8 @@ package com.hpccsystems.jdbcdriver;
 import java.sql.SQLException;
 import java.util.*;
 
-
-/**
- * Class is used for parsing sql statements.
- *
- * @author Zoran Milakovic <-- just noticed this need to ask Arjuna about this
- */
-public class SQLParser {
-
-	private static final String COMMA_ESCAPE = "~#####1~";
-
+public class SQLParser
+{
 	public final static short 	SQL_TYPE_UNKNOWN = -1;
 	public final static short 	SQL_TYPE_SELECT = 1;
 	public final static short 	SQL_TYPE_SELECTCONST = 2;
@@ -31,18 +23,9 @@ public class SQLParser {
 	private boolean columnsVerified;
 	private String indexHint;
 
-	/**
-	 * Parse sql statement.
-	 *
-	 * @param sql
-	 *            defines SQL statement
-	 * @exception Exception
-	 *                Description of Exception
-	 * @since
-	 */
-	public void parse(String sql) throws Exception
-	{
-		System.out.println("#####INCOMING SQL: " + sql);
+	public void process(String insql) throws SQLException
+	{
+		System.out.println("INCOMING SQL: " + insql);
 		columnsVerified = false;
 		limit = -1;
 		tableName = null;
@@ -52,69 +35,43 @@ public class SQLParser {
 		procInParamValues = new String[0];
 		storedProcName = null;
 		sqlType = SQL_TYPE_UNKNOWN;
-		sql = sql.trim();
-		sql = HPCCJDBCUtils.removeAllNewLines(sql);
 		indexHint = null;
 
-		// replace comma(,) in values between quotes(')
-		StringTokenizer tokQuote = new StringTokenizer(sql.toString(), "'",	true);
-		StringBuffer sb = new StringBuffer();
-		boolean openParent1 = false;
-		while (tokQuote.hasMoreTokens()) {
-			String next = tokQuote.nextToken();
-			if (openParent1) {
-				next = HPCCJDBCUtils.replaceAll(next, ",", COMMA_ESCAPE);
-			}
-			sb.append(next);
-			if (next.equalsIgnoreCase("'")) {
-				if (openParent1 == true) {
-					openParent1 = false;
-				} else {
-					openParent1 = true;
-				}
-			}
-		}
-		// END replacement
-		sql = sb.toString();
-		String upperSql = sql.toUpperCase();
+		insql = HPCCJDBCUtils.removeAllNewLines(insql);
+		String insqlupcase = insql.toUpperCase();
 
-		// handle unsupported statements
-		if (upperSql.startsWith("ALTER ")) {
-			throw new Exception("ALTER TABLE statements are not supported.");
-		}
-		if (upperSql.startsWith("DROP ")) {
-			throw new Exception("DROP statements are not supported.");
-		}
-		if (upperSql.startsWith("INSERT ")) {
-			throw new Exception("INSERT statements are not supported.");
+		if (insql.matches("^(?i)alter\\s+(.*?)"))
+		{
+			throw new SQLException("ALTER TABLE statements are not supported.");
 		}
-		if (upperSql.startsWith("UPDATE ")) {
-			throw new Exception("UPDATE statements are not supported.");
+		else if (insql.matches("^(?i)drop\\s+(.*?)"))
+		{
+			throw new SQLException("DROP statements are not supported.");
 		}
-		if (upperSql.contains(" UNION ")) {
-			throw new Exception("UNIONS are not supported.");
+		else if (insql.matches("^(?i)insert\\s+(.*?)"))
+		{
+			throw new SQLException("INSERT statements are not supported.");
 		}
-		if (upperSql.contains(" JOIN ")) {
-			throw new Exception("JOINS are not supported.");
+		else if (insql.matches("^(?i)update\\s+(.*?)"))
+		{
+			throw new SQLException("UPDATE statements are not supported.");
 		}
-
-		// CALL SP
-		if (upperSql.startsWith("CALL"))
+		else if (insql.matches("^(?i)call\\s+(.*?)"))
 		{
 			sqlType = SQL_TYPE_CALL;
-			int callPos = upperSql.lastIndexOf("CALL ");
-			int storedProcPos = sql.lastIndexOf("(");
-			int paramlistend =  sql.lastIndexOf(")");
+			int callstrpos = insqlupcase.lastIndexOf("CALL ");
+			int storedprocstrpos = insql.lastIndexOf("(");
+			int paramlistend =  insql.lastIndexOf(")");
 			String paramToken = "";
 
-			if (storedProcPos == -1)
-				storedProcName = sql.substring(callPos+5);
+			if (storedprocstrpos == -1)
+				storedProcName = insql.substring(callstrpos+5);
 			else
 			{
 				if (paramlistend == -1)
-					throw new SQLException("Missing closing param in: " + sql);
-				storedProcName = sql.substring(callPos+5, storedProcPos);
-				paramToken = sql.substring(storedProcPos+1, paramlistend);
+					throw new SQLException("Missing closing param in: " + insql);
+				storedProcName = insql.substring(callstrpos+5, storedprocstrpos);
+				paramToken = insql.substring(storedprocstrpos+1, paramlistend);
 			}
 
 			if(paramToken.length() >0 )
@@ -124,34 +81,37 @@ public class SQLParser {
 				int i = 0;
 				while (tokenizer.hasMoreTokens())
 				{
-					procInParamValues[i] = tokenizer.nextToken().trim();
+					procInParamValues[i++] = tokenizer.nextToken().trim();
 				}
-
 			}
 		}
-		// SELECT
-		if (upperSql.startsWith("SELECT "))
+		else if (insql.matches("^(?i)select\\s+(.*?)"))
 		{
+			if (insql.matches("^(?i)select(.*?)\\s+(?i)union\\s+.*"))
+				throw new SQLException("SELECT UNIONS are not supported.");
+			if (insql.matches("^(?i)select(.*?)\\s+(?i)join\\s+.*"))
+				throw new SQLException("SELECT JOINS are not supported.");
+
 			sqlType = SQL_TYPE_SELECT;
-			int fromPos  = upperSql.lastIndexOf(" FROM ");
+			int fromstrpos  = insqlupcase.lastIndexOf(" FROM ");
 
-			if (fromPos == -1)
+			if (fromstrpos == -1)
 			{
-				if (parseConstantSelect(sql))
+				if (parseConstantSelect(insql))
 				{
 					System.out.println("Found Select <constant>");
 					sqlType = SQL_TYPE_SELECTCONST;
 					return;
 				}
 				else
-					throw new Exception("Malformed SQL. Missing FROM statement.");
+					throw new SQLException("Malformed SQL. Missing FROM statement.");
 			}
 
-			int useindexPos = upperSql.lastIndexOf(" USE INDEX(");
+			int useindexstrpos = insqlupcase.lastIndexOf(" USE INDEX(");
 			/* Allow multiple spaces between USE and INDEX?
-			 * if (useindexPos > 0)
+			 * if (useindexstrpos > 0)
 			{
-				String afterUSE = upperSql.substring(useindexPos+4);
+				String afterUSE = upperSql.substring(useindexstrpos+4);
 				int inderelativepos = afterUSE.indexOf("INDEX(");
 
 				if(inderelativepos < 0)
@@ -165,28 +125,28 @@ public class SQLParser {
 
 			}*/
 
-			int wherePos = upperSql.lastIndexOf(" WHERE ");
-			int groupPos = upperSql.lastIndexOf(" GROUP BY ");
-			int orderPos = upperSql.lastIndexOf(" ORDER BY ");
-			int limitPos = upperSql.lastIndexOf(" LIMIT ");
+			int wherePos = insqlupcase.lastIndexOf(" WHERE ");
+			int groupPos = insqlupcase.lastIndexOf(" GROUP BY ");
+			int orderPos = insqlupcase.lastIndexOf(" ORDER BY ");
+			int limitPos = insqlupcase.lastIndexOf(" LIMIT ");
 
-			if ( useindexPos != -1 && useindexPos < fromPos)
-				throw new Exception("Malformed SQL. USING clause placement.");
+			if ( useindexstrpos != -1 && useindexstrpos < fromstrpos)
+				throw new SQLException("Malformed SQL. USING clause placement.");
 
-			if (wherePos != -1 && wherePos < fromPos)
-				throw new Exception("Malformed SQL. WHERE clause placement.");
+			if (wherePos != -1 && wherePos < fromstrpos)
+				throw new SQLException("Malformed SQL. WHERE clause placement.");
 
 			try
 			{
 				if (limitPos != -1)
 				{
-					limit = Integer.valueOf(upperSql.substring(limitPos+6).trim());
-					upperSql = upperSql.substring(0, limitPos);
+					limit = Integer.valueOf(insqlupcase.substring(limitPos+6).trim());
+					insqlupcase = insqlupcase.substring(0, limitPos);
 				}
 			}
 			catch (NumberFormatException ne)
 			{
-				throw new Exception("Error near :\'" + upperSql.substring(limitPos)+"\'");
+				throw new SQLException("Error near :\'" + insqlupcase.substring(limitPos)+"\'");
 			}
 
 			String orderByToken = "";
@@ -196,30 +156,29 @@ public class SQLParser {
 			{
 				if (orderPos == -1)
 				{
-					groupByToken = upperSql.substring(groupPos + 10);
+					groupByToken = insqlupcase.substring(groupPos + 10);
 				}
 				else
 				{
-					groupByToken = upperSql.substring(groupPos + 10, orderPos);
-					orderByToken = upperSql.substring(orderPos + 10);
+					groupByToken = insqlupcase.substring(groupPos + 10, orderPos);
+					orderByToken = insqlupcase.substring(orderPos + 10);
 				}
 
-				upperSql = upperSql.substring(0, groupPos);
+				insqlupcase = insqlupcase.substring(0, groupPos);
 
 			}
 			else if (orderPos != -1 && (groupPos == -1 || orderPos < groupPos))
 			{
 				if (groupPos == -1)
 				{
-					orderByToken = upperSql.substring(orderPos + 10);
+					orderByToken = insqlupcase.substring(orderPos + 10);
 				}
 				else
 				{
-					orderByToken = upperSql.substring(orderPos + 10, groupPos);
-					groupByToken = upperSql.substring(groupPos + 10);
-
+					orderByToken = insqlupcase.substring(orderPos + 10, groupPos);
+					groupByToken = insqlupcase.substring(groupPos + 10);
 				}
-				upperSql = upperSql.substring(0, orderPos);
+				insqlupcase = insqlupcase.substring(0, orderPos);
 			}
 
 			if (orderByToken.length()>0)
@@ -258,15 +217,15 @@ public class SQLParser {
 				}
 			}
 
-			sql = sql.substring(0,upperSql.length());
+			insql = insql.substring(0,insqlupcase.length());
 			String fullTableName = null;
-			if (wherePos == -1 && useindexPos == -1)
+			if (wherePos == -1 && useindexstrpos == -1)
 			{
-				fullTableName = sql.substring(fromPos + 6).trim();
+				fullTableName = insql.substring(fromstrpos + 6).trim();
 			}
 			else
 			{
-				fullTableName = sql.substring(fromPos + 6, (useindexPos == -1) ? wherePos : useindexPos).trim();
+				fullTableName = insql.substring(fromstrpos + 6, (useindexstrpos == -1) ? wherePos : useindexstrpos).trim();
 			}
 
 			String splittablefromalias [] = fullTableName.split("\\s+(?i)as(\\s+|$)");
@@ -275,18 +234,18 @@ public class SQLParser {
 				if (!splittablefromalias[0].contains(" "))
 					tableName = splittablefromalias[0].trim();
 				else
-					throw new Exception("Invalid SQL: " + splittablefromalias[0]);
+					throw new SQLException("Invalid SQL: " + splittablefromalias[0]);
 			}
 			else
-				throw new Exception("Invalid SQL: Missing table name.");
+				throw new SQLException("Invalid SQL: Missing table name.");
 
 			if (splittablefromalias.length > 1)
 				tableAlias = splittablefromalias[1].trim();
 
-			if (fromPos <= 7)
-				throw new Exception("Invalid SQL: Missing select column(s).");
+			if (fromstrpos <= 7)
+				throw new SQLException("Invalid SQL: Missing select column(s).");
 
-			StringTokenizer comatokens = new StringTokenizer(sql.substring(7, fromPos), ",");
+			StringTokenizer comatokens = new StringTokenizer(insql.substring(7, fromstrpos), ",");
 
 			for(int sqlcolpos = 1; comatokens.hasMoreTokens();)
 			{
@@ -302,7 +261,7 @@ public class SQLParser {
 					ECLFunction func= ECLFunctions.getEclFunction(funcname.toUpperCase());
 
 					if (func == null)
-						throw new Exception("ECL Function " + funcname + "is not currently supported");
+						throw new SQLException("ECL Function " + funcname + "is not currently supported");
 
 					col = col.substring(col.indexOf('(')+1).trim();
 
@@ -341,7 +300,7 @@ public class SQLParser {
 					if (ECLFunctions.verifyEclFunction(funcname, funccols))
 						colmetadata = new HPCCColumnMetaData(funcname, sqlcolpos++, funccols);
 					else
-						throw new Exception("Funtion " + funcname + " does not map to ECL as written");
+						throw new SQLException("Function " + funcname + " does not map to ECL as written");
 				}
 				else if(col.contains("."))
 				{
@@ -359,9 +318,9 @@ public class SQLParser {
 				selectColumns.add(colmetadata);
 			}
 
-			if (useindexPos != -1)
+			if (useindexstrpos != -1)
 			{
-				String useindexstr = sql.substring(useindexPos + 11);
+				String useindexstr = insql.substring(useindexstrpos + 11);
 				int useindexend = useindexstr.indexOf(")");
 				if (useindexend < 0 )
 					throw new SQLException("Malformed USE INDEX() clause.");
@@ -371,7 +330,7 @@ public class SQLParser {
 
 			if (wherePos != -1)
 			{
-				String strWhere = sql.substring(wherePos + 7);
+				String strWhere = insql.substring(wherePos + 7);
 				String splitedwhereands [] = strWhere.split(" and | AND |,");
 
 				for (int i = 0; i < splitedwhereands.length; i++)
@@ -440,10 +399,11 @@ public class SQLParser {
 					if (i < splitedwhereands.length-1)
 						whereclause.addExpression(andoperator);
 				}
-
-				System.out.println(whereclause);
+				//System.out.println(whereclause);
 			}
 		}
+		else
+			throw new SQLException("Invalid SQL found - only supports CALL and/or SELECT statements.");
 	}
 
 	private String handleComplexField(String fullFieldName) throws SQLException
@@ -458,7 +418,7 @@ public class SQLParser {
 		}
 		return null;
 	}
-	private boolean parseConstantSelect(String sql) throws Exception
+	private boolean parseConstantSelect(String sql) throws SQLException
 	{
 		String sqlUpper = sql.toUpperCase();
 		int wherePos = sqlUpper.lastIndexOf(" WHERE ");
@@ -478,7 +438,7 @@ public class SQLParser {
 		}
 		catch (NumberFormatException ne)
 		{
-			throw new Exception("Error near :\'" + sql.substring(limitPos)+"\'");
+			throw new SQLException("Error near :\'" + sql.substring(limitPos)+"\'");
 		}
 
 		//At this point we have select <something>
@@ -610,27 +570,33 @@ public class SQLParser {
 		return limit == -1 ? false : true;
 	}
 
-	public String [] getStoredProcInParamVals() {
+	public String [] getStoredProcInParamVals()
+	{
 		return procInParamValues;
 	}
 
-	public String getStoredProcName() {
+	public String getStoredProcName()
+	{
 		return storedProcName;
 	}
 
-	public String getTableAlias() {
+	public String getTableAlias()
+	{
 		return tableAlias;
 	}
 
-	public int getSqlType() {
+	public int getSqlType()
+	{
 		return sqlType;
 	}
 
-	public int getLimit() {
+	public int getLimit()
+	{
 		return limit;
 	}
 
-	public String getTableName() {
+	public String getTableName()
+	{
 		return tableName;
 	}
 
@@ -707,7 +673,8 @@ public class SQLParser {
 		return whereclause.getExpressionFromName(name);
 	}
 
-	public boolean whereClauseContainsKey(String name) {
+	public boolean whereClauseContainsKey(String name)
+	{
 		return whereclause.containsKey(name);
 	}
 
@@ -788,7 +755,6 @@ public class SQLParser {
 					{
 						verifyColumn(funccols.get(i), dfufile);
 					}
-
 				}
 				else if(HPCCJDBCUtils.isLiteralString(fieldName))
 				{
@@ -816,15 +782,18 @@ public class SQLParser {
 		}
 	}
 
-	public String getIndexHint() {
+	public String getIndexHint()
+	{
 		return indexHint;
 	}
 
 	public static void main(String[] args) throws Exception
 	{
 		SQLParser parser = new SQLParser();
-		parser.parse("select tg.people as mypeeps, zip as myzip from asfetchpeoplebyzipservice  as tg  USE INDEX(tutorial::rp::peoplebyzipindex2) where tg.zipvalue=?");
+		parser.process("select city, zip, count(*) from tutorial::rp::tutorialperson where zip ='33445' limit 1000");
+		System.out.println(parser.getWhereClauseString());
 		System.out.println(parser.getTableName());
+		if (parser.hasLimitBy())
+			System.out.println(parser.getLimit());
 	}
-
 }

roxie/roxiemem/roxiemem.cpp

@@ -1781,11 +1781,11 @@ protected:
 //Constants are here to ensure they can all be constant folded
 const unsigned roundupDoubleLimit = 2048;  // Values up to this limit are rounded to the nearest power of 2
 const unsigned roundupStepSize = 4096;  // Above the roundupDoubleLimit memory for a row is allocated in this size step
-const unsigned limitStepBlock = FixedSizeHeaplet::maxHeapSize()/20;  // until it gets to this size
+const unsigned limitStepBlock = FixedSizeHeaplet::maxHeapSize()/MAX_FRAC_ALLOCATOR;  // until it gets to this size
 const unsigned numStepBlocks = PAGES(limitStepBlock, roundupStepSize); // how many step blocks are there?
 const unsigned maxStepSize = numStepBlocks * roundupStepSize;
 const bool hasAnyStepBlocks = roundupDoubleLimit <= roundupStepSize;
-const unsigned firstFractionalHeap = (FixedSizeHeaplet::dataAreaSize()/(maxStepSize+1))+1;
+const unsigned firstFractionalHeap = (FixedSizeHeaplet::dataAreaSize()/(maxStepSize+ALLOC_ALIGNMENT))+1;
 
 class CChunkingRowManager : public CInterface, implements IRowManager
 {
@@ -2055,10 +2055,19 @@ public:
         if (hasAnyStepBlocks)
             baseBlock += numStepBlocks;
 
-        // Round up to nearest whole fraction of available heap space
-        unsigned frac = FixedSizeHeaplet::dataAreaSize()/size;
-        unsigned usesize = FixedSizeHeaplet::dataAreaSize()/frac;
-        return ROUNDED(baseBlock + (firstFractionalHeap-frac), usesize);
+        // Ensure the size is rounded up so it is correctly aligned. (Note the size already includes chunk overhead.)
+        unsigned minsize = align_pow2(size, ALLOC_ALIGNMENT);
+
+        // What fraction of the data area would this allocation take up?
+        unsigned frac = FixedSizeHeaplet::dataAreaSize()/minsize;
+
+        // Calculate the size of that fraction of the data.  Round down the divided size to ensure it is correctly aligned.
+        // (Round down to guarantee that frac allocations will fit in the data area).
+        size32_t usesize = FixedSizeHeaplet::dataAreaSize()/frac;
+        size32_t alignedsize = usesize &~ (ALLOC_ALIGNMENT-1);
+        dbgassertex(alignedsize >= size);
+
+        return ROUNDED(baseBlock + (firstFractionalHeap-frac), alignedsize);
     }
 
     inline void beforeAllocate(unsigned _size, unsigned activityId)
@@ -2371,7 +2380,7 @@ protected:
         dbgassertex(flags & RHFpacked);
         //Must be 4 byte aligned otherwise the atomic increments on the counts may not be atomic
         //Should this have a larger granularity (e.g., sizeof(void * *) if size is above a threshold?
-        size32_t chunkSize = align_pow2(size + PackedFixedSizeHeaplet::chunkHeaderSize, 4);
+        size32_t chunkSize = align_pow2(size + PackedFixedSizeHeaplet::chunkHeaderSize, PACKED_ALIGNMENT);
 
         //Not time critical, so don't worry about the scope of the spinblock around the new
         SpinBlock block(fixedCrit);
@@ -3106,6 +3115,7 @@ public:
 class RoxieMemTests : public CppUnit::TestFixture  
 {
     CPPUNIT_TEST_SUITE( RoxieMemTests );
+        CPPUNIT_TEST(testRoundup);
         CPPUNIT_TEST(testBitmapThreading);
         CPPUNIT_TEST(testAllocSize);
         CPPUNIT_TEST(testHuge);
@@ -3964,12 +3974,49 @@ protected:
             unsigned base = seed & 32767;
             unsigned shift = (seed / 32768) % 12;
             unsigned size = (base >> shift) + 4;
-            total += CChunkingRowManager::roundup(size+FixedSizeHeaplet::chunkHeaderSize);
+            size32_t thisSize = CChunkingRowManager::roundup(size+FixedSizeHeaplet::chunkHeaderSize);
+            ASSERT((thisSize & (ALLOC_ALIGNMENT-1)) == 0);
+            total += thisSize;
         }
         unsigned endTime = msTick();
 
         DBGLOG("Time to calculate %u = %d", total, endTime-startTime);
     }
+    void testRoundup()
+    {
+        Owned<IRowManager> rowManager = createRowManager(1, NULL, logctx, NULL);
+        const unsigned maxFrac = firstFractionalHeap;
+
+        const void * tempRow[MAX_FRAC_ALLOCATOR];
+        for (unsigned frac=1; frac < maxFrac; frac++)
+        {
+            size32_t fracSize = FixedSizeHeaplet::dataAreaSize() / frac;
+            for (unsigned j = 0; j < ALLOC_ALIGNMENT; j++)
+            {
+                size32_t allocSize = fracSize - j;
+                size32_t thisSize = CChunkingRowManager::roundup(allocSize);
+                ASSERT((thisSize & (ALLOC_ALIGNMENT-1)) == 0);
+            }
+
+            //Ensure you can allocate frac allocations in a single page (The row manager only has 1 page.)
+            size32_t testSize = (fracSize - FixedSizeHeaplet::chunkHeaderSize) & ~(ALLOC_ALIGNMENT-1);
+            for (unsigned i1=0; i1 < frac; i1++)
+            {
+                tempRow[i1] = rowManager->allocate(testSize, 0);
+                ASSERT(((memsize_t)tempRow[i1] & (ALLOC_ALIGNMENT-1)) == 0);
+            }
+            for (unsigned i2=0; i2 < frac; i2++)
+                ReleaseRoxieRow(tempRow[i2]);
+        }
+
+        //Check small allocations are also aligned
+        size32_t limitSize = firstFractionalHeap;
+        for (size_t nextSize = 1; nextSize < limitSize; nextSize++)
+        {
+            OwnedRoxieRow row = rowManager->allocate(nextSize, 0);
+            ASSERT(((memsize_t)row.get() & (ALLOC_ALIGNMENT-1)) == 0);
+        }
+    }
     void testFixedRelease()
     {
         //Ensure that row heaps (and therefore row allocators) can be destroyed before and after the rowManager

roxie/roxiemem/roxiemem.hpp

@@ -54,6 +54,11 @@
 #define ACTIVITY_FLAG_ISREGISTERED      0x00400000
 #define MAX_ACTIVITY_ID                 0x003fffff
 
+#define ALLOC_ALIGNMENT                 sizeof(void *)          // Minimum alignment of data allocated from the heap manager
+#define PACKED_ALIGNMENT                4                       // Minimum alignment of packed blocks
+
+#define MAX_FRAC_ALLOCATOR              20
+
 //================================================================================
 // Roxie heap