Reorganization of the SQL statement handling. Methods

to gather SQL statements have been implemented to speed up
the registration and unregistration of massive amounts of maps (>10000)


git-svn-id: https://svn.osgeo.org/grass/grass/trunk@51245 15284696-431f-4ddb-bdfa-cd5b030d7da7

lib/python/temporal/abstract_dataset.py

@@ -181,18 +181,11 @@ class abstract_dataset(object):
         bottom = self.spatial_extent.get_bottom()
         
         return (north, south, east, west, top, bottom)
-
+    
     def select(self, dbif=None):
 	"""!Select temporal dataset entry from database and fill up the internal structure"""
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
-
-        dbif.cursor.execute("BEGIN TRANSACTION")
+        dbif, connect = init_dbif(dbif)
 
 	self.base.select(dbif)
 	if self.is_time_absolute():
@@ -202,102 +195,88 @@ class abstract_dataset(object):
 	self.spatial_extent.select(dbif)
 	self.metadata.select(dbif)
 
-        dbif.cursor.execute("COMMIT TRANSACTION")
-
         if connect:
             dbif.close()
-        
+
     def is_in_db(self, dbif=None):
-	"""!Check if the temporal dataset entry is in the database"""
-	return self.base.is_in_db(dbif)
+        """!Check if the temporal dataset entry is in the database"""
+        return self.base.is_in_db(dbif)
 
     def delete(self):
 	"""!Delete temporal dataset entry from database if it exists"""
         raise IOError("This method must be implemented in the subclasses")
 
-    def insert(self, dbif=None):
-	"""!Insert temporal dataset entry into database from the internal structure"""
-
-        connect = False
+    def insert(self, dbif=None, execute=True):
+        """!Insert temporal dataset entry into database from the internal structure
 
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
-
-        dbif.cursor.execute("BEGIN TRANSACTION")
 
+           @param dbif: The database interface to be used
+           @param execute: If True the SQL statements will be executed.
+                           If False the prepared SQL statements are returned and must be executed by the caller.
+        """
 
-	self.base.insert(dbif)
-	if self.is_time_absolute():
-	    self.absolute_time.insert(dbif)
+        # Build the INSERT SQL statement
+        statement = self.base.get_insert_statement_mogrified(dbif)
+        if self.is_time_absolute():
+            statement += self.absolute_time.get_insert_statement_mogrified(dbif)
         if self.is_time_relative():
-	    self.relative_time.insert(dbif)
-	self.spatial_extent.insert(dbif)
-	self.metadata.insert(dbif)
+            statement += self.relative_time.get_insert_statement_mogrified(dbif)
+        statement += self.spatial_extent.get_insert_statement_mogrified(dbif)
+        statement += self.metadata.get_insert_statement_mogrified(dbif)
 
-        dbif.cursor.execute("COMMIT TRANSACTION")
+        if execute == True:
+            execute_transaction(statement, dbif)
+            return ""
 
-        if connect:
-            dbif.close()
- 
-    def update(self, dbif=None):
+        return statement
+
+    def update(self, dbif=None, execute=True):
 	"""!Update temporal dataset entry of database from the internal structure
 	   excluding None variables
-	"""
-
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
-
-        dbif.cursor.execute("BEGIN TRANSACTION")
 
+           @param dbif: The database interface to be used
+           @param execute: If True the SQL statements will be executed.
+                           If False the prepared SQL statements are returned and must be executed by the caller.
+	"""
 
-	self.base.update(dbif)
+        # Build the UPDATE SQL statement
+        statement = self.base.get_update_statement_mogrified(dbif)
 	if self.is_time_absolute():
-	    self.absolute_time.update(dbif)
+            statement += self.absolute_time.get_update_statement_mogrified(dbif)
         if self.is_time_relative():
-	    self.relative_time.update(dbif)
-	self.spatial_extent.update(dbif)
-	self.metadata.update(dbif)
+            statement += self.relative_time.get_update_statement_mogrified(dbif)
+        statement += self.spatial_extent.get_update_statement_mogrified(dbif)
+        statement += self.metadata.get_update_statement_mogrified(dbif)
 
-        dbif.cursor.execute("COMMIT TRANSACTION")
-
-        if connect:
-            dbif.close()
+        if execute == True:
+            execute_transaction(statement, dbif)
+            return ""
+ 
+        return statement
  
-    def update_all(self, dbif=None):
+    def update_all(self, dbif=None, execute=True):
 	"""!Update temporal dataset entry of database from the internal structure
 	   and include None varuables.
 
            @param dbif: The database interface to be used
+           @param execute: If True the SQL statements will be executed.
+                           If False the prepared SQL statements are returned and must be executed by the caller.
 	"""
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
-
-        dbif.cursor.execute("BEGIN TRANSACTION")
-
-
-	self.base.update_all(dbif)
+        # Build the UPDATE SQL statement
+        statement = self.base.get_update_all_statement_mogrified(dbif)
 	if self.is_time_absolute():
-	    self.absolute_time.update_all(dbif)
+            statement += self.absolute_time.get_update_all_statement_mogrified(dbif)
         if self.is_time_relative():
-	    self.relative_time.update_all(dbif)
-	self.spatial_extent.update_all(dbif)
-	self.metadata.update_all(dbif)
+            statement += self.relative_time.get_update_all_statement_mogrified(dbif)
+        statement += self.spatial_extent.get_update_all_statement_mogrified(dbif)
+        statement += self.metadata.get_update_all_statement_mogrified(dbif)
 
-        dbif.cursor.execute("COMMIT TRANSACTION")
+        if execute == True:
+            execute_transaction(statement, dbif)
+            return ""
 
-        if connect:
-            dbif.close()
+        return statement
 
     def set_time_to_absolute(self):
 	self.base.set_ttype("absolute")

lib/python/temporal/abstract_map_dataset.py

@@ -211,12 +211,7 @@ class abstract_map_dataset(abstract_dataset):
            @param end_time: a datetime object specifying the end time of the map
            @param timezone: Thee timezone of the map
         """
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         self.set_absolute_time(start_time, end_time, timezone)
         self.absolute_time.update_all(dbif)
@@ -290,12 +285,7 @@ class abstract_map_dataset(abstract_dataset):
            @param end_time: A double value 
            @param dbif: The database interface to be used
         """
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         if self.set_relative_time(start_time, end_time, unit):
             self.relative_time.update_all(dbif)
@@ -366,14 +356,9 @@ class abstract_map_dataset(abstract_dataset):
            @return The SQL statements if execute == False, else an empty string, None in case of a failure
         """
 
-        connect = False
+        dbif, connect = init_dbif(dbif)
         statement = ""
 
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
-
         if self.is_in_db(dbif):
  
             # SELECT all needed information from the database
@@ -389,26 +374,10 @@ class abstract_map_dataset(abstract_dataset):
             core.verbose(_("Delete %s dataset <%s> from temporal database") % (self.get_type(), self.get_id()))
 
             # Delete yourself from the database, trigger functions will take care of dependencies
-            statement += self.base.get_delete_statement() + ";\n"
+            statement += self.base.get_delete_statement()
 
         if execute == True:
-            sql_script = ""
-            sql_script += "BEGIN TRANSACTION;\n"
-            sql_script += statement
-            sql_script += "END TRANSACTION;"
-            # print sql_script
-            try:
-		if dbmi.__name__ == "sqlite3":
-		    dbif.cursor.executescript(statement)
-		else:
-		    dbif.cursor.execute(statement)
-            except:
-                if connect == True:
-                    dbif.close()
-                core.error(_("Unable to correctly delete %s map <%s>") % (self.get_type(), self.get_id()))
-                raise
-
-            dbif.connection.commit()
+            execute_transaction(statement, dbif)
 
         # Remove the timestamp from the file system
         if self.get_type() == "vect":
@@ -428,7 +397,6 @@ class abstract_map_dataset(abstract_dataset):
             return ""
 
         return statement
-            
 
     def unregister(self, dbif=None, update=True, execute=True):
 	"""! Remove the map entry in each space time dataset in which this map is registered
@@ -449,13 +417,8 @@ class abstract_map_dataset(abstract_dataset):
 	    core.verbose(_("Unregister %s map <%s> from space time datasets") % (self.get_type(), self.get_map_id()))
         
         statement = ""
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
-
+        dbif, connect = init_dbif(dbif)
+        
         # Get all datasets in which this map is registered
         rows = self.get_registered_datasets(dbif)
 
@@ -477,24 +440,9 @@ class abstract_map_dataset(abstract_dataset):
                     stds.update_from_registered_maps(dbif)
 
             core.percent(1, 1, 1)
+            
         if execute == True:
-            sql_script = ""
-            sql_script += "BEGIN TRANSACTION;\n"
-            sql_script += statement
-            sql_script += "END TRANSACTION;"
-            # print sql_script
-            try:
-		if dbmi.__name__ == "sqlite3":
-		    dbif.cursor.executescript(statement)
-		else:
-		    dbif.cursor.execute(statement)
-            except:
-                if connect == True:
-                    dbif.close()
-                core.error(_("Unable to correctly unregister %s <%s>") % (self.get_type(), self.get_id()))
-                raise
-
-            dbif.connection.commit()
+            execute_transaction(statement, dbif)
 
         if connect == True:
             dbif.close()
@@ -511,12 +459,7 @@ class abstract_map_dataset(abstract_dataset):
 
            @param dbif: The database interface to be used
         """
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         rows = None
 

lib/python/temporal/abstract_space_time_dataset.py

@@ -480,12 +480,7 @@ class abstract_space_time_dataset(abstract_dataset):
             use_contain = False
             use_equal = False
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         obj_list = []
         sample_maps = stds.get_registered_maps_as_objects_with_gaps(where=None, dbif=dbif)
@@ -559,12 +554,7 @@ class abstract_space_time_dataset(abstract_dataset):
            In case nothing found None is returned
         """
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         obj_list = []
 
@@ -623,12 +613,7 @@ class abstract_space_time_dataset(abstract_dataset):
            In case nothing found None is returned
         """
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         obj_list = []
         
@@ -675,12 +660,7 @@ class abstract_space_time_dataset(abstract_dataset):
            In case nothing found None is returned
         """
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         obj_list = []
         
@@ -718,12 +698,7 @@ class abstract_space_time_dataset(abstract_dataset):
            In case nothing found None is returned
         """
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         rows = None
 
@@ -775,19 +750,13 @@ class abstract_space_time_dataset(abstract_dataset):
         core.verbose(_("Delete space time %s  dataset <%s> from temporal database") % (self.get_new_map_instance(ident=None).get_type(), self.get_id()))
 
         statement = ""
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         # SELECT all needed information from the database
         self.metadata.select(dbif)
 
-        core.verbose(_("Drop map register table: %s") %  (self.get_map_register()))
-        
         if self.get_map_register():
+            core.verbose(_("Drop map register table: %s") %  (self.get_map_register()))
             rows = self.get_registered_maps("id", None, None, dbif)
             # Unregister each registered map in the table
             if rows:
@@ -800,39 +769,26 @@ class abstract_space_time_dataset(abstract_dataset):
                     statement += self.unregister_map(map=map, dbif=dbif, execute=False)
                     count += 1
 	        core.percent(1, 1, 1)
-                # Safe the DROP table statement
-                statement += "DROP TABLE " + self.get_map_register() + ";\n"
+
+            # Safe the DROP table statement
+            statement += "DROP TABLE " + self.get_map_register() + ";\n"
 
         # Remove the primary key, the foreign keys will be removed by trigger
-        statement += self.base.get_delete_statement() + ";\n"
-        
+        statement += self.base.get_delete_statement()
+
         if execute == True:
-            sql_script = ""
-            sql_script += "BEGIN TRANSACTION;\n"
-            sql_script += statement
-            sql_script += "END TRANSACTION;"
-            print sql_script
-            try:
-		if dbmi.__name__ == "sqlite3":
-		    dbif.cursor.executescript(statement)
-		else:
-		    dbif.cursor.execute(statement)
-            except:
-                if connect == True:
-                    dbif.close()
-                core.error(_("Unable to correctly delete %s <%s>") % (self.get_type(), self.get_id()))
-                raise
+            execute_transaction(statement, dbif)
 
         self.reset(None)
 
         if connect == True:
             dbif.close()
-    
+
         if execute:
             return ""
 
         return statement
-            
+
     def register_map(self, map, dbif=None):
         """!Register a map in the space time dataset.
 
@@ -844,12 +800,7 @@ class abstract_space_time_dataset(abstract_dataset):
 
            @param dbif: The database interface to be used
         """
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         if map.is_in_db(dbif) == False:
             dbif.close()
@@ -882,6 +833,9 @@ class abstract_space_time_dataset(abstract_dataset):
         stds_mapset = self.base.get_mapset()
         stds_register_table = self.get_map_register()
         stds_ttype = self.get_temporal_type()
+        
+        # The gathered SQL statemets are stroed here
+        statement = ""
 
         # Check temporal types
         if stds_ttype != map_ttype:
@@ -895,7 +849,7 @@ class abstract_space_time_dataset(abstract_dataset):
             self.map_counter == 0 and self.is_time_relative():
 
             self.set_relative_time_unit(map_rel_time_unit)
-            self.relative_time.update()
+            statement += self.relative_time.get_update_all_statement_mogrified(dbif)
             core.verbose(_("Set temporal unit for space time %s dataset <%s> to %s") %  (map.get_type(), self.get_id(), map_rel_time_unit))
 
         stds_rel_time_unit = self.get_relative_time_unit()
@@ -919,18 +873,23 @@ class abstract_space_time_dataset(abstract_dataset):
 		sql = "SELECT id FROM " + stds_register_table + " WHERE id = (?)"
 	    else:
 		sql = "SELECT id FROM " + stds_register_table + " WHERE id = (%s)"
-            dbif.cursor.execute(sql, (map_id,))
-            row = dbif.cursor.fetchone()
-            # In case of no entry make a new one
+            try:
+                dbif.cursor.execute(sql, (map_id,))
+                row = dbif.cursor.fetchone()
+            except:
+                row = None
+                core.warning(_("Error in strds_register_table request"))
+                raise
+
             if row and row[0] == map_id:
                 if connect == True:
                     dbif.close()
-                    
+
 		if map.get_layer():
 		    core.warning(_("Map <%s> with layer %s is already registered.") % (map.get_map_id(), map.get_layer()))
 		else:
 		    core.warning(_("Map <%s> is already registered.") % (map.get_map_id()))
-                return False
+                return ""
 
         # Create tables
         sql_path = get_sql_template_path()
@@ -950,25 +909,12 @@ class abstract_space_time_dataset(abstract_dataset):
             sql = sql.replace("TABLE_NAME", uuid_rand )
             sql = sql.replace("MAP_ID", map_id)
             sql = sql.replace("STDS", self.get_type())
-            try:
-		if dbmi.__name__ == "sqlite3":
-		    dbif.cursor.executescript(sql)
-		else:
-		    dbif.cursor.execute(sql)
-            except:
-                if connect == True:
-                    dbif.close()
-		if map.get_layer():
-		    core.error(_("Unable to create the space time %s dataset register table for map <%s> with layer %s") % \
-                            (map.get_type(), map.get_map_id(), map.get_layer()))
-                else:
-		    core.error(_("Unable to create the space time %s dataset register table for <%s>") % \
-                            (map.get_type(), map.get_map_id()))
-                raise
+            
+            statement += sql
 
             # Set the stds register table name and put it into the DB
             map.set_stds_register(map_register_table)
-            map.metadata.update(dbif)
+            statement += map.metadata.get_update_statement_mogrified(dbif)
             
             if map.get_layer():
 		core.verbose(_("Created register table <%s> for %s map <%s> with layer %s") % \
@@ -988,70 +934,59 @@ class abstract_space_time_dataset(abstract_dataset):
             sql = sql.replace("SPACETIME_NAME", stds_name + "_" + stds_mapset )
             sql = sql.replace("SPACETIME_ID", self.base.get_id())
             sql = sql.replace("STDS", self.get_type())
-
-            sql_script = ""
-            sql_script += "BEGIN TRANSACTION;\n"
-            sql_script += sql
-            sql_script += "\n"
-            sql_script += "END TRANSACTION;"
-            try:
-		if dbmi.__name__ == "sqlite3":
-		    dbif.cursor.executescript(sql_script)
-		else:
-		    dbif.cursor.execute(sql_script)
-                dbif.connection.commit()
-            except:
-                if connect == True:
-                    dbif.close()
-		if map.get_layer():
-		    core.error(_("Unable to create the space time %s dataset register table for map <%s> with layer %s") % \
-                            (map.get_type(), map.get_map_id(), map.get_layer()))
-                else:
-		    core.error(_("Unable to create the space time %s dataset register table for <%s>") % \
-                            (map.get_type(), map.get_map_id()))
-                raise
+            statement += sql
 
             # Set the map register table name and put it into the DB
             self.set_map_register(stds_register_table)
-            self.metadata.update(dbif)
+            statement += self.metadata.get_update_statement_mogrified(dbif)
 
             core.verbose(_("Created register table <%s> for space time %s  dataset <%s>") % \
                           (stds_register_table, map.get_type(), self.get_id()))
 
+        # We need to execute the statement at this time
+        if statement != "":
+            execute_transaction(statement, dbif)
+
+        statement = ""
+
         # Register the stds in the map stds register table
         # Check if the entry is already there
 	if dbmi.paramstyle == "qmark":
 	    sql = "SELECT id FROM " + map_register_table + " WHERE id = ?"
 	else:
 	    sql = "SELECT id FROM " + map_register_table + " WHERE id = %s"
-        dbif.cursor.execute(sql, (self.base.get_id(),))
-      	row = dbif.cursor.fetchone()
+        try:
+            dbif.cursor.execute(sql, (self.base.get_id(),))
+            row = dbif.cursor.fetchone()
+        except:
+            row = None
 
         # In case of no entry make a new one
         if row == None:
 	    if dbmi.paramstyle == "qmark":
-		sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (?)"
+		sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (?);\n"
 	    else:
-		sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (%s)"
-            #print sql
-            dbif.cursor.execute(sql, (self.base.get_id(),))
+		sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (%s);\n"
+
+            statement += dbif._mogrify_sql_statement((sql, (self.base.get_id(),)), dbif)
 
         # Now put the raster name in the stds map register table
 	if dbmi.paramstyle == "qmark":
-	    sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (?)"
+	    sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (?);\n"
 	else:
-	    sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (%s)"
-        #print sql
-        dbif.cursor.execute(sql, (map_id,))
+	    sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (%s);\n"
+
+        statement += dbif._mogrify_sql_statement((sql, (map_id,)), dbif)
+
+        # Now execute the insert transaction
+        execute_transaction(statement, dbif)
 
         if connect == True:
             dbif.close()
-            
+
         # increase the counter
         self.map_counter += 1
 
-        return True
-
     def unregister_map(self, map, dbif = None, execute=True):
         """!Unregister a map from the space time dataset.
 
@@ -1068,12 +1003,7 @@ class abstract_space_time_dataset(abstract_dataset):
 
         statement = ""
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         # First select needed data from the database
         map.metadata.select(dbif)
@@ -1092,8 +1022,11 @@ class abstract_space_time_dataset(abstract_dataset):
 	    sql = "SELECT id FROM " + map_register_table + " WHERE id = ?"
 	else:
 	    sql = "SELECT id FROM " + map_register_table + " WHERE id = %s"
-        dbif.cursor.execute(sql, (self.base.get_id(),))
-      	row = dbif.cursor.fetchone()
+        try:
+            dbif.cursor.execute(sql, (self.base.get_id(),))
+            row = dbif.cursor.fetchone()
+        except:
+            row = None
 
         # Break if the map is not registered
         if row == None:
@@ -1103,34 +1036,29 @@ class abstract_space_time_dataset(abstract_dataset):
 		core.warning(_("Map <%s> is not registered in space time dataset <%s>") %(map.get_map_id(), self.base.get_id()))
             if connect == True:
                 dbif.close()
-            return None
+            return ""
 
         # Remove the space time raster dataset from the raster dataset register
         if map_register_table != None:
+            if dbmi.paramstyle == "qmark":
+                sql = "DELETE FROM " + map_register_table + " WHERE id = ?;\n"
+            else:
+                sql = "DELETE FROM " + map_register_table + " WHERE id = %s;\n"
 
-            statement += "DELETE FROM " + map_register_table + " WHERE id = \'%s\';\n"%(self.base.get_id())
-
-            if execute == True:
-                if dbmi.paramstyle == "qmark":
-		    sql = "DELETE FROM " + map_register_table + " WHERE id = ?"
-	        else:
-		    sql = "DELETE FROM " + map_register_table + " WHERE id = %s"
-
-                dbif.cursor.execute(sql, (self.base.get_id(),))
+            statement += dbif._mogrify_sql_statement((sql, (self.base.get_id(),)), dbif)
 
         # Remove the raster map from the space time raster dataset register
         if stds_register_table != None:
+            if dbmi.paramstyle == "qmark":
+                sql = "DELETE FROM " + stds_register_table + " WHERE id = ?;\n"
+            else:
+                sql = "DELETE FROM " + stds_register_table + " WHERE id = %s;\n"
 
-            statement += "DELETE FROM " + stds_register_table + " WHERE id = \'%s\';\n"%(map_id)
-
-            if execute == True:
-	        if dbmi.paramstyle == "qmark":
-		    sql = "DELETE FROM " + stds_register_table + " WHERE id = ?"
-	        else:
-		    sql = "DELETE FROM " + stds_register_table + " WHERE id = %s"
-
-                dbif.cursor.execute(sql, (map_id,))
+            statement += dbif._mogrify_sql_statement((sql, (map_id,)), dbif)
 
+        if execute == True:
+            execute_transaction(statement, dbif)
+            
         if connect == True:
             dbif.close()
 
@@ -1164,12 +1092,7 @@ class abstract_space_time_dataset(abstract_dataset):
         if not self.get_map_register():
             return
 
-        connect = False
-
-        if dbif == None:
-            dbif = sql_database_interface()
-            dbif.connect()
-            connect = True
+        dbif, connect = init_dbif(dbif)
 
         map_time = None
 
@@ -1182,7 +1105,6 @@ class abstract_space_time_dataset(abstract_dataset):
 
         #We create a transaction
         sql_script = ""
-        sql_script += "BEGIN TRANSACTION;\n"
         
         # Update the spatial and temporal extent from registered maps
         # Read the SQL template
@@ -1205,12 +1127,7 @@ class abstract_space_time_dataset(abstract_dataset):
         sql_script += sql
         sql_script += "\n"
 
-        sql_script += "END TRANSACTION;"
-
-	if dbmi.__name__ == "sqlite3":
-	    dbif.cursor.executescript(sql_script)
-	else:
-	    dbif.cursor.execute(sql_script)
+	execute_transaction(sql_script, dbif)
 	    
         # Read and validate the selected end time
         self.select()
@@ -1280,10 +1197,7 @@ class abstract_space_time_dataset(abstract_dataset):
                 sql = sql.replace("SPACETIME_ID", self.base.get_id())
                 sql = sql.replace("STDS", self.get_type())
 
-	    if dbmi.__name__ == "sqlite3":
-		dbif.cursor.executescript(sql)
-	    else:
-		dbif.cursor.execute(sql)
+	    execute_transaction(sql, dbif)
 
         # Count the temporal map types
         maps = self.get_registered_maps_as_objects(dbif=dbif)

lib/python/temporal/base.py

@@ -59,6 +59,7 @@ class dict_sql_serializer(object):
             sql += ' FROM ' + table + ' '
 	    if where:
 	        sql += where
+            sql += ";\n"
 
 	# Create insert statement
 	if type =="INSERT":
@@ -90,6 +91,7 @@ class dict_sql_serializer(object):
 
 	    if where:
 	        sql += where
+            sql += ";\n"
 
 	# Create update statement for existing entries
 	if type =="UPDATE":
@@ -114,6 +116,7 @@ class dict_sql_serializer(object):
 	            args.append(self.D[key])
 	    if where:
 	        sql += where
+            sql += ";\n"
 
 	# Create update statement for all entries
 	if type =="UPDATE ALL":
@@ -136,6 +139,7 @@ class dict_sql_serializer(object):
                 args.append(self.D[key])
 	    if where:
 	        sql += where
+            sql += ";\n"
 
     	return sql, tuple(args)
 
@@ -219,7 +223,7 @@ class sql_database_interface(dict_sql_serializer):
             self.cursor.execute("PRAGMA journal_mode = MEMORY")
         elif dbmi.__name__ == "psycopg2":
 	    self.connection = dbmi.connect(init)
-	    self.connection.set_isolation_level(dbmi.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+	    #self.connection.set_isolation_level(dbmi.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
 	    self.cursor = self.connection.cursor(cursor_factory=dbmi.extras.DictCursor)
 
     def close(self):
@@ -228,8 +232,65 @@ class sql_database_interface(dict_sql_serializer):
 	self.connection.commit()
         self.cursor.close()
 
+
+    def _mogrify_sql_statement(self, content, dbif=None):
+        """!Return the SQL statement and arguments as executable SQL string
+        """
+        sql = content[0]
+        args = content[1]
+
+        if dbmi.__name__ == "psycopg2":
+            if len(args) == 0:
+                return sql
+            else:
+                if dbif:
+                    try:
+                        return dbif.cursor.mogrify(sql, args)
+                    except:
+                        print sql, args
+                        raise
+                else:
+                    self.connect()
+                    statement = self.cursor.mogrify(sql, args)
+                    self.close()
+                    return statement
+                    
+        elif dbmi.__name__ == "sqlite3":
+            if len(args) == 0:
+                return sql
+            else:
+                # Unfortunately as sqlite does not support 
+                # the transformation of sql strings and qmarked or
+                # named arguments we must make our hands dirty
+                # and do it by ourself. :(
+                # Doors are open for SQL injection because of the 
+                # limited python sqlite3 implementation!!!
+                pos = 0
+                count = 0
+                maxcount = 100
+                statement = sql
+
+                while count < maxcount:
+                    pos = statement.find("?", pos + 1)
+                    if pos == -1:
+                        break
+                    
+                    if args[count] == None:
+                        statement = "%sNULL%s"%(statement[0:pos], statement[pos+1:])
+                    elif isinstance(args[count], (int, long)):
+                        statement = "%s%d%s"%(statement[0:pos], args[count],statement[pos+1:])
+                    elif isinstance(args[count], float):
+                        statement = "%s%f%s"%(statement[0:pos], args[count],statement[pos+1:])
+                    else:
+                        # Default is a string, this works for datetime objects too
+                        statement = "%s\'%s\'%s"%(statement[0:pos], str(args[count]),statement[pos+1:])
+                    count += 1
+
+                return statement
+
     def get_delete_statement(self):
-	return "DELETE FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\'"
+        """!Return the delete string"""
+	return "DELETE FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\';\n"
 
     def delete(self, dbif=None):
         """!Delete the entry of this object from the temporal database"""
@@ -244,7 +305,8 @@ class sql_database_interface(dict_sql_serializer):
             self.close()
 
     def get_is_in_db_statement(self):
-	return "SELECT id FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\'"
+        """Return the selection string"""
+	return "SELECT id FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\';\n"
 
     def is_in_db(self, dbif=None):
         """!Check if this object is present in the temporal database
@@ -271,8 +333,13 @@ class sql_database_interface(dict_sql_serializer):
 	return True
 
     def get_select_statement(self):
+        """!Return the sql statement and the argument list in database specific style"""
 	return self.serialize("SELECT", self.get_table_name(), "WHERE id = \'" + str(self.ident) + "\'")
-
+    
+    def get_select_statement_mogrified(self, dbif=None):
+        """!Return the select statement as mogrified string"""
+        return self._mogrify_sql_statement(self.get_select_statement(), dbif)
+                
     def select(self, dbif=None):
         """!Select the content from the temporal database and store it
            in the internal dictionary structure
@@ -310,7 +377,12 @@ class sql_database_interface(dict_sql_serializer):
 	return True
 
     def get_insert_statement(self):
+        """!Return the sql statement and the argument list in database specific style"""
 	return self.serialize("INSERT", self.get_table_name())
+    
+    def get_insert_statement_mogrified(self, dbif=None):
+        """!Return the insert statement as mogrified string"""
+        return self._mogrify_sql_statement(self.get_insert_statement(), dbif)
 
     def insert(self, dbif=None):
         """!Serialize the content of this object and store it in the temporal
@@ -330,8 +402,13 @@ class sql_database_interface(dict_sql_serializer):
             self.close()
 
     def get_update_statement(self):
+        """!Return the sql statement and the argument list in database specific style"""
 	return self.serialize("UPDATE", self.get_table_name(), "WHERE id = \'" + str(self.ident) + "\'")
 
+    def get_update_statement_mogrified(self,dbif=None):
+        """!Return the update statement as mogrified string"""
+        return self._mogrify_sql_statement(self.get_update_statement(), dbif)
+
     def update(self, dbif=None):
         """!Serialize the content of this object and update it in the temporal
            database using the internal identifier
@@ -355,8 +432,13 @@ class sql_database_interface(dict_sql_serializer):
             self.close()
 
     def get_update_all_statement(self):
+        """!Return the sql statement and the argument list in database specific style"""
 	return self.serialize("UPDATE ALL", self.get_table_name(), "WHERE id = \'" + str(self.ident) + "\'")
 
+    def get_update_all_statement_mogrified(self, dbif=None):
+        """!Return the update all statement as mogrified string"""
+        return self._mogrify_sql_statement(self.get_update_all_statement(), dbif)
+
     def update_all(self, dbif=None):
         """!Serialize the content of this object, including None objects, and update it in the temporal
            database using the internal identifier
@@ -672,3 +754,52 @@ class stvds_base(stds_base):
         stds_base.__init__(self, "stvds_base", ident, name, mapset, semantic_type, creator, creation_time,\
 	            modification_time, temporal_type, revision)
 
+###############################################################################
+
+def init_dbif(dbif):
+    """!This method checks if the database interface exists, if not a new one 
+        will be created and True will be returned
+
+        Usage code sample:
+        dbif, connect = self._init_dbif(dbif)
+        if connect:
+            dbif.close()
+    """
+    if dbif == None:
+        dbif = sql_database_interface()
+        dbif.connect()
+        return dbif, True
+
+    return dbif, False
+
+###############################################################################
+
+def execute_transaction(statement, dbif=None):
+    """!Execute a transactional SQL statement
+
+        The BEGIN and END TRANSACTION statements will be added automatically
+        to the sql statement
+
+        @param statement The executable SQL statement or SQL script
+        @param dbif The database interface, if None a new db interface will be created temporary
+    """
+    dbif, connect = init_dbif(dbif)
+
+    sql_script = ""
+    sql_script += "BEGIN TRANSACTION;\n"
+    sql_script += statement
+    sql_script += "END TRANSACTION;"
+    try:
+        if dbmi.__name__ == "sqlite3":
+            dbif.cursor.executescript(statement)
+        else:
+            dbif.cursor.execute(statement)
+        dbif.connection.commit()
+    except:
+        if connect == True:
+            dbif.close()
+        core.error(_("Unable to execute transaction:\n %s") % (statement))
+        raise
+
+    if connect:
+        dbif.close()

lib/python/temporal/space_time_datasets_tools.py

@@ -86,12 +86,7 @@ def register_maps_in_space_time_dataset(type, name, maps=None, file=None, start=
 	    core.fatal(_("Unkown map type: %s")%(type))
 
         
-    connect = False
-
-    if dbif == None:
-        dbif = sql_database_interface()
-        dbif.connect()
-        connect = True
+    dbif, connect = init_dbif(None)
 
     if name:
 	# Read content from temporal database
@@ -163,6 +158,11 @@ def register_maps_in_space_time_dataset(type, name, maps=None, file=None, start=
             maplist.append(row)
     
     num_maps = len(maplist)
+    map_object_list = []
+    statement = ""
+    
+    core.message(_("Gathering map informations"))
+    
     for count in range(len(maplist)):
 	core.percent(count, num_maps, 1)
 
@@ -174,9 +174,12 @@ def register_maps_in_space_time_dataset(type, name, maps=None, file=None, start=
             start = maplist[count]["start"]
         if maplist[count].has_key("end"):
             end = maplist[count]["end"]
+            
+        is_in_db = False
 
         # Put the map into the database
         if map.is_in_db(dbif) == False:
+            is_in_db = False
             # Break in case no valid time is provided
             if start == "" or start == None:
                 dbif.close()
@@ -186,17 +189,14 @@ def register_maps_in_space_time_dataset(type, name, maps=None, file=None, start=
 		else:
 		    core.fatal(_("Unable to register %s map <%s>. The map has no valid time and the start time is not set.") % \
 				(map.get_type(), map.get_map_id() ))
-            # Load the data from the grass file database
-            map.load()
 	    
 	    if unit:
                 map.set_time_to_relative()
             else:
                 map.set_time_to_absolute()
-                
-            #  Put it into the temporal database
-            map.insert(dbif)
+ 
         else:
+            is_in_db = True
             map.select(dbif)
             if name and map.get_temporal_type() != sp.get_temporal_type():
                 dbif.close()
@@ -206,20 +206,55 @@ def register_maps_in_space_time_dataset(type, name, maps=None, file=None, start=
 		    core.fatal(_("Unable to register %s map <%s>. The temporal types are different.") %  \
 		                 (map.get_type(), map.get_map_id()))
 
-        # In case the time is in the input file we ignore the increment counter
-        if start_time_in_file:
-            count = 1
+        # Load the data from the grass file database
+        map.load()
 
         # Set the valid time
         if start:
-            assign_valid_time_to_map(ttype=map.get_temporal_type(), map=map, start=start, end=end, unit=unit, increment=increment, mult=count, dbif=dbif, interval=interval)
-
-        # Finally Register map in the space time dataset
+            # In case the time is in the input file we ignore the increment counter
+            if start_time_in_file:
+                count = 1
+            assign_valid_time_to_map(ttype=map.get_temporal_type(), map=map, start=start, end=end, unit=unit, increment=increment, mult=count, interval=interval)
+
+        if is_in_db:
+           #  Gather the SQL update statement
+           statement += map.update_all(dbif=dbif, execute=True)
+        else:
+           #  Gather the SQL insert statement
+           statement += map.insert(dbif=dbif, execute=False)
+
+        # Sqlite3 performace better for huge datasets when committing in small chunks
+        if dbmi.__name__ == "sqlite3":
+            if count % 100 == 0:
+                if statement != None and statement != "":
+                    core.message(_("Registering 100 maps in the temporal database"))
+                    execute_transaction(statement, dbif)
+                    statement = ""
+
+        # Store the maps in a list to register in a space time dataset
         if name:
-	    sp.register_map(map, dbif)
+            map_object_list.append(map)
+
+    core.percent(num_maps, num_maps, 1)
 
+    if statement != None and statement != "":
+        core.message(_("Register maps in the temporal database"))
+        execute_transaction(statement, dbif)
+
+    # Finally Register the maps in the space time dataset
+    if name:
+        statement = ""
+        count = 0
+        num_maps = len(map_object_list)
+        core.message(_("Register maps in the space time raster dataset"))
+        for map in map_object_list:
+	    core.percent(count, num_maps, 1)
+	    sp.register_map(map=map, dbif=dbif)
+            count += 1
+        
     # Update the space time tables
     if name:
+        core.message(_("Update space time raster dataset"))
 	sp.update_from_registered_maps(dbif)
 
     if connect == True:
@@ -230,7 +265,7 @@ def register_maps_in_space_time_dataset(type, name, maps=None, file=None, start=
 
 ###############################################################################
 
-def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=1, dbif = None, interval=False):
+def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=1, interval=False):
     """!Assign the valid time to a map dataset
 
        @param ttype: The temporal type which should be assigned and which the time format is of
@@ -240,16 +275,8 @@ def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=
        @param unit: The unit of the relative time: years, months, days, hours, minutes, seconds
        @param increment: Time increment between maps for time stamp creation (format absolute: NNN seconds, minutes, hours, days, weeks, months, years; format relative is integer 1)
        @param multi: A multiplier for the increment
-       @param dbif: The database interface to use for sql queries
        @param interval: If True, time intervals are created in case the start time and an increment is provided
     """
-    
-    connect = False
-
-    if dbif == None:
-        dbif = sql_database_interface()
-        dbif.connect()
-        connect = True
 
     if ttype == "absolute":
         start_time = string_to_datetime(start)
@@ -257,7 +284,7 @@ def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=
             dbif.close()
             core.fatal(_("Unable to convert string \"%s\"into a datetime object")%(start))
         end_time = None
-        
+
         if end:
             end_time = string_to_datetime(end)
             if end_time == None:
@@ -274,7 +301,7 @@ def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=
         else:
 	    core.verbose(_("Set absolute valid time for map <%s> to %s - %s") % (map.get_map_id(), str(start_time), str(end_time)))
         
-        map.update_absolute_time(start_time, end_time, None, dbif)
+        map.set_absolute_time(start_time, end_time, None)
     else:
         start_time = int(start)
         end_time = None
@@ -292,10 +319,7 @@ def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=
         else:
 	    core.verbose(_("Set relative valid time for map <%s> to %i - %s with unit %s") % (map.get_map_id(), start_time,  str(end_time), unit))
 	    
-        map.update_relative_time(start_time, end_time, unit, dbif)
-
-    if connect == True:
-        dbif.close()
+        map.set_relative_time(start_time, end_time, unit)
 
 ###############################################################################
 

temporal/t.register/test.t.register.vector.file.layer.sh

@@ -51,19 +51,19 @@ t.create --v --o type=stvds temporaltype=absolute output=lidar_abs_ds1 title="A
 # Test with input files
 # File 1
 t.register --v type=vect input=lidar_abs_ds1 file="${n1}" start="2001-01-01" increment="1 months"
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
 # File 1
 t.register --v type=vect input=lidar_abs_ds1 file="${n1}" start="2001-01-01"
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
 # File 2
 t.register --v type=vect input=lidar_abs_ds1 file="${n2}" 
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
 # File 2
 t.register --v type=vect input=lidar_abs_ds1 -i file="${n2}" start=file increment="1 months"
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
 # File 3
 t.register --v type=vect input=lidar_abs_ds1 file="${n3}" start=file 
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
 
 t.unregister --v type=vect maps=lidar_abs_1:1,lidar_abs_1:2,lidar_abs_1:3,lidar_abs_1:4,lidar_abs_1:5,lidar_abs_1:6
 t.remove type=stvds input=lidar_abs_ds1

temporal/t.register/test.t.register.vector.sh

@@ -62,3 +62,6 @@ t.info type=stvds input=lidar_abs_ds7
 t.unregister --v type=vect maps=lidar_abs_1,lidar_abs_2,lidar_abs_3
 t.remove --v type=stvds input=lidar_abs_ds1,lidar_abs_ds2,lidar_abs_ds3,lidar_abs_ds4,lidar_abs_ds5,lidar_abs_ds6,lidar_abs_ds7
 t.unregister --v type=vect maps=lidar_abs_4,lidar_abs_5,lidar_abs_6
+g.remove vect=lidar_abs_1,lidar_abs_2,lidar_abs_3,lidar_abs_4,lidar_abs_5,lidar_abs_6
+
+