Geografic-Information-System/gui/wxpython/gmodeler/model.py

"""!
@package gmodeler.model

@brief wxGUI Graphical Modeler (base classes)

Classes:
 - Model

(C) 2010-2011 by the GRASS Development Team
This program is free software under the GNU General Public License
(>=v2). Read the file COPYING that comes with GRASS for details.

@author Martin Landa <landa.martin gmail.com>
"""

import os
import getpass
import copy
import re
try:
    import xml.etree.ElementTree as etree
except ImportError:
    import elementtree.ElementTree as etree # Python <= 2.4

import wx

from core.globalvar import ETCWXDIR
from core.gcmd      import GMessage, GException, GError, RunCommand

from grass.script import core as grass
from grass.script import task as gtask

class Model(object):
    """!Class representing the model"""
    def __init__(self, canvas = None):
        self.items      = list() # list of actions/loops/...
        
        # model properties
        self.properties = { 'name'        : _("model"),
                            'description' : _("Script generated by wxGUI Graphical Modeler."),
                            'author'      : getpass.getuser() }
        # model variables
        self.variables = dict()
        self.variablesParams = dict()
        
        self.canvas  = canvas
        
    def GetCanvas(self):
        """!Get canvas or None"""
        return self.canvas
    
    def GetItems(self, objType = None):
        """!Get list of model items

        @param objType Object type to filter model objects
        """
        if not objType:
            return self.items
        
        result = list()
        for item in self.items:
            if isinstance(item, objType):
                result.append(item)
        
        return result

    def GetItem(self, aId):
        """!Get item of given id

        @param aId item id
        
        @return Model* instance
        @return None if no item found
        """
        ilist = self.GetItems()
        for item in ilist:
            if item.GetId() == aId:
                return item
        
        return None

    def GetNumItems(self, actionOnly = False):
        """!Get number of items"""
        if actionOnly:
            return len(self.GetItems(objType = ModelAction))
        
        return len(self.GetItems())

    def GetNextId(self):
        """!Get next id (data ignored)

        @return next id to be used (default: 1)
        """
        if len(self.items) < 1:
            return 1
        
        currId = self.items[-1].GetId()
        if currId > 0:
            return currId + 1
        
        return 1

    def GetProperties(self):
        """!Get model properties"""
        return self.properties

    def GetVariables(self, params = False):
        """!Get model variables"""
        if params:
            return self.variablesParams
        
        return self.variables
    
    def SetVariables(self, data):
        """!Set model variables"""
        self.variables = data
    
    def Reset(self):
        """!Reset model"""
        self.items = list()
        
    def RemoveItem(self, item):
        """!Remove item from model
        
        @return list of related items to remove/update
        """
        relList = list()
        upList = list()
        
        if not isinstance(item, ModelData):
            self.items.remove(item)
        
        if isinstance(item, ModelAction):
            for rel in item.GetRelations():
                relList.append(rel)
                data = rel.GetData()
                if len(data.GetRelations()) < 2:
                    relList.append(data)
                else:
                    upList.append(data)
            
        elif isinstance(item, ModelData):
            for rel in item.GetRelations():
                relList.append(rel)
                if rel.GetFrom() == self:
                    relList.append(rel.GetTo())
                else:
                    relList.append(rel.GetFrom())
        
        elif isinstance(item, ModelLoop):
            for rel in item.GetRelations():
                relList.append(rel)
            for action in self.GetItems():
                action.UnSetBlock(item)
        
        return relList, upList
    
    def FindAction(self, aId):
        """!Find action by id"""
        alist = self.GetItems(objType = ModelAction)
        for action in alist:
            if action.GetId() == aId:
                return action
        
        return None

    def GetData(self):
        """!Get list of data items"""
        result = list()
        dataItems = self.GetItems(objType = ModelData)
        
        for action in self.GetItems(objType = ModelAction):
            for rel in action.GetRelations():
                dataItem = rel.GetData()
                if dataItem not in result:
                    result.append(dataItem)
                if dataItem in dataItems:
                    dataItems.remove(dataItem)
        
        # standalone data
        if dataItems:
            result += dataItems
        
        return result

    def FindData(self, value, prompt):
        """!Find data item in the model

        @param value value
        @param prompt prompt

        @return ModelData instance
        @return None if not found
        """
        for data in self.GetData():
            if data.GetValue() == value and \
                    data.GetPrompt() == prompt:
                return data
        
        return None
                
    def LoadModel(self, filename):
        """!Load model definition stored in GRASS Model XML file (gxm)
        
        @todo Validate against DTD
        
        Raise exception on error.
        """
        dtdFilename = os.path.join(ETCWXDIR, "xml", "grass-gxm.dtd")
        
        # parse workspace file
        try:
            gxmXml = ProcessModelFile(etree.parse(filename))
        except StandardError, e:
            raise GException(e)
        
        if self.canvas:
            win = self.canvas.parent
            if gxmXml.pos:
                win.SetPosition(gxmXml.pos)
            if gxmXml.size:
                win.SetSize(gxmXml.size)
        
        # load properties
        self.properties = gxmXml.properties
        self.variables  = gxmXml.variables
        
        # load model.GetActions()
        for action in gxmXml.actions:
            actionItem = ModelAction(parent = self, 
                                     x = action['pos'][0],
                                     y = action['pos'][1],
                                     width = action['size'][0],
                                     height = action['size'][1],
                                     task = action['task'],
                                     id = action['id'])
            
            if action['disabled']:
                actionItem.Enable(False)
            
            self.AddItem(actionItem)
            
            actionItem.SetValid(actionItem.GetTask().get_options())
            actionItem.GetLog() # substitute variables (-> valid/invalid)
        
        # load data & relations
        for data in gxmXml.data:
            dataItem = ModelData(parent = self, 
                                 x = data['pos'][0],
                                 y = data['pos'][1],
                                 width = data['size'][0],
                                 height = data['size'][1],
                                 prompt = data['prompt'],
                                 value = data['value'])
            dataItem.SetIntermediate(data['intermediate'])
            
            for rel in data['rels']:
                actionItem = self.FindAction(rel['id'])
                if rel['dir'] == 'from':
                    relation = ModelRelation(parent = self, fromShape = dataItem,
                                             toShape = actionItem, param = rel['name'])
                else:
                    relation = ModelRelation(parent = self, fromShape = actionItem,
                                             toShape = dataItem, param = rel['name'])
                relation.SetControlPoints(rel['points'])
                actionItem.AddRelation(relation)
                dataItem.AddRelation(relation)

            if self.canvas:
                dataItem.Update()
           
        # load loops
        for loop in gxmXml.loops:
            loopItem = ModelLoop(parent = self, 
                                 x = loop['pos'][0],
                                 y = loop['pos'][1],
                                 width = loop['size'][0],
                                 height = loop['size'][1],
                                 text = loop['text'],
                                 id = loop['id'])
            self.AddItem(loopItem)

        # load conditions
        for condition in gxmXml.conditions:
            conditionItem = ModelCondition(parent = self, 
                                           x = condition['pos'][0],
                                           y = condition['pos'][1],
                                           width = condition['size'][0],
                                           height = condition['size'][1],
                                           text = condition['text'],
                                           id = condition['id'])
            self.AddItem(conditionItem)

        # define loops & if/else items
        for loop in gxmXml.loops:
            alist = list()
            for aId in loop['items']:
                action = self.GetItem(aId)
                alist.append(action)
            
            loopItem = self.GetItem(loop['id'])
            loopItem.SetItems(alist)
            
            for action in loopItem.GetItems():
                action.SetBlock(loopItem)
        
        for condition in gxmXml.conditions:
            conditionItem = self.GetItem(condition['id'])
            for b in condition['items'].keys():
                alist = list()
                for aId in condition['items'][b]:
                    action = self.GetItem(aId)
                    alist.append(action)
                conditionItem.SetItems(alist, branch = b)
            
            items = conditionItem.GetItems()
            for b in items.keys():
                for action in items[b]:
                    action.SetBlock(conditionItem)
        
    def AddItem(self, newItem):
        """!Add item to the list"""
        iId = newItem.GetId()
        
        i  = 0
        for item in self.items:
            if item.GetId() > iId:
                self.items.insert(i, newItem)
                return
            i += 1
        
        self.items.append(newItem)
        
    def IsValid(self):
        """Return True if model is valid"""
        if self.Validate():
            return False
        
        return True
    
    def Validate(self):
        """!Validate model, return None if model is valid otherwise
        error string"""
        errList = list()

        variables = self.GetVariables().keys()
        pattern = re.compile(r'(.*)(%.+\s?)(.*)')
        for action in self.GetItems(objType = ModelAction):
            cmd = action.GetLog(string = False)
            
            task = menuform.GUI(show = None).ParseCommand(cmd = cmd)
            errList += map(lambda x: cmd[0] + ': ' + x, task.get_cmd_error())
            
            # check also variables
            for opt in cmd[1:]:
                if '=' not in opt:
                    continue
                key, value = opt.split('=', 1)
                sval = pattern.search(value)
                if sval:
                    var = sval.group(2).strip()[1:] # ignore '%'
                    if var not in variables:
                        report = True
                        for item in filter(lambda x: isinstance(x, ModelLoop), action.GetBlock()):
                            if var in item.GetText():
                                report = False
                                break
                        if report:
                            errList.append(_("%s: undefined variable '%s'") % (cmd[0], var))
            ### TODO: check variables in file only optionally
            ### errList += self._substituteFile(action, checkOnly = True)
        
        return errList

    def _substituteFile(self, item, params = None, checkOnly = False):
        """!Subsitute variables in command file inputs

        @param checkOnly tuble - True to check variable, don't touch files
        
        @return list of undefined variables
        """
        errList = list()
        
        self.fileInput = dict()
        
        # collect ascii inputs
        for p in item.GetParams()['params']:
            if p.get('element', '') == 'file' and \
                    p.get('prompt', '') == 'input' and \
                    p.get('age', '') == 'old':
                filename = p.get('value', p.get('default', ''))
                if filename and \
                        mimetypes.guess_type(filename)[0] == 'text/plain':
                    self.fileInput[filename] = None
        
        for finput in self.fileInput:
            # read lines
            fd = open(finput, "r")
            try:
                data = self.fileInput[finput] = fd.read()
            finally:
                fd.close()
            
            # substitute variables
            write = False
            variables = self.GetVariables()
            for variable in variables:
                pattern = re.compile('%' + variable)
                value = ''
                if params and 'variables' in params:
                    for p in params['variables']['params']:
                        if variable == p.get('name', ''):
                            if p.get('type', 'string') == 'string':
                                value = p.get('value', '')
                            else:
                                value = str(p.get('value', ''))
                            break
                
                if not value:
                    value = variables[variable].get('value', '')
                
                data = pattern.sub(value, data)
                if not checkOnly:
                    write = True
            
            pattern = re.compile(r'(.*)(%.+\s?)(.*)')
            sval = pattern.search(data)
            if sval:
                var = sval.group(2).strip()[1:] # ignore '%'
                cmd = item.GetLog(string = False)[0]
                errList.append(_("%s: undefined variable '%s'") % (cmd, var))
            
            if not checkOnly:
                if write:
                    fd = open(finput, "w")
                    try:
                        fd.write(data)
                    finally:
                        fd.close()
                else:
                    self.fileInput[finput] = None
        
        return errList
    
    def OnPrepare(self, item, params):
        self._substituteFile(item, params, checkOnly = False)

    def RunAction(self, item, params, log, onDone, onPrepare = None, statusbar = None):
        """!Run given action

        @param item action item
        @param params parameters dict
        @param log logging window
        @param onDone on-done method
        @param onPrepare on-prepare method
        @param statusbar wx.StatusBar instance or None
        """
        name = item.GetName()
        if name in params:
            paramsOrig = item.GetParams(dcopy = True)
            item.MergeParams(params[name])
        
        if statusbar:
            statusbar.SetStatusText(_('Running model...'), 0)
            
        data = { 'item' : item,
                 'params' : copy.deepcopy(params) }
        log.RunCmd(command = item.GetLog(string = False, substitute = params),
                   onDone = onDone, onPrepare = self.OnPrepare, userData = data)
        
        if name in params:
            item.SetParams(paramsOrig)

    def Run(self, log, onDone, parent = None):
        """!Run model

        @param log logging window (see goutput.GMConsole)
        @param onDone on-done method
        @param parent window for messages or None
        """
        if self.GetNumItems() < 1:
            GMessage(parent = parent,
                     message = _('Model is empty. Nothing to run.'))
            return
        
        statusbar = None
        if isinstance(parent, wx.Frame):
            statusbar = parent.GetStatusBar()
        
        # validation
        if statusbar:
            statusbar.SetStatusText(_('Validating model...'), 0)
        errList = self.Validate()
        if statusbar:
            statusbar.SetStatusText('', 0)
        if errList:
            dlg = wx.MessageDialog(parent = parent,
                                   message = _('Model is not valid. Do you want to '
                                               'run the model anyway?\n\n%s') % '\n'.join(errList),
                                   caption = _("Run model?"),
                                   style = wx.YES_NO | wx.NO_DEFAULT |
                                   wx.ICON_QUESTION | wx.CENTRE)
            ret = dlg.ShowModal()
            if ret != wx.ID_YES:
                return
        
        # parametrization
        params = self.Parameterize()
        if params:
            dlg = ModelParamDialog(parent = parent,
                                   params = params)
            dlg.CenterOnParent()
            
            ret = dlg.ShowModal()
            if ret != wx.ID_OK:
                dlg.Destroy()
                return
            
            err = dlg.GetErrors()
            if err:
                GError(parent = parent, message = unicode('\n'.join(err)))
                return
        
            err = list()
            for key, item in params.iteritems():
                for p in item['params']:
                    if p.get('value', '') == '':
                        err.append((key, p.get('name', ''), p.get('description', '')))
            if err:
                GError(parent = parent,
                       message = _("Variables below not defined:") + \
                           "\n\n" + unicode('\n'.join(map(lambda x: "%s: %s (%s)" % (x[0], x[1], x[2]), err))))
                return
        
        log.cmdThread.SetId(-1)
        for item in self.GetItems():
            if not item.IsEnabled():
                continue
            if isinstance(item, ModelAction):
                if item.GetBlockId():
                    continue
                self.RunAction(item, params, log, onDone)
            elif isinstance(item, ModelLoop):
                cond = item.GetText()
                # substitute variables in condition
                variables = self.GetVariables()
                for variable in variables:
                    pattern = re.compile('%' + variable)
                    if pattern.search(cond):
                        value = ''
                        if params and 'variables' in params:
                            for p in params['variables']['params']:
                                if variable == p.get('name', ''):
                                    value = p.get('value', '')
                                    break
                        
                        if not value:
                            value = variables[variable].get('value', '')
                        
                        if not value:
                            continue
                        
                        vtype = variables[variable].get('type', 'string')
                        if vtype == 'string':
                            value = '"' + value + '"'
                        cond = pattern.sub(value, cond)
                
                # split condition
                condVar, condText = map(lambda x: x.strip(), re.split('\s*in\s*', cond))
                pattern = re.compile('%' + condVar)
                ### for vars()[condVar] in eval(condText): ?
                if condText[0] == '`' and condText[-1] == '`':
                    # run command
                    cmd, dcmd = utils.CmdToTuple(condText[1:-1].split(' '))
                    ret = RunCommand(cmd,
                                     read = True,
                                     **dcmd)
                    if ret:
                        vlist = ret.splitlines()
                else:
                    vlist = eval(condText)
                
                if 'variables' not in params:
                    params['variables'] = { 'params' : [] }
                varDict = { 'name' : condVar, 'value' : '' }
                params['variables']['params'].append(varDict)
                                
                for var in vlist:
                    for action in item.GetItems():
                        if not isinstance(action, ModelAction) or \
                                not action.IsEnabled():
                            continue
                        
                        varDict['value'] = var
                                                
                        self.RunAction(item = action, params = params,
                                       log = log, onDone = onDone)
                params['variables']['params'].remove(varDict)
                
        # discard values
        if params:
            for item in params.itervalues():
                for p in item['params']:
                    p['value'] = ''
        
        if params:
            dlg.Destroy()
        
    def DeleteIntermediateData(self, log):
        """!Detele intermediate data"""
        rast, vect, rast3d, msg = self.GetIntermediateData()
        
        if rast:
            log.RunCmd(['g.remove', 'rast=%s' %','.join(rast)])
        if rast3d:
            log.RunCmd(['g.remove', 'rast3d=%s' %','.join(rast3d)])
        if vect:
            log.RunCmd(['g.remove', 'vect=%s' %','.join(vect)])
        
    def GetIntermediateData(self):
        """!Get info about intermediate data"""
        rast = list()
        rast3d = list()
        vect = list()
        for data in self.GetData():
            if not data.IsIntermediate():
                continue
            name = data.GetValue()
            prompt = data.GetPrompt()
            if prompt == 'raster':
                rast.append(name)
            elif prompt == 'vector':
                vect.append(name)
            elif prompt == 'rast3d':
                rast3d.append(name)
        
        msg = ''
        if rast:
            msg += '\n\n%s: ' % _('Raster maps')
            msg += ', '.join(rast)
        if rast3d:
            msg += '\n\n%s: ' % _('3D raster maps')
            msg += ', '.join(rast3d)
        if vect:
            msg += '\n\n%s: ' % _('Vector maps')
            msg += ', '.join(vect)
        
        return rast, vect, rast3d, msg

    def Update(self):
        """!Update model"""
        for item in self.items:
            item.Update()
        
    def IsParameterized(self):
        """!Return True if model is parameterized"""
        if self.Parameterize():
            return True
        
        return False
    
    def Parameterize(self):
        """!Return parameterized options"""
        result = dict()
        idx = 0
        if self.variables:
            params = list()
            result["variables"] = { 'flags'  : list(),
                                    'params' : params,
                                    'idx'    : idx }
            for name, values in self.variables.iteritems():
                gtype = values.get('type', 'string')
                if gtype in ('raster', 'vector', 'mapset', 'file'):
                    gisprompt = True
                    prompt = gtype
                    if gtype == 'raster':
                        element = 'cell'
                    else:
                        element = gtype
                    ptype = 'string'
                else:
                    gisprompt = False
                    prompt = None
                    element = None
                    ptype = gtype
                params.append({ 'gisprompt' : gisprompt,
                                'multiple'  : False,
                                'description' : values.get('description', ''),
                                'guidependency' : '',
                                'default' : '',
                                'age' : None,
                                'required' : True,
                                'value' : values.get('value', ''),
                                'label' : '',
                                'guisection' : '',
                                'key_desc' : '',
                                'values' : list(),
                                'parameterized' : False,
                                'values_desc' : list(),
                                'prompt' : prompt,
                                'element' : element,
                                'type' : ptype,
                                'name' : name })
            
            idx += 1
        
        for action in self.GetItems(objType = ModelAction):
            if not action.IsEnabled():
                continue
            name   = action.GetName()
            params = action.GetParams()
            for f in params['flags']:
                if f.get('parameterized', False):
                    if name not in result:
                        result[name] = { 'flags' : list(),
                                         'params': list(),
                                         'idx'   : idx }
                    result[name]['flags'].append(f)
            for p in params['params']:
                if p.get('parameterized', False):
                    if name not in result:
                        result[name] = { 'flags' : list(),
                                         'params': list(),
                                         'idx'   : idx }
                    result[name]['params'].append(p)
            if name in result:
                idx += 1
        
        self.variablesParams = result # record parameters
        
        return result