core.py 60 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014
  1. """
  2. Core functions to be used in Python scripts.
  3. Usage:
  4. ::
  5. from grass.script import core as grass
  6. grass.parser()
  7. (C) 2008-2021 by the GRASS Development Team
  8. This program is free software under the GNU General Public
  9. License (>=v2). Read the file COPYING that comes with GRASS
  10. for details.
  11. .. sectionauthor:: Glynn Clements
  12. .. sectionauthor:: Martin Landa <landa.martin gmail.com>
  13. .. sectionauthor:: Michael Barton <michael.barton asu.edu>
  14. """
  15. from __future__ import absolute_import, print_function
  16. import os
  17. import sys
  18. import atexit
  19. import subprocess
  20. import shutil
  21. import codecs
  22. import string
  23. import random
  24. import pipes
  25. from tempfile import NamedTemporaryFile
  26. from .utils import KeyValue, parse_key_val, basename, encode, decode, try_remove
  27. from grass.exceptions import ScriptError, CalledModuleError
  28. # PY2/PY3 compat
  29. if sys.version_info.major >= 3:
  30. unicode = str
  31. # subprocess wrapper that uses shell on Windows
  32. class Popen(subprocess.Popen):
  33. _builtin_exts = set([".com", ".exe", ".bat", ".cmd"])
  34. @staticmethod
  35. def _escape_for_shell(arg):
  36. # TODO: what are cmd.exe's parsing rules?
  37. return arg
  38. def __init__(self, args, **kwargs):
  39. if (
  40. sys.platform == "win32"
  41. and isinstance(args, list)
  42. and not kwargs.get("shell", False)
  43. and kwargs.get("executable") is None
  44. ):
  45. cmd = shutil_which(args[0])
  46. if cmd is None:
  47. raise OSError(_("Cannot find the executable {0}").format(args[0]))
  48. args = [cmd] + args[1:]
  49. name, ext = os.path.splitext(cmd)
  50. if ext.lower() not in self._builtin_exts:
  51. kwargs["shell"] = True
  52. args = [self._escape_for_shell(arg) for arg in args]
  53. subprocess.Popen.__init__(self, args, **kwargs)
  54. PIPE = subprocess.PIPE
  55. STDOUT = subprocess.STDOUT
  56. raise_on_error = False # raise exception instead of calling fatal()
  57. _capture_stderr = False # capture stderr of subprocesses if possible
  58. def call(*args, **kwargs):
  59. return Popen(*args, **kwargs).wait()
  60. # GRASS-oriented interface to subprocess module
  61. _popen_args = [
  62. "bufsize",
  63. "executable",
  64. "stdin",
  65. "stdout",
  66. "stderr",
  67. "preexec_fn",
  68. "close_fds",
  69. "cwd",
  70. "env",
  71. "universal_newlines",
  72. "startupinfo",
  73. "creationflags",
  74. "encoding",
  75. ]
  76. def _make_val(val):
  77. """Convert value to unicode"""
  78. if isinstance(val, (bytes, str, unicode)):
  79. return decode(val)
  80. if isinstance(val, (int, float)):
  81. return unicode(val)
  82. try:
  83. return ",".join(map(_make_val, iter(val)))
  84. except TypeError:
  85. pass
  86. return unicode(val)
  87. def _make_unicode(val, enc):
  88. """Convert value to unicode with given encoding
  89. :param val: value to be converted
  90. :param enc: encoding to be used
  91. """
  92. if val is None or enc is None:
  93. return val
  94. else:
  95. if enc == "default":
  96. return decode(val)
  97. else:
  98. return decode(val, encoding=enc)
  99. def get_commands():
  100. """Create list of available GRASS commands to use when parsing
  101. string from the command line
  102. :return: list of commands (set) and directory of scripts (collected
  103. by extension - MS Windows only)
  104. >>> cmds = list(get_commands()[0])
  105. >>> cmds.sort()
  106. >>> cmds[:5]
  107. ['d.barscale', 'd.colorlist', 'd.colortable', 'd.correlate', 'd.erase']
  108. """
  109. gisbase = os.environ["GISBASE"]
  110. cmd = list()
  111. scripts = {".py": list()} if sys.platform == "win32" else {}
  112. def scan(gisbase, directory):
  113. dir_path = os.path.join(gisbase, directory)
  114. if os.path.exists(dir_path):
  115. for fname in os.listdir(os.path.join(gisbase, directory)):
  116. if scripts: # win32
  117. name, ext = os.path.splitext(fname)
  118. if ext != ".manifest":
  119. cmd.append(name)
  120. if ext in scripts.keys():
  121. scripts[ext].append(name)
  122. else:
  123. cmd.append(fname)
  124. for directory in ("bin", "scripts"):
  125. scan(gisbase, directory)
  126. # scan gui/scripts/
  127. gui_path = os.path.join(gisbase, "etc", "gui", "scripts")
  128. if os.path.exists(gui_path):
  129. os.environ["PATH"] = os.getenv("PATH") + os.pathsep + gui_path
  130. cmd = cmd + os.listdir(gui_path)
  131. return set(cmd), scripts
  132. # TODO: Please replace this function with shutil.which() before 8.0 comes out
  133. # replacement for which function from shutil (not available in all versions)
  134. # from http://hg.python.org/cpython/file/6860263c05b3/Lib/shutil.py#l1068
  135. # added because of Python scripts running Python scripts on MS Windows
  136. # see also ticket #2008 which is unrelated but same function was proposed
  137. def shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None):
  138. """Given a command, mode, and a PATH string, return the path which
  139. conforms to the given mode on the PATH, or None if there is no such
  140. file.
  141. `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
  142. of os.environ.get("PATH"), or can be overridden with a custom search
  143. path.
  144. :param cmd: the command
  145. :param mode:
  146. :param path:
  147. """
  148. # Check that a given file can be accessed with the correct mode.
  149. # Additionally check that `file` is not a directory, as on Windows
  150. # directories pass the os.access check.
  151. def _access_check(fn, mode):
  152. return os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)
  153. # If we're given a path with a directory part, look it up directly rather
  154. # than referring to PATH directories. This includes checking relative to the
  155. # current directory, e.g. ./script
  156. if os.path.dirname(cmd):
  157. if _access_check(cmd, mode):
  158. return cmd
  159. return None
  160. if path is None:
  161. path = os.environ.get("PATH", os.defpath)
  162. if not path:
  163. return None
  164. path = path.split(os.pathsep)
  165. if sys.platform == "win32":
  166. # The current directory takes precedence on Windows.
  167. if os.curdir not in path:
  168. path.insert(0, os.curdir)
  169. # PATHEXT is necessary to check on Windows (force lowercase)
  170. pathext = list(
  171. map(lambda x: x.lower(), os.environ.get("PATHEXT", "").split(os.pathsep))
  172. )
  173. if ".py" not in pathext:
  174. # we assume that PATHEXT contains always '.py'
  175. pathext.insert(0, ".py")
  176. # See if the given file matches any of the expected path extensions.
  177. # This will allow us to short circuit when given "python3.exe".
  178. # If it does match, only test that one, otherwise we have to try
  179. # others.
  180. if any(cmd.lower().endswith(ext) for ext in pathext):
  181. files = [cmd]
  182. else:
  183. files = [cmd + ext for ext in pathext]
  184. else:
  185. # On other platforms you don't have things like PATHEXT to tell you
  186. # what file suffixes are executable, so just pass on cmd as-is.
  187. files = [cmd]
  188. seen = set()
  189. for dir in path:
  190. normdir = os.path.normcase(dir)
  191. if normdir not in seen:
  192. seen.add(normdir)
  193. for thefile in files:
  194. name = os.path.join(dir, thefile)
  195. if _access_check(name, mode):
  196. return name
  197. return None
  198. if sys.version_info.major >= 3:
  199. # Use shutil.which in Python 3, not the custom implementation.
  200. shutil_which = shutil.which # noqa: F811
  201. # Added because of scripts calling scripts on MS Windows.
  202. # Module name (here cmd) differs from the file name (does not have extension).
  203. # Additionally, we don't run scripts using system executable mechanism,
  204. # so we need the full path name.
  205. # However, scripts are on the PATH and '.PY' in in PATHEXT, so we can use
  206. # shutil.which to get the full file path. Addons are on PATH too.
  207. # An alternative to which function call would be to check the script path and
  208. # addons path. This is proposed improvement for the future.
  209. # Another alternative is to check some global list of scripts but this list
  210. # needs to be created first. The question is what is less expensive.
  211. # Note that getting the full path is only part of the solution,
  212. # the other part is to use the right Python as an executable and pass the full
  213. # script path as a parameter.
  214. # Nevertheless, it is unclear on which places which extensions are added.
  215. # This function also could skip the check for platform but depends
  216. # how will be used, this is most general but not most effective.
  217. def get_real_command(cmd):
  218. """Returns the real file command for a module (cmd)
  219. For Python scripts on MS Windows it returns full path to the script
  220. and adds a '.py' extension.
  221. For other cases it just returns a module (name).
  222. So, you can just use this function for all without further check.
  223. >>> get_real_command('g.region')
  224. 'g.region'
  225. :param cmd: the command
  226. """
  227. if sys.platform == "win32":
  228. # we in fact expect pure module name (without extension)
  229. # so, lets remove extension
  230. if os.path.splitext(cmd)[1] == ".py":
  231. cmd = cmd[:-3]
  232. # PATHEXT is necessary to check on Windows (force lowercase)
  233. pathext = list(
  234. map(lambda x: x.lower(), os.environ["PATHEXT"].split(os.pathsep))
  235. )
  236. if ".py" not in pathext:
  237. # we assume that PATHEXT contains always '.py'
  238. os.environ["PATHEXT"] = ".py;" + os.environ["PATHEXT"]
  239. full_path = shutil_which(cmd + ".py")
  240. if full_path:
  241. return full_path
  242. return cmd
  243. def make_command(
  244. prog,
  245. flags="",
  246. overwrite=False,
  247. quiet=False,
  248. verbose=False,
  249. superquiet=False,
  250. errors=None,
  251. **options,
  252. ):
  253. """Return a list of strings suitable for use as the args parameter to
  254. Popen() or call(). Example:
  255. >>> make_command("g.message", flags = 'w', message = 'this is a warning')
  256. ['g.message', '-w', 'message=this is a warning']
  257. :param str prog: GRASS module
  258. :param str flags: flags to be used (given as a string)
  259. :param bool overwrite: True to enable overwriting the output (<tt>--o</tt>)
  260. :param bool quiet: True to run quietly (<tt>--q</tt>)
  261. :param bool verbose: True to run verbosely (<tt>--v</tt>)
  262. :param options: module's parameters
  263. :return: list of arguments
  264. """
  265. args = [_make_val(prog)]
  266. if overwrite:
  267. args.append("--o")
  268. if quiet:
  269. args.append("--q")
  270. if verbose:
  271. args.append("--v")
  272. if superquiet:
  273. args.append("--qq")
  274. if flags:
  275. flags = _make_val(flags)
  276. if "-" in flags:
  277. raise ScriptError("'-' is not a valid flag")
  278. args.append("-" + flags)
  279. for opt, val in options.items():
  280. if opt in _popen_args:
  281. continue
  282. # convert string to bytes
  283. if val is not None:
  284. if opt.startswith("_"):
  285. opt = opt[1:]
  286. warning(
  287. _(
  288. "To run the module <%s> add underscore at the end"
  289. " of the option <%s> to avoid conflict with Python"
  290. " keywords. Underscore at the beginning is"
  291. " deprecated in GRASS GIS 7.0 and has been removed"
  292. " in version 7.1."
  293. )
  294. % (prog, opt)
  295. )
  296. elif opt.endswith("_"):
  297. opt = opt[:-1]
  298. args.append(opt + "=" + _make_val(val))
  299. return args
  300. def handle_errors(returncode, result, args, kwargs):
  301. """Error handler for :func:`run_command()` and similar functions
  302. The functions which are using this function to handle errors,
  303. can be typically called with an *errors* parameter.
  304. This function can handle one of the following values: raise,
  305. fatal, status, exit, and ignore. The value raise is a default.
  306. If returncode is 0, *result* is returned, unless
  307. ``errors="status"`` is set.
  308. If *kwargs* dictionary contains key ``errors``, the value is used
  309. to determine the return value and the behavior on error.
  310. The value ``errors="raise"`` is a default in which case a
  311. ``CalledModuleError`` exception is raised.
  312. For ``errors="fatal"``, the function calls :func:`fatal()`
  313. which has its own rules on what happens next.
  314. For ``errors="status"``, the *returncode* will be returned.
  315. This is useful, e.g., for cases when the exception-based error
  316. handling mechanism is not desirable or the return code has some
  317. meaning not necessarily interpreted as an error by the caller.
  318. For ``errors="exit"``, ``sys.exit()`` is called with the
  319. *returncode*, so it behaves similarly to a Bash script with
  320. ``set -e``. No additional error message or exception is produced.
  321. This might be useful for a simple script where error message
  322. produced by the called module provides sufficient information about
  323. what happened to the end user.
  324. Finally, for ``errors="ignore"``, the value of *result* will be
  325. passed in any case regardless of the *returncode*.
  326. """
  327. def get_module_and_code(args, kwargs):
  328. """Get module name and formatted command"""
  329. # TODO: construction of the whole command is far from perfect
  330. args = make_command(*args, **kwargs)
  331. # Since we are in error handler, let's be extra cautious
  332. # about an empty command.
  333. if args:
  334. module = args[0]
  335. else:
  336. module = None
  337. code = " ".join(args)
  338. return module, code
  339. handler = kwargs.get("errors", "raise")
  340. if handler.lower() == "status":
  341. return returncode
  342. if returncode == 0:
  343. return result
  344. if handler.lower() == "ignore":
  345. return result
  346. elif handler.lower() == "fatal":
  347. module, code = get_module_and_code(args, kwargs)
  348. fatal(
  349. _(
  350. "Module {module} ({code}) failed with"
  351. " non-zero return code {returncode}"
  352. ).format(module=module, code=code, returncode=returncode)
  353. )
  354. elif handler.lower() == "exit":
  355. sys.exit(returncode)
  356. else:
  357. module, code = get_module_and_code(args, kwargs)
  358. raise CalledModuleError(module=module, code=code, returncode=returncode)
  359. def start_command(
  360. prog,
  361. flags="",
  362. overwrite=False,
  363. quiet=False,
  364. verbose=False,
  365. superquiet=False,
  366. **kwargs,
  367. ):
  368. """Returns a Popen object with the command created by make_command.
  369. Accepts any of the arguments which Popen() accepts apart from "args"
  370. and "shell".
  371. >>> p = start_command("g.gisenv", stdout=subprocess.PIPE)
  372. >>> print(p) # doctest: +ELLIPSIS
  373. <...Popen object at 0x...>
  374. >>> print(p.communicate()[0]) # doctest: +SKIP
  375. GISDBASE='/opt/grass-data';
  376. LOCATION_NAME='spearfish60';
  377. MAPSET='glynn';
  378. GUI='text';
  379. MONITOR='x0';
  380. If the module parameter is the same as Python keyword, add
  381. underscore at the end of the parameter. For example, use
  382. ``lambda_=1.6`` instead of ``lambda=1.6``.
  383. :param str prog: GRASS module
  384. :param str flags: flags to be used (given as a string)
  385. :param bool overwrite: True to enable overwriting the output (<tt>--o</tt>)
  386. :param bool quiet: True to run quietly (<tt>--q</tt>)
  387. :param bool verbose: True to run verbosely (<tt>--v</tt>)
  388. :param kwargs: module's parameters
  389. :return: Popen object
  390. """
  391. options = {}
  392. popts = {}
  393. for opt, val in kwargs.items():
  394. if opt in _popen_args:
  395. popts[opt] = val
  396. else:
  397. options[opt] = val
  398. args = make_command(prog, flags, overwrite, quiet, verbose, **options)
  399. if debug_level() > 0:
  400. sys.stderr.write(
  401. "D1/{}: {}.start_command(): {}\n".format(
  402. debug_level(), __name__, " ".join(args)
  403. )
  404. )
  405. sys.stderr.flush()
  406. return Popen(args, **popts)
  407. def run_command(*args, **kwargs):
  408. """Execute a module synchronously
  409. This function passes all arguments to ``start_command()``,
  410. then waits for the process to complete. It is similar to
  411. ``subprocess.check_call()``, but with the :func:`make_command()`
  412. interface. By default, an exception is raised in case of a non-zero
  413. return code by default.
  414. >>> run_command('g.region', raster='elevation')
  415. See :func:`start_command()` for details about parameters and usage.
  416. The behavior on error can be changed using *errors* parameter
  417. which is passed to the :func:`handle_errors()` function.
  418. :param *args: unnamed arguments passed to :func:`start_command()`
  419. :param **kwargs: named arguments passed to :func:`start_command()`
  420. :param str errors: passed to :func:`handle_errors()`
  421. .. versionchanged:: 8.0
  422. Before 8.0, the function was returning 0 when no error occurred
  423. for backward compatibility with code which was checking that
  424. value. Now the function returns None, unless ``errors="status"``
  425. is specified.
  426. .. versionchanged:: 7.2
  427. In 7.0.0, this function was returning the error code. However,
  428. it was rarely checked especially outside of the core code.
  429. Additionally, :func:`read_command()` needed a mechanism to
  430. report errors as it was used more and more in context which
  431. required error handling, Thus, exceptions were introduced as a
  432. more expected default behavior for Python programmers. The
  433. change was backported to 7.0 series.
  434. :raises: ``CalledModuleError`` when module returns non-zero return code
  435. """
  436. encoding = "default"
  437. if "encoding" in kwargs:
  438. encoding = kwargs["encoding"]
  439. if _capture_stderr and "stderr" not in kwargs.keys():
  440. kwargs["stderr"] = PIPE
  441. ps = start_command(*args, **kwargs)
  442. if _capture_stderr:
  443. stdout, stderr = ps.communicate()
  444. if encoding is not None:
  445. stdout = _make_unicode(stdout, encoding)
  446. stderr = _make_unicode(stderr, encoding)
  447. returncode = ps.poll()
  448. if returncode:
  449. sys.stderr.write(stderr)
  450. else:
  451. returncode = ps.wait()
  452. return handle_errors(returncode, result=None, args=args, kwargs=kwargs)
  453. def pipe_command(*args, **kwargs):
  454. """Passes all arguments to start_command(), but also adds
  455. "stdout = PIPE". Returns the Popen object.
  456. >>> p = pipe_command("g.gisenv")
  457. >>> print(p) # doctest: +ELLIPSIS
  458. <....Popen object at 0x...>
  459. >>> print(p.communicate()[0]) # doctest: +SKIP
  460. GISDBASE='/opt/grass-data';
  461. LOCATION_NAME='spearfish60';
  462. MAPSET='glynn';
  463. GUI='text';
  464. MONITOR='x0';
  465. :param list args: list of unnamed arguments (see start_command() for details)
  466. :param list kwargs: list of named arguments (see start_command() for details)
  467. :return: Popen object
  468. """
  469. kwargs["stdout"] = PIPE
  470. return start_command(*args, **kwargs)
  471. def feed_command(*args, **kwargs):
  472. """Passes all arguments to start_command(), but also adds
  473. "stdin = PIPE". Returns the Popen object.
  474. :param list args: list of unnamed arguments (see start_command() for details)
  475. :param list kwargs: list of named arguments (see start_command() for details)
  476. :return: Popen object
  477. """
  478. kwargs["stdin"] = PIPE
  479. return start_command(*args, **kwargs)
  480. def read_command(*args, **kwargs):
  481. """Passes all arguments to pipe_command, then waits for the process to
  482. complete, returning its stdout (i.e. similar to shell `backticks`).
  483. The behavior on error can be changed using *errors* parameter
  484. which is passed to the :func:`handle_errors()` function.
  485. :param list args: list of unnamed arguments (see start_command() for details)
  486. :param list kwargs: list of named arguments (see start_command() for details)
  487. :return: stdout
  488. """
  489. encoding = "default"
  490. if "encoding" in kwargs:
  491. encoding = kwargs["encoding"]
  492. if _capture_stderr and "stderr" not in kwargs.keys():
  493. kwargs["stderr"] = PIPE
  494. process = pipe_command(*args, **kwargs)
  495. stdout, stderr = process.communicate()
  496. if encoding is not None:
  497. stdout = _make_unicode(stdout, encoding)
  498. stderr = _make_unicode(stderr, encoding)
  499. returncode = process.poll()
  500. if _capture_stderr and returncode:
  501. sys.stderr.write(stderr)
  502. return handle_errors(returncode, stdout, args, kwargs)
  503. def parse_command(*args, **kwargs):
  504. """Passes all arguments to read_command, then parses the output
  505. by parse_key_val().
  506. Parsing function can be optionally given by <em>parse</em> parameter
  507. including its arguments, e.g.
  508. ::
  509. parse_command(..., parse = (grass.parse_key_val, { 'sep' : ':' }))
  510. or you can simply define <em>delimiter</em>
  511. ::
  512. parse_command(..., delimiter = ':')
  513. :param args: list of unnamed arguments (see start_command() for details)
  514. :param kwargs: list of named arguments (see start_command() for details)
  515. :return: parsed module output
  516. """
  517. parse = None
  518. parse_args = {}
  519. if "parse" in kwargs:
  520. if isinstance(kwargs["parse"], tuple):
  521. parse = kwargs["parse"][0]
  522. parse_args = kwargs["parse"][1]
  523. del kwargs["parse"]
  524. if "delimiter" in kwargs:
  525. parse_args = {"sep": kwargs["delimiter"]}
  526. del kwargs["delimiter"]
  527. if not parse:
  528. parse = parse_key_val # use default fn
  529. res = read_command(*args, **kwargs)
  530. return parse(res, **parse_args)
  531. def write_command(*args, **kwargs):
  532. """Execute a module with standard input given by *stdin* parameter.
  533. Passes all arguments to ``feed_command()``, with the string specified
  534. by the *stdin* argument fed to the process' standard input.
  535. >>> write_command(
  536. ... 'v.in.ascii', input='-',
  537. ... stdin='%s|%s' % (635818.8, 221342.4),
  538. ... output='view_point')
  539. 0
  540. See ``start_command()`` for details about parameters and usage.
  541. The behavior on error can be changed using *errors* parameter
  542. which is passed to the :func:`handle_errors()` function.
  543. :param *args: unnamed arguments passed to ``start_command()``
  544. :param **kwargs: named arguments passed to ``start_command()``
  545. :returns: 0 with default parameters for backward compatibility only
  546. :raises: ``CalledModuleError`` when module returns non-zero return code
  547. """
  548. encoding = "default"
  549. if "encoding" in kwargs:
  550. encoding = kwargs["encoding"]
  551. # TODO: should we delete it from kwargs?
  552. stdin = kwargs["stdin"]
  553. if encoding is None or encoding == "default":
  554. stdin = encode(stdin)
  555. else:
  556. stdin = encode(stdin, encoding=encoding)
  557. if _capture_stderr and "stderr" not in kwargs.keys():
  558. kwargs["stderr"] = PIPE
  559. process = feed_command(*args, **kwargs)
  560. unused, stderr = process.communicate(stdin)
  561. if encoding is not None:
  562. unused = _make_unicode(unused, encoding)
  563. stderr = _make_unicode(stderr, encoding)
  564. returncode = process.poll()
  565. if _capture_stderr and returncode:
  566. sys.stderr.write(stderr)
  567. return handle_errors(returncode, None, args, kwargs)
  568. def exec_command(
  569. prog,
  570. flags="",
  571. overwrite=False,
  572. quiet=False,
  573. verbose=False,
  574. superquiet=False,
  575. env=None,
  576. **kwargs,
  577. ):
  578. """Interface to os.execvpe(), but with the make_command() interface.
  579. :param str prog: GRASS module
  580. :param str flags: flags to be used (given as a string)
  581. :param bool overwrite: True to enable overwriting the output (<tt>--o</tt>)
  582. :param bool quiet: True to run quietly (<tt>--q</tt>)
  583. :param bool verbose: True to run verbosely (<tt>--v</tt>)
  584. :param env: directory with environmental variables
  585. :param list kwargs: module's parameters
  586. """
  587. args = make_command(prog, flags, overwrite, quiet, verbose, **kwargs)
  588. if env is None:
  589. env = os.environ
  590. os.execvpe(prog, args, env)
  591. # interface to g.message
  592. def message(msg, flag=None):
  593. """Display a message using `g.message`
  594. :param str msg: message to be displayed
  595. :param str flag: flags (given as string)
  596. """
  597. run_command("g.message", flags=flag, message=msg, errors="ignore")
  598. def debug(msg, debug=1):
  599. """Display a debugging message using `g.message -d`
  600. :param str msg: debugging message to be displayed
  601. :param str debug: debug level (0-5)
  602. """
  603. if debug_level() >= debug:
  604. # TODO: quite a random hack here, do we need it somewhere else too?
  605. if sys.platform == "win32":
  606. msg = msg.replace("&", "^&")
  607. run_command("g.message", flags="d", message=msg, debug=debug)
  608. def verbose(msg):
  609. """Display a verbose message using `g.message -v`
  610. :param str msg: verbose message to be displayed
  611. """
  612. message(msg, flag="v")
  613. def info(msg):
  614. """Display an informational message using `g.message -i`
  615. :param str msg: informational message to be displayed
  616. """
  617. message(msg, flag="i")
  618. def percent(i, n, s):
  619. """Display a progress info message using `g.message -p`
  620. ::
  621. message(_("Percent complete..."))
  622. n = 100
  623. for i in range(n):
  624. percent(i, n, 1)
  625. percent(1, 1, 1)
  626. :param int i: current item
  627. :param int n: total number of items
  628. :param int s: increment size
  629. """
  630. message("%d %d %d" % (i, n, s), flag="p")
  631. def warning(msg):
  632. """Display a warning message using `g.message -w`
  633. :param str msg: warning message to be displayed
  634. """
  635. message(msg, flag="w")
  636. def error(msg):
  637. """Display an error message using `g.message -e`
  638. This function does not end the execution of the program.
  639. The right action after the error is up to the caller.
  640. For error handling using the standard mechanism use :func:`fatal()`.
  641. :param str msg: error message to be displayed
  642. """
  643. message(msg, flag="e")
  644. def fatal(msg):
  645. """Display an error message using `g.message -e`, then abort or raise
  646. Raises exception when module global raise_on_error is 'True', abort
  647. (calls exit) otherwise.
  648. Use :func:`set_raise_on_error()` to set the behavior.
  649. :param str msg: error message to be displayed
  650. """
  651. global raise_on_error
  652. if raise_on_error:
  653. raise ScriptError(msg)
  654. error(msg)
  655. sys.exit(1)
  656. def set_raise_on_error(raise_exp=True):
  657. """Define behaviour on fatal error (fatal() called)
  658. :param bool raise_exp: True to raise ScriptError instead of calling
  659. sys.exit(1) in fatal()
  660. :return: current status
  661. """
  662. global raise_on_error
  663. tmp_raise = raise_on_error
  664. raise_on_error = raise_exp
  665. return tmp_raise
  666. def get_raise_on_error():
  667. """Return True if a ScriptError exception is raised instead of calling
  668. sys.exit(1) in case a fatal error was invoked with fatal()
  669. """
  670. global raise_on_error
  671. return raise_on_error
  672. # TODO: solve also warnings (not printed now)
  673. def set_capture_stderr(capture=True):
  674. """Enable capturing standard error output of modules and print it.
  675. By default, standard error output (stderr) of child processes shows
  676. in the same place as output of the parent process. This may not
  677. always be the same place as ``sys.stderr`` is written.
  678. After calling this function, functions in the ``grass.script``
  679. package will capture the stderr of child processes and pass it
  680. to ``sys.stderr`` if there is an error.
  681. .. note::
  682. This is advantages for interactive shells such as the one in GUI
  683. and interactive notebooks such as Jupyer Notebook.
  684. The capturing can be applied only in certain cases, for example
  685. in case of run_command() it is applied because run_command() nor
  686. its callers do not handle the streams, however feed_command()
  687. cannot do capturing because its callers handle the streams.
  688. The previous state is returned. Passing ``False`` disables the
  689. capturing.
  690. .. versionadded:: 7.4
  691. """
  692. global _capture_stderr
  693. tmp = _capture_stderr
  694. _capture_stderr = capture
  695. return tmp
  696. def get_capture_stderr():
  697. """Return True if stderr is captured, False otherwise.
  698. See set_capture_stderr().
  699. """
  700. global _capture_stderr
  701. return _capture_stderr
  702. # interface to g.parser
  703. def _parse_opts(lines):
  704. options = {}
  705. flags = {}
  706. for line in lines:
  707. if not line:
  708. break
  709. try:
  710. var, val = line.split(b"=", 1)
  711. except ValueError:
  712. raise SyntaxError("invalid output from g.parser: {}".format(line))
  713. try:
  714. var = decode(var)
  715. val = decode(val)
  716. except UnicodeError as error:
  717. raise SyntaxError(
  718. "invalid output from g.parser ({error}): {line}".format(
  719. error=error, line=line
  720. )
  721. )
  722. if var.startswith("flag_"):
  723. flags[var[5:]] = bool(int(val))
  724. elif var.startswith("opt_"):
  725. options[var[4:]] = val
  726. elif var in ["GRASS_OVERWRITE", "GRASS_VERBOSE"]:
  727. os.environ[var] = val
  728. else:
  729. raise SyntaxError(
  730. "unexpected output variable from g.parser: {}".format(line)
  731. )
  732. return (options, flags)
  733. def parser():
  734. """Interface to g.parser, intended to be run from the top-level, e.g.:
  735. ::
  736. if __name__ == "__main__":
  737. options, flags = grass.parser()
  738. main()
  739. Thereafter, the global variables "options" and "flags" will be
  740. dictionaries containing option/flag values, keyed by lower-case
  741. option/flag names. The values in "options" are strings, those in
  742. "flags" are Python booleans.
  743. Overview table of parser standard options:
  744. https://grass.osgeo.org/grass80/manuals/parser_standard_options.html
  745. """
  746. if not os.getenv("GISBASE"):
  747. print("You must be in GRASS GIS to run this program.", file=sys.stderr)
  748. sys.exit(1)
  749. cmdline = [basename(sys.argv[0])]
  750. cmdline += [pipes.quote(a) for a in sys.argv[1:]]
  751. os.environ["CMDLINE"] = " ".join(cmdline)
  752. argv = sys.argv[:]
  753. name = argv[0]
  754. if not os.path.isabs(name):
  755. if os.sep in name or (os.altsep and os.altsep in name):
  756. argv[0] = os.path.abspath(name)
  757. else:
  758. argv[0] = os.path.join(sys.path[0], name)
  759. prog = "g.parser.exe" if sys.platform == "win32" else "g.parser"
  760. p = subprocess.Popen([prog, "-n"] + argv, stdout=subprocess.PIPE)
  761. s = p.communicate()[0]
  762. lines = s.split(b"\0")
  763. if not lines or lines[0] != b"@ARGS_PARSED@":
  764. stdout = os.fdopen(sys.stdout.fileno(), "wb")
  765. stdout.write(s)
  766. sys.exit(p.returncode)
  767. return _parse_opts(lines[1:])
  768. # interface to g.tempfile
  769. def tempfile(create=True, env=None):
  770. """Returns the name of a temporary file, created with g.tempfile.
  771. :param bool create: True to create a file
  772. :param env: environment
  773. :return: path to a tmp file
  774. """
  775. flags = ""
  776. if not create:
  777. flags += "d"
  778. return read_command("g.tempfile", flags=flags, pid=os.getpid(), env=env).strip()
  779. def tempdir(env=None):
  780. """Returns the name of a temporary dir, created with g.tempfile."""
  781. tmp = tempfile(create=False, env=env)
  782. os.mkdir(tmp)
  783. return tmp
  784. def tempname(length, lowercase=False):
  785. """Generate a GRASS and SQL compliant random name starting with tmp_
  786. followed by a random part of length "length"
  787. :param int length: length of the random part of the name to generate
  788. :param bool lowercase: use only lowercase characters to generate name
  789. :returns: String with a random name of length "length" starting with a letter
  790. :rtype: str
  791. :Example:
  792. >>> tempname(12)
  793. 'tmp_MxMa1kAS13s9'
  794. .. seealso:: functions :func:`append_uuid()`, :func:`append_random()`
  795. """
  796. chars = string.ascii_lowercase + string.digits
  797. if not lowercase:
  798. chars += string.ascii_uppercase
  799. random_part = "".join(random.choice(chars) for _ in range(length))
  800. randomname = "tmp_" + random_part
  801. return randomname
  802. def _compare_projection(dic):
  803. """Check if projection has some possibility of duplicate names like
  804. Universal Transverse Mercator and Universe Transverse Mercator and
  805. unify them
  806. :param dic: The dictionary containing information about projection
  807. :return: The dictionary with the new values if needed
  808. """
  809. # the lookup variable is a list of list, each list contains all the
  810. # possible name for a projection system
  811. lookup = [["Universal Transverse Mercator", "Universe Transverse Mercator"]]
  812. for lo in lookup:
  813. for n in range(len(dic["name"])):
  814. if dic["name"][n] in lo:
  815. dic["name"][n] = lo[0]
  816. return dic
  817. def _compare_units(dic):
  818. """Check if units has some possibility of duplicate names like
  819. meter and metre and unify them
  820. :param dic: The dictionary containing information about units
  821. :return: The dictionary with the new values if needed
  822. """
  823. # the lookup variable is a list of list, each list contains all the
  824. # possible name for a units
  825. lookup = [
  826. ["meter", "metre"],
  827. ["meters", "metres"],
  828. ["kilometer", "kilometre"],
  829. ["kilometers", "kilometres"],
  830. ]
  831. for item in lookup:
  832. for n in range(len(dic["unit"])):
  833. if dic["unit"][n].lower() in item:
  834. dic["unit"][n] = item[0]
  835. for n in range(len(dic["units"])):
  836. if dic["units"][n].lower() in item:
  837. dic["units"][n] = item[0]
  838. return dic
  839. def _text_to_key_value_dict(
  840. filename, sep=":", val_sep=",", checkproj=False, checkunits=False
  841. ):
  842. """Convert a key-value text file, where entries are separated by newlines
  843. and the key and value are separated by `sep', into a key-value dictionary
  844. and discover/use the correct data types (float, int or string) for values.
  845. :param str filename: The name or name and path of the text file to convert
  846. :param str sep: The character that separates the keys and values, default
  847. is ":"
  848. :param str val_sep: The character that separates the values of a single
  849. key, default is ","
  850. :param bool checkproj: True if it has to check some information about
  851. projection system
  852. :param bool checkproj: True if it has to check some information about units
  853. :return: The dictionary
  854. A text file with this content:
  855. ::
  856. a: Hello
  857. b: 1.0
  858. c: 1,2,3,4,5
  859. d : hello,8,0.1
  860. Will be represented as this dictionary:
  861. ::
  862. {'a': ['Hello'], 'c': [1, 2, 3, 4, 5], 'b': [1.0], 'd': ['hello', 8, 0.1]}
  863. """
  864. text = open(filename, "r").readlines()
  865. kvdict = KeyValue()
  866. for line in text:
  867. if line.find(sep) >= 0:
  868. key, value = line.split(sep)
  869. key = key.strip()
  870. value = value.strip()
  871. else:
  872. # Jump over empty values
  873. continue
  874. values = value.split(val_sep)
  875. value_list = []
  876. for value in values:
  877. not_float = False
  878. not_int = False
  879. # Convert values into correct types
  880. # We first try integer then float
  881. try:
  882. value_converted = int(value)
  883. except ValueError:
  884. not_int = True
  885. if not_int:
  886. try:
  887. value_converted = float(value)
  888. except ValueError:
  889. not_float = True
  890. if not_int and not_float:
  891. value_converted = value.strip()
  892. value_list.append(value_converted)
  893. kvdict[key] = value_list
  894. if checkproj:
  895. kvdict = _compare_projection(kvdict)
  896. if checkunits:
  897. kvdict = _compare_units(kvdict)
  898. return kvdict
  899. def compare_key_value_text_files(
  900. filename_a,
  901. filename_b,
  902. sep=":",
  903. val_sep=",",
  904. precision=0.000001,
  905. proj=False,
  906. units=False,
  907. ):
  908. """Compare two key-value text files
  909. This method will print a warning in case keys that are present in the first
  910. file are not present in the second one.
  911. The comparison method tries to convert the values into their native format
  912. (float, int or string) to allow correct comparison.
  913. An example key-value text file may have this content:
  914. ::
  915. a: Hello
  916. b: 1.0
  917. c: 1,2,3,4,5
  918. d : hello,8,0.1
  919. :param str filename_a: name of the first key-value text file
  920. :param str filenmae_b: name of the second key-value text file
  921. :param str sep: character that separates the keys and values, default is ":"
  922. :param str val_sep: character that separates the values of a single key,
  923. default is ","
  924. :param double precision: precision with which the floating point values are compared
  925. :param bool proj: True if it has to check some information about projection system
  926. :param bool units: True if it has to check some information about units
  927. :return: True if full or almost identical, False if different
  928. """
  929. dict_a = _text_to_key_value_dict(filename_a, sep, checkproj=proj, checkunits=units)
  930. dict_b = _text_to_key_value_dict(filename_b, sep, checkproj=proj, checkunits=units)
  931. if sorted(dict_a.keys()) != sorted(dict_b.keys()):
  932. return False
  933. # We compare matching keys
  934. for key in dict_a.keys():
  935. # Floating point values must be handled separately
  936. if isinstance(dict_a[key], float) and isinstance(dict_b[key], float):
  937. if abs(dict_a[key] - dict_b[key]) > precision:
  938. return False
  939. elif isinstance(dict_a[key], float) or isinstance(dict_b[key], float):
  940. warning(
  941. _("Mixing value types. Will try to compare after " "integer conversion")
  942. )
  943. return int(dict_a[key]) == int(dict_b[key])
  944. elif key == "+towgs84":
  945. # We compare the sum of the entries
  946. if abs(sum(dict_a[key]) - sum(dict_b[key])) > precision:
  947. return False
  948. else:
  949. if dict_a[key] != dict_b[key]:
  950. return False
  951. return True
  952. # interface to g.gisenv
  953. def gisenv(env=None):
  954. """Returns the output from running g.gisenv (with no arguments), as a
  955. dictionary. Example:
  956. >>> env = gisenv()
  957. >>> print(env['GISDBASE']) # doctest: +SKIP
  958. /opt/grass-data
  959. :param env run with different environment
  960. :return: list of GRASS variables
  961. """
  962. s = read_command("g.gisenv", flags="n", env=env)
  963. return parse_key_val(s)
  964. # interface to g.region
  965. def locn_is_latlong(env=None):
  966. """Tests if location is lat/long. Value is obtained
  967. by checking the "g.region -pu" projection code.
  968. :return: True for a lat/long region, False otherwise
  969. """
  970. s = read_command("g.region", flags="pu", env=env)
  971. kv = parse_key_val(s, ":")
  972. if kv["projection"].split(" ")[0] == "3":
  973. return True
  974. else:
  975. return False
  976. def region(region3d=False, complete=False, env=None):
  977. """Returns the output from running "g.region -gu", as a
  978. dictionary. Example:
  979. :param bool region3d: True to get 3D region
  980. :param bool complete:
  981. :param env env
  982. >>> curent_region = region()
  983. >>> # obtain n, s, e and w values
  984. >>> [curent_region[key] for key in "nsew"] # doctest: +ELLIPSIS
  985. [..., ..., ..., ...]
  986. >>> # obtain ns and ew resulutions
  987. >>> (curent_region['nsres'], curent_region['ewres']) # doctest: +ELLIPSIS
  988. (..., ...)
  989. :return: dictionary of region values
  990. """
  991. flgs = "gu"
  992. if region3d:
  993. flgs += "3"
  994. if complete:
  995. flgs += "cep"
  996. s = read_command("g.region", flags=flgs, env=env)
  997. reg = parse_key_val(s, val_type=float)
  998. for k in [
  999. "projection",
  1000. "zone",
  1001. "rows",
  1002. "cols",
  1003. "cells",
  1004. "rows3",
  1005. "cols3",
  1006. "cells3",
  1007. "depths",
  1008. ]:
  1009. if k not in reg:
  1010. continue
  1011. reg[k] = int(reg[k])
  1012. return reg
  1013. def region_env(region3d=False, flags=None, env=None, **kwargs):
  1014. """Returns region settings as a string which can used as
  1015. GRASS_REGION environmental variable.
  1016. If no 'kwargs' are given then the current region is used. Note
  1017. that this function doesn't modify the current region!
  1018. See also :func:`use_temp_region()` for alternative method how to define
  1019. temporary region used for raster-based computation.
  1020. :param bool region3d: True to get 3D region
  1021. :param string flags: for example 'a'
  1022. :param env: different environment than current
  1023. :param kwargs: g.region's parameters like 'raster', 'vector' or 'region'
  1024. ::
  1025. os.environ['GRASS_REGION'] = grass.region_env(region='detail')
  1026. grass.mapcalc('map=1', overwrite=True)
  1027. os.environ.pop('GRASS_REGION')
  1028. :return: string with region values
  1029. :return: empty string on error
  1030. """
  1031. # read proj/zone from WIND file
  1032. gis_env = gisenv(env)
  1033. windfile = os.path.join(
  1034. gis_env["GISDBASE"], gis_env["LOCATION_NAME"], gis_env["MAPSET"], "WIND"
  1035. )
  1036. with open(windfile, "r") as fd:
  1037. grass_region = ""
  1038. for line in fd.readlines():
  1039. key, value = map(lambda x: x.strip(), line.split(":", 1))
  1040. if kwargs and key not in ("proj", "zone"):
  1041. continue
  1042. if (
  1043. not kwargs
  1044. and not region3d
  1045. and key
  1046. in (
  1047. "top",
  1048. "bottom",
  1049. "cols3",
  1050. "rows3",
  1051. "depths",
  1052. "e-w resol3",
  1053. "n-s resol3",
  1054. "t-b resol",
  1055. )
  1056. ):
  1057. continue
  1058. grass_region += "%s: %s;" % (key, value)
  1059. if not kwargs: # return current region
  1060. return grass_region
  1061. # read other values from `g.region -gu`
  1062. flgs = "ug"
  1063. if region3d:
  1064. flgs += "3"
  1065. if flags:
  1066. flgs += flags
  1067. s = read_command("g.region", flags=flgs, env=env, **kwargs)
  1068. if not s:
  1069. return ""
  1070. reg = parse_key_val(s)
  1071. kwdata = [
  1072. ("north", "n"),
  1073. ("south", "s"),
  1074. ("east", "e"),
  1075. ("west", "w"),
  1076. ("cols", "cols"),
  1077. ("rows", "rows"),
  1078. ("e-w resol", "ewres"),
  1079. ("n-s resol", "nsres"),
  1080. ]
  1081. if region3d:
  1082. kwdata += [
  1083. ("top", "t"),
  1084. ("bottom", "b"),
  1085. ("cols3", "cols3"),
  1086. ("rows3", "rows3"),
  1087. ("depths", "depths"),
  1088. ("e-w resol3", "ewres3"),
  1089. ("n-s resol3", "nsres3"),
  1090. ("t-b resol", "tbres"),
  1091. ]
  1092. for wkey, rkey in kwdata:
  1093. grass_region += "%s: %s;" % (wkey, reg[rkey])
  1094. return grass_region
  1095. def use_temp_region():
  1096. """Copies the current region to a temporary region with "g.region save=",
  1097. then sets WIND_OVERRIDE to refer to that region. Installs an atexit
  1098. handler to delete the temporary region upon termination.
  1099. """
  1100. name = "tmp.%s.%d" % (os.path.basename(sys.argv[0]), os.getpid())
  1101. run_command("g.region", flags="u", save=name, overwrite=True)
  1102. os.environ["WIND_OVERRIDE"] = name
  1103. atexit.register(del_temp_region)
  1104. def del_temp_region():
  1105. """Unsets WIND_OVERRIDE and removes any region named by it."""
  1106. try:
  1107. name = os.environ.pop("WIND_OVERRIDE")
  1108. run_command("g.remove", flags="f", quiet=True, type="region", name=name)
  1109. except (KeyError, CalledModuleError):
  1110. # The function succeeds even when called more than once.
  1111. pass
  1112. # interface to g.findfile
  1113. def find_file(name, element="cell", mapset=None, env=None):
  1114. """Returns the output from running g.findfile as a
  1115. dictionary. Example:
  1116. >>> result = find_file('elevation', element='cell')
  1117. >>> print(result['fullname'])
  1118. elevation@PERMANENT
  1119. >>> print(result['file']) # doctest: +ELLIPSIS
  1120. /.../PERMANENT/cell/elevation
  1121. :param str name: file name
  1122. :param str element: element type (default 'cell')
  1123. :param str mapset: mapset name (default all mapsets in search path)
  1124. :param env: environment
  1125. :return: parsed output of g.findfile
  1126. """
  1127. if element == "raster" or element == "rast":
  1128. verbose(_('Element type should be "cell" and not "%s"') % element)
  1129. element = "cell"
  1130. # g.findfile returns non-zero when file was not found
  1131. # se we ignore return code and just focus on stdout
  1132. process = start_command(
  1133. "g.findfile",
  1134. flags="n",
  1135. element=element,
  1136. file=name,
  1137. mapset=mapset,
  1138. stdout=PIPE,
  1139. env=env,
  1140. )
  1141. stdout = process.communicate()[0]
  1142. return parse_key_val(stdout)
  1143. # interface to g.list
  1144. def list_strings(type, pattern=None, mapset=None, exclude=None, flag="", env=None):
  1145. """List of elements as strings.
  1146. Returns the output from running g.list, as a list of qualified
  1147. names.
  1148. :param str type: element type (raster, vector, raster_3d, region, ...)
  1149. :param str pattern: pattern string
  1150. :param str mapset: mapset name (if not given use search path)
  1151. :param str exclude: pattern string to exclude maps from the research
  1152. :param str flag: pattern type: 'r' (basic regexp), 'e' (extended regexp),
  1153. or '' (glob pattern)
  1154. :param env: environment
  1155. :return: list of elements
  1156. """
  1157. if type == "cell":
  1158. verbose(_('Element type should be "raster" and not "%s"') % type)
  1159. result = list()
  1160. for line in read_command(
  1161. "g.list",
  1162. quiet=True,
  1163. flags="m" + flag,
  1164. type=type,
  1165. pattern=pattern,
  1166. exclude=exclude,
  1167. mapset=mapset,
  1168. env=env,
  1169. ).splitlines():
  1170. result.append(line.strip())
  1171. return result
  1172. def list_pairs(type, pattern=None, mapset=None, exclude=None, flag="", env=None):
  1173. """List of elements as pairs
  1174. Returns the output from running g.list, as a list of
  1175. (name, mapset) pairs
  1176. :param str type: element type (raster, vector, raster_3d, region, ...)
  1177. :param str pattern: pattern string
  1178. :param str mapset: mapset name (if not given use search path)
  1179. :param str exclude: pattern string to exclude maps from the research
  1180. :param str flag: pattern type: 'r' (basic regexp), 'e' (extended regexp),
  1181. or '' (glob pattern)
  1182. :param env: environment
  1183. :return: list of elements
  1184. """
  1185. return [
  1186. tuple(map.split("@", 1))
  1187. for map in list_strings(type, pattern, mapset, exclude, flag, env)
  1188. ]
  1189. def list_grouped(
  1190. type, pattern=None, check_search_path=True, exclude=None, flag="", env=None
  1191. ):
  1192. """List of elements grouped by mapsets.
  1193. Returns the output from running g.list, as a dictionary where the
  1194. keys are mapset names and the values are lists of maps in that
  1195. mapset. Example:
  1196. >>> list_grouped('vect', pattern='*roads*')['PERMANENT']
  1197. ['railroads', 'roadsmajor']
  1198. :param str type: element type (raster, vector, raster_3d, region, ...)
  1199. or list of elements
  1200. :param str pattern: pattern string
  1201. :param str check_search_path: True to add mapsets for the search path
  1202. with no found elements
  1203. :param str exclude: pattern string to exclude maps from the research
  1204. :param str flag: pattern type: 'r' (basic regexp), 'e' (extended regexp),
  1205. or '' (glob pattern)
  1206. :param env: environment
  1207. :return: directory of mapsets/elements
  1208. """
  1209. if isinstance(type, str) or len(type) == 1:
  1210. types = [type]
  1211. store_types = False
  1212. else:
  1213. types = type
  1214. store_types = True
  1215. flag += "t"
  1216. for i in range(len(types)):
  1217. if types[i] == "cell":
  1218. verbose(_('Element type should be "raster" and not "%s"') % types[i])
  1219. types[i] = "raster"
  1220. result = {}
  1221. if check_search_path:
  1222. for mapset in mapsets(search_path=True, env=env):
  1223. if store_types:
  1224. result[mapset] = {}
  1225. else:
  1226. result[mapset] = []
  1227. mapset = None
  1228. for line in read_command(
  1229. "g.list",
  1230. quiet=True,
  1231. flags="m" + flag,
  1232. type=types,
  1233. pattern=pattern,
  1234. exclude=exclude,
  1235. env=env,
  1236. ).splitlines():
  1237. try:
  1238. name, mapset = line.split("@")
  1239. except ValueError:
  1240. warning(_("Invalid element '%s'") % line)
  1241. continue
  1242. if store_types:
  1243. type_, name = name.split("/")
  1244. if mapset in result:
  1245. if type_ in result[mapset]:
  1246. result[mapset][type_].append(name)
  1247. else:
  1248. result[mapset][type_] = [
  1249. name,
  1250. ]
  1251. else:
  1252. result[mapset] = {
  1253. type_: [
  1254. name,
  1255. ]
  1256. }
  1257. else:
  1258. if mapset in result:
  1259. result[mapset].append(name)
  1260. else:
  1261. result[mapset] = [
  1262. name,
  1263. ]
  1264. return result
  1265. # color parsing
  1266. named_colors = {
  1267. "white": (1.00, 1.00, 1.00),
  1268. "black": (0.00, 0.00, 0.00),
  1269. "red": (1.00, 0.00, 0.00),
  1270. "green": (0.00, 1.00, 0.00),
  1271. "blue": (0.00, 0.00, 1.00),
  1272. "yellow": (1.00, 1.00, 0.00),
  1273. "magenta": (1.00, 0.00, 1.00),
  1274. "cyan": (0.00, 1.00, 1.00),
  1275. "aqua": (0.00, 0.75, 0.75),
  1276. "grey": (0.75, 0.75, 0.75),
  1277. "gray": (0.75, 0.75, 0.75),
  1278. "orange": (1.00, 0.50, 0.00),
  1279. "brown": (0.75, 0.50, 0.25),
  1280. "purple": (0.50, 0.00, 1.00),
  1281. "violet": (0.50, 0.00, 1.00),
  1282. "indigo": (0.00, 0.50, 1.00),
  1283. }
  1284. def parse_color(val, dflt=None):
  1285. """Parses the string "val" as a GRASS colour, which can be either one of
  1286. the named colours or an R:G:B tuple e.g. 255:255:255. Returns an
  1287. (r,g,b) triple whose components are floating point values between 0
  1288. and 1. Example:
  1289. >>> parse_color("red")
  1290. (1.0, 0.0, 0.0)
  1291. >>> parse_color("255:0:0")
  1292. (1.0, 0.0, 0.0)
  1293. :param val: color value
  1294. :param dflt: default color value
  1295. :return: tuple RGB
  1296. """
  1297. if val in named_colors:
  1298. return named_colors[val]
  1299. vals = val.split(":")
  1300. if len(vals) == 3:
  1301. return tuple(float(v) / 255 for v in vals)
  1302. return dflt
  1303. # check GRASS_OVERWRITE
  1304. def overwrite():
  1305. """Return True if existing files may be overwritten"""
  1306. owstr = "GRASS_OVERWRITE"
  1307. return owstr in os.environ and os.environ[owstr] != "0"
  1308. # check GRASS_VERBOSE
  1309. def verbosity():
  1310. """Return the verbosity level selected by GRASS_VERBOSE
  1311. Currently, there are 5 levels of verbosity:
  1312. -1 nothing will be printed (also fatal errors and warnings will be discarded)
  1313. 0 only errors and warnings are printed, triggered by "--q" or "--quiet" flag.
  1314. 1 progress information (percent) and important messages will be printed
  1315. 2 all messages will be printed
  1316. 3 also verbose messages will be printed. Triggered by "--v" or "--verbose" flag.
  1317. """
  1318. vbstr = os.getenv("GRASS_VERBOSE")
  1319. if vbstr:
  1320. return int(vbstr)
  1321. else:
  1322. return 2
  1323. # Various utilities, not specific to GRASS
  1324. def find_program(pgm, *args):
  1325. """Attempt to run a program, with optional arguments.
  1326. You must call the program in a way that will return a successful
  1327. exit code. For GRASS modules this means you need to pass it some
  1328. valid CLI option, like "--help". For other programs a common
  1329. valid do-little option is usually "--version".
  1330. Example:
  1331. >>> find_program('r.sun', '--help')
  1332. True
  1333. >>> find_program('ls', '--version')
  1334. True
  1335. :param str pgm: program name
  1336. :param args: list of arguments
  1337. :return: False if the attempt failed due to a missing executable
  1338. or non-zero return code
  1339. :return: True otherwise
  1340. """
  1341. nuldev = open(os.devnull, "w+")
  1342. try:
  1343. # TODO: the doc or impl is not correct, any return code is accepted
  1344. call([pgm] + list(args), stdin=nuldev, stdout=nuldev, stderr=nuldev)
  1345. found = True
  1346. except Exception:
  1347. found = False
  1348. nuldev.close()
  1349. return found
  1350. # interface to g.mapsets
  1351. def mapsets(search_path=False, env=None):
  1352. """List available mapsets
  1353. :param bool search_path: True to list mapsets only in search path
  1354. :return: list of mapsets
  1355. """
  1356. if search_path:
  1357. flags = "p"
  1358. else:
  1359. flags = "l"
  1360. mapsets = read_command("g.mapsets", flags=flags, sep="newline", quiet=True, env=env)
  1361. if not mapsets:
  1362. fatal(_("Unable to list mapsets"))
  1363. return mapsets.splitlines()
  1364. # interface to `g.proj -c`
  1365. def create_location(
  1366. dbase,
  1367. location,
  1368. epsg=None,
  1369. proj4=None,
  1370. filename=None,
  1371. wkt=None,
  1372. datum=None,
  1373. datum_trans=None,
  1374. desc=None,
  1375. overwrite=False,
  1376. ):
  1377. """Create new location
  1378. Raise ScriptError on error.
  1379. :param str dbase: path to GRASS database
  1380. :param str location: location name to create
  1381. :param epsg: if given create new location based on EPSG code
  1382. :param proj4: if given create new location based on Proj4 definition
  1383. :param str filename: if given create new location based on georeferenced file
  1384. :param str wkt: if given create new location based on WKT definition
  1385. (can be path to PRJ file or WKT string)
  1386. :param datum: GRASS format datum code
  1387. :param datum_trans: datum transformation parameters (used for epsg and proj4)
  1388. :param desc: description of the location (creates MYNAME file)
  1389. :param bool overwrite: True to overwrite location if exists(WARNING:
  1390. ALL DATA from existing location ARE DELETED!)
  1391. """
  1392. # create dbase if not exists
  1393. if not os.path.exists(dbase):
  1394. os.mkdir(dbase)
  1395. if epsg or proj4 or filename or wkt:
  1396. # here the location shouldn't really matter
  1397. tmp_gisrc, env = create_environment(
  1398. dbase, gisenv()["LOCATION_NAME"], "PERMANENT"
  1399. )
  1400. # check if location already exists
  1401. if os.path.exists(os.path.join(dbase, location)):
  1402. if not overwrite:
  1403. warning(_("Location <%s> already exists. Operation canceled.") % location)
  1404. try_remove(tmp_gisrc)
  1405. return
  1406. else:
  1407. warning(
  1408. _("Location <%s> already exists and will be overwritten") % location
  1409. )
  1410. shutil.rmtree(os.path.join(dbase, location))
  1411. stdin = None
  1412. kwargs = dict()
  1413. if datum:
  1414. kwargs["datum"] = datum
  1415. if datum_trans:
  1416. kwargs["datum_trans"] = datum_trans
  1417. if epsg:
  1418. ps = pipe_command(
  1419. "g.proj",
  1420. quiet=True,
  1421. flags="t",
  1422. epsg=epsg,
  1423. location=location,
  1424. stderr=PIPE,
  1425. env=env,
  1426. **kwargs,
  1427. )
  1428. elif proj4:
  1429. ps = pipe_command(
  1430. "g.proj",
  1431. quiet=True,
  1432. flags="t",
  1433. proj4=proj4,
  1434. location=location,
  1435. stderr=PIPE,
  1436. env=env,
  1437. **kwargs,
  1438. )
  1439. elif filename:
  1440. ps = pipe_command(
  1441. "g.proj",
  1442. quiet=True,
  1443. georef=filename,
  1444. location=location,
  1445. stderr=PIPE,
  1446. env=env,
  1447. )
  1448. elif wkt:
  1449. if os.path.isfile(wkt):
  1450. ps = pipe_command(
  1451. "g.proj", quiet=True, wkt=wkt, location=location, stderr=PIPE, env=env
  1452. )
  1453. else:
  1454. ps = pipe_command(
  1455. "g.proj",
  1456. quiet=True,
  1457. wkt="-",
  1458. location=location,
  1459. stderr=PIPE,
  1460. stdin=PIPE,
  1461. env=env,
  1462. )
  1463. stdin = encode(wkt)
  1464. else:
  1465. _create_location_xy(dbase, location)
  1466. if epsg or proj4 or filename or wkt:
  1467. error = ps.communicate(stdin)[1]
  1468. try_remove(tmp_gisrc)
  1469. if ps.returncode != 0 and error:
  1470. raise ScriptError(repr(error))
  1471. try:
  1472. fd = codecs.open(
  1473. os.path.join(dbase, location, "PERMANENT", "MYNAME"),
  1474. encoding="utf-8",
  1475. mode="w",
  1476. )
  1477. if desc:
  1478. fd.write(desc + os.linesep)
  1479. else:
  1480. fd.write(os.linesep)
  1481. fd.close()
  1482. except OSError as e:
  1483. raise ScriptError(repr(e))
  1484. def _create_location_xy(database, location):
  1485. """Create unprojected location
  1486. Raise ScriptError on error.
  1487. :param database: GRASS database where to create new location
  1488. :param location: location name
  1489. """
  1490. cur_dir = os.getcwd()
  1491. try:
  1492. os.chdir(database)
  1493. os.mkdir(location)
  1494. os.mkdir(os.path.join(location, "PERMANENT"))
  1495. # create DEFAULT_WIND and WIND files
  1496. regioninfo = [
  1497. "proj: 0",
  1498. "zone: 0",
  1499. "north: 1",
  1500. "south: 0",
  1501. "east: 1",
  1502. "west: 0",
  1503. "cols: 1",
  1504. "rows: 1",
  1505. "e-w resol: 1",
  1506. "n-s resol: 1",
  1507. "top: 1",
  1508. "bottom: 0",
  1509. "cols3: 1",
  1510. "rows3: 1",
  1511. "depths: 1",
  1512. "e-w resol3: 1",
  1513. "n-s resol3: 1",
  1514. "t-b resol: 1",
  1515. ]
  1516. defwind = open(os.path.join(location, "PERMANENT", "DEFAULT_WIND"), "w")
  1517. for param in regioninfo:
  1518. defwind.write(param + "%s" % os.linesep)
  1519. defwind.close()
  1520. shutil.copy(
  1521. os.path.join(location, "PERMANENT", "DEFAULT_WIND"),
  1522. os.path.join(location, "PERMANENT", "WIND"),
  1523. )
  1524. os.chdir(cur_dir)
  1525. except OSError as e:
  1526. raise ScriptError(repr(e))
  1527. # interface to g.version
  1528. def version():
  1529. """Get GRASS version as dictionary
  1530. ::
  1531. >>> print(version())
  1532. {'proj4': '4.8.0', 'geos': '3.3.5', 'libgis_revision': '52468',
  1533. 'libgis_date': '2012-07-27 22:53:30 +0200 (Fri, 27 Jul 2012)',
  1534. 'version': '7.0.svn', 'date': '2012', 'gdal': '2.0dev',
  1535. 'revision': '53670'}
  1536. """
  1537. data = parse_command("g.version", flags="rge", errors="ignore")
  1538. for k, v in data.items():
  1539. data[k.strip()] = v.replace('"', "").strip()
  1540. return data
  1541. # get debug_level
  1542. _debug_level = None
  1543. def debug_level(force=False):
  1544. global _debug_level
  1545. if not force and _debug_level is not None:
  1546. return _debug_level
  1547. _debug_level = 0
  1548. if find_program("g.gisenv", "--help"):
  1549. try:
  1550. _debug_level = int(gisenv().get("DEBUG", 0))
  1551. if _debug_level < 0 or _debug_level > 5:
  1552. raise ValueError(_("Debug level {0}").format(_debug_level))
  1553. except ValueError as e:
  1554. _debug_level = 0
  1555. sys.stderr.write(
  1556. _(
  1557. "WARNING: Ignoring unsupported debug level (must be >=0 and <=5):"
  1558. " {}\n"
  1559. ).format(e)
  1560. )
  1561. return _debug_level
  1562. # TODO: Move legal_name() to utils or a new dedicated "name" module.
  1563. # TODO: Remove the pygrass backwards compatibility version of it?
  1564. def legal_name(s):
  1565. """Checks if the string contains only allowed characters.
  1566. This is the Python implementation of :func:`G_legal_filename()` function.
  1567. ..note::
  1568. It is not clear when exactly use this function, but it might be
  1569. useful anyway for checking map names and column names.
  1570. """
  1571. if not s or s[0] == ".":
  1572. warning(_("Illegal filename <%s>. Cannot be 'NULL' or start with " "'.'.") % s)
  1573. return False
  1574. illegal = [c for c in s if c in "/\"'@,=*~" or c <= " " or c >= "\177"]
  1575. if illegal:
  1576. illegal = "".join(sorted(set(illegal)))
  1577. warning(
  1578. _("Illegal filename <%(s)s>. <%(il)s> not allowed.\n")
  1579. % {"s": s, "il": illegal}
  1580. )
  1581. return False
  1582. return True
  1583. def sanitize_mapset_environment(env):
  1584. """Remove environmental variables relevant only
  1585. for a specific mapset. This should be called
  1586. when a copy of environment is used with a different mapset."""
  1587. if "WIND_OVERRIDE" in env:
  1588. del env["WIND_OVERRIDE"]
  1589. if "GRASS_REGION" in env:
  1590. del env["GRASS_REGION"]
  1591. return env
  1592. def create_environment(gisdbase, location, mapset):
  1593. """Creates environment to be passed in run_command for example.
  1594. Returns tuple with temporary file path and the environment. The user
  1595. of this function is responsible for deleting the file."""
  1596. with NamedTemporaryFile(mode="w", delete=False) as f:
  1597. f.write("MAPSET: {mapset}\n".format(mapset=mapset))
  1598. f.write("GISDBASE: {g}\n".format(g=gisdbase))
  1599. f.write("LOCATION_NAME: {l}\n".format(l=location))
  1600. f.write("GUI: text\n")
  1601. env = os.environ.copy()
  1602. env["GISRC"] = f.name
  1603. # remove mapset-specific env vars
  1604. env = sanitize_mapset_environment(env)
  1605. return f.name, env
  1606. if __name__ == "__main__":
  1607. import doctest
  1608. doctest.testmod()