libgis: fix and simplify G_tokenize()

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@62747 15284696-431f-4ddb-bdfa-cd5b030d7da7

general/g.version/main.c

@@ -119,8 +119,9 @@ int main(int argc, char *argv[])
     if (gish_rev->answer) {
 	char **rev_ver = G_tokenize(GIS_H_VERSION, "$");
 	char **rev_time = G_tokenize(GIS_H_DATE, "$");
-	const int tokens_expected = 2;
+	const int tokens_expected = 3;
 	int no_libgis = FALSE;
+
 	/* if number of tokes is right, print it */
 	if (G_number_of_tokens(rev_ver) == tokens_expected &&
 	    G_number_of_tokens(rev_time) == tokens_expected) {

lib/gis/token.c

@@ -87,39 +87,40 @@ char **G_tokenize2(const char *buf, const char *delim, const char *valchar)
 
 char **tokenize(const char *buf, const char *delim, const char *inchar)
 {
-    int i, invalue;
+    int i;
     char **tokens;
     char *p;
 
-    /* needed for G_free () */
-    while (!strchr(delim, *buf) && (*buf == ' ' || *buf == '\t'))
-	buf++;
-
+    /* do not modify buf, make a copy */
     p = G_store(buf);
 
-    tokens = (char **)G_malloc(sizeof(char *));
-
     i = 0;
-    invalue = FALSE;
+    tokens = (char **)G_malloc(2 * sizeof(char *));
+
+    /* always one token */
+    tokens[i++] = p;
+
     while (TRUE) {
-	while (!(strchr(delim, *p) && !invalue) && (*p == ' ' || *p == '\t')) {
-	    if (inchar && *p == *inchar)
-		invalue = invalue ? FALSE : TRUE;
+	/* find next delimiter */
+	while (!strchr(delim, *p)) {
+	    /* opening border ? */
+	    if (inchar && *p == *inchar) {
+		p++;
+		/* find closing border */
+		while (*p != *inchar && *p != 0)
+		    p++;
+		if (*p == 0)
+		    break;
+	    }
 	    p++;
 	}
 	if (*p == 0)
 	    break;
+	/* replace delim with '\0' */
+	*p++ = 0;
+	/* set next token */
 	tokens[i++] = p;
 	tokens = (char **)G_realloc((char *)tokens, (i + 1) * sizeof(char *));
-
-	while (*p && !(strchr(delim, *p) && !invalue)) {
-	    if (inchar && *p == *inchar)
-		invalue = invalue ? FALSE : TRUE;
-	    p++;
-	}
-	if (*p == 0)
-	    break;
-	*p++ = 0;
     }
     tokens[i] = NULL;