Allow suppressing missing driver warnings

Signed-off-by: Vadim Markovtsev <vadim@sourced.tech>

internal/plumbing/uast/uast.go

@@ -32,11 +32,12 @@ import (
 // It is a PipelineItem.
 type Extractor struct {
 	core.NoopMerger
-	Endpoint       string
-	Context        func() (context.Context, context.CancelFunc)
-	PoolSize       int
-	FailOnErrors   bool
-	ProcessedFiles map[string]int
+	Endpoint              string
+	Context               func() (context.Context, context.CancelFunc)
+	PoolSize              int
+	FailOnErrors          bool
+	ProcessedFiles        map[string]int
+	IgnoredMissingDrivers map[string]bool
 
 	clients []*bblfsh.Client
 	pool    *tunny.Pool
@@ -45,22 +46,30 @@ type Extractor struct {
 const (
 	// ConfigUASTEndpoint is the name of the configuration option (Extractor.Configure())
 	// which sets the Babelfish server address.
-	ConfigUASTEndpoint = "ConfigUASTEndpoint"
+	ConfigUASTEndpoint = "UAST.Endpoint"
 	// ConfigUASTTimeout is the name of the configuration option (Extractor.Configure())
 	// which sets the maximum amount of time to wait for a Babelfish server response.
-	ConfigUASTTimeout = "ConfigUASTTimeout"
+	ConfigUASTTimeout = "UAST.Timeout"
 	// ConfigUASTPoolSize is the name of the configuration option (Extractor.Configure())
 	// which sets the number of goroutines to run for UAST parse queries.
-	ConfigUASTPoolSize = "ConfigUASTPoolSize"
+	ConfigUASTPoolSize = "UAST.PoolSize"
 	// ConfigUASTFailOnErrors is the name of the configuration option (Extractor.Configure())
 	// which enables early exit in case of any Babelfish UAST parsing errors.
-	ConfigUASTFailOnErrors = "ConfigUASTFailOnErrors"
+	ConfigUASTFailOnErrors = "UAST.FailOnErrors"
+	// ConfigUASTIgnoreMissingDrivers is the name of the configuration option (Extractor.Configure())
+	// which sets the ignored missing driver names.
+	ConfigUASTIgnoreMissingDrivers = "UAST.IgnoreMissingDrivers"
 	// FeatureUast is the name of the Pipeline feature which activates all the items related to UAST.
 	FeatureUast = "uast"
 	// DependencyUasts is the name of the dependency provided by Extractor.
 	DependencyUasts = "uasts"
 )
 
+var (
+	// DefaultIgnoredMissingDrivers is the languages which are ignored if the Babelfish driver is missing.
+	DefaultIgnoredMissingDrivers = []string{"markdown", "text", "yaml", "json"}
+)
+
 type uastTask struct {
 	Lock   *sync.RWMutex
 	Dest   map[plumbing.Hash]nodes.Node
@@ -132,7 +141,12 @@ func (exr *Extractor) ListConfigurationOptions() []core.ConfigurationOption {
 		Description: "Panic if there is a UAST extraction error.",
 		Flag:        "bblfsh-fail-on-error",
 		Type:        core.BoolConfigurationOption,
-		Default:     false},
+		Default:     false}, {
+		Name:        ConfigUASTIgnoreMissingDrivers,
+		Description: "Do not warn about missing drivers for the specified languages.",
+		Flag:        "bblfsh-ignored-drivers",
+		Type:        core.StringsConfigurationOption,
+		Default:     DefaultIgnoredMissingDrivers},
 	}
 	return options[:]
 }
@@ -154,6 +168,12 @@ func (exr *Extractor) Configure(facts map[string]interface{}) error {
 	if val, exists := facts[ConfigUASTFailOnErrors].(bool); exists {
 		exr.FailOnErrors = val
 	}
+	if val, exists := facts[ConfigUASTIgnoreMissingDrivers].([]string); exists {
+		exr.IgnoredMissingDrivers = map[string]bool{}
+		for _, name := range val {
+			exr.IgnoredMissingDrivers[name] = true
+		}
+	}
 	return nil
 }
 
@@ -196,6 +216,12 @@ func (exr *Extractor) Initialize(repository *git.Repository) error {
 		panic("UAST goroutine pool was not created")
 	}
 	exr.ProcessedFiles = map[string]int{}
+	if exr.IgnoredMissingDrivers == nil {
+		exr.IgnoredMissingDrivers = map[string]bool{}
+		for _, name := range DefaultIgnoredMissingDrivers {
+			exr.IgnoredMissingDrivers[name] = true
+		}
+	}
 	return nil
 }
 
@@ -250,7 +276,7 @@ func (exr *Extractor) Consume(deps map[string]interface{}) (map[string]interface
 		if exr.FailOnErrors {
 			return nil, errors.New(joined)
 		}
-		fmt.Fprintln(os.Stderr, joined)
+		log.Println(joined)
 	}
 	return map[string]interface{}{DependencyUasts: uasts}, nil
 }
@@ -284,6 +310,11 @@ func (exr *Extractor) extractTask(client *bblfsh.Client, data interface{}) inter
 	task.Lock.Lock()
 	defer task.Lock.Unlock()
 	if err != nil {
+		for lang := range exr.IgnoredMissingDrivers {
+			if strings.HasSuffix(err.Error(), "\""+lang+"\"") {
+				return nil
+			}
+		}
 		*task.Errors = append(*task.Errors,
 			fmt.Errorf("\nfile %s, blob %s: %v", task.Name, task.Hash.String(), err))
 		return nil

internal/plumbing/uast/uast_test.go

@@ -50,11 +50,12 @@ func TestUASTExtractorMeta(t *testing.T) {
 	assert.Equal(t, exr.Requires()[0], items.DependencyTreeChanges)
 	assert.Equal(t, exr.Requires()[1], items.DependencyBlobCache)
 	opts := exr.ListConfigurationOptions()
-	assert.Len(t, opts, 4)
+	assert.Len(t, opts, 5)
 	assert.Equal(t, opts[0].Name, ConfigUASTEndpoint)
 	assert.Equal(t, opts[1].Name, ConfigUASTTimeout)
 	assert.Equal(t, opts[2].Name, ConfigUASTPoolSize)
 	assert.Equal(t, opts[3].Name, ConfigUASTFailOnErrors)
+	assert.Equal(t, opts[4].Name, ConfigUASTIgnoreMissingDrivers)
 	feats := exr.Features()
 	assert.Len(t, feats, 1)
 	assert.Equal(t, feats[0], FeatureUast)
@@ -68,11 +69,13 @@ func TestUASTExtractorConfiguration(t *testing.T) {
 	facts[ConfigUASTTimeout] = 15
 	facts[ConfigUASTPoolSize] = 7
 	facts[ConfigUASTFailOnErrors] = true
+	facts[ConfigUASTIgnoreMissingDrivers] = []string{"test"}
 	exr.Configure(facts)
 	assert.Equal(t, exr.Endpoint, facts[ConfigUASTEndpoint])
 	assert.NotNil(t, exr.Context)
 	assert.Equal(t, exr.PoolSize, facts[ConfigUASTPoolSize])
 	assert.Equal(t, exr.FailOnErrors, true)
+	assert.Equal(t, exr.IgnoredMissingDrivers, map[string]bool{"test": true})
 }
 
 func TestUASTExtractorRegistration(t *testing.T) {
@@ -92,7 +95,7 @@ func TestUASTExtractorNoBabelfish(t *testing.T) {
 
 func TestUASTExtractorConsume(t *testing.T) {
 	exr := fixtureUASTExtractor()
-	changes := make(object.Changes, 3)
+	changes := make(object.Changes, 4)
 	// 2b1ed978194a94edeabbca6de7ff3b5771d4d665
 	treeFrom, _ := test.Repository.TreeObject(plumbing.NewHash(
 		"96c6ece9b2f3c7c51b83516400d278dea5605100"))
@@ -136,6 +139,16 @@ func TestUASTExtractorConsume(t *testing.T) {
 		},
 	},
 	}
+	changes[3] = &object.Change{From: object.ChangeEntry{}, To: object.ChangeEntry{
+		Name: "README.md",
+		Tree: treeTo,
+		TreeEntry: object.TreeEntry{
+			Name: "README.md",
+			Mode: 0100644,
+			Hash: plumbing.NewHash("5248c86995f6d60eb57730da18b5e020a4341863"),
+		},
+	},
+	}
 	cache := map[plumbing.Hash]*items.CachedBlob{}
 	for _, hash := range []string{
 		"baa64828831d174f40140e4b3cfa77d1e917a2c1",
@@ -143,6 +156,7 @@ func TestUASTExtractorConsume(t *testing.T) {
 		"c29112dbd697ad9b401333b80c18a63951bc18d9",
 		"f7d918ec500e2f925ecde79b51cc007bac27de72",
 		"81f2b6d1fa5357f90e9dead150cd515720897545",
+		"5248c86995f6d60eb57730da18b5e020a4341863",
 	} {
 		AddHash(t, cache, hash)
 	}
@@ -158,6 +172,12 @@ func TestUASTExtractorConsume(t *testing.T) {
 	assert.Len(t, res[DependencyUasts], 1)
 	assert.Nil(t, err)
 
+	exr.FailOnErrors = true
+	res, err = exr.Consume(deps)
+	assert.Nil(t, res)
+	assert.NotNil(t, err)
+	exr.FailOnErrors = false
+
 	hash := plumbing.NewHash("5d78f57d732aed825764347ec6f3ab74d50d0619")
 	changes[1] = &object.Change{From: object.ChangeEntry{}, To: object.ChangeEntry{
 		Name: "labours.py",
@@ -176,6 +196,17 @@ func TestUASTExtractorConsume(t *testing.T) {
 	uasts := res[DependencyUasts].(map[plumbing.Hash]nodes.Node)
 	assert.Equal(t, len(uasts), 1)
 	assert.Equal(t, len(uasts[hash].(nodes.Object)["body"].(nodes.Array)), 24)
+
+	exr.IgnoredMissingDrivers = map[string]bool{}
+	changes[2] = changes[3]
+	deps[items.DependencyTreeChanges] = changes[:3]
+	res, err = exr.Consume(deps)
+	assert.Nil(t, err)
+	exr.FailOnErrors = true
+	res, err = exr.Consume(deps)
+	assert.Nil(t, res)
+	assert.NotNil(t, err)
+	exr.FailOnErrors = false
 }
 
 func TestUASTExtractorFork(t *testing.T) {