Merge pull request #23 from vmarkovtsev/master

Continue working on plugins

.travis.yml

@@ -1,5 +1,8 @@
 dist: trusty
 
+git:
+  depth: 9999999
+
 language: go
 
 services:
@@ -33,6 +36,7 @@ install:
   - docker exec -it bblfshd bblfshctl driver install --all
   
 script:
+  - go vet .
   - go test -v -cpu=1,2 -coverprofile=coverage.txt -covermode=count gopkg.in/src-d/hercules.v3
   - $GOPATH/bin/hercules -version
   - $GOPATH/bin/hercules -burndown -burndown-files -burndown-people -couples -quiet https://github.com/src-d/hercules | python3 labours.py -m all -o out --backend Agg --disable-projector

burndown.go

@@ -1,7 +1,6 @@
 package hercules
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -334,31 +333,6 @@ func checkClose(c io.Closer) {
 	}
 }
 
-func countLines(file *object.Blob) (int, error) {
-	reader, err := file.Reader()
-	if err != nil {
-		return 0, err
-	}
-	defer checkClose(reader)
-	var scanner *bufio.Scanner
-	buffer := make([]byte, bufio.MaxScanTokenSize)
-	counter := 0
-	for scanner == nil || scanner.Err() == bufio.ErrTooLong {
-		if scanner != nil && !utf8.Valid(scanner.Bytes()) {
-			return -1, errors.New("binary")
-		}
-		scanner = bufio.NewScanner(reader)
-		scanner.Buffer(buffer, 0)
-		for scanner.Scan() {
-			if !utf8.Valid(scanner.Bytes()) {
-				return -1, errors.New("binary")
-			}
-			counter++
-		}
-	}
-	return counter, nil
-}
-
 func (analyser *BurndownAnalysis) packPersonWithDay(person int, day int) int {
 	if analyser.PeopleNumber == 0 {
 		return day
@@ -441,7 +415,7 @@ func (analyser *BurndownAnalysis) newFile(
 func (analyser *BurndownAnalysis) handleInsertion(
 	change *object.Change, author int, cache map[plumbing.Hash]*object.Blob) error {
 	blob := cache[change.To.TreeEntry.Hash]
-	lines, err := countLines(blob)
+	lines, err := CountLines(blob)
 	if err != nil {
 		if err.Error() == "binary" {
 			return nil
@@ -463,7 +437,7 @@ func (analyser *BurndownAnalysis) handleDeletion(
 	change *object.Change, author int, cache map[plumbing.Hash]*object.Blob) error {
 
 	blob := cache[change.From.TreeEntry.Hash]
-	lines, err := countLines(blob)
+	lines, err := CountLines(blob)
 	if err != nil {
 		if err.Error() == "binary" {
 			return nil

cmd/hercules-generate-plugin/main.go

@@ -4,6 +4,8 @@ import (
   "flag"
   "fmt"
   "os"
+  "path"
+  "runtime"
   "strings"
   "text/template"
 
@@ -13,6 +15,13 @@ import (
 
 //go:generate go run embed.go
 
+var SHLIB_EXT = map[string]string {
+  "window": "dll",
+  "linux": "so",
+  "darwin": "dylib",
+  "freebsd": "dylib",
+}
+
 func main() {
   var outputPath, name, varname, _flag, pkg string
   var printVersion bool
@@ -24,7 +33,7 @@ func main() {
   flag.StringVar(&_flag, "flag", "", "Name of the plugin activation cmdline flag, If not " +
       "specified, inferred from -varname.")
   flag.BoolVar(&printVersion, "version", false, "Print version information and exit.")
-  flag.StringVar(&pkg, "package", "contrib", "Name of the package.")
+  flag.StringVar(&pkg, "package", "main", "Name of the package.")
   flag.Parse()
   if printVersion {
 		fmt.Printf("Version: 3\nGit:     %s\n", hercules.GIT_HASH)
@@ -38,6 +47,8 @@ func main() {
   splitted := camelcase.Split(name)
   if outputPath == "" {
     outputPath = strings.ToLower(strings.Join(splitted, "_")) + ".go"
+  } else if !strings.HasSuffix(outputPath, ".go") {
+    panic("-o must end with \".go\"")
   }
   gen := template.Must(template.New("plugin").Parse(PLUGIN_TEMPLATE_SOURCE))
   outFile, err := os.Create(outputPath)
@@ -49,9 +60,13 @@ func main() {
     varname = strings.ToLower(splitted[0])
   }
   if _flag == "" {
-    _flag = varname
+    _flag = strings.Join(splitted, "-")
   }
-  dict := map[string]string{"name": name, "varname": varname, "flag": _flag, "package": pkg}
+  outputBase := path.Base(outputPath)
+  shlib := outputBase[:len(outputBase)-2] + SHLIB_EXT[runtime.GOOS]
+  dict := map[string]string{
+    "name": name, "varname": varname, "flag": _flag, "package": pkg,
+    "output": outputPath, "shlib": shlib}
   err = gen.Execute(outFile, dict)
   if err != nil {
     panic(err)

cmd/hercules-generate-plugin/plugin.template

@@ -1,3 +1,8 @@
+// Hercules plugin
+// How to build: go build -buildmode=plugin {{.output}}
+// This command creates ./{{.shlib}}
+// Usage: hercules -plugin {{.shlib}} -{{.flag}}
+
 package {{.package}}
 
 import (
@@ -16,36 +21,48 @@ type {{.name}}Result struct {
 
 // Analysis' name in the graph is usually the same as the type's name, however, does not have to.
 func ({{.varname}} *{{.name}}) Name() string {
-	return "{{.name}}"
+  return "{{.name}}"
 }
 
 // LeafPipelineItem-s normally do not act as intermediate nodes and thus we return an empty slice.
 func ({{.varname}} *{{.name}}) Provides() []string {
-	return []string{}
+  return []string{}
 }
 
 // Requires returns the list of dependencies which must be supplied in Consume().
 func ({{.varname}} *{{.name}}) Requires() []string {
-	arr := [...]string{/* insert dependencies here */}
-	return arr[:]
+  arr := [...]string{/* insert dependencies here */}
+  return arr[:]
 }
 
+// ListConfigurationOptions tells the engine which parameters can be changed through the command
+// line.
 func ({{.varname}} *{{.name}}) ListConfigurationOptions() []hercules.ConfigurationOption {
-	return []hercules.ConfigurationOption{}
+  opts := [...]hercules.ConfigurationOption{ /* {
+    Name:        "ParameterName",
+    Description: "Parameter's description.",
+    Flag:        "my-cmdline-flag",
+    Type:        hercules.BoolConfigurationOption,
+    Default:     false}, */
+  }
+  return opts[:]
 }
 
+// Flag returns the command line switch which activates the analysis.
 func ({{.varname}} *{{.name}}) Flag() string {
-	return "{{.flag}}"
+  return "{{.flag}}"
 }
 
+// Configure applies the parameters specified in the command line. Map keys correspond to "Name".
 func ({{.varname}} *{{.name}}) Configure(facts map[string]interface{}) {
 }
 
+// Initialize resets the internal temporary data structures and prepares the object for Consume().
 func ({{.varname}} *{{.name}}) Initialize(repository *git.Repository) {
 }
 
 func ({{.varname}} *{{.name}}) Consume(deps map[string]interface{}) (map[string]interface{}, error) {
-	return nil, nil
+  return nil, nil
 }
 
 func ({{.varname}} *{{.name}}) Finalize() interface{} {
@@ -55,5 +72,5 @@ func ({{.varname}} *{{.name}}) Finalize() interface{} {
 }
 
 func init() {
-	hercules.Registry.Register(&{{.name}}{})
+  hercules.Registry.Register(&{{.name}}{})
 }

cmd/hercules/main.go

@@ -173,7 +173,7 @@ func main() {
 	}
 	if len(flag.Args()) == 0 || len(flag.Args()) > 3 {
 		fmt.Fprint(os.Stderr,
-			"Usage: hercules <path to repo or URL> [<disk cache path>]\n")
+			"Usage: hercules [options] <path to repo or URL> [<disk cache path>]\n")
 		os.Exit(1)
 	}
 	uri := flag.Arg(0)

diff.go

@@ -1,8 +1,10 @@
 package hercules
 
 import (
+	"bufio"
 	"bytes"
 	"errors"
+	"unicode/utf8"
 
 	"github.com/sergi/go-diff/diffmatchpatch"
 	"gopkg.in/src-d/go-git.v4"
@@ -58,11 +60,11 @@ func (diff *FileDiff) Consume(deps map[string]interface{}) (map[string]interface
 			blob_to := cache[change.To.TreeEntry.Hash]
 			// we are not validating UTF-8 here because for example
 			// git/git 4f7770c87ce3c302e1639a7737a6d2531fe4b160 fetch-pack.c is invalid UTF-8
-			str_from, err := blobToString(blob_from)
+			str_from, err := BlobToString(blob_from)
 			if err != nil {
 				return nil, err
 			}
-			str_to, err := blobToString(blob_to)
+			str_to, err := BlobToString(blob_to)
 			if err != nil {
 				return nil, err
 			}
@@ -81,9 +83,37 @@ func (diff *FileDiff) Consume(deps map[string]interface{}) (map[string]interface
 	return map[string]interface{}{"file_diff": result}, nil
 }
 
-func blobToString(file *object.Blob) (string, error) {
+func CountLines(file *object.Blob) (int, error) {
 	if file == nil {
-		return "", errors.New("Blob not cached.")
+		return -1, errors.New("Blob is nil: probably not cached.")
+	}
+	reader, err := file.Reader()
+	if err != nil {
+		return -1, err
+	}
+	defer checkClose(reader)
+	var scanner *bufio.Scanner
+	buffer := make([]byte, bufio.MaxScanTokenSize)
+	counter := 0
+	for scanner == nil || scanner.Err() == bufio.ErrTooLong {
+		if scanner != nil && !utf8.Valid(scanner.Bytes()) {
+			return -1, errors.New("binary")
+		}
+		scanner = bufio.NewScanner(reader)
+		scanner.Buffer(buffer, 0)
+		for scanner.Scan() {
+			if !utf8.Valid(scanner.Bytes()) {
+				return -1, errors.New("binary")
+			}
+			counter++
+		}
+	}
+	return counter, nil
+}
+
+func BlobToString(file *object.Blob) (string, error) {
+	if file == nil {
+		return "", errors.New("Blob is nil: probably not cached.")
 	}
 	reader, err := file.Reader()
 	if err != nil {

diff_test.go

@@ -177,3 +177,53 @@ func TestFileDiffConsumeInvalidBlob(t *testing.T) {
 	assert.Nil(t, res)
 	assert.NotNil(t, err)
 }
+
+func TestCountLines(t *testing.T) {
+	blob, _ := testRepository.BlobObject(
+		plumbing.NewHash("291286b4ac41952cbd1389fda66420ec03c1a9fe"))
+	lines, err := CountLines(blob)
+	assert.Equal(t, lines, 12)
+	assert.Nil(t, err)
+	lines, err = CountLines(nil)
+	assert.Equal(t, lines, -1)
+	assert.NotNil(t, err)
+	blob, _ = createDummyBlob(plumbing.NewHash("291286b4ac41952cbd1389fda66420ec03c1a9fe"), true)
+	lines, err = CountLines(blob)
+	assert.Equal(t, lines, -1)
+	assert.NotNil(t, err)
+	// test_data/blob
+	blob, err = testRepository.BlobObject(
+		plumbing.NewHash("c86626638e0bc8cf47ca49bb1525b40e9737ee64"))
+	assert.Nil(t, err)
+	lines, err = CountLines(blob)
+	assert.Equal(t, lines, -1)
+	assert.NotNil(t, err)
+	assert.EqualError(t, err, "binary")
+}
+
+func TestBlobToString(t *testing.T) {
+	blob, _ := testRepository.BlobObject(
+		plumbing.NewHash("291286b4ac41952cbd1389fda66420ec03c1a9fe"))
+	str, err := BlobToString(blob)
+	assert.Nil(t, err)
+	assert.Equal(t, str, `language: go
+
+go:
+  - 1.7
+
+go_import_path: gopkg.in/src-d/hercules.v1
+`+"  "+`
+script:
+  - go test -v -cpu=1,2 ./...
+
+notifications:
+  email: false
+`)
+	str, err = BlobToString(nil)
+	assert.Equal(t, str, "")
+	assert.NotNil(t, err)
+	blob, _ = createDummyBlob(plumbing.NewHash("291286b4ac41952cbd1389fda66420ec03c1a9fe"), true)
+	str, err = BlobToString(blob)
+	assert.Equal(t, str, "")
+	assert.NotNil(t, err)
+}

dummies.go

@@ -3,6 +3,7 @@ package hercules
 import (
 	"io"
 
+	"github.com/pkg/errors"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
@@ -24,6 +25,7 @@ func (dummyIO) Close() error {
 
 type dummyEncodedObject struct {
 	FakeHash plumbing.Hash
+	Fails    bool
 }
 
 func (obj dummyEncodedObject) Hash() plumbing.Hash {
@@ -45,13 +47,26 @@ func (obj dummyEncodedObject) SetSize(int64) {
 }
 
 func (obj dummyEncodedObject) Reader() (io.ReadCloser, error) {
-	return dummyIO{}, nil
+	if !obj.Fails {
+		return dummyIO{}, nil
+	}
+	return nil, errors.New("dummy failure")
 }
 
 func (obj dummyEncodedObject) Writer() (io.WriteCloser, error) {
-	return dummyIO{}, nil
+	if !obj.Fails {
+		return dummyIO{}, nil
+	}
+	return nil, errors.New("dummy failure")
 }
 
-func createDummyBlob(hash plumbing.Hash) (*object.Blob, error) {
-	return object.DecodeBlob(dummyEncodedObject{hash})
+func createDummyBlob(hash plumbing.Hash, fails ...bool) (*object.Blob, error) {
+	if len(fails) > 1 {
+		panic("invalid usage of createDummyBlob() - this is a bug")
+	}
+	var realFails bool
+	if len(fails) == 1 {
+		realFails = fails[0]
+	}
+	return object.DecodeBlob(dummyEncodedObject{FakeHash: hash, Fails: realFails})
 }

dummies_test.go

@@ -24,6 +24,17 @@ func TestCreateDummyBlob(t *testing.T) {
 	reader.Close()
 }
 
+func TestCreateDummyBlobFails(t *testing.T) {
+	dummy, err := createDummyBlob(plumbing.NewHash("334cde09da4afcb74f8d2b3e6fd6cce61228b485"), true)
+	assert.Nil(t, err)
+	reader, err := dummy.Reader()
+	assert.Nil(t, reader)
+	assert.NotNil(t, err)
+	assert.Panics(t, func() {
+		createDummyBlob(plumbing.NewHash("334cde09da4afcb74f8d2b3e6fd6cce61228b485"), true, true)
+	})
+}
+
 func TestNotUsedDummyStuff(t *testing.T) {
 	dio := dummyIO{}
 	n, err := dio.Write([]byte{})

file.go

@@ -246,14 +246,14 @@ func (file *File) Update(time int, pos int, ins_length int, del_length int) {
 
 	if ins_length > 0 {
 		if origin.Value != time {
-			tree.Insert(rbtree.Item{pos + ins_length, origin.Value})
+			tree.Insert(rbtree.Item{Key: pos + ins_length, Value: origin.Value})
 		} else if pos == 0 {
 			// recover the beginning
-			tree.Insert(rbtree.Item{pos, time})
+			tree.Insert(rbtree.Item{Key: pos, Value: time})
 		}
 	} else if (pos > origin.Key && previous.Value != origin.Value) || pos == origin.Key || pos == 0 {
 		// continue the original interval
-		tree.Insert(rbtree.Item{pos, origin.Value})
+		tree.Insert(rbtree.Item{Key: pos, Value: origin.Value})
 	}
 }
 

file_test.go

@@ -391,7 +391,7 @@ func TestUpdatePanic(t *testing.T) {
 	vals := [...]int{-1}
 	file := NewFileFromTree(keys[:], vals[:])
 	file.tree.DeleteWithKey(0)
-	file.tree.Insert(rbtree.Item{-1, -1})
+	file.tree.Insert(rbtree.Item{Key: -1, Value: -1})
 	var paniced interface{}
 	func() {
 		defer func() {
@@ -413,18 +413,18 @@ func TestFileValidate(t *testing.T) {
 	vals := [...]int{-1}
 	file := NewFileFromTree(keys[:], vals[:])
 	file.tree.DeleteWithKey(0)
-	file.tree.Insert(rbtree.Item{-1, -1})
+	file.tree.Insert(rbtree.Item{Key: -1, Value: -1})
 	assert.Panics(t, func() { file.Validate() })
 	file.tree.DeleteWithKey(-1)
-	file.tree.Insert(rbtree.Item{0, -1})
+	file.tree.Insert(rbtree.Item{Key: 0, Value: -1})
 	file.Validate()
 	file.tree.DeleteWithKey(0)
-	file.tree.Insert(rbtree.Item{0, 0})
+	file.tree.Insert(rbtree.Item{Key: 0, Value: 0})
 	assert.Panics(t, func() { file.Validate() })
 	file.tree.DeleteWithKey(0)
-	file.tree.Insert(rbtree.Item{0, 1})
-	file.tree.Insert(rbtree.Item{1, 1})
-	file.tree.Insert(rbtree.Item{2, -1})
+	file.tree.Insert(rbtree.Item{Key: 0, Value: 1})
+	file.tree.Insert(rbtree.Item{Key: 1, Value: 1})
+	file.tree.Insert(rbtree.Item{Key: 2, Value: -1})
 	file.Validate()
 	file.tree.FindGE(2).Item().Key = 1
 	assert.Panics(t, func() { file.Validate() })

identity_test.go

@@ -125,7 +125,7 @@ func TestIdentityDetectorRegistration(t *testing.T) {
 
 func TestIdentityDetectorConfigureEmpty(t *testing.T) {
 	id := IdentityDetector{}
-	assert.Panics(t, func() {id.Configure(map[string]interface{}{})})
+	assert.Panics(t, func() { id.Configure(map[string]interface{}{}) })
 }
 
 func TestIdentityDetectorConsume(t *testing.T) {

pipeline.go

@@ -33,7 +33,7 @@ const (
 
 const (
 	ConfigPipelineDumpPath = "Pipeline.DumpPath"
-	ConfigPipelineDryRun = "Pipeline.DryRun"
+	ConfigPipelineDryRun   = "Pipeline.DryRun"
 )
 
 // ConfigurationOption allows for the unified, retrospective way to setup PipelineItem-s.
@@ -203,8 +203,8 @@ func (registry *PipelineItemRegistry) AddFlags() (map[string]interface{}, map[st
 		flags[ConfigPipelineDumpPath] = iface
 		iface = interface{}(true)
 		ptr2 := (**bool)(unsafe.Pointer(uintptr(unsafe.Pointer(&iface)) + unsafe.Sizeof(&iface)))
-		*ptr2 = flag.Bool("dry-run", false, "Do not run any analyses - only resolve the DAG. " +
-				"Useful for -dump-dag.")
+		*ptr2 = flag.Bool("dry-run", false, "Do not run any analyses - only resolve the DAG. "+
+			"Useful for -dump-dag.")
 		flags[ConfigPipelineDryRun] = iface
 	}
 	features := []string{}
@@ -413,7 +413,7 @@ func (pipeline *Pipeline) resolve(dumpPath string) {
 						fmt.Fprint(os.Stderr, item2.Name(), " [")
 						for i, key2 := range item2.Provides() {
 							fmt.Fprint(os.Stderr, key2)
-							if i < len(item.Provides()) - 1 {
+							if i < len(item.Provides())-1 {
 								fmt.Fprint(os.Stderr, ", ")
 							}
 						}

pipeline_test.go

@@ -2,9 +2,9 @@ package hercules
 
 import (
 	"errors"
+	"flag"
 	"io"
 	"io/ioutil"
-	"flag"
 	"os"
 	"path"
 	"reflect"

renames.go

@@ -186,11 +186,11 @@ func (ra *RenameAnalysis) sizesAreClose(size1 int64, size2 int64) bool {
 
 func (ra *RenameAnalysis) blobsAreClose(
 	blob1 *object.Blob, blob2 *object.Blob) (bool, error) {
-	str_from, err := blobToString(blob1)
+	str_from, err := BlobToString(blob1)
 	if err != nil {
 		return false, err
 	}
-	str_to, err := blobToString(blob2)
+	str_to, err := BlobToString(blob2)
 	if err != nil {
 		return false, err
 	}

uast.go

@@ -29,7 +29,7 @@ import (
 
 type UASTExtractor struct {
 	Endpoint       string
-	Context        func() context.Context
+	Context        func() (context.Context, context.CancelFunc)
 	PoolSize       int
 	Languages      map[string]bool
 	FailOnErrors   bool
@@ -128,10 +128,9 @@ func (exr *UASTExtractor) Configure(facts map[string]interface{}) {
 		exr.Endpoint = val
 	}
 	if val, exists := facts[ConfigUASTTimeout].(int); exists {
-		exr.Context = func() context.Context {
-			ctx, _ := context.WithTimeout(context.Background(),
+		exr.Context = func() (context.Context, context.CancelFunc) {
+			return context.WithTimeout(context.Background(),
 				time.Duration(val)*time.Second)
-			return ctx
 		}
 	}
 	if val, exists := facts[ConfigUASTPoolSize].(int); exists {
@@ -150,7 +149,9 @@ func (exr *UASTExtractor) Configure(facts map[string]interface{}) {
 
 func (exr *UASTExtractor) Initialize(repository *git.Repository) {
 	if exr.Context == nil {
-		exr.Context = func() context.Context { return context.Background() }
+		exr.Context = func() (context.Context, context.CancelFunc) {
+			return context.Background(), nil
+		}
 	}
 	poolSize := exr.PoolSize
 	if poolSize == 0 {
@@ -259,7 +260,11 @@ func (exr *UASTExtractor) extractUAST(
 	}
 	request.Content(contents)
 	request.Filename(file.Name)
-	response, err := request.DoWithContext(exr.Context())
+	ctx, cancel := exr.Context()
+	if cancel != nil {
+		defer cancel()
+	}
+	response, err := request.DoWithContext(ctx)
 	if err != nil {
 		if strings.Contains("missing driver", err.Error()) {
 			return nil, nil