Add pipeline tests

blob_cache_test.go

@@ -5,22 +5,21 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/storage/memory"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
 
-var repository *git.Repository
+var testRepository *git.Repository
 
 func fixtureBlobCache() *BlobCache {
 	cache := &BlobCache{}
-	cache.Initialize(repository)
+	cache.Initialize(testRepository)
 	return cache
 }
 
 func TestBlobCacheInitialize(t *testing.T) {
-  cache := fixtureBlobCache()
-	assert.Equal(t, repository, cache.repository)
+	cache := fixtureBlobCache()
+	assert.Equal(t, testRepository, cache.repository)
 }
 
 func TestBlobCacheMetadata(t *testing.T) {
@@ -34,13 +33,13 @@ func TestBlobCacheMetadata(t *testing.T) {
 }
 
 func TestBlobCacheConsumeModification(t *testing.T) {
-	commit, _ := repository.CommitObject(plumbing.NewHash(
+	commit, _ := testRepository.CommitObject(plumbing.NewHash(
 		"af2d8db70f287b52d2428d9887a69a10bc4d1f46"))
 	changes := make(object.Changes, 1)
-	treeFrom, _ := repository.TreeObject(plumbing.NewHash(
+	treeFrom, _ := testRepository.TreeObject(plumbing.NewHash(
 		"80fe25955b8e725feee25c08ea5759d74f8b670d"))
-	treeTo, _ := repository.TreeObject(plumbing.NewHash(
-	  "63076fa0dfd93e94b6d2ef0fc8b1fdf9092f83c4"))
+	treeTo, _ := testRepository.TreeObject(plumbing.NewHash(
+		"63076fa0dfd93e94b6d2ef0fc8b1fdf9092f83c4"))
 	changes[0] = &object.Change{From: object.ChangeEntry{
 		Name: "labours.py",
 		Tree: treeFrom,
@@ -77,13 +76,13 @@ func TestBlobCacheConsumeModification(t *testing.T) {
 }
 
 func TestBlobCacheConsumeInsertionDeletion(t *testing.T) {
-	commit, _ := repository.CommitObject(plumbing.NewHash(
+	commit, _ := testRepository.CommitObject(plumbing.NewHash(
 		"2b1ed978194a94edeabbca6de7ff3b5771d4d665"))
 	changes := make(object.Changes, 2)
-	treeFrom, _ := repository.TreeObject(plumbing.NewHash(
+	treeFrom, _ := testRepository.TreeObject(plumbing.NewHash(
 		"96c6ece9b2f3c7c51b83516400d278dea5605100"))
-	treeTo, _ := repository.TreeObject(plumbing.NewHash(
-	  "251f2094d7b523d5bcc60e663b6cf38151bf8844"))
+	treeTo, _ := testRepository.TreeObject(plumbing.NewHash(
+		"251f2094d7b523d5bcc60e663b6cf38151bf8844"))
 	changes[0] = &object.Change{From: object.ChangeEntry{
 		Name: "analyser.go",
 		Tree: treeFrom,
@@ -94,15 +93,15 @@ func TestBlobCacheConsumeInsertionDeletion(t *testing.T) {
 		},
 	}, To: object.ChangeEntry{},
 	}
-  changes[1] = &object.Change{From: object.ChangeEntry{}, To: object.ChangeEntry{
+	changes[1] = &object.Change{From: object.ChangeEntry{}, To: object.ChangeEntry{
+		Name: "pipeline.go",
+		Tree: treeTo,
+		TreeEntry: object.TreeEntry{
 			Name: "pipeline.go",
-			Tree: treeTo,
-			TreeEntry: object.TreeEntry{
-				Name: "pipeline.go",
-				Mode: 0100644,
-				Hash: plumbing.NewHash("db99e1890f581ad69e1527fe8302978c661eb473"),
-			},
+			Mode: 0100644,
+			Hash: plumbing.NewHash("db99e1890f581ad69e1527fe8302978c661eb473"),
 		},
+	},
 	}
 	deps := map[string]interface{}{}
 	deps["commit"] = commit
@@ -123,20 +122,20 @@ func TestBlobCacheConsumeInsertionDeletion(t *testing.T) {
 }
 
 func TestBlobCacheConsumeNoAction(t *testing.T) {
-  commit, _ := repository.CommitObject(plumbing.NewHash(
+	commit, _ := testRepository.CommitObject(plumbing.NewHash(
 		"af2d8db70f287b52d2428d9887a69a10bc4d1f46"))
 	changes := make(object.Changes, 1)
-	treeFrom, _ := repository.TreeObject(plumbing.NewHash(
+	treeFrom, _ := testRepository.TreeObject(plumbing.NewHash(
 		"80fe25955b8e725feee25c08ea5759d74f8b670d"))
-	treeTo, _ := repository.TreeObject(plumbing.NewHash(
-	  "63076fa0dfd93e94b6d2ef0fc8b1fdf9092f83c4"))
+	treeTo, _ := testRepository.TreeObject(plumbing.NewHash(
+		"63076fa0dfd93e94b6d2ef0fc8b1fdf9092f83c4"))
 	changes[0] = &object.Change{From: object.ChangeEntry{
-		Name: "labours.py",
-		Tree: treeFrom,
+		Name:      "labours.py",
+		Tree:      treeFrom,
 		TreeEntry: object.TreeEntry{},
 	}, To: object.ChangeEntry{
-		Name: "labours.py",
-		Tree: treeTo,
+		Name:      "labours.py",
+		Tree:      treeTo,
 		TreeEntry: object.TreeEntry{},
 	}}
 	deps := map[string]interface{}{}
@@ -148,13 +147,13 @@ func TestBlobCacheConsumeNoAction(t *testing.T) {
 }
 
 func TestBlobCacheConsumeInvalidHash(t *testing.T) {
-  commit, _ := repository.CommitObject(plumbing.NewHash(
+	commit, _ := testRepository.CommitObject(plumbing.NewHash(
 		"af2d8db70f287b52d2428d9887a69a10bc4d1f46"))
 	changes := make(object.Changes, 1)
-	treeFrom, _ := repository.TreeObject(plumbing.NewHash(
+	treeFrom, _ := testRepository.TreeObject(plumbing.NewHash(
 		"80fe25955b8e725feee25c08ea5759d74f8b670d"))
-	treeTo, _ := repository.TreeObject(plumbing.NewHash(
-	  "63076fa0dfd93e94b6d2ef0fc8b1fdf9092f83c4"))
+	treeTo, _ := testRepository.TreeObject(plumbing.NewHash(
+		"63076fa0dfd93e94b6d2ef0fc8b1fdf9092f83c4"))
 	changes[0] = &object.Change{From: object.ChangeEntry{
 		Name: "labours.py",
 		Tree: treeFrom,
@@ -164,8 +163,8 @@ func TestBlobCacheConsumeInvalidHash(t *testing.T) {
 			Hash: plumbing.NewHash("ffffffffffffffffffffffffffffffffffffffff"),
 		},
 	}, To: object.ChangeEntry{
-		Name: "labours.py",
-		Tree: treeTo,
+		Name:      "labours.py",
+		Tree:      treeTo,
 		TreeEntry: object.TreeEntry{},
 	}}
 	deps := map[string]interface{}{}
@@ -180,9 +179,3 @@ func TestBlobCacheFinalize(t *testing.T) {
 	outcome := fixtureBlobCache().Finalize()
 	assert.Nil(t, outcome)
 }
-
-func init() {
-	repository, _ = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-		URL: "https://github.com/src-d/hercules",
-	})
-}

cmd/hercules/main.go

@@ -253,7 +253,10 @@ func main() {
 	if commitsFile == "" {
 		commits = pipeline.Commits()
 	} else {
-		commits = hercules.LoadCommitsFromFile(commitsFile, repository)
+		commits, err = hercules.LoadCommitsFromFile(commitsFile, repository)
+		if err != nil {
+			panic(err)
+		}
 	}
 
 	pipeline.AddItem(&hercules.BlobCache{})

couples.go

@@ -41,11 +41,11 @@ func (couples *Couples) Requires() []string {
 }
 
 func (couples *Couples) Initialize(repository *git.Repository) {
-	couples.people = make([]map[string]int, couples.PeopleNumber + 1)
+	couples.people = make([]map[string]int, couples.PeopleNumber+1)
 	for i := range couples.people {
 		couples.people[i] = map[string]int{}
 	}
-	couples.people_commits = make([]int, couples.PeopleNumber + 1)
+	couples.people_commits = make([]int, couples.PeopleNumber+1)
 	couples.files = map[string]map[string]int{}
 }
 
@@ -107,8 +107,8 @@ func (couples *Couples) Consume(deps map[string]interface{}) (map[string]interfa
 }
 
 func (couples *Couples) Finalize() interface{} {
-	peopleMatrix := make([]map[int]int64, couples.PeopleNumber + 1)
-	peopleFiles := make([][]string, couples.PeopleNumber + 1)
+	peopleMatrix := make([]map[int]int64, couples.PeopleNumber+1)
+	peopleFiles := make([][]string, couples.PeopleNumber+1)
 	for i := range peopleMatrix {
 		peopleMatrix[i] = map[int]int64{}
 		for file, commits := range couples.people[i] {

pipeline.go

@@ -1,6 +1,7 @@
 package hercules
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -84,7 +85,7 @@ func (pipeline *Pipeline) Commits() []*object.Commit {
 	if err != nil {
 		panic(err)
 	}
-	result = append(result, commit)
+	// the first parent matches the head
 	for ; err != io.EOF; commit, err = commit.Parents().Next() {
 		if err != nil {
 			panic(err)
@@ -175,12 +176,13 @@ func (pipeline *Pipeline) Run(commits []*object.Commit) (map[PipelineItem]interf
 	return result, nil
 }
 
-func LoadCommitsFromFile(path string, repository *git.Repository) []*object.Commit {
-	var file io.Reader
+func LoadCommitsFromFile(path string, repository *git.Repository) ([]*object.Commit, error) {
+	var file io.ReadCloser
 	if path != "-" {
-		file, err := os.Open(path)
+		var err error
+		file, err = os.Open(path)
 		if err != nil {
-			panic(err)
+			return nil, err
 		}
 		defer file.Close()
 	} else {
@@ -191,13 +193,13 @@ func LoadCommitsFromFile(path string, repository *git.Repository) []*object.Comm
 	for scanner.Scan() {
 		hash := plumbing.NewHash(scanner.Text())
 		if len(hash) != 20 {
-			panic("invalid commit hash " + scanner.Text())
+			return nil, errors.New("invalid commit hash " + scanner.Text())
 		}
 		commit, err := repository.CommitObject(hash)
 		if err != nil {
-			panic(err)
+			return nil, err
 		}
 		commits = append(commits, commit)
 	}
-	return commits
+	return commits, nil
 }

pipeline_test.go

@@ -0,0 +1,152 @@
+package hercules
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/object"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+type testPipelineItem struct {
+	Initialized   bool
+	DepsConsumed  bool
+	CommitMatches bool
+	IndexMatches  bool
+}
+
+func (item *testPipelineItem) Name() string {
+	return "Test"
+}
+
+func (item *testPipelineItem) Provides() []string {
+	arr := [...]string{"test"}
+	return arr[:]
+}
+
+func (item *testPipelineItem) Requires() []string {
+	return []string{}
+}
+
+func (item *testPipelineItem) Initialize(repository *git.Repository) {
+	item.Initialized = repository != nil
+}
+
+func (item *testPipelineItem) Consume(deps map[string]interface{}) (map[string]interface{}, error) {
+	obj, exists := deps["commit"]
+	item.DepsConsumed = exists
+	if item.DepsConsumed {
+		commit := obj.(*object.Commit)
+		item.CommitMatches = commit.Hash == plumbing.NewHash(
+			"af9ddc0db70f09f3f27b4b98e415592a7485171c")
+		obj, item.DepsConsumed = deps["index"]
+		if item.DepsConsumed {
+			item.IndexMatches = obj.(int) == 0
+		}
+	}
+	return map[string]interface{}{"test": item}, nil
+}
+
+func (item *testPipelineItem) Finalize() interface{} {
+	return item
+}
+
+func TestPipelineRun(t *testing.T) {
+	pipeline := NewPipeline(testRepository)
+	item := &testPipelineItem{}
+	pipeline.AddItem(item)
+	pipeline.Initialize()
+	assert.True(t, item.Initialized)
+	commits := make([]*object.Commit, 1)
+	commits[0], _ = testRepository.CommitObject(plumbing.NewHash(
+		"af9ddc0db70f09f3f27b4b98e415592a7485171c"))
+	result, err := pipeline.Run(commits)
+	assert.Nil(t, err)
+	assert.Equal(t, item, result[item].(*testPipelineItem))
+	assert.True(t, item.DepsConsumed)
+	assert.True(t, item.CommitMatches)
+	assert.True(t, item.IndexMatches)
+	pipeline.RemoveItem(item)
+	result, err = pipeline.Run(commits)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(result))
+}
+
+func TestPipelineOnProgress(t *testing.T) {
+	pipeline := NewPipeline(testRepository)
+	var progressOk1, progressOk2 bool
+
+	onProgress := func(step int, total int) {
+		if step == 0 && total == 1 {
+			progressOk1 = true
+		}
+		if step == 1 && total == 1 && progressOk1 {
+			progressOk2 = true
+		}
+	}
+
+	pipeline.OnProgress = onProgress
+	commits := make([]*object.Commit, 1)
+	commits[0], _ = testRepository.CommitObject(plumbing.NewHash(
+		"af9ddc0db70f09f3f27b4b98e415592a7485171c"))
+	result, err := pipeline.Run(commits)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(result))
+	assert.True(t, progressOk1)
+	assert.True(t, progressOk2)
+}
+
+func TestPipelineCommits(t *testing.T) {
+	pipeline := NewPipeline(testRepository)
+	commits := pipeline.Commits()
+	assert.True(t, len(commits) >= 90)
+	assert.Equal(t, commits[0].Hash, plumbing.NewHash(
+		"cce947b98a050c6d356bc6ba95030254914027b1"))
+	assert.Equal(t, commits[89].Hash, plumbing.NewHash(
+		"6db8065cdb9bb0758f36a7e75fc72ab95f9e8145"))
+	assert.NotEqual(t, commits[len(commits)-1], commits[len(commits)-2])
+}
+
+func TestLoadCommitsFromFile(t *testing.T) {
+	tmp, err := ioutil.TempFile("", "hercules-test-")
+	assert.Nil(t, err)
+	tmp.WriteString("cce947b98a050c6d356bc6ba95030254914027b1\n6db8065cdb9bb0758f36a7e75fc72ab95f9e8145")
+	tmp.Close()
+	defer os.Remove(tmp.Name())
+	commits, err := LoadCommitsFromFile(tmp.Name(), testRepository)
+	assert.Nil(t, err)
+	assert.Equal(t, len(commits), 2)
+	assert.Equal(t, commits[0].Hash, plumbing.NewHash(
+		"cce947b98a050c6d356bc6ba95030254914027b1"))
+	assert.Equal(t, commits[1].Hash, plumbing.NewHash(
+		"6db8065cdb9bb0758f36a7e75fc72ab95f9e8145"))
+	commits, err = LoadCommitsFromFile("/WAT?xxx!", testRepository)
+	assert.Nil(t, commits)
+	assert.NotNil(t, err)
+	tmp, err = ioutil.TempFile("", "hercules-test-")
+	assert.Nil(t, err)
+	tmp.WriteString("WAT")
+	tmp.Close()
+	defer os.Remove(tmp.Name())
+	commits, err = LoadCommitsFromFile(tmp.Name(), testRepository)
+	assert.Nil(t, commits)
+	assert.NotNil(t, err)
+	tmp, err = ioutil.TempFile("", "hercules-test-")
+	assert.Nil(t, err)
+	tmp.WriteString("ffffffffffffffffffffffffffffffffffffffff")
+	tmp.Close()
+	defer os.Remove(tmp.Name())
+	commits, err = LoadCommitsFromFile(tmp.Name(), testRepository)
+	assert.Nil(t, commits)
+	assert.NotNil(t, err)
+}
+
+func init() {
+	testRepository, _ = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
+		URL: "https://github.com/src-d/hercules",
+	})
+}