Add the siva file support

Fixes #59

Signed-off-by: Vadim Markovtsev <vadim@sourced.tech>

cmd/hercules/root.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"path/filepath"
 	"plugin"
 	"runtime/pprof"
 	"strings"
@@ -19,7 +20,9 @@ import (
 	"github.com/spf13/pflag"
 	"golang.org/x/crypto/ssh/terminal"
 	progress "gopkg.in/cheggaaa/pb.v1"
+	"gopkg.in/src-d/go-billy.v4/memfs"
 	"gopkg.in/src-d/go-billy.v4/osfs"
+	"gopkg.in/src-d/go-billy-siva.v4"
 	"gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/go-git.v4/storage"
@@ -75,6 +78,19 @@ func loadRepository(uri string, cachePath string, disableStatus bool) *git.Repos
 		if !disableStatus {
 			fmt.Fprint(os.Stderr, strings.Repeat(" ", 80)+"\r")
 		}
+	} else if stat, err2 := os.Stat(uri); err2 == nil && !stat.IsDir() {
+		localFs := osfs.New(filepath.Dir(uri))
+		tmpFs := memfs.New()
+		basePath := filepath.Base(uri)
+		fs, err2 := sivafs.NewFilesystem(localFs, basePath, tmpFs)
+		if err2 != nil {
+			log.Panicf("unable to create a siva filesystem from %s: %v", uri, err2)
+		}
+		sivaStorage, err2 := filesystem.NewStorage(fs)
+		if err2 != nil {
+			log.Panicf("unable to create a new storage backend for siva file %s: %v", uri, err2)
+		}
+		repository, err = git.Open(sivaStorage, tmpFs)
 	} else {
 		if uri[len(uri)-1] == os.PathSeparator {
 			uri = uri[:len(uri)-1]
@@ -82,7 +98,7 @@ func loadRepository(uri string, cachePath string, disableStatus bool) *git.Repos
 		repository, err = git.PlainOpen(uri)
 	}
 	if err != nil {
-		panic(err)
+		log.Panicf("failed to open %s: %v", uri, err)
 	}
 	return repository
 }
@@ -90,7 +106,7 @@ func loadRepository(uri string, cachePath string, disableStatus bool) *git.Repos
 type arrayPluginFlags map[string]bool
 
 func (apf *arrayPluginFlags) String() string {
-	list := []string{}
+	var list []string
 	for key := range *apf {
 		list = append(list, key)
 	}
@@ -180,19 +196,17 @@ targets can be added using the --plugin system.`,
 		}
 
 		var commits []*object.Commit
+		var err error
 		if commitsFile == "" {
-			// list of commits belonging to the default branch, from oldest to newest
-			// rev-list --first-parent
 			fmt.Fprint(os.Stderr, "git log...\r")
-			commits = pipeline.Commits()
+			commits, err = pipeline.Commits()
 		} else {
-			var err error
 			commits, err = hercules.LoadCommitsFromFile(commitsFile, repository)
-			if err != nil {
-				panic(err)
-			}
 		}
-		cmdlineFacts["commits"] = commits
+		if err != nil {
+			log.Panicf("failed to list the commits: %v", err)
+		}
+		cmdlineFacts[hercules.ConfigPipelineCommits] = commits
 		var deployed []hercules.LeafPipelineItem
 		for name, valPtr := range cmdlineDeployed {
 			if *valPtr {

cmd/hercules/root_test.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/src-d/go-billy.v4/osfs"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
+	"gopkg.in/src-d/go-git.v4"
+)
+
+func TestLoadRepository(t *testing.T) {
+	repo := loadRepository("https://github.com/src-d/hercules", "", true)
+	assert.NotNil(t, repo)
+	log.Println("TestLoadRepository: 1/3")
+
+	tempdir, err := ioutil.TempDir("", "hercules-")
+	assert.Nil(t, err)
+	if err != nil {
+		assert.FailNow(t, "ioutil.TempDir")
+	}
+	defer os.RemoveAll(tempdir)
+	backend, err := filesystem.NewStorage(osfs.New(tempdir))
+	assert.Nil(t, err)
+	if err != nil {
+		assert.FailNow(t, "filesystem.NewStorage")
+	}
+	cloneOptions := &git.CloneOptions{URL: "https://github.com/src-d/hercules"}
+	_, err = git.Clone(backend, nil, cloneOptions)
+	assert.Nil(t, err)
+	if err != nil {
+		assert.FailNow(t, "filesystem.NewStorage")
+	}
+
+	repo = loadRepository(tempdir, "", true)
+	assert.NotNil(t, repo)
+	log.Println("TestLoadRepository: 2/3")
+
+	_, filename, _, _ := runtime.Caller(0)
+	sivafile := filepath.Join(filepath.Dir(filename), "test_data", "hercules.siva")
+	repo = loadRepository(sivafile, "", true)
+	assert.NotNil(t, repo)
+	log.Println("TestLoadRepository: 3/3")
+
+	assert.Panics(t, func() { loadRepository("https://github.com/src-d/porn", "", true) })
+	assert.Panics(t, func() { loadRepository(filepath.Dir(filename), "", true) })
+	assert.Panics(t, func() { loadRepository("/xxx", "", true) })
+}

contrib/_plugin_example/churn_analysis.go

@@ -14,7 +14,6 @@ import (
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/go-git.v4/utils/merkletrie"
 	"gopkg.in/src-d/hercules.v4"
-	"gopkg.in/src-d/hercules.v4/yaml"
 )
 
 // ChurnAnalysis contains the intermediate state which is mutated by Consume(). It should implement
@@ -206,7 +205,7 @@ func (churn *ChurnAnalysis) serializeText(result *ChurnAnalysisResult, writer io
 	fmt.Fprintln(writer, "  global:")
 	printEdits(result.Global, writer, 4)
 	for key, val := range result.People {
-		fmt.Fprintf(writer, "  %s:\n", yaml.SafeString(key))
+		fmt.Fprintf(writer, "  %s:\n", hercules.SafeYamlString(key))
 		printEdits(val, writer, 4)
 	}
 }

core.go

@@ -8,6 +8,7 @@ import (
 	"gopkg.in/src-d/hercules.v4/internal/plumbing/identity"
 	"gopkg.in/src-d/hercules.v4/internal/plumbing/uast"
 	"gopkg.in/src-d/hercules.v4/leaves"
+	"gopkg.in/src-d/hercules.v4/internal/yaml"
 )
 
 // ConfigurationOptionType represents the possible types of a ConfigurationOption's value.
@@ -145,6 +146,11 @@ func CountLines(file *object.Blob) (int, error) {
 	return plumbing.CountLines(file)
 }
 
+// SafeYamlString escapes the string so that it can be reliably used in YAML.
+func SafeYamlString(str string) string {
+	return yaml.SafeString(str)
+}
+
 func init() {
 	// hack to link with .leaves
 	_ = leaves.BurndownAnalysis{}

internal/core/pipeline.go

@@ -2,7 +2,6 @@ package core
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,11 +12,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/hercules.v4/internal/pb"
 	"gopkg.in/src-d/hercules.v4/internal/toposort"
+	"gopkg.in/src-d/go-git.v4/plumbing/storer"
 )
 
 // ConfigurationOptionType represents the possible types of a ConfigurationOption's value.
@@ -363,10 +364,29 @@ func (pipeline *Pipeline) Len() int {
 }
 
 // Commits returns the list of commits from the history similar to `git log` over the HEAD.
-func (pipeline *Pipeline) Commits() []*object.Commit {
+func (pipeline *Pipeline) Commits() ([]*object.Commit, error) {
 	cit, err := pipeline.repository.Log(&git.LogOptions{From: plumbing.ZeroHash})
 	if err != nil {
-		log.Fatalf("unable to collect the commit history: %v", err)
+		if err == plumbing.ErrReferenceNotFound {
+			refs, errr := pipeline.repository.References()
+			if errr != nil {
+				return nil, errors.Wrap(errr, "unable to list the references")
+			}
+			var head *plumbing.Reference
+			refs.ForEach(func(ref *plumbing.Reference) error {
+				if strings.HasPrefix(ref.Name().String(), "refs/heads/HEAD/") {
+					head = ref
+					return storer.ErrStop
+				}
+				return nil
+			})
+			if head != nil {
+				cit, err = pipeline.repository.Log(&git.LogOptions{From: head.Hash()})
+			}
+		}
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to collect the commit history")
+		}
 	}
 	defer cit.Close()
 	var result []*object.Commit
@@ -374,7 +394,7 @@ func (pipeline *Pipeline) Commits() []*object.Commit {
 		result = append(result, commit)
 		return nil
 	})
-	return result
+	return result, nil
 }
 
 type sortablePipelineItems []PipelineItem
@@ -524,7 +544,11 @@ func (pipeline *Pipeline) Initialize(facts map[string]interface{}) {
 		facts = map[string]interface{}{}
 	}
 	if _, exists := facts[ConfigPipelineCommits]; !exists {
-		facts[ConfigPipelineCommits] = pipeline.Commits()
+		var err error
+		facts[ConfigPipelineCommits], err = pipeline.Commits()
+		if err != nil {
+			log.Panicf("failed to list the commits: %v", err)
+		}
 	}
 	dumpPath, _ := facts[ConfigPipelineDumpPath].(string)
 	pipeline.resolve(dumpPath)

internal/core/pipeline_test.go

@@ -287,7 +287,8 @@ func TestPipelineOnProgress(t *testing.T) {
 
 func TestPipelineCommits(t *testing.T) {
 	pipeline := NewPipeline(test.Repository)
-	commits := pipeline.Commits()
+	commits, err := pipeline.Commits()
+	assert.Nil(t, err)
 	assert.True(t, len(commits) >= 100)
 	hashMap := map[plumbing.Hash]bool{}
 	for _, c := range commits {

leaves/burndown.go

@@ -20,7 +20,7 @@ import (
 	"gopkg.in/src-d/hercules.v4/internal/pb"
 	items "gopkg.in/src-d/hercules.v4/internal/plumbing"
 	"gopkg.in/src-d/hercules.v4/internal/plumbing/identity"
-	"gopkg.in/src-d/hercules.v4/yaml"
+	"gopkg.in/src-d/hercules.v4/internal/yaml"
 )
 
 // BurndownAnalysis allows to gather the line burndown statistics for a Git repository.

leaves/couples.go

@@ -13,7 +13,7 @@ import (
 	"gopkg.in/src-d/hercules.v4/internal/pb"
 	items "gopkg.in/src-d/hercules.v4/internal/plumbing"
 	"gopkg.in/src-d/hercules.v4/internal/plumbing/identity"
-	"gopkg.in/src-d/hercules.v4/yaml"
+	"gopkg.in/src-d/hercules.v4/internal/yaml"
 )
 
 // CouplesAnalysis calculates the number of common commits for files and authors.