Merge pull request #100 from vmarkovtsev/master

Fix days comparison

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length=99
+exclude=.git,internal/pb/pb_pb2.py

.travis.yml

@@ -54,7 +54,7 @@ before_install:
   - export PATH=~/usr/bin:$PATH
   - make --version
   - pip3 --version
-  - pip3 install --user --no-build-isolation -r requirements.txt tensorflow
+  - pip3 install --user --no-build-isolation -r requirements.txt tensorflow flake8
   - docker run -d --privileged -p 9432:9432 --name bblfshd bblfsh/bblfshd
   - docker exec -it bblfshd bblfshctl driver install python bblfsh/python-driver:latest
   - docker exec -it bblfshd bblfshctl driver install go bblfsh/go-driver:latest
@@ -68,6 +68,7 @@ script:
   - set -e
   - go vet -tags tensorflow ./...
   - golint -set_exit_status ./...
+  - flake8
   - if [[ $TRAVIS_GO_VERSION != 1.9.* ]]; then go test -coverpkg=all -v -cpu=1,2 -coverprofile=coverage.txt -covermode=count gopkg.in/src-d/hercules.v4/... && sed -i '/cmd\/hercules\|core.go/d' coverage.txt; fi
   - if [[ $TRAVIS_GO_VERSION = 1.9.* ]]; then go test -v -cpu=1,2 gopkg.in/src-d/hercules.v4/...; fi
   - $GOPATH/bin/hercules version

contrib/_plugin_example/plot_churn.py

@@ -59,7 +59,8 @@ def plot_churn(name, data, url, beginTime, endTime, output, fmt, tick_interval):
         spine.set_visible(False)
     pyplot.gca().xaxis.set_major_locator(mdates.DayLocator(interval=tick_interval))
     pyplot.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d"))
-    pyplot.tick_params(top="off", bottom="off", left="off", right="off", labelleft="off", labelbottom="on")
+    pyplot.tick_params(top="off", bottom="off", left="off", right="off",
+                       labelleft="off", labelbottom="on")
     pyplot.bar(df.index, df["additions"], label="additions")
     pyplot.bar(df.index, df["removals"], label="removals")
     pyplot.plot(df.index, effective, "black", label="effective")

internal/burndown/file.go

@@ -254,16 +254,12 @@ func (file *File) Merge(day int, others... *File) {
 			if ol & TreeMergeMark == TreeMergeMark {
 				continue
 			}
-			// the following should happen rarely:
-			// l & TreeMergeMark != TreeMergeMark && l != ol
-			if l & TreeMergeMark == TreeMergeMark || l > ol {
-				// 1 - the line is merged in myself and exists in other
-				// 2 - the same line introduced in different branches,
-				// consider the oldest version as the ground truth
-				//
-				// in case with (2) we should decrease the "future" counter,
-				// but that really poisons the analysis
+			if l & TreeMergeMark == TreeMergeMark || l & TreeMergeMark > ol & TreeMergeMark {
+				// the line is merged in myself and exists in other
+				// OR the same line introduced in different branches
+				// consider the oldest version as the ground truth in that case
 				myself[i] = ol
+				continue
 			}
 		}
 	}

internal/core/pipeline.go

@@ -145,14 +145,16 @@ type ResultMergeablePipelineItem interface {
 
 // CommonAnalysisResult holds the information which is always extracted at Pipeline.Run().
 type CommonAnalysisResult struct {
-	// Time of the first commit in the analysed sequence.
+	// BeginTime is the time of the first commit in the analysed sequence.
 	BeginTime int64
-	// Time of the last commit in the analysed sequence.
+	// EndTime is the time of the last commit in the analysed sequence.
 	EndTime int64
-	// The number of commits in the analysed sequence.
+	// CommitsNumber is the number of commits in the analysed sequence.
 	CommitsNumber int
-	// The duration of Pipeline.Run().
+	// RunTime is the duration of Pipeline.Run().
 	RunTime time.Duration
+	// RunTimePerItem is the time elapsed by each PipelineItem.
+	RunTimePerItem map[string]float64
 }
 
 // BeginTimeAsTime converts the UNIX timestamp of the beginning to Go time.
@@ -180,6 +182,9 @@ func (car *CommonAnalysisResult) Merge(other *CommonAnalysisResult) {
 	}
 	car.CommitsNumber += other.CommitsNumber
 	car.RunTime += other.RunTime
+	for key, val := range other.RunTimePerItem {
+		car.RunTimePerItem[key] += val
+	}
 }
 
 // FillMetadata copies the data to a Protobuf message.
@@ -188,6 +193,7 @@ func (car *CommonAnalysisResult) FillMetadata(meta *pb.Metadata) *pb.Metadata {
 	meta.EndUnixTime = car.EndTime
 	meta.Commits = int32(car.CommitsNumber)
 	meta.RunTime = car.RunTime.Nanoseconds() / 1e6
+	meta.RunTimePerItem = car.RunTimePerItem
 	return meta
 }
 
@@ -197,10 +203,11 @@ type Metadata = pb.Metadata
 // MetadataToCommonAnalysisResult copies the data from a Protobuf message.
 func MetadataToCommonAnalysisResult(meta *Metadata) *CommonAnalysisResult {
 	return &CommonAnalysisResult{
-		BeginTime:     meta.BeginUnixTime,
-		EndTime:       meta.EndUnixTime,
-		CommitsNumber: int(meta.Commits),
-		RunTime:       time.Duration(meta.RunTime * 1e6),
+		BeginTime:      meta.BeginUnixTime,
+		EndTime:        meta.EndUnixTime,
+		CommitsNumber:  int(meta.Commits),
+		RunTime:        time.Duration(meta.RunTime * 1e6),
+		RunTimePerItem: meta.RunTimePerItem,
 	}
 }
 
@@ -608,6 +615,7 @@ func (pipeline *Pipeline) Run(commits []*object.Commit) (map[LeafPipelineItem]in
 	// we will need rootClone if there is more than one root branch
 	rootClone := cloneItems(pipeline.items, 1)[0]
 	var newestTime int64
+	runTimePerItem := map[string]float64{}
 
 	commitIndex := 0
 	for index, step := range plan {
@@ -627,7 +635,9 @@ func (pipeline *Pipeline) Run(commits []*object.Commit) (map[LeafPipelineItem]in
 					plan[index+1].Commit.Hash == step.Commit.Hash),
 			}
 			for _, item := range branches[firstItem] {
+				startTime := time.Now()
 				update, err := item.Consume(state)
+				runTimePerItem[item.Name()] += time.Now().Sub(startTime).Seconds()
 				if err != nil {
 					log.Printf("%s failed on commit #%d (%d) %s\n",
 						item.Name(), commitIndex + 1, index + 1, step.Commit.Hash.String())
@@ -675,10 +685,11 @@ func (pipeline *Pipeline) Run(commits []*object.Commit) (map[LeafPipelineItem]in
 	}
 	onProgress(progressSteps, progressSteps)
 	result[nil] = &CommonAnalysisResult{
-		BeginTime:     plan[0].Commit.Committer.When.Unix(),
-		EndTime:       newestTime,
-		CommitsNumber: len(commits),
-		RunTime:       time.Since(startRunTime),
+		BeginTime:      plan[0].Commit.Committer.When.Unix(),
+		EndTime:        newestTime,
+		CommitsNumber:  len(commits),
+		RunTime:        time.Since(startRunTime),
+		RunTimePerItem: runTimePerItem,
 	}
 	return result, nil
 }

internal/core/pipeline_test.go

@@ -224,6 +224,10 @@ func TestPipelineRun(t *testing.T) {
 	assert.Equal(t, common.EndTime, int64(1481719198))
 	assert.Equal(t, common.CommitsNumber, 1)
 	assert.True(t, common.RunTime.Nanoseconds()/1e6 < 100)
+	assert.Len(t, common.RunTimePerItem, 1)
+	for key, val := range common.RunTimePerItem {
+		assert.True(t, val >= 0, key)
+	}
 	assert.True(t, item.DepsConsumed)
 	assert.True(t, item.CommitMatches)
 	assert.True(t, item.IndexMatches)
@@ -405,27 +409,32 @@ func TestPipelineError(t *testing.T) {
 
 func TestCommonAnalysisResultMerge(t *testing.T) {
 	c1 := CommonAnalysisResult{
-		BeginTime: 1513620635, EndTime: 1513720635, CommitsNumber: 1, RunTime: 100}
+		BeginTime: 1513620635, EndTime: 1513720635, CommitsNumber: 1, RunTime: 100,
+	    RunTimePerItem: map[string]float64{"one": 1, "two": 2}}
 	assert.Equal(t, c1.BeginTimeAsTime().Unix(), int64(1513620635))
 	assert.Equal(t, c1.EndTimeAsTime().Unix(), int64(1513720635))
 	c2 := CommonAnalysisResult{
-		BeginTime: 1513620535, EndTime: 1513730635, CommitsNumber: 2, RunTime: 200}
+		BeginTime: 1513620535, EndTime: 1513730635, CommitsNumber: 2, RunTime: 200,
+		RunTimePerItem: map[string]float64{"two": 4, "three": 8}}
 	c1.Merge(&c2)
 	assert.Equal(t, c1.BeginTime, int64(1513620535))
 	assert.Equal(t, c1.EndTime, int64(1513730635))
 	assert.Equal(t, c1.CommitsNumber, 3)
 	assert.Equal(t, c1.RunTime.Nanoseconds(), int64(300))
+	assert.Equal(t, c1.RunTimePerItem, map[string]float64{"one": 1, "two": 6, "three": 8})
 }
 
 func TestCommonAnalysisResultMetadata(t *testing.T) {
 	c1 := &CommonAnalysisResult{
-		BeginTime: 1513620635, EndTime: 1513720635, CommitsNumber: 1, RunTime: 100 * 1e6}
+		BeginTime: 1513620635, EndTime: 1513720635, CommitsNumber: 1, RunTime: 100 * 1e6,
+		RunTimePerItem: map[string]float64{"one": 1, "two": 2}}
 	meta := &pb.Metadata{}
 	c1 = MetadataToCommonAnalysisResult(c1.FillMetadata(meta))
 	assert.Equal(t, c1.BeginTimeAsTime().Unix(), int64(1513620635))
 	assert.Equal(t, c1.EndTimeAsTime().Unix(), int64(1513720635))
 	assert.Equal(t, c1.CommitsNumber, 1)
 	assert.Equal(t, c1.RunTime.Nanoseconds(), int64(100*1e6))
+	assert.Equal(t, c1.RunTimePerItem, map[string]float64{"one": 1, "two": 2})
 }
 
 func TestConfigurationOptionTypeString(t *testing.T) {

internal/pb/pb.pb.go

@@ -58,6 +58,8 @@ type Metadata struct {
 	Commits int32 `protobuf:"varint,6,opt,name=commits,proto3" json:"commits,omitempty"`
 	// duration of the analysis in milliseconds
 	RunTime int64 `protobuf:"varint,7,opt,name=run_time,json=runTime,proto3" json:"run_time,omitempty"`
+	// time taken by each pipeline item in seconds
+	RunTimePerItem map[string]float64 `protobuf:"bytes,8,rep,name=run_time_per_item,json=runTimePerItem" json:"run_time_per_item,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"fixed64,2,opt,name=value,proto3"`
 }
 
 func (m *Metadata) Reset()                    { *m = Metadata{} }
@@ -114,6 +116,13 @@ func (m *Metadata) GetRunTime() int64 {
 	return 0
 }
 
+func (m *Metadata) GetRunTimePerItem() map[string]float64 {
+	if m != nil {
+		return m.RunTimePerItem
+	}
+	return nil
+}
+
 type BurndownSparseMatrixRow struct {
 	// the first `len(column)` elements are stored,
 	// the rest `number_of_columns - len(column)` values are zeros
@@ -615,71 +624,74 @@ func init() {
 func init() { proto.RegisterFile("pb.proto", fileDescriptorPb) }
 
 var fileDescriptorPb = []byte{
-	// 1053 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0xdf, 0x6e, 0x1b, 0xc5,
-	0x17, 0xd6, 0x66, 0xfd, 0xf7, 0xac, 0x9d, 0xb4, 0xf3, 0xeb, 0xaf, 0xd9, 0x06, 0xb5, 0x98, 0x25,
-	0x80, 0xa1, 0x65, 0x8b, 0xdc, 0x1b, 0x08, 0x37, 0x24, 0x2e, 0x15, 0xbd, 0x08, 0x48, 0xe3, 0x14,
-	0x2e, 0xad, 0xf1, 0xee, 0x24, 0x5e, 0x58, 0xcf, 0xac, 0x66, 0x76, 0x93, 0xf8, 0x65, 0xb8, 0x43,
-	0x42, 0x48, 0x88, 0x0b, 0x5e, 0x80, 0xd7, 0xe0, 0x19, 0x78, 0x09, 0x34, 0xff, 0xec, 0xb5, 0xe5,
-	0x54, 0xdc, 0xcd, 0x39, 0xe7, 0xfb, 0xce, 0x9c, 0xf9, 0xce, 0x99, 0xd9, 0x85, 0x4e, 0x31, 0x8b,
-	0x0b, 0xc1, 0x4b, 0x1e, 0xfd, 0xed, 0x41, 0xe7, 0x9c, 0x96, 0x24, 0x25, 0x25, 0x41, 0x21, 0xb4,
-	0xaf, 0xa9, 0x90, 0x19, 0x67, 0xa1, 0x37, 0xf0, 0x86, 0x4d, 0xec, 0x4c, 0x84, 0xa0, 0x31, 0x27,
-	0x72, 0x1e, 0xee, 0x0d, 0xbc, 0x61, 0x17, 0xeb, 0x35, 0x7a, 0x02, 0x20, 0x68, 0xc1, 0x65, 0x56,
-	0x72, 0xb1, 0x0c, 0x7d, 0x1d, 0xa9, 0x79, 0xd0, 0x87, 0x70, 0x30, 0xa3, 0x57, 0x19, 0x9b, 0x56,
-	0x2c, 0xbb, 0x9d, 0x96, 0xd9, 0x82, 0x86, 0x8d, 0x81, 0x37, 0xf4, 0x71, 0x5f, 0xbb, 0xdf, 0xb0,
-	0xec, 0xf6, 0x22, 0x5b, 0x50, 0x14, 0x41, 0x9f, 0xb2, 0xb4, 0x86, 0x6a, 0x6a, 0x54, 0x40, 0x59,
-	0xba, 0xc2, 0x84, 0xd0, 0x4e, 0xf8, 0x62, 0x91, 0x95, 0x32, 0x6c, 0x99, 0xca, 0xac, 0x89, 0x1e,
-	0x41, 0x47, 0x54, 0xcc, 0x10, 0xdb, 0x9a, 0xd8, 0x16, 0x15, 0x53, 0xa4, 0xe8, 0x05, 0x1c, 0x9e,
-	0x55, 0x82, 0xa5, 0xfc, 0x86, 0x4d, 0x0a, 0x22, 0x24, 0x3d, 0x27, 0xa5, 0xc8, 0x6e, 0x31, 0xbf,
-	0x31, 0xf9, 0xf2, 0x6a, 0xc1, 0x64, 0xe8, 0x0d, 0xfc, 0x61, 0x1f, 0x3b, 0x33, 0xfa, 0xcd, 0x83,
-	0x07, 0xbb, 0x58, 0x4a, 0x02, 0x46, 0x16, 0x54, 0x2b, 0xd3, 0xc5, 0x7a, 0x8d, 0x8e, 0x61, 0x9f,
-	0x55, 0x8b, 0x19, 0x15, 0x53, 0x7e, 0x39, 0x15, 0xfc, 0x46, 0x6a, 0x81, 0x9a, 0xb8, 0x67, 0xbc,
-	0xdf, 0x5d, 0x62, 0x7e, 0x23, 0xd1, 0x27, 0x70, 0x7f, 0x8d, 0x72, 0xdb, 0xfa, 0x1a, 0x78, 0xe0,
-	0x80, 0x63, 0xe3, 0x46, 0xcf, 0xa0, 0xa1, 0xf3, 0x34, 0x06, 0xfe, 0x30, 0x18, 0x85, 0xf1, 0x1d,
-	0x07, 0xc0, 0x1a, 0x15, 0xfd, 0xb1, 0xb7, 0x3e, 0xe2, 0x29, 0x23, 0xf9, 0x52, 0x66, 0x12, 0x53,
-	0x59, 0xe5, 0xa5, 0x44, 0x03, 0x08, 0xae, 0x04, 0x61, 0x55, 0x4e, 0x44, 0x56, 0x2e, 0x6d, 0x43,
-	0xeb, 0x2e, 0x74, 0x04, 0x1d, 0x49, 0x16, 0x45, 0x9e, 0xb1, 0x2b, 0x5b, 0xf7, 0xca, 0x46, 0xcf,
-	0xa1, 0x5d, 0x08, 0xfe, 0x23, 0x4d, 0x4a, 0x5d, 0x69, 0x30, 0xfa, 0xff, 0xee, 0x52, 0x1c, 0x0a,
-	0x3d, 0x85, 0xe6, 0x65, 0x96, 0x53, 0x57, 0xf9, 0x1d, 0x70, 0x83, 0x41, 0x9f, 0x42, 0xab, 0xa0,
-	0xbc, 0xc8, 0x55, 0xaf, 0xdf, 0x82, 0xb6, 0x20, 0xf4, 0x1a, 0x90, 0x59, 0x4d, 0x33, 0x56, 0x52,
-	0x41, 0x92, 0x52, 0x8d, 0x68, 0x4b, 0xd7, 0x75, 0x14, 0x8f, 0xf9, 0xa2, 0x10, 0x54, 0x4a, 0x9a,
-	0x1a, 0x32, 0xe6, 0x37, 0x96, 0x7f, 0xdf, 0xb0, 0x5e, 0xaf, 0x49, 0xd1, 0x9f, 0x1e, 0x3c, 0xba,
-	0x93, 0xb0, 0xa3, 0x9f, 0xde, 0x7f, 0xed, 0xe7, 0xde, 0xee, 0x7e, 0x22, 0x68, 0xa8, 0xab, 0x15,
-	0xfa, 0x03, 0x7f, 0xe8, 0xe3, 0x86, 0xbb, 0x66, 0x19, 0x4b, 0xb3, 0xc4, 0x8a, 0xd5, 0xc4, 0xce,
-	0x44, 0x0f, 0xa1, 0x95, 0xb1, 0xb4, 0x28, 0x85, 0xd6, 0xc5, 0xc7, 0xd6, 0x8a, 0x26, 0xd0, 0x1e,
-	0xf3, 0xaa, 0x50, 0xd2, 0x3d, 0x80, 0x66, 0xc6, 0x52, 0x7a, 0xab, 0xe7, 0xb6, 0x8b, 0x8d, 0x81,
-	0x46, 0xd0, 0x5a, 0xe8, 0x23, 0xe8, 0x3a, 0xde, 0xae, 0x8a, 0x45, 0x46, 0xc7, 0xd0, 0xbb, 0xe0,
-	0x55, 0x32, 0xa7, 0xe9, 0xab, 0xcc, 0x66, 0x36, 0x1d, 0xf4, 0x74, 0x51, 0xc6, 0x88, 0x7e, 0xf5,
-	0xe0, 0xa1, 0xdd, 0x7b, 0x7b, 0xc2, 0x9e, 0x42, 0x4f, 0x61, 0xa6, 0x89, 0x09, 0xdb, 0x86, 0x74,
-	0x62, 0x0b, 0xc7, 0x81, 0x8a, 0xba, 0xba, 0x9f, 0xc3, 0xbe, 0xed, 0xa1, 0x83, 0xb7, 0xb7, 0xe0,
-	0x7d, 0x13, 0x77, 0x84, 0xcf, 0xa0, 0x67, 0x09, 0xa6, 0xaa, 0x8e, 0x9e, 0x94, 0x7e, 0x5c, 0xaf,
-	0x19, 0x07, 0x06, 0xa2, 0x8d, 0xe8, 0x17, 0x0f, 0xe0, 0xcd, 0xe9, 0xe4, 0x62, 0x3c, 0x27, 0xec,
-	0x8a, 0xa2, 0x77, 0xa0, 0xab, 0xcb, 0xab, 0xdd, 0xda, 0x8e, 0x72, 0x7c, 0xab, 0x6e, 0xee, 0x63,
-	0x00, 0x29, 0x92, 0xe9, 0x8c, 0x5e, 0x72, 0x41, 0xed, 0xb3, 0xd6, 0x95, 0x22, 0x39, 0xd3, 0x0e,
-	0xc5, 0x55, 0x61, 0x72, 0x59, 0x52, 0x61, 0x9f, 0xb6, 0x8e, 0x14, 0xc9, 0xa9, 0xb2, 0xd1, 0xbb,
-	0x10, 0x54, 0x44, 0x96, 0x8e, 0xdc, 0x30, 0x2f, 0x9f, 0x72, 0x59, 0xf6, 0x63, 0xd0, 0x96, 0xa5,
-	0x37, 0x4d, 0x72, 0xe5, 0xd1, 0xfc, 0xe8, 0x2b, 0x38, 0x5c, 0x97, 0x29, 0x27, 0xe4, 0x9a, 0x0a,
-	0x27, 0xe9, 0x07, 0xd0, 0x4e, 0x8c, 0x5b, 0x77, 0x21, 0x18, 0x05, 0xf1, 0x1a, 0x8a, 0x5d, 0x2c,
-	0xfa, 0xc7, 0x83, 0xfd, 0xc9, 0x9c, 0x97, 0x8c, 0x4a, 0x89, 0x69, 0xc2, 0x45, 0x8a, 0xde, 0x87,
-	0xbe, 0xbe, 0x1c, 0x8c, 0xe4, 0x53, 0xc1, 0x73, 0x77, 0xe2, 0x9e, 0x73, 0x62, 0x9e, 0x53, 0xd5,
-	0x62, 0x15, 0x53, 0xd3, 0xaa, 0x5b, 0xac, 0x8d, 0xd5, 0xcb, 0xe6, 0xd7, 0x5e, 0x36, 0x04, 0x0d,
-	0xa5, 0x95, 0x3d, 0x9c, 0x5e, 0xa3, 0x2f, 0xa0, 0x93, 0xf0, 0x4a, 0xe5, 0x93, 0xf6, 0xde, 0x3e,
-	0x8e, 0x37, 0xab, 0x50, 0xbd, 0xd4, 0xf1, 0xaf, 0x59, 0x29, 0x96, 0x78, 0x05, 0x3f, 0xfa, 0x12,
-	0xfa, 0x1b, 0x21, 0x74, 0x0f, 0xfc, 0x9f, 0xa8, 0x7b, 0x95, 0xd4, 0x52, 0xd5, 0x76, 0x4d, 0xf2,
-	0x8a, 0xda, 0x9b, 0x64, 0x8c, 0x93, 0xbd, 0xcf, 0xbd, 0xe8, 0x25, 0x1c, 0xba, 0x6d, 0xb6, 0x47,
-	0xf0, 0x63, 0x68, 0x0b, 0xbd, 0xb3, 0xd3, 0xeb, 0x60, 0xab, 0x22, 0xec, 0xe2, 0xd1, 0x47, 0x10,
-	0xa8, 0x31, 0xf9, 0x26, 0x93, 0xfa, 0xeb, 0x54, 0xfb, 0xa2, 0x98, 0x9b, 0xe4, 0xcc, 0xe8, 0x67,
-	0x0f, 0xc2, 0x1a, 0xd2, 0x6c, 0x75, 0x4e, 0xa5, 0x24, 0x57, 0x14, 0x9d, 0xd4, 0x2f, 0x49, 0x30,
-	0x3a, 0x8e, 0xef, 0x42, 0xea, 0x80, 0xd5, 0xc1, 0x50, 0x8e, 0x5e, 0x01, 0xac, 0x9d, 0x75, 0x05,
-	0xba, 0x46, 0x81, 0xa8, 0xae, 0x40, 0x30, 0xea, 0x6d, 0xe4, 0xae, 0xe9, 0xf1, 0x03, 0x74, 0x27,
-	0x94, 0xa9, 0x2f, 0x1e, 0x2b, 0xd7, 0xb2, 0xa9, 0x44, 0x7b, 0x16, 0xa6, 0x9e, 0x76, 0x75, 0x1c,
-	0xca, 0x4a, 0xd3, 0xeb, 0x2e, 0x5e, 0xd9, 0xf5, 0x93, 0xfb, 0x9b, 0x27, 0xff, 0xcb, 0x83, 0xc3,
-	0xb1, 0x81, 0xad, 0x36, 0x70, 0x4a, 0x7f, 0x0f, 0xf7, 0xa4, 0xf3, 0x4d, 0x67, 0xcb, 0x69, 0x4a,
-	0x96, 0x56, 0x83, 0x67, 0xf1, 0x1d, 0x9c, 0x78, 0xe5, 0x38, 0x5b, 0xbe, 0x24, 0x4b, 0xa3, 0xc5,
-	0xbe, 0xdc, 0x70, 0x1e, 0x9d, 0xc3, 0xff, 0x76, 0xc0, 0x76, 0xcc, 0xc7, 0x60, 0x53, 0x1d, 0x58,
-	0x67, 0xaf, 0x6b, 0xf3, 0xbb, 0x07, 0x07, 0xdb, 0x43, 0xf2, 0x1e, 0xb4, 0xe6, 0x94, 0xa4, 0x54,
-	0xe8, 0x74, 0xc1, 0xa8, 0x1b, 0xbb, 0x3f, 0x1e, 0x6c, 0x03, 0xe8, 0x44, 0xe9, 0xc5, 0xca, 0x95,
-	0x5e, 0xc1, 0xe8, 0x49, 0xbc, 0x95, 0x26, 0x1e, 0x5b, 0xc0, 0x6a, 0xb6, 0x8d, 0x69, 0x66, 0xbb,
-	0x16, 0xda, 0xd1, 0xd9, 0x8d, 0xd9, 0xee, 0xd5, 0xea, 0x9d, 0xb5, 0xf4, 0x6f, 0xd8, 0x8b, 0x7f,
-	0x03, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x55, 0x2d, 0x51, 0x92, 0x09, 0x00, 0x00,
+	// 1098 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0xcd, 0x92, 0xdb, 0xc4,
+	0x13, 0x2f, 0x59, 0xfe, 0x6c, 0xd9, 0xbb, 0xc9, 0x24, 0xff, 0xac, 0xe2, 0x7f, 0x6d, 0x30, 0x22,
+	0x80, 0x21, 0x41, 0xa1, 0x9c, 0x0b, 0x84, 0x0b, 0x1b, 0x87, 0x54, 0x72, 0x58, 0xa0, 0xc6, 0x1b,
+	0x38, 0xaa, 0x64, 0x69, 0x76, 0x2d, 0xb0, 0x66, 0x54, 0x33, 0xd2, 0xee, 0xfa, 0x65, 0xb8, 0x51,
+	0x45, 0x51, 0x45, 0x71, 0xe0, 0x05, 0x78, 0x1a, 0x2e, 0xbc, 0x04, 0x35, 0x5f, 0xb6, 0xec, 0xf2,
+	0x06, 0x6e, 0xea, 0xee, 0x5f, 0xf7, 0x74, 0xff, 0xba, 0xa7, 0x47, 0xd0, 0x2d, 0xe6, 0x61, 0xc1,
+	0x59, 0xc9, 0x82, 0xbf, 0x1a, 0xd0, 0x3d, 0x25, 0x65, 0x9c, 0xc6, 0x65, 0x8c, 0x7c, 0xe8, 0x5c,
+	0x12, 0x2e, 0x32, 0x46, 0x7d, 0x67, 0xe4, 0x8c, 0x5b, 0xd8, 0x8a, 0x08, 0x41, 0x73, 0x11, 0x8b,
+	0x85, 0xdf, 0x18, 0x39, 0xe3, 0x1e, 0x56, 0xdf, 0xe8, 0x01, 0x00, 0x27, 0x05, 0x13, 0x59, 0xc9,
+	0xf8, 0xca, 0x77, 0x95, 0xa5, 0xa6, 0x41, 0x1f, 0xc0, 0xe1, 0x9c, 0x5c, 0x64, 0x34, 0xaa, 0x68,
+	0x76, 0x1d, 0x95, 0x59, 0x4e, 0xfc, 0xe6, 0xc8, 0x19, 0xbb, 0x78, 0xa0, 0xd4, 0x6f, 0x68, 0x76,
+	0x7d, 0x96, 0xe5, 0x04, 0x05, 0x30, 0x20, 0x34, 0xad, 0xa1, 0x5a, 0x0a, 0xe5, 0x11, 0x9a, 0xae,
+	0x31, 0x3e, 0x74, 0x12, 0x96, 0xe7, 0x59, 0x29, 0xfc, 0xb6, 0xce, 0xcc, 0x88, 0xe8, 0x3e, 0x74,
+	0x79, 0x45, 0xb5, 0x63, 0x47, 0x39, 0x76, 0x78, 0x45, 0x95, 0xd3, 0x2b, 0xb8, 0x6d, 0x4d, 0x51,
+	0x41, 0x78, 0x94, 0x95, 0x24, 0xf7, 0xbb, 0x23, 0x77, 0xec, 0x4d, 0x8e, 0x43, 0x5b, 0x74, 0x88,
+	0x35, 0xfa, 0x5b, 0xc2, 0x5f, 0x97, 0x24, 0xff, 0x8a, 0x96, 0x7c, 0x85, 0x0f, 0xf8, 0x96, 0x72,
+	0x78, 0x02, 0x77, 0xf6, 0xc0, 0xd0, 0x2d, 0x70, 0x7f, 0x24, 0x2b, 0xc5, 0x55, 0x0f, 0xcb, 0x4f,
+	0x74, 0x17, 0x5a, 0x97, 0xf1, 0xb2, 0x22, 0x8a, 0x28, 0x07, 0x6b, 0xe1, 0x59, 0xe3, 0x33, 0x27,
+	0x78, 0x0a, 0x47, 0xcf, 0x2b, 0x4e, 0x53, 0x76, 0x45, 0x67, 0x45, 0xcc, 0x05, 0x39, 0x8d, 0x4b,
+	0x9e, 0x5d, 0x63, 0x76, 0xa5, 0x8b, 0x5b, 0x56, 0x39, 0x15, 0xbe, 0x33, 0x72, 0xc7, 0x03, 0x6c,
+	0xc5, 0xe0, 0x57, 0x07, 0xee, 0xee, 0xf3, 0x92, 0xfd, 0xa0, 0x71, 0x4e, 0xcc, 0xd1, 0xea, 0x1b,
+	0x3d, 0x84, 0x03, 0x5a, 0xe5, 0x73, 0xc2, 0x23, 0x76, 0x1e, 0x71, 0x76, 0x25, 0x54, 0x12, 0x2d,
+	0xdc, 0xd7, 0xda, 0x6f, 0xce, 0x31, 0xbb, 0x12, 0xe8, 0x63, 0xb8, 0xbd, 0x41, 0xd9, 0x63, 0x5d,
+	0x05, 0x3c, 0xb4, 0xc0, 0xa9, 0x56, 0xa3, 0xc7, 0xd0, 0x54, 0x71, 0x9a, 0x8a, 0x33, 0x3f, 0xbc,
+	0xa1, 0x00, 0xac, 0x50, 0xc1, 0xef, 0x8d, 0x4d, 0x89, 0x27, 0x34, 0x5e, 0xae, 0x44, 0x26, 0x30,
+	0x11, 0xd5, 0xb2, 0x14, 0x68, 0x04, 0xde, 0x05, 0x8f, 0x69, 0xb5, 0x8c, 0x79, 0x56, 0xae, 0xcc,
+	0x74, 0xd5, 0x55, 0x68, 0x08, 0x5d, 0x11, 0xe7, 0xc5, 0x32, 0xa3, 0x17, 0x26, 0xef, 0xb5, 0x8c,
+	0x9e, 0x40, 0xa7, 0xe0, 0xec, 0x07, 0x92, 0x94, 0x2a, 0x53, 0x6f, 0xf2, 0xbf, 0xfd, 0xa9, 0x58,
+	0x14, 0x7a, 0x04, 0xad, 0xf3, 0x6c, 0x49, 0x6c, 0xe6, 0x37, 0xc0, 0x35, 0x06, 0x7d, 0x02, 0xed,
+	0x82, 0xb0, 0x62, 0x29, 0x07, 0xef, 0x2d, 0x68, 0x03, 0x42, 0xaf, 0x01, 0xe9, 0xaf, 0x28, 0xa3,
+	0x25, 0xe1, 0x71, 0x52, 0xca, 0xfb, 0xd2, 0x56, 0x79, 0x0d, 0xc3, 0x29, 0xcb, 0x0b, 0x4e, 0x84,
+	0x20, 0xa9, 0x76, 0xc6, 0xec, 0xca, 0xf8, 0xdf, 0xd6, 0x5e, 0xaf, 0x37, 0x4e, 0xc1, 0x1f, 0x0e,
+	0xdc, 0xbf, 0xd1, 0x61, 0x4f, 0x3f, 0x9d, 0xff, 0xda, 0xcf, 0xc6, 0xfe, 0x7e, 0x22, 0x68, 0xca,
+	0x91, 0xf7, 0xdd, 0x91, 0x3b, 0x76, 0x71, 0xd3, 0xde, 0xf9, 0x8c, 0xa6, 0x59, 0x62, 0xc8, 0x6a,
+	0x61, 0x2b, 0xa2, 0x7b, 0xd0, 0xce, 0x68, 0x5a, 0x94, 0x5c, 0xf1, 0xe2, 0x62, 0x23, 0x05, 0x33,
+	0xe8, 0x4c, 0x59, 0x55, 0x48, 0xea, 0xee, 0x42, 0x2b, 0xa3, 0x29, 0xb9, 0x56, 0x73, 0xdb, 0xc3,
+	0x5a, 0x40, 0x13, 0x68, 0xe7, 0xaa, 0x04, 0x95, 0xc7, 0xdb, 0x59, 0x31, 0xc8, 0xe0, 0x21, 0xf4,
+	0xcf, 0x58, 0x95, 0x2c, 0x48, 0xfa, 0x32, 0x33, 0x91, 0x75, 0x07, 0x1d, 0x95, 0x94, 0x16, 0x82,
+	0x5f, 0x1c, 0xb8, 0x67, 0xce, 0xde, 0x9d, 0xb0, 0x47, 0xd0, 0x97, 0x98, 0x28, 0xd1, 0x66, 0xd3,
+	0x90, 0x6e, 0x68, 0xe0, 0xd8, 0x93, 0x56, 0x9b, 0xf7, 0x13, 0x38, 0x30, 0x3d, 0xb4, 0xf0, 0xce,
+	0x0e, 0x7c, 0xa0, 0xed, 0xd6, 0xe1, 0x53, 0xe8, 0x1b, 0x07, 0x9d, 0x95, 0xde, 0x22, 0x83, 0xb0,
+	0x9e, 0x33, 0xf6, 0x34, 0x44, 0x09, 0xc1, 0xcf, 0x0e, 0xc0, 0x9b, 0x93, 0xd9, 0xd9, 0x74, 0x11,
+	0xd3, 0x0b, 0x82, 0xfe, 0x0f, 0x3d, 0x95, 0x5e, 0xed, 0xd6, 0x76, 0xa5, 0xe2, 0x6b, 0x79, 0x73,
+	0x8f, 0x01, 0x04, 0x4f, 0xa2, 0x39, 0x39, 0x67, 0x9c, 0x98, 0x1d, 0xdb, 0x13, 0x3c, 0x79, 0xae,
+	0x14, 0xd2, 0x57, 0x9a, 0xe3, 0xf3, 0x92, 0x70, 0xb3, 0x67, 0xbb, 0x82, 0x27, 0x27, 0x52, 0x46,
+	0xef, 0x80, 0x57, 0xc5, 0xa2, 0xb4, 0xce, 0x4d, 0xbd, 0x86, 0xa5, 0xca, 0x78, 0x1f, 0x83, 0x92,
+	0x8c, 0x7b, 0x4b, 0x07, 0x97, 0x1a, 0xe5, 0x1f, 0x7c, 0x09, 0x47, 0x9b, 0x34, 0xc5, 0x2c, 0xbe,
+	0x24, 0xdc, 0x52, 0xfa, 0x3e, 0x74, 0x12, 0xad, 0x56, 0x5d, 0xf0, 0x26, 0x5e, 0xb8, 0x81, 0x62,
+	0x6b, 0x0b, 0xfe, 0x76, 0xe0, 0x60, 0xb6, 0x60, 0x25, 0x25, 0x42, 0x60, 0x92, 0x30, 0x9e, 0xa2,
+	0xf7, 0x60, 0xa0, 0x2e, 0x07, 0x8d, 0x97, 0x11, 0x67, 0x4b, 0x5b, 0x71, 0xdf, 0x2a, 0x31, 0x5b,
+	0x12, 0xd9, 0x62, 0x69, 0x93, 0xd3, 0xaa, 0x5a, 0xac, 0x84, 0xf5, 0x66, 0x73, 0x6b, 0x9b, 0x0d,
+	0x41, 0x53, 0x72, 0x65, 0x8a, 0x53, 0xdf, 0xe8, 0x73, 0xe8, 0x26, 0xac, 0x92, 0xf1, 0x84, 0xb9,
+	0xb7, 0xc7, 0xe1, 0x76, 0x16, 0xb2, 0x97, 0xca, 0xae, 0x77, 0xfa, 0x1a, 0x3e, 0xfc, 0x02, 0x06,
+	0x5b, 0xa6, 0xfa, 0x1e, 0x6f, 0xed, 0xd9, 0xe3, 0xad, 0xfa, 0x1e, 0x7f, 0x01, 0x47, 0xf6, 0x98,
+	0xdd, 0x11, 0xfc, 0x08, 0x3a, 0x5c, 0x9d, 0x6c, 0xf9, 0x3a, 0xdc, 0xc9, 0x08, 0x5b, 0x7b, 0xf0,
+	0x21, 0x78, 0x72, 0x4c, 0x5e, 0x65, 0x42, 0x3d, 0x95, 0xb5, 0xe7, 0x4d, 0xdf, 0x24, 0x2b, 0x06,
+	0x3f, 0x39, 0xe0, 0xd7, 0x90, 0xfa, 0xa8, 0x53, 0x22, 0x44, 0x7c, 0x41, 0xd0, 0xb3, 0xfa, 0x25,
+	0xf1, 0x26, 0x0f, 0xc3, 0x9b, 0x90, 0xca, 0x60, 0x78, 0xd0, 0x2e, 0xc3, 0x97, 0x00, 0x1b, 0xe5,
+	0x9e, 0x97, 0x2c, 0xa8, 0x33, 0xe0, 0x4d, 0xfa, 0x5b, 0xb1, 0x6b, 0x7c, 0x7c, 0x0f, 0xbd, 0x19,
+	0xa1, 0xf2, 0x8d, 0xa5, 0xe5, 0x86, 0x36, 0x19, 0xa8, 0x61, 0x60, 0x72, 0xb5, 0xcb, 0x72, 0x08,
+	0x2d, 0x75, 0xaf, 0x7b, 0x78, 0x2d, 0xd7, 0x2b, 0x77, 0xb7, 0x2b, 0xff, 0xd3, 0x81, 0xa3, 0xa9,
+	0x86, 0xad, 0x0f, 0xb0, 0x4c, 0x7f, 0x07, 0xb7, 0x84, 0xd5, 0x45, 0xf3, 0x55, 0x94, 0xc6, 0x2b,
+	0xc3, 0xc1, 0xe3, 0xf0, 0x06, 0x9f, 0x70, 0xad, 0x78, 0xbe, 0x7a, 0x11, 0xaf, 0xcc, 0x3b, 0x2f,
+	0xb6, 0x94, 0xc3, 0x53, 0xb8, 0xb3, 0x07, 0xb6, 0x67, 0x3e, 0x46, 0xdb, 0xec, 0xc0, 0x26, 0x7a,
+	0x9d, 0x9b, 0xdf, 0x1c, 0x38, 0xdc, 0x1d, 0x92, 0x77, 0xa1, 0xbd, 0x20, 0x71, 0x4a, 0xb8, 0x0a,
+	0xe7, 0x4d, 0x7a, 0xeb, 0x3f, 0x11, 0x6c, 0x0c, 0xe8, 0x99, 0xe4, 0x8b, 0x96, 0x6b, 0xbe, 0xbc,
+	0xc9, 0x83, 0x70, 0x27, 0x4c, 0x38, 0x35, 0x80, 0xf5, 0x6c, 0x6b, 0x51, 0xcf, 0x76, 0xcd, 0xf4,
+	0x6f, 0xff, 0x28, 0xfd, 0x5a, 0xbe, 0xf3, 0xb6, 0xfa, 0x27, 0x7c, 0xfa, 0x4f, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0x0a, 0xd8, 0x24, 0x36, 0x1f, 0x0a, 0x00, 0x00,
 }

internal/pb/pb.proto

@@ -15,6 +15,8 @@ message Metadata {
     int32 commits = 6;
     // duration of the analysis in milliseconds
     int64 run_time = 7;
+    // time taken by each pipeline item in seconds
+    map<string, double> run_time_per_item = 8;
 }
 
 message BurndownSparseMatrixRow {

labours.py

@@ -24,7 +24,7 @@ import yaml
 
 if sys.version_info[0] < 3:
     # OK, ancients, I will support Python 2, but you owe me a beer
-    input = raw_input
+    input = raw_input  # noqa: F821
 
 
 PB_MESSAGES = {
@@ -54,7 +54,7 @@ def parse_args():
     parser.add_argument("--couples-tmp-dir", help="Temporary directory to work with couples.")
     parser.add_argument("-m", "--mode",
                         choices=["project", "file", "person", "churn_matrix", "ownership",
-                                 "couples", "shotness", "sentiment", "all"],
+                                 "couples", "shotness", "sentiment", "all", "run_times"],
                         help="What to plot.")
     parser.add_argument(
         "--resample", default="year",
@@ -131,6 +131,9 @@ class YamlReader(Reader):
             sys.exit(1)
         self.data = data
 
+    def get_run_times(self):
+        return {}
+
     def get_name(self):
         return self.data["hercules"]["repository"]
 
@@ -249,6 +252,9 @@ class ProtobufReader(Reader):
             self.contents[key] = msg = cls()
             msg.ParseFromString(val)
 
+    def get_run_times(self):
+        return {key: val for key, val in self.data.header.run_time_per_item.items()}
+
     def get_name(self):
         return self.data.header.repository
 
@@ -495,6 +501,7 @@ def load_burndown(header, name, matrix, resample):
     print(name, "lifetime index:", calculate_average_lifetime(matrix))
     finish = start + timedelta(days=matrix.shape[1] * sampling)
     if resample not in ("no", "raw"):
+        print("resampling to %s, please wait..." % resample)
         # Interpolate the day x day matrix.
         # Each day brings equal weight in the granularity.
         # Sampling's interpolation is linear.
@@ -641,7 +648,7 @@ def deploy_plot(title, output, style):
             pyplot.title(title, color=style)
         try:
             pyplot.tight_layout()
-        except:
+        except:  # noqa: E722
             print("Warning: failed to set the tight layout")
         pyplot.savefig(output, transparent=True)
     pyplot.clf()
@@ -831,6 +838,7 @@ def plot_ownership(args, repo, names, people, date_range, last):
         output = args.output
     deploy_plot("%s code ownership through time" % repo, output, args.style)
 
+
 IDEAL_SHARD_SIZE = 4096
 
 
@@ -965,7 +973,7 @@ class CORSWebServer(object):
 
         try:
             from http.server import HTTPServer, SimpleHTTPRequestHandler, test
-        except ImportError: # Python 2
+        except ImportError:  # Python 2
             from BaseHTTPServer import HTTPServer, test
             from SimpleHTTPServer import SimpleHTTPRequestHandler
 
@@ -974,9 +982,8 @@ class CORSWebServer(object):
                 HTTPServer.__init__(self, *args, **kwargs)
                 outer.server = self
 
-
         class CORSRequestHandler(SimpleHTTPRequestHandler):
-            def end_headers (self):
+            def end_headers(self):
                 self.send_header("Access-Control-Allow-Origin", "*")
                 SimpleHTTPRequestHandler.end_headers(self)
 
@@ -1118,7 +1125,6 @@ def show_sentiment_stats(args, name, resample, start, data):
     overall_neg = sum(2 * (d[1].Value - 0.5) for d in data if d[1].Value > 0.5)
     title = "%s sentiment +%.1f -%.1f δ=%.1f" % (
         name, overall_pos, overall_neg, overall_pos - overall_neg)
-    output = args.output
     deploy_plot(title, args.output, args.style)
 
 
@@ -1140,6 +1146,11 @@ def main():
                        "--shotness. Also check --languages - the output may be empty."
     sentiment_warning = "Sentiment stats were not collected. Re-run hercules with --sentiment."
 
+    def run_times():
+        rt = reader.get_run_times()
+        import pandas
+        print(pandas.to_timedelta(pandas.Series(rt).sort_values(ascending=False), unit="s"))
+
     def project_burndown():
         try:
             full_header = header + reader.get_burndown_parameters()
@@ -1224,7 +1235,9 @@ def main():
             return
         show_sentiment_stats(args, reader.get_name(), args.resample, reader.get_header()[0], data)
 
-    if args.mode == "project":
+    if args.mode == "run_times":
+        run_times()
+    elif args.mode == "project":
         project_burndown()
     elif args.mode == "file":
         files_burndown()
@@ -1260,5 +1273,6 @@ def main():
             pass
         web_server.stop()
 
+
 if __name__ == "__main__":
     sys.exit(main())