Merge pull request #150 from vmarkovtsev/master

Hibernation of Burndown

internal/core/pipeline.go

@@ -147,6 +147,16 @@ type ResultMergeablePipelineItem interface {
 	MergeResults(r1, r2 interface{}, c1, c2 *CommonAnalysisResult) interface{}
 }
 
+// HibernateablePipelineItem is the interface to allow pipeline items to be frozen (compacted, unloaded)
+// while they are not needed in the hosting branch.
+type HibernateablePipelineItem interface {
+	PipelineItem
+	// Hibernate signals that the item is temporarily not needed and it's memory can be optimized.
+	Hibernate()
+	// Boot signals that the item is needed again and must be de-hibernate-d.
+	Boot()
+}
+
 // CommonAnalysisResult holds the information which is always extracted at Pipeline.Run().
 type CommonAnalysisResult struct {
 	// BeginTime is the time of the first commit in the analysed sequence.
@@ -639,7 +649,10 @@ func (pipeline *Pipeline) Run(commits []*object.Commit) (map[LeafPipelineItem]in
 	progressSteps := len(plan) + 2
 	branches := map[int][]PipelineItem{}
 	// we will need rootClone if there is more than one root branch
-	rootClone := cloneItems(pipeline.items, 1)[0]
+	var rootClone []PipelineItem
+	if !pipeline.DryRun {
+		rootClone = cloneItems(pipeline.items, 1)[0]
+	}
 	var newestTime int64
 	runTimePerItem := map[string]float64{}
 

internal/rbtree/rbtree.go

@@ -64,7 +64,7 @@ func (allocator *Allocator) Clone() *Allocator {
 
 // Hibernate compresses the allocated memory.
 func (allocator *Allocator) Hibernate() {
-	if allocator.HibernationThreshold > 0 && len(allocator.storage) < allocator.HibernationThreshold {
+	if len(allocator.storage) < allocator.HibernationThreshold {
 		return
 	}
 	allocator.hibernatedLen = len(allocator.storage)

internal/rbtree/rbtree_test.go

@@ -419,3 +419,18 @@ func TestAllocatorHibernateBootEmpty(t *testing.T) {
 	assert.Equal(t, alloc.Size(), 0)
 	assert.Equal(t, alloc.Used(), 0)
 }
+
+func TestAllocatorHibernateBootThreshold(t *testing.T) {
+	alloc := NewAllocator()
+	alloc.malloc()
+	alloc.HibernationThreshold = 3
+	alloc.Hibernate()
+	assert.Equal(t, alloc.hibernatedLen, 0)
+	alloc.Boot()
+	alloc.malloc()
+	alloc.Hibernate()
+	assert.Equal(t, alloc.hibernatedLen, 3)
+	alloc.Boot()
+	assert.Equal(t, alloc.Size(), 3)
+	assert.Equal(t, alloc.Used(), 3)
+}

leaves/burndown.go

@@ -40,9 +40,14 @@ type BurndownAnalysis struct {
 	// It does not change the project level burndown results.
 	TrackFiles bool
 
-	// The number of developers for which to collect the burndown stats. 0 disables it.
+	// PeopleNumber is the number of developers for which to collect the burndown stats. 0 disables it.
 	PeopleNumber int
 
+	// HibernationThreshold sets the hibernation threshold for the underlying
+	// RBTree allocator. It is useful to trade CPU time for reduced peak memory consumption
+	// if there are many branches.
+	HibernationThreshold int
+
 	// Debug activates the debugging mode. Analyse() runs slower in this mode
 	// but it accurately checks all the intermediate states for invariant
 	// violations.
@@ -126,6 +131,10 @@ const (
 	ConfigBurndownTrackFiles = "Burndown.TrackFiles"
 	// ConfigBurndownTrackPeople enables burndown collection for authors.
 	ConfigBurndownTrackPeople = "Burndown.TrackPeople"
+	// ConfigBurndownHibernationThreshold sets the hibernation threshold for the underlying
+	// RBTree allocator. It is useful to trade CPU time for reduced peak memory consumption
+	// if there are many branches.
+	ConfigBurndownHibernationThreshold = "Burndown.HibernationThreshold"
 	// ConfigBurndownDebug enables some extra debug assertions.
 	ConfigBurndownDebug = "Burndown.Debug"
 	// DefaultBurndownGranularity is the default number of days for BurndownAnalysis.Granularity
@@ -186,6 +195,12 @@ func (analyser *BurndownAnalysis) ListConfigurationOptions() []core.Configuratio
 		Flag:        "burndown-people",
 		Type:        core.BoolConfigurationOption,
 		Default:     false}, {
+		Name: ConfigBurndownHibernationThreshold,
+		Description: "The minimum size for the allocated memory in each branch to be compressed." +
+			"0 disables this optimization. Lower values trade CPU time more. Sane examples: Nx1000.",
+		Flag:    "burndown-hibernation-threshold",
+		Type:    core.IntConfigurationOption,
+		Default: 0}, {
 		Name:        ConfigBurndownDebug,
 		Description: "Validate the trees on each step.",
 		Flag:        "burndown-debug",
@@ -217,6 +232,9 @@ func (analyser *BurndownAnalysis) Configure(facts map[string]interface{}) error
 	} else if exists {
 		analyser.PeopleNumber = 0
 	}
+	if val, exists := facts[ConfigBurndownHibernationThreshold].(int); exists {
+		analyser.HibernationThreshold = val
+	}
 	if val, exists := facts[ConfigBurndownDebug].(bool); exists {
 		analyser.Debug = val
 	}
@@ -261,6 +279,7 @@ func (analyser *BurndownAnalysis) Initialize(repository *git.Repository) error {
 	analyser.peopleHistories = make([]sparseHistory, analyser.PeopleNumber)
 	analyser.files = map[string]*burndown.File{}
 	analyser.fileAllocator = rbtree.NewAllocator()
+	analyser.fileAllocator.HibernationThreshold = analyser.HibernationThreshold
 	analyser.mergedFiles = map[string]bool{}
 	analyser.mergedAuthor = identity.AuthorMissing
 	analyser.renames = map[string]string{}
@@ -276,6 +295,9 @@ func (analyser *BurndownAnalysis) Initialize(repository *git.Repository) error {
 // This function returns the mapping with analysis results. The keys must be the same as
 // in Provides(). If there was an error, nil is returned.
 func (analyser *BurndownAnalysis) Consume(deps map[string]interface{}) (map[string]interface{}, error) {
+	if analyser.fileAllocator.Size() == 0 && len(analyser.files) > 0 {
+		panic("BurndownAnalysis.Consume() was called on a hibernated instance")
+	}
 	author := deps[identity.DependencyAuthor].(int)
 	day := deps[items.DependencyDay].(int)
 	if !deps[core.DependencyIsMerge].(bool) {
@@ -380,6 +402,16 @@ func (analyser *BurndownAnalysis) Merge(branches []core.PipelineItem) {
 	analyser.onNewDay()
 }
 
+// Hibernate compresses the bound RBTree memory with the files.
+func (analyser *BurndownAnalysis) Hibernate() {
+	analyser.fileAllocator.Hibernate()
+}
+
+// Boot decompresses the bound RBTree memory with the files.
+func (analyser *BurndownAnalysis) Boot() {
+	analyser.fileAllocator.Boot()
+}
+
 // Finalize returns the result of the analysis. Further Consume() calls are not expected.
 func (analyser *BurndownAnalysis) Finalize() interface{} {
 	globalHistory, lastDay := analyser.groupSparseHistory(analyser.globalHistory, -1)

leaves/burndown_test.go

@@ -45,7 +45,7 @@ func TestBurndownMeta(t *testing.T) {
 	for _, opt := range opts {
 		switch opt.Name {
 		case ConfigBurndownGranularity, ConfigBurndownSampling, ConfigBurndownTrackFiles,
-			ConfigBurndownTrackPeople, ConfigBurndownDebug:
+			ConfigBurndownTrackPeople, ConfigBurndownHibernationThreshold, ConfigBurndownDebug:
 			matches++
 		}
 	}
@@ -61,6 +61,7 @@ func TestBurndownConfigure(t *testing.T) {
 	facts[ConfigBurndownTrackFiles] = true
 	facts[ConfigBurndownTrackPeople] = true
 	facts[ConfigBurndownDebug] = true
+	facts[ConfigBurndownHibernationThreshold] = 100
 	facts[identity.FactIdentityDetectorPeopleCount] = 5
 	facts[identity.FactIdentityDetectorReversedPeopleDict] = burndown.Requires()
 	burndown.Configure(facts)
@@ -68,6 +69,7 @@ func TestBurndownConfigure(t *testing.T) {
 	assert.Equal(t, burndown.Sampling, 200)
 	assert.Equal(t, burndown.TrackFiles, true)
 	assert.Equal(t, burndown.PeopleNumber, 5)
+	assert.Equal(t, burndown.HibernationThreshold, 100)
 	assert.Equal(t, burndown.Debug, true)
 	assert.Equal(t, burndown.reversedPeopleDict, burndown.Requires())
 	facts[ConfigBurndownTrackPeople] = false
@@ -103,9 +105,11 @@ func TestBurndownInitialize(t *testing.T) {
 	burndown := BurndownAnalysis{}
 	burndown.Sampling = -10
 	burndown.Granularity = DefaultBurndownGranularity
+	burndown.HibernationThreshold = 10
 	burndown.Initialize(test.Repository)
 	assert.Equal(t, burndown.Sampling, DefaultBurndownGranularity)
 	assert.Equal(t, burndown.Granularity, DefaultBurndownGranularity)
+	assert.Equal(t, burndown.fileAllocator.HibernationThreshold, 10)
 	burndown.Sampling = 0
 	burndown.Granularity = DefaultBurndownGranularity - 1
 	burndown.Initialize(test.Repository)
@@ -342,7 +346,8 @@ func TestBurndownConsumeFinalize(t *testing.T) {
 	}
 }
 
-func bakeBurndownForSerialization(t *testing.T, firstAuthor, secondAuthor int) BurndownResult {
+func bakeBurndownForSerialization(t *testing.T, firstAuthor, secondAuthor int) (
+	BurndownResult, *BurndownAnalysis) {
 	burndown := BurndownAnalysis{
 		Granularity:  30,
 		Sampling:     30,
@@ -481,11 +486,11 @@ func bakeBurndownForSerialization(t *testing.T, firstAuthor, secondAuthor int) B
 	burndown.reversedPeopleDict = people[:]
 	burndown.Consume(deps)
 	out := burndown.Finalize().(BurndownResult)
-	return out
+	return out, &burndown
 }
 
 func TestBurndownSerialize(t *testing.T) {
-	out := bakeBurndownForSerialization(t, 0, 1)
+	out, _ := bakeBurndownForSerialization(t, 0, 1)
 	burndown := &BurndownAnalysis{}
 
 	buffer := &bytes.Buffer{}
@@ -564,7 +569,7 @@ func TestBurndownSerialize(t *testing.T) {
 }
 
 func TestBurndownSerializeAuthorMissing(t *testing.T) {
-	out := bakeBurndownForSerialization(t, 0, identity.AuthorMissing)
+	out, _ := bakeBurndownForSerialization(t, 0, identity.AuthorMissing)
 	burndown := &BurndownAnalysis{}
 
 	buffer := &bytes.Buffer{}
@@ -1201,3 +1206,14 @@ func TestBurndownNegativePeople(t *testing.T) {
 	err = burndown.Configure(facts)
 	assert.Equal(t, err.Error(), "PeopleNumber is negative: -1")
 }
+
+func TestBurndownHibernateBoot(t *testing.T) {
+	_, burndown := bakeBurndownForSerialization(t, 0, 1)
+	assert.Equal(t, burndown.fileAllocator.Size(), 157)
+	assert.Equal(t, burndown.fileAllocator.Used(), 155)
+	burndown.Hibernate()
+	assert.Equal(t, burndown.fileAllocator.Size(), 0)
+	burndown.Boot()
+	assert.Equal(t, burndown.fileAllocator.Size(), 157)
+	assert.Equal(t, burndown.fileAllocator.Used(), 155)
+}