Add the ability to serialize RBTree allocators to disk

Signed-off-by: Vadim Markovtsev <vadim@sourced.tech>

Gopkg.lock

@@ -593,6 +593,7 @@
     "github.com/Masterminds/sprig",
     "github.com/fatih/camelcase",
     "github.com/gogo/protobuf/proto",
+    "github.com/gogo/protobuf/sortkeys",
     "github.com/jeffail/tunny",
     "github.com/minio/highwayhash",
     "github.com/mitchellh/go-homedir",
@@ -621,6 +622,7 @@
     "gopkg.in/src-d/go-git.v4/storage",
     "gopkg.in/src-d/go-git.v4/storage/filesystem",
     "gopkg.in/src-d/go-git.v4/storage/memory",
+    "gopkg.in/src-d/go-git.v4/utils/binary",
     "gopkg.in/src-d/go-git.v4/utils/merkletrie",
     "gopkg.in/vmarkovtsev/BiDiSentiment.v1",
   ]

internal/core/pipeline.go

@@ -152,9 +152,9 @@ type ResultMergeablePipelineItem interface {
 type HibernateablePipelineItem interface {
 	PipelineItem
 	// Hibernate signals that the item is temporarily not needed and it's memory can be optimized.
-	Hibernate()
+	Hibernate() error
 	// Boot signals that the item is needed again and must be de-hibernate-d.
-	Boot()
+	Boot() error
 }
 
 // CommonAnalysisResult holds the information which is always extracted at Pipeline.Run().
@@ -738,7 +738,10 @@ func (pipeline *Pipeline) Run(commits []*object.Commit) (map[LeafPipelineItem]in
 				for _, item := range branches[item] {
 					if hi, ok := item.(HibernateablePipelineItem); ok {
 						startTime := time.Now()
-						hi.Hibernate()
+						err := hi.Hibernate()
+						if err != nil {
+							log.Panicf("Failed to hibernate %s: %v\n", item.Name(), err)
+						}
 						runTimePerItem[item.Name()+".Hibernation"] += time.Now().Sub(startTime).Seconds()
 					}
 				}
@@ -748,7 +751,10 @@ func (pipeline *Pipeline) Run(commits []*object.Commit) (map[LeafPipelineItem]in
 				for _, item := range branches[item] {
 					if hi, ok := item.(HibernateablePipelineItem); ok {
 						startTime := time.Now()
-						hi.Boot()
+						err := hi.Boot()
+						if err != nil {
+							log.Panicf("Failed to boot %s: %v\n", item.Name(), err)
+						}
 						runTimePerItem[item.Name()+".Hibernation"] += time.Now().Sub(startTime).Seconds()
 					}
 				}

internal/core/pipeline_test.go

@@ -136,6 +136,8 @@ type dependingTestPipelineItem struct {
 	TestNilConsumeReturn bool
 	Hibernated           bool
 	Booted               bool
+	RaiseHibernateError  bool
+	RaiseBootError       bool
 }
 
 func (item *dependingTestPipelineItem) Name() string {
@@ -199,12 +201,20 @@ func (item *dependingTestPipelineItem) Fork(n int) []PipelineItem {
 func (item *dependingTestPipelineItem) Merge(branches []PipelineItem) {
 }
 
-func (item *dependingTestPipelineItem) Hibernate() {
+func (item *dependingTestPipelineItem) Hibernate() error {
 	item.Hibernated = true
+	if item.RaiseHibernateError {
+		return errors.New("error")
+	}
+	return nil
 }
 
-func (item *dependingTestPipelineItem) Boot() {
+func (item *dependingTestPipelineItem) Boot() error {
 	item.Booted = true
+	if item.RaiseBootError {
+		return errors.New("error")
+	}
+	return nil
 }
 
 func (item *dependingTestPipelineItem) Finalize() interface{} {
@@ -834,4 +844,10 @@ func TestPipelineRunHibernation(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, item.Hibernated)
 	assert.True(t, item.Booted)
+	item.RaiseHibernateError = true
+	assert.Panics(t, func() { pipeline.Run(commits) })
+	item.RaiseHibernateError = false
+	pipeline.Run(commits)
+	item.RaiseBootError = true
+	assert.Panics(t, func() { pipeline.Run(commits) })
 }

internal/rbtree/rbtree.go

@@ -1,8 +1,13 @@
 package rbtree
 
 import (
+	"fmt"
 	"math"
+	"os"
 	"sync"
+
+	"github.com/gogo/protobuf/sortkeys"
+	"gopkg.in/src-d/go-git.v4/utils/binary"
 )
 
 //
@@ -19,10 +24,11 @@ type Item struct {
 type Allocator struct {
 	HibernationThreshold int
 
-	storage           []node
-	gaps              map[uint32]bool
-	hibernatedStorage [6][]byte
-	hibernatedLen     int
+	storage              []node
+	gaps                 map[uint32]bool
+	hibernatedData       [7][]byte
+	hibernatedStorageLen int
+	hibernatedGapsLen    int
 }
 
 // NewAllocator creates a new allocator for RBTree's nodes.
@@ -64,11 +70,14 @@ func (allocator *Allocator) Clone() *Allocator {
 
 // Hibernate compresses the allocated memory.
 func (allocator *Allocator) Hibernate() {
+	if allocator.hibernatedStorageLen > 0 {
+		panic("cannot hibernate an already hibernated Allocator")
+	}
 	if len(allocator.storage) < allocator.HibernationThreshold {
 		return
 	}
-	allocator.hibernatedLen = len(allocator.storage)
-	if allocator.hibernatedLen == 0 {
+	allocator.hibernatedStorageLen = len(allocator.storage)
+	if allocator.hibernatedStorageLen == 0 {
 		return
 	}
 	buffers := [6][]uint32{}
@@ -88,36 +97,69 @@ func (allocator *Allocator) Hibernate() {
 	}
 	allocator.storage = nil
 	wg := &sync.WaitGroup{}
-	wg.Add(len(buffers))
+	wg.Add(len(buffers) + 1)
 	for i, buffer := range buffers {
 		go func(i int, buffer []uint32) {
-			allocator.hibernatedStorage[i] = CompressUInt32Slice(buffer)
+			allocator.hibernatedData[i] = CompressUInt32Slice(buffer)
 			buffers[i] = nil
 			wg.Done()
 		}(i, buffer)
 	}
+	// compress gaps
+	go func() {
+		if len(allocator.gaps) > 0 {
+			allocator.hibernatedGapsLen = len(allocator.gaps)
+			gapsBuffer := make([]uint32, len(allocator.gaps))
+			i := 0
+			for key := range allocator.gaps {
+				gapsBuffer[i] = key
+				i++
+			}
+			sortkeys.Uint32s(gapsBuffer)
+			allocator.hibernatedData[len(buffers)] = CompressUInt32Slice(gapsBuffer)
+		}
+		allocator.gaps = nil
+		wg.Done()
+	}()
 	wg.Wait()
 }
 
 // Boot performs the opposite of Hibernate() - decompresses and restores the allocated memory.
 func (allocator *Allocator) Boot() {
-	if allocator.hibernatedLen == 0 {
+	if allocator.hibernatedStorageLen == 0 {
 		// not hibernated
 		return
 	}
+	if allocator.hibernatedData[0] == nil {
+		panic("cannot boot a serialized Allocator")
+	}
+	allocator.gaps = map[uint32]bool{}
 	buffers := [6][]uint32{}
 	wg := &sync.WaitGroup{}
-	wg.Add(len(buffers))
+	wg.Add(len(buffers) + 1)
 	for i := 0; i < len(buffers); i++ {
 		go func(i int) {
-			buffers[i] = make([]uint32, allocator.hibernatedLen)
-			DecompressUInt32Slice(allocator.hibernatedStorage[i], buffers[i])
-			allocator.hibernatedStorage[i] = nil
+			buffers[i] = make([]uint32, allocator.hibernatedStorageLen)
+			DecompressUInt32Slice(allocator.hibernatedData[i], buffers[i])
+			allocator.hibernatedData[i] = nil
 			wg.Done()
 		}(i)
 	}
+	go func() {
+		if allocator.hibernatedGapsLen > 0 {
+			gapData := allocator.hibernatedData[len(buffers)]
+			buffer := make([]uint32, allocator.hibernatedGapsLen)
+			DecompressUInt32Slice(gapData, buffer)
+			for _, key := range buffer {
+				allocator.gaps[key] = true
+			}
+			allocator.hibernatedData[len(buffers)] = nil
+			allocator.hibernatedGapsLen = 0
+		}
+		wg.Done()
+	}()
 	wg.Wait()
-	allocator.storage = make([]node, allocator.hibernatedLen, (allocator.hibernatedLen*3)/2)
+	allocator.storage = make([]node, allocator.hibernatedStorageLen, (allocator.hibernatedStorageLen*3)/2)
 	for i := range allocator.storage {
 		n := &allocator.storage[i]
 		n.item.Key = buffers[0][i]
@@ -127,7 +169,76 @@ func (allocator *Allocator) Boot() {
 		n.right = buffers[4][i]
 		n.color = buffers[5][i] > 0
 	}
-	allocator.hibernatedLen = 0
+	allocator.hibernatedStorageLen = 0
+}
+
+// Serialize writes the hibernated allocator on disk.
+func (allocator *Allocator) Serialize(path string) error {
+	if allocator.storage != nil {
+		panic("serialization requires the hibernated state")
+	}
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	err = binary.WriteVariableWidthInt(file, int64(allocator.hibernatedStorageLen))
+	if err != nil {
+		return err
+	}
+	err = binary.WriteVariableWidthInt(file, int64(allocator.hibernatedGapsLen))
+	if err != nil {
+		return err
+	}
+	for i, hse := range allocator.hibernatedData {
+		err = binary.WriteVariableWidthInt(file, int64(len(hse)))
+		if err != nil {
+			return err
+		}
+		_, err = file.Write(hse)
+		if err != nil {
+			return err
+		}
+		allocator.hibernatedData[i] = nil
+	}
+	return nil
+}
+
+// Deserialize reads a hibernated allocator from disk.
+func (allocator *Allocator) Deserialize(path string) error {
+	if allocator.storage != nil {
+		panic("deserialization requires the hibernated state")
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	x, err := binary.ReadVariableWidthInt(file)
+	if err != nil {
+		return err
+	}
+	allocator.hibernatedStorageLen = int(x)
+	x, err = binary.ReadVariableWidthInt(file)
+	if err != nil {
+		return err
+	}
+	allocator.hibernatedGapsLen = int(x)
+	for i := range allocator.hibernatedData {
+		x, err = binary.ReadVariableWidthInt(file)
+		if err != nil {
+			return err
+		}
+		allocator.hibernatedData[i] = make([]byte, int(x))
+		n, err := file.Read(allocator.hibernatedData[i])
+		if err != nil {
+			return err
+		}
+		if n != int(x) {
+			return fmt.Errorf("incomplete read %d: %d instead of %d", i, n, x)
+		}
+	}
+	return nil
 }
 
 func (allocator *Allocator) malloc() uint32 {

internal/rbtree/rbtree_test.go

@@ -2,7 +2,9 @@ package rbtree
 
 import (
 	"fmt"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"sort"
 	"testing"
 
@@ -321,15 +323,17 @@ func TestCloneShallow(t *testing.T) {
 	alloc1.malloc()
 	tree := NewRBTree(alloc1)
 	tree.Insert(Item{7, 7})
-	assert.Equal(t, alloc1.storage, []node{{}, {}, {color: black, item: Item{7, 7}}})
+	tree.Insert(Item{8, 8})
+	tree.DeleteWithKey(8)
+	assert.Equal(t, alloc1.storage, []node{{}, {}, {color: black, item: Item{7, 7}}, {}})
 	assert.Equal(t, tree.minNode, uint32(2))
 	assert.Equal(t, tree.maxNode, uint32(2))
 	alloc2 := alloc1.Clone()
 	clone := tree.CloneShallow(alloc2)
-	assert.Equal(t, alloc2.storage, []node{{}, {}, {color: black, item: Item{7, 7}}})
+	assert.Equal(t, alloc2.storage, []node{{}, {}, {color: black, item: Item{7, 7}}, {}})
 	assert.Equal(t, clone.minNode, uint32(2))
 	assert.Equal(t, clone.maxNode, uint32(2))
-	assert.Equal(t, alloc2.Size(), 3)
+	assert.Equal(t, alloc2.Size(), 4)
 	tree.Insert(Item{10, 10})
 	alloc3 := alloc1.Clone()
 	clone = tree.CloneShallow(alloc3)
@@ -340,7 +344,7 @@ func TestCloneShallow(t *testing.T) {
 	assert.Equal(t, clone.minNode, uint32(2))
 	assert.Equal(t, clone.maxNode, uint32(3))
 	assert.Equal(t, alloc3.Size(), 4)
-	assert.Equal(t, alloc2.Size(), 3)
+	assert.Equal(t, alloc2.Size(), 4)
 }
 
 func TestCloneDeep(t *testing.T) {
@@ -392,16 +396,23 @@ func TestAllocatorHibernateBoot(t *testing.T) {
 		alloc.storage[n].parent = uint32(i)
 		alloc.storage[n].color = i%2 == 0
 	}
+	for i := 0; i < 10000; i++ {
+		alloc.gaps[uint32(i)] = true // makes no sense, only to test
+	}
 	alloc.Hibernate()
+	assert.PanicsWithValue(t, "cannot hibernate an already hibernated Allocator", alloc.Hibernate)
 	assert.Nil(t, alloc.storage)
+	assert.Nil(t, alloc.gaps)
 	assert.Equal(t, alloc.Size(), 0)
-	assert.Equal(t, alloc.hibernatedLen, 10001)
+	assert.Equal(t, alloc.hibernatedStorageLen, 10001)
+	assert.Equal(t, alloc.hibernatedGapsLen, 10000)
 	assert.PanicsWithValue(t, "hibernated allocators cannot be used", func() { alloc.Used() })
 	assert.PanicsWithValue(t, "hibernated allocators cannot be used", func() { alloc.malloc() })
 	assert.PanicsWithValue(t, "hibernated allocators cannot be used", func() { alloc.free(0) })
 	assert.PanicsWithValue(t, "cannot clone a hibernated allocator", func() { alloc.Clone() })
 	alloc.Boot()
-	assert.Equal(t, alloc.hibernatedLen, 0)
+	assert.Equal(t, alloc.hibernatedStorageLen, 0)
+	assert.Equal(t, alloc.hibernatedGapsLen, 0)
 	for n := 1; n <= 10000; n++ {
 		assert.Equal(t, alloc.storage[n].item.Key, uint32(n-1))
 		assert.Equal(t, alloc.storage[n].item.Value, uint32(n-1))
@@ -409,6 +420,7 @@ func TestAllocatorHibernateBoot(t *testing.T) {
 		assert.Equal(t, alloc.storage[n].right, uint32(n-1))
 		assert.Equal(t, alloc.storage[n].parent, uint32(n-1))
 		assert.Equal(t, alloc.storage[n].color, (n-1)%2 == 0)
+		assert.True(t, alloc.gaps[uint32(n-1)])
 	}
 }
 
@@ -416,6 +428,7 @@ func TestAllocatorHibernateBootEmpty(t *testing.T) {
 	alloc := NewAllocator()
 	alloc.Hibernate()
 	alloc.Boot()
+	assert.NotNil(t, alloc.gaps)
 	assert.Equal(t, alloc.Size(), 0)
 	assert.Equal(t, alloc.Used(), 0)
 }
@@ -425,12 +438,77 @@ func TestAllocatorHibernateBootThreshold(t *testing.T) {
 	alloc.malloc()
 	alloc.HibernationThreshold = 3
 	alloc.Hibernate()
-	assert.Equal(t, alloc.hibernatedLen, 0)
+	assert.Equal(t, alloc.hibernatedStorageLen, 0)
 	alloc.Boot()
 	alloc.malloc()
 	alloc.Hibernate()
-	assert.Equal(t, alloc.hibernatedLen, 3)
+	assert.Equal(t, alloc.hibernatedGapsLen, 0)
+	assert.Equal(t, alloc.hibernatedStorageLen, 3)
 	alloc.Boot()
 	assert.Equal(t, alloc.Size(), 3)
 	assert.Equal(t, alloc.Used(), 3)
+	assert.NotNil(t, alloc.gaps)
+}
+
+func TestAllocatorSerializeDeserialize(t *testing.T) {
+	alloc := NewAllocator()
+	for i := 0; i < 10000; i++ {
+		n := alloc.malloc()
+		alloc.storage[n].item.Key = uint32(i)
+		alloc.storage[n].item.Value = uint32(i)
+		alloc.storage[n].left = uint32(i)
+		alloc.storage[n].right = uint32(i)
+		alloc.storage[n].parent = uint32(i)
+		alloc.storage[n].color = i%2 == 0
+	}
+	for i := 0; i < 10000; i++ {
+		alloc.gaps[uint32(i)] = true // makes no sense, only to test
+	}
+	assert.PanicsWithValue(t, "serialization requires the hibernated state",
+		func() { alloc.Serialize("...") })
+	assert.PanicsWithValue(t, "deserialization requires the hibernated state",
+		func() { alloc.Deserialize("...") })
+	alloc.Hibernate()
+	file, err := ioutil.TempFile("", "")
+	assert.Nil(t, err)
+	name := file.Name()
+	defer os.Remove(name)
+	assert.Nil(t, file.Close())
+	assert.NotNil(t, alloc.Serialize("/tmp/xxx/yyy"))
+	assert.Nil(t, alloc.Serialize(name))
+	assert.Nil(t, alloc.storage)
+	assert.Nil(t, alloc.gaps)
+	for _, d := range alloc.hibernatedData {
+		assert.Nil(t, d)
+	}
+	assert.Equal(t, alloc.hibernatedStorageLen, 10001)
+	assert.Equal(t, alloc.hibernatedGapsLen, 10000)
+	assert.PanicsWithValue(t, "cannot boot a serialized Allocator", alloc.Boot)
+	assert.NotNil(t, alloc.Deserialize("/tmp/xxx/yyy"))
+	assert.Nil(t, alloc.Deserialize(name))
+	for _, d := range alloc.hibernatedData {
+		assert.True(t, len(d) > 0)
+	}
+	alloc.Boot()
+	assert.Equal(t, alloc.hibernatedStorageLen, 0)
+	assert.Equal(t, alloc.hibernatedGapsLen, 0)
+	for _, d := range alloc.hibernatedData {
+		assert.Nil(t, d)
+	}
+	for n := 1; n <= 10000; n++ {
+		assert.Equal(t, alloc.storage[n].item.Key, uint32(n-1))
+		assert.Equal(t, alloc.storage[n].item.Value, uint32(n-1))
+		assert.Equal(t, alloc.storage[n].left, uint32(n-1))
+		assert.Equal(t, alloc.storage[n].right, uint32(n-1))
+		assert.Equal(t, alloc.storage[n].parent, uint32(n-1))
+		assert.Equal(t, alloc.storage[n].color, (n-1)%2 == 0)
+		assert.True(t, alloc.gaps[uint32(n-1)])
+	}
+	alloc.Hibernate()
+	assert.Nil(t, os.Truncate(name, 100))
+	assert.NotNil(t, alloc.Deserialize(name))
+	assert.Nil(t, os.Truncate(name, 4))
+	assert.NotNil(t, alloc.Deserialize(name))
+	assert.Nil(t, os.Truncate(name, 0))
+	assert.NotNil(t, alloc.Deserialize(name))
 }

leaves/burndown.go

@@ -4,7 +4,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
+	"os"
 	"sort"
 	"sync"
 	"unicode/utf8"
@@ -48,6 +50,14 @@ type BurndownAnalysis struct {
 	// if there are many branches.
 	HibernationThreshold int
 
+	// HibernationToDisk specifies whether the hibernated RBTree allocator must be saved on disk
+	// rather than kept in memory.
+	HibernationToDisk bool
+
+	// HibernationDirectory is the name of the temporary directory to use for saving hibernated
+	// RBTree allocators.
+	HibernationDirectory string
+
 	// Debug activates the debugging mode. Analyse() runs slower in this mode
 	// but it accurately checks all the intermediate states for invariant
 	// violations.
@@ -74,6 +84,8 @@ type BurndownAnalysis struct {
 	files map[string]*burndown.File
 	// fileAllocator is the allocator for RBTree-s in `files`.
 	fileAllocator *rbtree.Allocator
+	// hibernatedFileName is the path to the serialized `fileAllocator`.
+	hibernatedFileName string
 	// mergedFiles is used during merges to record the real file hashes
 	mergedFiles map[string]bool
 	// mergedAuthor of the processed merge commit
@@ -135,6 +147,12 @@ const (
 	// RBTree allocator. It is useful to trade CPU time for reduced peak memory consumption
 	// if there are many branches.
 	ConfigBurndownHibernationThreshold = "Burndown.HibernationThreshold"
+	// ConfigBurndownHibernationToDisk sets whether the hibernated RBTree allocator must be saved
+	// on disk rather than kept in memory.
+	ConfigBurndownHibernationToDisk = "Burndown.HibernationOnDisk"
+	// ConfigBurndownHibernationDirectory sets the name of the temporary directory to use for
+	// saving hibernated RBTree allocators.
+	ConfigBurndownHibernationDirectory = "Burndown.HibernationDirectory"
 	// ConfigBurndownDebug enables some extra debug assertions.
 	ConfigBurndownDebug = "Burndown.Debug"
 	// DefaultBurndownGranularity is the default number of days for BurndownAnalysis.Granularity
@@ -201,6 +219,18 @@ func (analyser *BurndownAnalysis) ListConfigurationOptions() []core.Configuratio
 		Flag:    "burndown-hibernation-threshold",
 		Type:    core.IntConfigurationOption,
 		Default: 0}, {
+		Name: ConfigBurndownHibernationToDisk,
+		Description: "Save hibernated RBTree allocators to disk rather than keep it in memory; " +
+			"requires --burndown-hibernation-threshold to be greater than zero.",
+		Flag:    "burndown-hibernation-disk",
+		Type:    core.BoolConfigurationOption,
+		Default: false}, {
+		Name: ConfigBurndownHibernationDirectory,
+		Description: "Temporary directory where to save the hibernated RBTree allocators; " +
+			"requires --burndown-hibernation-disk.",
+		Flag:    "burndown-hibernation-dir",
+		Type:    core.StringConfigurationOption,
+		Default: ""}, {
 		Name:        ConfigBurndownDebug,
 		Description: "Validate the trees on each step.",
 		Flag:        "burndown-debug",
@@ -235,6 +265,12 @@ func (analyser *BurndownAnalysis) Configure(facts map[string]interface{}) error
 	if val, exists := facts[ConfigBurndownHibernationThreshold].(int); exists {
 		analyser.HibernationThreshold = val
 	}
+	if val, exists := facts[ConfigBurndownHibernationToDisk].(bool); exists {
+		analyser.HibernationToDisk = val
+	}
+	if val, exists := facts[ConfigBurndownHibernationDirectory].(string); exists {
+		analyser.HibernationDirectory = val
+	}
 	if val, exists := facts[ConfigBurndownDebug].(bool); exists {
 		analyser.Debug = val
 	}
@@ -394,7 +430,9 @@ func (analyser *BurndownAnalysis) Merge(branches []core.PipelineItem) {
 		}
 		for _, burn := range all {
 			if burn.files[key] != files[0] {
-				burn.files[key].Delete()
+				if burn.files[key] != nil {
+					burn.files[key].Delete()
+				}
 				burn.files[key] = files[0].CloneDeep(burn.fileAllocator)
 			}
 		}
@@ -403,13 +441,32 @@ func (analyser *BurndownAnalysis) Merge(branches []core.PipelineItem) {
 }
 
 // Hibernate compresses the bound RBTree memory with the files.
-func (analyser *BurndownAnalysis) Hibernate() {
+func (analyser *BurndownAnalysis) Hibernate() error {
 	analyser.fileAllocator.Hibernate()
+	if analyser.HibernationToDisk {
+		file, err := ioutil.TempFile(analyser.HibernationDirectory, "*-hercules.bin")
+		if err != nil {
+			return err
+		}
+		analyser.hibernatedFileName = file.Name()
+		file.Close()
+		analyser.fileAllocator.Serialize(analyser.hibernatedFileName)
+	}
+	return nil
 }
 
 // Boot decompresses the bound RBTree memory with the files.
-func (analyser *BurndownAnalysis) Boot() {
+func (analyser *BurndownAnalysis) Boot() error {
+	if analyser.hibernatedFileName != "" {
+		analyser.fileAllocator.Deserialize(analyser.hibernatedFileName)
+		err := os.Remove(analyser.hibernatedFileName)
+		if err != nil {
+			return err
+		}
+		analyser.hibernatedFileName = ""
+	}
 	analyser.fileAllocator.Boot()
+	return nil
 }
 
 // Finalize returns the result of the analysis. Further Consume() calls are not expected.

leaves/burndown_test.go

@@ -45,7 +45,9 @@ func TestBurndownMeta(t *testing.T) {
 	for _, opt := range opts {
 		switch opt.Name {
 		case ConfigBurndownGranularity, ConfigBurndownSampling, ConfigBurndownTrackFiles,
-			ConfigBurndownTrackPeople, ConfigBurndownHibernationThreshold, ConfigBurndownDebug:
+			ConfigBurndownTrackPeople, ConfigBurndownHibernationThreshold,
+			ConfigBurndownHibernationToDisk, ConfigBurndownHibernationDirectory,
+			ConfigBurndownDebug:
 			matches++
 		}
 	}
@@ -62,6 +64,8 @@ func TestBurndownConfigure(t *testing.T) {
 	facts[ConfigBurndownTrackPeople] = true
 	facts[ConfigBurndownDebug] = true
 	facts[ConfigBurndownHibernationThreshold] = 100
+	facts[ConfigBurndownHibernationToDisk] = true
+	facts[ConfigBurndownHibernationDirectory] = "xxx"
 	facts[identity.FactIdentityDetectorPeopleCount] = 5
 	facts[identity.FactIdentityDetectorReversedPeopleDict] = burndown.Requires()
 	burndown.Configure(facts)
@@ -70,6 +74,8 @@ func TestBurndownConfigure(t *testing.T) {
 	assert.Equal(t, burndown.TrackFiles, true)
 	assert.Equal(t, burndown.PeopleNumber, 5)
 	assert.Equal(t, burndown.HibernationThreshold, 100)
+	assert.True(t, burndown.HibernationToDisk)
+	assert.Equal(t, burndown.HibernationDirectory, "xxx")
 	assert.Equal(t, burndown.Debug, true)
 	assert.Equal(t, burndown.reversedPeopleDict, burndown.Requires())
 	facts[ConfigBurndownTrackPeople] = false
@@ -1211,11 +1217,27 @@ func TestBurndownHibernateBoot(t *testing.T) {
 	_, burndown := bakeBurndownForSerialization(t, 0, 1)
 	assert.Equal(t, burndown.fileAllocator.Size(), 157)
 	assert.Equal(t, burndown.fileAllocator.Used(), 155)
-	burndown.Hibernate()
+	assert.Nil(t, burndown.Hibernate())
 	assert.PanicsWithValue(t, "BurndownAnalysis.Consume() was called on a hibernated instance",
 		func() { burndown.Consume(nil) })
 	assert.Equal(t, burndown.fileAllocator.Size(), 0)
-	burndown.Boot()
+	assert.Nil(t, burndown.Boot())
 	assert.Equal(t, burndown.fileAllocator.Size(), 157)
 	assert.Equal(t, burndown.fileAllocator.Used(), 155)
 }
+
+func TestBurndownHibernateBootSerialize(t *testing.T) {
+	_, burndown := bakeBurndownForSerialization(t, 0, 1)
+	assert.Equal(t, burndown.fileAllocator.Size(), 157)
+	assert.Equal(t, burndown.fileAllocator.Used(), 155)
+	burndown.HibernationToDisk = true
+	assert.Nil(t, burndown.Hibernate())
+	assert.NotEmpty(t, burndown.hibernatedFileName)
+	assert.PanicsWithValue(t, "BurndownAnalysis.Consume() was called on a hibernated instance",
+		func() { burndown.Consume(nil) })
+	assert.Equal(t, burndown.fileAllocator.Size(), 0)
+	assert.Nil(t, burndown.Boot())
+	assert.Equal(t, burndown.fileAllocator.Size(), 157)
+	assert.Equal(t, burndown.fileAllocator.Used(), 155)
+	assert.Empty(t, burndown.hibernatedFileName)
+}