Add disk storage ability

car.go

@@ -20,12 +20,30 @@ type Builder struct {
 }
 
 // NewBuilder creates a new car builder.
+// Files processed by this builder are only stored in memory.
 func NewBuilder() *Builder {
 	return &Builder{
 		di: NewDataImporter(),
 	}
 }
 
+// NewBuilder creates a new car builder, but files are backed by storage on the disk
+// instead of in memory.
+//
+// path doesn't have to exist, but must be a directory if it does exist.
+//
+// It's up to the caller to remove the files stored under path once they are no longer
+// using the Builder or the car structs it can build.
+func NewBuilderDisk(path string) (*Builder, error) {
+	di, err := NewDataImporterDisk(path)
+	if err != nil {
+		return nil, err
+	}
+	return &Builder{
+		di: di,
+	}, nil
+}
+
 type CarV1 struct {
 	root cid.Cid
 	car  *carv1.SelectiveCar

importer.go

@@ -2,6 +2,7 @@ package car
 
 import (
 	"context"
+	"errors"
 	"io"
 	"os"
 	"path/filepath"
@@ -18,6 +19,8 @@ import (
 	coreiface "github.com/ipfs/kubo/core/coreiface"
 	"github.com/ipfs/kubo/core/coreiface/options"
 	"github.com/ipfs/kubo/core/coreunix"
+
+	exampleds "github.com/ipfs/go-datastore/examples"
 )
 
 // DataImporter creates a new importer that imports data (can be byte slice,
@@ -38,6 +41,30 @@ func NewDataImporter() *DataImporter {
 	}
 }
 
+func NewDataImporterDisk(path string) (*DataImporter, error) {
+	_, err := os.Stat(path)
+	if errors.Is(err, os.ErrNotExist) {
+		err = os.Mkdir(path, 0755)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	dstore, err := exampleds.NewDatastore(path)
+	if err != nil {
+		return nil, err
+	}
+	bstore := blockstore.NewBlockstore(dssync.MutexWrap(dstore))
+	return &DataImporter{
+		bstore: bstore,
+		dagServ: merkledag.NewDAGService(
+			blockservice.New(bstore, newNoopExchg()),
+		),
+	}, nil
+}
+
 // Import imports the given input.
 func (di *DataImporter) Import(
 	ctx context.Context,