Documentation
¶
Index ¶
- Constants
- Variables
- func LoadMHList(filePath string) ([]multihash.Multihash, error)
- func SaveMHList(filePath string, list []multihash.Multihash) error
- type BSSTIndex
- type CarLog
- func (j *CarLog) Close() error
- func (j *CarLog) Commit() (int64, error)
- func (j *CarLog) Delete(c []mh.Multihash) error
- func (j *CarLog) FinDataReload(ctx context.Context, blkEnts int64, carSz int64) error
- func (j *CarLog) Finalize(ctx context.Context) error
- func (j *CarLog) HashSample() ([]mh.Multihash, error)
- func (j *CarLog) LoadData(ctx context.Context, car io.Reader, sz int64) error
- func (j *CarLog) MarkReadOnly() error
- func (j *CarLog) Offload() error
- func (j *CarLog) OffloadData() error
- func (j *CarLog) Put(c []mh.Multihash, b []blocks.Block) error
- func (j *CarLog) View(c []mh.Multihash, cb func(cidx int, found bool, data []byte) error) error
- func (j *CarLog) WriteCar(w io.Writer) (int64, cid.Cid, error)
- type CarStorageProvider
- type Head
- type IndexSource
- type LevelDBIndex
- func (l *LevelDBIndex) Close() error
- func (l *LevelDBIndex) Del(c []multihash.Multihash) error
- func (l *LevelDBIndex) Entries() (int64, error)
- func (l *LevelDBIndex) Get(c []multihash.Multihash) ([]int64, error)
- func (l *LevelDBIndex) Has(c []multihash.Multihash) ([]bool, error)
- func (l *LevelDBIndex) List(f func(c multihash.Multihash, offs []int64) error) error
- func (l *LevelDBIndex) Put(c []multihash.Multihash, offs []int64) error
- func (l *LevelDBIndex) ToTruncate(atOrAbove int64) ([]multihash.Multihash, error)
- type ReadableIndex
- type TruncCleanup
- type WritableIndex
Constants ¶
View Source
const ( HeadName = "head" HeadSize = 512 LevelIndex = "index.level" BsstIndex = "index.bsst" BsstIndexCanon = "fil.bsst" HashSample = "sample.mhlist" BlockLog = "blklog.car" FilCar = "fil.car" )
View Source
const MaxEntryLen = 1 << (64 - 40)
Variables ¶
View Source
var ErrAlreadyOffloaded = xerrors.Errorf("group already offloaded")
View Source
var ErrReadOnly = errors.New("already read-only")
Functions ¶
Types ¶
type BSSTIndex ¶
type BSSTIndex struct {
// contains filtered or unexported fields
}
func CreateBSSTIndex ¶
func CreateBSSTIndex(path string, index IndexSource) (*BSSTIndex, error)
func OpenBSSTIndex ¶
type CarLog ¶
type CarLog struct {
// index = dir, data = file
IndexPath, DataPath string
// contains filtered or unexported fields
}
CarLog is a .car file which is storing a flat layer of blocks, with a wide top tree and a single root * Before the CarLog is finalized, the root CID is a placeholher Identity hash * Blocks are stored layer-by-layer, from the bottom, left to right * Transforming into depth-first .car is very cheap with the head file * NOT THREAD SAFE FOR WRITING!!
- Reads can happen in parallel with writing
* One tx at a time * Not considered written until committed
func Create ¶
func Create(staging CarStorageProvider, indexPath, dataPath string, _ TruncCleanup) (*CarLog, error)
func Open ¶
func Open(staging CarStorageProvider, indexPath, dataPath string, tc TruncCleanup) (*CarLog, error)
func (*CarLog) FinDataReload ¶
func (*CarLog) MarkReadOnly ¶
func (*CarLog) OffloadData ¶
type CarStorageProvider ¶
type Head ¶
type Head struct {
Version int64
// something that's not zero
Valid bool
// byte offset just after the last retired op. If finalized, but layers aren't set
// this points to the end of first (bottom) layer
RetiredAt int64
DataStart int64
// byte offset of the start of the second layer
DataEnd int64
ReadOnly bool // if true, no more writes are allowed
Finalized bool // if true, no more writes are allowed, and the bsst index is finalized
Offloaded bool // if true, the data file is offloaded to external storage, only hash samples are kept
External bool // if true, the data is moved to external storage on finalize
// Layer stats, set after Finalized (Finalized can be true and layers may still not be set)
LayerOffsets []int64 // byte offsets of the start of each layer
}
Head is the on-disk head object. CBOR-map-serialized. Must fit in
HeadSize bytes. Null-Padded to exactly HeadSize
type LevelDBIndex ¶
func OpenLevelDBIndex ¶
func OpenLevelDBIndex(path string, create bool) (*LevelDBIndex, error)
func (*LevelDBIndex) Close ¶
func (l *LevelDBIndex) Close() error
func (*LevelDBIndex) Entries ¶
func (l *LevelDBIndex) Entries() (int64, error)
func (*LevelDBIndex) Get ¶
func (l *LevelDBIndex) Get(c []multihash.Multihash) ([]int64, error)
Get returns offsets to data, -1 if not found
func (*LevelDBIndex) ToTruncate ¶
func (l *LevelDBIndex) ToTruncate(atOrAbove int64) ([]multihash.Multihash, error)
type ReadableIndex ¶
type ReadableIndex interface {
// todo maybe callback calling with sequential indexes of what we don't have
// to pipeline better?
Has(c []mh.Multihash) ([]bool, error)
// Get returns offsets to data, -1 if not found
Get(c []mh.Multihash) ([]int64, error)
// bsst creation
Entries() (int64, error)
bsst.Source
Close() error
}
type WritableIndex ¶
type WritableIndex interface {
// Put records entries in the index
// sync for now, todo
// -1 offset means 'skip'
Put(c []mh.Multihash, offs []int64) error
Del(c []mh.Multihash) error
// Truncate returns a list of multihashes to remove from the index
ToTruncate(atOrAbove int64) ([]mh.Multihash, error)
Close() error
}
Click to show internal directories.
Click to hide internal directories.