tendermint/p2p/pex/addrbook.go

814 lines
22 KiB
Go
Raw Normal View History

2015-10-25 18:21:51 -07:00
// Modified for Tendermint
// Originally Copyright (c) 2013-2014 Conformal Systems LLC.
// https://github.com/conformal/btcd/blob/master/LICENSE
2018-01-20 18:12:04 -08:00
package pex
2015-10-25 18:21:51 -07:00
import (
2018-01-13 14:25:51 -08:00
"crypto/sha256"
2015-10-25 18:21:51 -07:00
"encoding/binary"
"math"
"net"
"sync"
"time"
2017-04-14 12:59:22 -07:00
crypto "github.com/tendermint/go-crypto"
2018-01-20 21:33:53 -08:00
"github.com/tendermint/tendermint/p2p"
2017-05-01 19:12:14 -07:00
cmn "github.com/tendermint/tmlibs/common"
2015-10-25 18:21:51 -07:00
)
const (
2018-01-20 16:12:04 -08:00
bucketTypeNew = 0x01
bucketTypeOld = 0x02
)
2015-10-25 18:21:51 -07:00
2018-01-20 16:12:04 -08:00
// AddrBook is an address book used for tracking peers
// so we can gossip about them to others and select
// peers to dial.
2018-01-23 19:25:39 -08:00
// TODO: break this up?
2018-01-20 16:12:04 -08:00
type AddrBook interface {
cmn.Service
2015-10-25 18:21:51 -07:00
2018-01-20 18:12:04 -08:00
// Add our own addresses so we don't later add ourselves
2018-01-20 21:33:53 -08:00
AddOurAddress(*p2p.NetAddress)
// Check if it is our address
OurAddress(*p2p.NetAddress) bool
2018-01-20 16:12:04 -08:00
// Add and remove an address
2018-01-20 21:33:53 -08:00
AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error
RemoveAddress(*p2p.NetAddress)
// Check if the address is in the book
HasAddress(*p2p.NetAddress) bool
2015-10-25 18:21:51 -07:00
2018-01-20 16:12:04 -08:00
// Do we need more peers?
NeedMoreAddrs() bool
2015-10-25 18:21:51 -07:00
2018-01-20 16:12:04 -08:00
// Pick an address to dial
2018-03-24 09:49:42 -07:00
PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress
2015-10-25 18:21:51 -07:00
2018-01-20 16:12:04 -08:00
// Mark address
2018-01-20 21:33:53 -08:00
MarkGood(*p2p.NetAddress)
MarkAttempt(*p2p.NetAddress)
MarkBad(*p2p.NetAddress)
2015-10-25 18:21:51 -07:00
2018-03-24 09:49:42 -07:00
IsGood(*p2p.NetAddress) bool
2018-01-20 16:12:04 -08:00
// Send a selection of addresses to peers
2018-01-20 21:33:53 -08:00
GetSelection() []*p2p.NetAddress
2018-03-24 09:49:42 -07:00
// Send a selection of addresses with bias
GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress
2018-01-20 18:12:04 -08:00
// TODO: remove
ListOfKnownAddresses() []*knownAddress
2018-01-23 19:25:39 -08:00
// Persist to disk
Save()
2018-01-20 16:12:04 -08:00
}
2017-04-17 02:03:26 -07:00
2018-01-20 18:12:04 -08:00
var _ AddrBook = (*addrBook)(nil)
2018-01-20 16:12:04 -08:00
// addrBook - concurrency safe peer address manager.
// Implements AddrBook.
type addrBook struct {
2017-05-01 19:12:14 -07:00
cmn.BaseService
2015-10-25 18:21:51 -07:00
2017-11-15 18:25:00 -08:00
// immutable after creation
2016-11-30 19:57:21 -08:00
filePath string
routabilityStrict bool
2018-01-20 18:12:04 -08:00
key string // random prefix for bucket placement
2017-11-15 18:25:00 -08:00
// accessed concurrently
mtx sync.Mutex
rand *cmn.Rand
ourAddrs map[string]struct{}
2018-01-20 21:33:53 -08:00
addrLookup map[p2p.ID]*knownAddress // new & old
bucketsOld []map[string]*knownAddress
bucketsNew []map[string]*knownAddress
2017-11-15 18:25:00 -08:00
nOld int
nNew int
wg sync.WaitGroup
2015-10-25 18:21:51 -07:00
}
2017-04-17 02:03:26 -07:00
// NewAddrBook creates a new address book.
2015-10-25 18:21:51 -07:00
// Use Start to begin processing asynchronous address updates.
2018-01-20 16:12:04 -08:00
func NewAddrBook(filePath string, routabilityStrict bool) *addrBook {
2018-01-20 18:12:04 -08:00
am := &addrBook{
rand: cmn.NewRand(),
ourAddrs: make(map[string]struct{}),
2018-01-20 21:33:53 -08:00
addrLookup: make(map[p2p.ID]*knownAddress),
2016-11-30 19:57:21 -08:00
filePath: filePath,
routabilityStrict: routabilityStrict,
2015-10-25 18:21:51 -07:00
}
am.init()
2017-05-02 00:53:32 -07:00
am.BaseService = *cmn.NewBaseService(nil, "AddrBook", am)
2015-10-25 18:21:51 -07:00
return am
}
2018-01-20 16:12:04 -08:00
// Initialize the buckets.
2015-10-25 18:21:51 -07:00
// When modifying this, don't forget to update loadFromFile()
2018-01-20 16:12:04 -08:00
func (a *addrBook) init() {
2016-03-13 09:42:11 -07:00
a.key = crypto.CRandHex(24) // 24/2 * 8 = 96 bits
2015-10-25 18:21:51 -07:00
// New addr buckets
a.bucketsNew = make([]map[string]*knownAddress, newBucketCount)
for i := range a.bucketsNew {
a.bucketsNew[i] = make(map[string]*knownAddress)
2015-10-25 18:21:51 -07:00
}
// Old addr buckets
a.bucketsOld = make([]map[string]*knownAddress, oldBucketCount)
for i := range a.bucketsOld {
a.bucketsOld[i] = make(map[string]*knownAddress)
2015-10-25 18:21:51 -07:00
}
}
2017-04-17 02:03:26 -07:00
// OnStart implements Service.
2018-01-20 16:12:04 -08:00
func (a *addrBook) OnStart() error {
2017-09-21 09:38:48 -07:00
if err := a.BaseService.OnStart(); err != nil {
return err
}
2015-10-25 18:21:51 -07:00
a.loadFromFile(a.filePath)
// wg.Add to ensure that any invocation of .Wait()
// later on will wait for saveRoutine to terminate.
2017-04-17 02:03:26 -07:00
a.wg.Add(1)
2015-10-25 18:21:51 -07:00
go a.saveRoutine()
2015-10-25 18:21:51 -07:00
return nil
}
2017-04-17 02:03:26 -07:00
// OnStop implements Service.
2018-01-20 16:12:04 -08:00
func (a *addrBook) OnStop() {
2017-04-17 02:03:26 -07:00
a.BaseService.OnStop()
2015-10-25 18:21:51 -07:00
}
2018-01-20 16:12:04 -08:00
func (a *addrBook) Wait() {
2017-04-17 02:22:59 -07:00
a.wg.Wait()
}
2018-03-09 04:02:24 -08:00
func (a *addrBook) FilePath() string {
return a.filePath
}
2018-01-20 18:12:04 -08:00
//-------------------------------------------------------
// AddOurAddress one of our addresses.
2018-01-20 21:33:53 -08:00
func (a *addrBook) AddOurAddress(addr *p2p.NetAddress) {
2015-10-25 18:21:51 -07:00
a.mtx.Lock()
defer a.mtx.Unlock()
2017-05-02 00:53:32 -07:00
a.Logger.Info("Add our address to book", "addr", addr)
a.ourAddrs[addr.String()] = struct{}{}
}
// OurAddress returns true if it is our address.
func (a *addrBook) OurAddress(addr *p2p.NetAddress) bool {
a.mtx.Lock()
_, ok := a.ourAddrs[addr.String()]
a.mtx.Unlock()
return ok
2015-10-25 18:21:51 -07:00
}
2018-04-28 10:08:44 -07:00
// AddAddress implements AddrBook
// Add address to a "new" bucket. If it's already in one, only add it probabilistically.
// Returns error if the addr is non-routable. Does not add self.
2017-03-04 19:55:42 -08:00
// NOTE: addr must not be nil
2018-01-20 21:33:53 -08:00
func (a *addrBook) AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error {
2015-10-25 18:21:51 -07:00
a.mtx.Lock()
defer a.mtx.Unlock()
return a.addAddress(addr, src)
2015-10-25 18:21:51 -07:00
}
2018-01-20 16:12:04 -08:00
// RemoveAddress implements AddrBook - removes the address from the book.
2018-01-20 21:33:53 -08:00
func (a *addrBook) RemoveAddress(addr *p2p.NetAddress) {
2015-10-25 18:21:51 -07:00
a.mtx.Lock()
defer a.mtx.Unlock()
2018-01-20 16:12:04 -08:00
ka := a.addrLookup[addr.ID]
if ka == nil {
return
}
a.Logger.Info("Remove address from book", "addr", ka.Addr, "ID", ka.ID())
2018-01-20 16:12:04 -08:00
a.removeFromAllBuckets(ka)
2015-10-25 18:21:51 -07:00
}
2018-03-24 09:49:42 -07:00
// IsGood returns true if peer was ever marked as good and haven't
// done anything wrong since then.
func (a *addrBook) IsGood(addr *p2p.NetAddress) bool {
a.mtx.Lock()
defer a.mtx.Unlock()
return a.addrLookup[addr.ID].isOld()
}
// HasAddress returns true if the address is in the book.
func (a *addrBook) HasAddress(addr *p2p.NetAddress) bool {
a.mtx.Lock()
defer a.mtx.Unlock()
ka := a.addrLookup[addr.ID]
return ka != nil
}
2018-01-20 16:12:04 -08:00
// NeedMoreAddrs implements AddrBook - returns true if there are not have enough addresses in the book.
func (a *addrBook) NeedMoreAddrs() bool {
return a.Size() < needAddressThreshold
2015-10-25 18:21:51 -07:00
}
2018-01-20 16:12:04 -08:00
// PickAddress implements AddrBook. It picks an address to connect to.
2017-11-20 11:30:05 -08:00
// The address is picked randomly from an old or new bucket according
2018-03-24 09:49:42 -07:00
// to the biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to that range)
2017-11-20 11:30:05 -08:00
// and determines how biased we are to pick an address from a new bucket.
// PickAddress returns nil if the AddrBook is empty or if we try to pick
// from an empty bucket.
2018-03-24 09:49:42 -07:00
func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress {
2015-10-25 18:21:51 -07:00
a.mtx.Lock()
defer a.mtx.Unlock()
2018-04-30 05:19:19 -07:00
bookSize := a.size()
if bookSize <= 0 {
if bookSize < 0 {
a.Logger.Error("Addrbook size less than 0", "nNew", a.nNew, "nOld", a.nOld)
}
2015-10-25 18:21:51 -07:00
return nil
}
2018-03-24 09:49:42 -07:00
if biasTowardsNewAddrs > 100 {
biasTowardsNewAddrs = 100
2015-10-25 18:21:51 -07:00
}
2018-03-24 09:49:42 -07:00
if biasTowardsNewAddrs < 0 {
biasTowardsNewAddrs = 0
2015-10-25 18:21:51 -07:00
}
// Bias between new and old addresses.
2018-03-24 09:49:42 -07:00
oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(biasTowardsNewAddrs))
newCorrelation := math.Sqrt(float64(a.nNew)) * float64(biasTowardsNewAddrs)
2015-10-25 18:21:51 -07:00
2017-11-15 18:31:47 -08:00
// pick a random peer from a random bucket
var bucket map[string]*knownAddress
2017-11-15 18:25:00 -08:00
pickFromOldBucket := (newCorrelation+oldCorrelation)*a.rand.Float64() < oldCorrelation
if (pickFromOldBucket && a.nOld == 0) ||
(!pickFromOldBucket && a.nNew == 0) {
return nil
}
// loop until we pick a random non-empty bucket
2017-11-15 18:31:47 -08:00
for len(bucket) == 0 {
if pickFromOldBucket {
bucket = a.bucketsOld[a.rand.Intn(len(a.bucketsOld))]
2017-11-15 18:31:47 -08:00
} else {
bucket = a.bucketsNew[a.rand.Intn(len(a.bucketsNew))]
2015-10-25 18:21:51 -07:00
}
}
// pick a random index and loop over the map to return that index
2017-11-15 18:31:47 -08:00
randIndex := a.rand.Intn(len(bucket))
for _, ka := range bucket {
if randIndex == 0 {
return ka.Addr
}
randIndex--
}
return nil
2015-10-25 18:21:51 -07:00
}
2018-01-20 16:12:04 -08:00
// MarkGood implements AddrBook - it marks the peer as good and
// moves it into an "old" bucket.
2018-01-20 21:33:53 -08:00
func (a *addrBook) MarkGood(addr *p2p.NetAddress) {
2015-10-25 18:21:51 -07:00
a.mtx.Lock()
defer a.mtx.Unlock()
ka := a.addrLookup[addr.ID]
2015-10-25 18:21:51 -07:00
if ka == nil {
return
}
ka.markGood()
if ka.isNew() {
a.moveToOld(ka)
}
}
2018-01-20 16:12:04 -08:00
// MarkAttempt implements AddrBook - it marks that an attempt was made to connect to the address.
2018-01-20 21:33:53 -08:00
func (a *addrBook) MarkAttempt(addr *p2p.NetAddress) {
2015-10-25 18:21:51 -07:00
a.mtx.Lock()
defer a.mtx.Unlock()
ka := a.addrLookup[addr.ID]
2015-10-25 18:21:51 -07:00
if ka == nil {
return
}
ka.markAttempt()
}
2018-01-20 16:12:04 -08:00
// MarkBad implements AddrBook. Currently it just ejects the address.
// TODO: black list for some amount of time
2018-01-20 21:33:53 -08:00
func (a *addrBook) MarkBad(addr *p2p.NetAddress) {
a.RemoveAddress(addr)
}
2018-01-20 16:12:04 -08:00
// GetSelection implements AddrBook.
// It randomly selects some addresses (old & new). Suitable for peer-exchange protocols.
2018-04-28 12:01:33 -07:00
// Must never return a nil address.
2018-01-20 21:33:53 -08:00
func (a *addrBook) GetSelection() []*p2p.NetAddress {
2015-10-25 18:21:51 -07:00
a.mtx.Lock()
defer a.mtx.Unlock()
2018-04-28 10:08:44 -07:00
bookSize := a.size()
2018-04-30 05:19:19 -07:00
if bookSize <= 0 {
if bookSize < 0 {
a.Logger.Error("Addrbook size less than 0", "nNew", a.nNew, "nOld", a.nOld)
}
2015-10-25 18:21:51 -07:00
return nil
}
2018-04-28 10:08:44 -07:00
numAddresses := cmn.MaxInt(
cmn.MinInt(minGetSelection, bookSize),
bookSize*getSelectionPercent/100)
numAddresses = cmn.MinInt(maxGetSelection, numAddresses)
// XXX: instead of making a list of all addresses, shuffling, and slicing a random chunk,
// could we just select a random numAddresses of indexes?
allAddr := make([]*p2p.NetAddress, bookSize)
2015-10-25 18:21:51 -07:00
i := 0
for _, ka := range a.addrLookup {
allAddr[i] = ka.Addr
2015-10-25 18:21:51 -07:00
i++
}
// Fisher-Yates shuffle the array. We only need to do the first
// `numAddresses' since we are throwing the rest.
for i := 0; i < numAddresses; i++ {
// pick a number between current index and the end
j := cmn.RandIntn(len(allAddr)-i) + i
2015-10-25 18:21:51 -07:00
allAddr[i], allAddr[j] = allAddr[j], allAddr[i]
}
// slice off the limit we are willing to share.
return allAddr[:numAddresses]
}
2018-03-24 09:49:42 -07:00
// GetSelectionWithBias implements AddrBook.
// It randomly selects some addresses (old & new). Suitable for peer-exchange protocols.
2018-04-28 12:01:33 -07:00
// Must never return a nil address.
2018-03-24 09:49:42 -07:00
//
// Each address is picked randomly from an old or new bucket according to the
// biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to
// that range) and determines how biased we are to pick an address from a new
// bucket.
func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress {
a.mtx.Lock()
defer a.mtx.Unlock()
2018-04-28 10:08:44 -07:00
bookSize := a.size()
2018-04-30 05:19:19 -07:00
if bookSize <= 0 {
if bookSize < 0 {
a.Logger.Error("Addrbook size less than 0", "nNew", a.nNew, "nOld", a.nOld)
}
2018-03-24 09:49:42 -07:00
return nil
}
if biasTowardsNewAddrs > 100 {
biasTowardsNewAddrs = 100
}
if biasTowardsNewAddrs < 0 {
biasTowardsNewAddrs = 0
}
numAddresses := cmn.MaxInt(
2018-04-28 10:08:44 -07:00
cmn.MinInt(minGetSelection, bookSize),
bookSize*getSelectionPercent/100)
2018-03-24 09:49:42 -07:00
numAddresses = cmn.MinInt(maxGetSelection, numAddresses)
selection := make([]*p2p.NetAddress, numAddresses)
oldBucketToAddrsMap := make(map[int]map[string]struct{})
var oldIndex int
newBucketToAddrsMap := make(map[int]map[string]struct{})
var newIndex int
selectionIndex := 0
ADDRS_LOOP:
for selectionIndex < numAddresses {
pickFromOldBucket := int((float64(selectionIndex)/float64(numAddresses))*100) >= biasTowardsNewAddrs
pickFromOldBucket = (pickFromOldBucket && a.nOld > 0) || a.nNew == 0
bucket := make(map[string]*knownAddress)
// loop until we pick a random non-empty bucket
for len(bucket) == 0 {
if pickFromOldBucket {
oldIndex = a.rand.Intn(len(a.bucketsOld))
bucket = a.bucketsOld[oldIndex]
} else {
newIndex = a.rand.Intn(len(a.bucketsNew))
bucket = a.bucketsNew[newIndex]
}
}
// pick a random index
randIndex := a.rand.Intn(len(bucket))
// loop over the map to return that index
var selectedAddr *p2p.NetAddress
for _, ka := range bucket {
if randIndex == 0 {
selectedAddr = ka.Addr
break
}
randIndex--
}
// if we have selected the address before, restart the loop
// otherwise, record it and continue
if pickFromOldBucket {
if addrsMap, ok := oldBucketToAddrsMap[oldIndex]; ok {
if _, ok = addrsMap[selectedAddr.String()]; ok {
continue ADDRS_LOOP
}
} else {
oldBucketToAddrsMap[oldIndex] = make(map[string]struct{})
}
oldBucketToAddrsMap[oldIndex][selectedAddr.String()] = struct{}{}
} else {
if addrsMap, ok := newBucketToAddrsMap[newIndex]; ok {
if _, ok = addrsMap[selectedAddr.String()]; ok {
continue ADDRS_LOOP
}
} else {
newBucketToAddrsMap[newIndex] = make(map[string]struct{})
}
newBucketToAddrsMap[newIndex][selectedAddr.String()] = struct{}{}
}
selection[selectionIndex] = selectedAddr
selectionIndex++
}
return selection
}
2018-01-20 18:28:00 -08:00
// ListOfKnownAddresses returns the new and old addresses.
2018-01-20 18:12:04 -08:00
func (a *addrBook) ListOfKnownAddresses() []*knownAddress {
2018-01-20 18:28:00 -08:00
a.mtx.Lock()
defer a.mtx.Unlock()
addrs := []*knownAddress{}
for _, addr := range a.addrLookup {
addrs = append(addrs, addr.copy())
}
return addrs
}
2018-01-20 16:12:04 -08:00
//------------------------------------------------
2015-10-25 18:21:51 -07:00
2018-01-20 16:12:04 -08:00
// Size returns the number of addresses in the book.
func (a *addrBook) Size() int {
a.mtx.Lock()
defer a.mtx.Unlock()
return a.size()
2015-10-25 18:21:51 -07:00
}
2018-01-20 16:12:04 -08:00
func (a *addrBook) size() int {
return a.nNew + a.nOld
2017-01-13 08:48:42 -08:00
}
2018-01-20 16:12:04 -08:00
//----------------------------------------------------------
2015-10-25 18:21:51 -07:00
2018-01-23 19:25:39 -08:00
// Save persists the address book to disk.
func (a *addrBook) Save() {
a.saveToFile(a.filePath) // thread safe
}
2018-01-20 16:12:04 -08:00
func (a *addrBook) saveRoutine() {
defer a.wg.Done()
2017-11-15 20:30:23 -08:00
saveFileTicker := time.NewTicker(dumpAddressInterval)
2015-10-25 18:21:51 -07:00
out:
for {
select {
2017-11-15 20:30:23 -08:00
case <-saveFileTicker.C:
2015-10-25 18:21:51 -07:00
a.saveToFile(a.filePath)
case <-a.Quit():
2015-10-25 18:21:51 -07:00
break out
}
}
2017-11-15 20:30:23 -08:00
saveFileTicker.Stop()
2015-10-25 18:21:51 -07:00
a.saveToFile(a.filePath)
2017-05-02 00:53:32 -07:00
a.Logger.Info("Address handler done")
2015-10-25 18:21:51 -07:00
}
2018-01-20 18:12:04 -08:00
//----------------------------------------------------------
2018-01-20 16:12:04 -08:00
func (a *addrBook) getBucket(bucketType byte, bucketIdx int) map[string]*knownAddress {
2015-10-25 18:21:51 -07:00
switch bucketType {
case bucketTypeNew:
return a.bucketsNew[bucketIdx]
2015-10-25 18:21:51 -07:00
case bucketTypeOld:
return a.bucketsOld[bucketIdx]
2015-10-25 18:21:51 -07:00
default:
2017-05-01 19:12:14 -07:00
cmn.PanicSanity("Should not happen")
2015-10-25 18:21:51 -07:00
return nil
}
}
// Adds ka to new bucket. Returns false if it couldn't do it cuz buckets full.
// NOTE: currently it always returns true.
2018-04-28 14:14:58 -07:00
func (a *addrBook) addToNewBucket(ka *knownAddress, bucketIdx int) {
2015-10-25 18:21:51 -07:00
// Sanity check
if ka.isOld() {
2018-04-28 14:27:51 -07:00
a.Logger.Error("Failed Sanity Check! Cant add old address to new bucket", "ka", ka, "bucket", bucketIdx)
2018-04-28 14:14:58 -07:00
return
2015-10-25 18:21:51 -07:00
}
addrStr := ka.Addr.String()
bucket := a.getBucket(bucketTypeNew, bucketIdx)
// Already exists?
if _, ok := bucket[addrStr]; ok {
2018-04-28 14:14:58 -07:00
return
2015-10-25 18:21:51 -07:00
}
// Enforce max addresses.
if len(bucket) > newBucketSize {
2018-01-20 18:12:04 -08:00
a.Logger.Info("new bucket is full, expiring new")
2015-10-25 18:21:51 -07:00
a.expireNew(bucketIdx)
}
// Add to bucket.
bucket[addrStr] = ka
2018-01-20 18:12:04 -08:00
// increment nNew if the peer doesnt already exist in a bucket
2015-10-25 18:21:51 -07:00
if ka.addBucketRef(bucketIdx) == 1 {
a.nNew++
}
2018-01-20 18:12:04 -08:00
// Add it to addrLookup
a.addrLookup[ka.ID()] = ka
2015-10-25 18:21:51 -07:00
}
// Adds ka to old bucket. Returns false if it couldn't do it cuz buckets full.
2018-01-20 16:12:04 -08:00
func (a *addrBook) addToOldBucket(ka *knownAddress, bucketIdx int) bool {
2015-10-25 18:21:51 -07:00
// Sanity check
if ka.isNew() {
2017-05-02 00:53:32 -07:00
a.Logger.Error(cmn.Fmt("Cannot add new address to old bucket: %v", ka))
2015-10-25 18:21:51 -07:00
return false
}
if len(ka.Buckets) != 0 {
2017-05-02 00:53:32 -07:00
a.Logger.Error(cmn.Fmt("Cannot add already old address to another old bucket: %v", ka))
2015-10-25 18:21:51 -07:00
return false
}
addrStr := ka.Addr.String()
2017-11-15 20:08:46 -08:00
bucket := a.getBucket(bucketTypeOld, bucketIdx)
2015-10-25 18:21:51 -07:00
// Already exists?
if _, ok := bucket[addrStr]; ok {
return true
}
// Enforce max addresses.
if len(bucket) > oldBucketSize {
return false
}
// Add to bucket.
bucket[addrStr] = ka
if ka.addBucketRef(bucketIdx) == 1 {
a.nOld++
}
// Ensure in addrLookup
a.addrLookup[ka.ID()] = ka
2015-10-25 18:21:51 -07:00
return true
}
2018-01-20 16:12:04 -08:00
func (a *addrBook) removeFromBucket(ka *knownAddress, bucketType byte, bucketIdx int) {
2015-10-25 18:21:51 -07:00
if ka.BucketType != bucketType {
2017-05-02 00:53:32 -07:00
a.Logger.Error(cmn.Fmt("Bucket type mismatch: %v", ka))
2015-10-25 18:21:51 -07:00
return
}
bucket := a.getBucket(bucketType, bucketIdx)
delete(bucket, ka.Addr.String())
if ka.removeBucketRef(bucketIdx) == 0 {
if bucketType == bucketTypeNew {
a.nNew--
} else {
a.nOld--
}
delete(a.addrLookup, ka.ID())
2015-10-25 18:21:51 -07:00
}
}
2018-01-20 16:12:04 -08:00
func (a *addrBook) removeFromAllBuckets(ka *knownAddress) {
2015-10-25 18:21:51 -07:00
for _, bucketIdx := range ka.Buckets {
bucket := a.getBucket(ka.BucketType, bucketIdx)
delete(bucket, ka.Addr.String())
}
ka.Buckets = nil
if ka.BucketType == bucketTypeNew {
a.nNew--
} else {
a.nOld--
}
delete(a.addrLookup, ka.ID())
2015-10-25 18:21:51 -07:00
}
2018-01-20 18:12:04 -08:00
//----------------------------------------------------------
2018-01-20 16:12:04 -08:00
func (a *addrBook) pickOldest(bucketType byte, bucketIdx int) *knownAddress {
2015-10-25 18:21:51 -07:00
bucket := a.getBucket(bucketType, bucketIdx)
var oldest *knownAddress
for _, ka := range bucket {
if oldest == nil || ka.LastAttempt.Before(oldest.LastAttempt) {
oldest = ka
}
}
return oldest
}
2018-01-20 18:12:04 -08:00
// adds the address to a "new" bucket. if its already in one,
// it only adds it probabilistically
2018-01-20 21:33:53 -08:00
func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error {
2018-04-30 05:19:19 -07:00
if addr == nil || src == nil {
2018-04-28 10:08:44 -07:00
return ErrAddrBookNilAddr{addr, src}
}
2016-11-30 19:57:21 -08:00
if a.routabilityStrict && !addr.Routable() {
2018-04-28 10:08:44 -07:00
return ErrAddrBookNonRoutable{addr}
2015-10-25 18:21:51 -07:00
}
2018-04-28 10:08:44 -07:00
// TODO: we should track ourAddrs by ID and by IP:PORT and refuse both.
2015-10-25 18:21:51 -07:00
if _, ok := a.ourAddrs[addr.String()]; ok {
2018-04-28 12:39:09 -07:00
return ErrAddrBookSelf{addr}
2015-10-25 18:21:51 -07:00
}
ka := a.addrLookup[addr.ID]
2015-10-25 18:21:51 -07:00
if ka != nil {
// If its already old and the addr is the same, ignore it.
if ka.isOld() && ka.Addr.Equals(addr) {
return nil
2015-10-25 18:21:51 -07:00
}
// Already in max new buckets.
if len(ka.Buckets) == maxNewBucketsPerAddress {
return nil
2015-10-25 18:21:51 -07:00
}
// The more entries we have, the less likely we are to add more.
factor := int32(2 * len(ka.Buckets))
if a.rand.Int31n(factor) != 0 {
return nil
2015-10-25 18:21:51 -07:00
}
} else {
ka = newKnownAddress(addr, src)
}
bucket := a.calcNewBucket(addr, src)
2018-04-28 14:14:58 -07:00
a.addToNewBucket(ka, bucket)
return nil
2015-10-25 18:21:51 -07:00
}
// Make space in the new buckets by expiring the really bad entries.
// If no bad entries are available we remove the oldest.
2018-01-20 16:12:04 -08:00
func (a *addrBook) expireNew(bucketIdx int) {
for addrStr, ka := range a.bucketsNew[bucketIdx] {
2015-10-25 18:21:51 -07:00
// If an entry is bad, throw it away
if ka.isBad() {
2017-05-02 00:53:32 -07:00
a.Logger.Info(cmn.Fmt("expiring bad address %v", addrStr))
2015-10-25 18:21:51 -07:00
a.removeFromBucket(ka, bucketTypeNew, bucketIdx)
return
}
}
// If we haven't thrown out a bad entry, throw out the oldest entry
oldest := a.pickOldest(bucketTypeNew, bucketIdx)
a.removeFromBucket(oldest, bucketTypeNew, bucketIdx)
}
2018-01-20 18:12:04 -08:00
// Promotes an address from new to old. If the destination bucket is full,
// demote the oldest one to a "new" bucket.
// TODO: Demote more probabilistically?
2018-01-20 16:12:04 -08:00
func (a *addrBook) moveToOld(ka *knownAddress) {
2015-10-25 18:21:51 -07:00
// Sanity check
if ka.isOld() {
2017-05-02 00:53:32 -07:00
a.Logger.Error(cmn.Fmt("Cannot promote address that is already old %v", ka))
2015-10-25 18:21:51 -07:00
return
}
if len(ka.Buckets) == 0 {
2017-05-02 00:53:32 -07:00
a.Logger.Error(cmn.Fmt("Cannot promote address that isn't in any new buckets %v", ka))
2015-10-25 18:21:51 -07:00
return
}
// Remove from all (new) buckets.
a.removeFromAllBuckets(ka)
// It's officially old now.
ka.BucketType = bucketTypeOld
// Try to add it to its oldBucket destination.
oldBucketIdx := a.calcOldBucket(ka.Addr)
added := a.addToOldBucket(ka, oldBucketIdx)
if !added {
2018-04-28 14:14:58 -07:00
// No room; move the oldest to a new bucket
2015-10-25 18:21:51 -07:00
oldest := a.pickOldest(bucketTypeOld, oldBucketIdx)
a.removeFromBucket(oldest, bucketTypeOld, oldBucketIdx)
newBucketIdx := a.calcNewBucket(oldest.Addr, oldest.Src)
2018-04-28 14:14:58 -07:00
a.addToNewBucket(oldest, newBucketIdx)
// Finally, add our ka to old bucket again.
2015-10-25 18:21:51 -07:00
added = a.addToOldBucket(ka, oldBucketIdx)
if !added {
2017-05-02 00:53:32 -07:00
a.Logger.Error(cmn.Fmt("Could not re-add ka %v to oldBucketIdx %v", ka, oldBucketIdx))
2015-10-25 18:21:51 -07:00
}
}
}
2018-01-20 18:12:04 -08:00
//---------------------------------------------------------------------
// calculate bucket placements
2015-10-25 18:21:51 -07:00
// doublesha256( key + sourcegroup +
// int64(doublesha256(key + group + sourcegroup))%bucket_per_group ) % num_new_buckets
2018-01-20 21:33:53 -08:00
func (a *addrBook) calcNewBucket(addr, src *p2p.NetAddress) int {
2015-10-25 18:21:51 -07:00
data1 := []byte{}
data1 = append(data1, []byte(a.key)...)
2016-11-30 19:57:21 -08:00
data1 = append(data1, []byte(a.groupKey(addr))...)
data1 = append(data1, []byte(a.groupKey(src))...)
2015-10-25 18:21:51 -07:00
hash1 := doubleSha256(data1)
hash64 := binary.BigEndian.Uint64(hash1)
hash64 %= newBucketsPerGroup
var hashbuf [8]byte
binary.BigEndian.PutUint64(hashbuf[:], hash64)
data2 := []byte{}
data2 = append(data2, []byte(a.key)...)
2016-11-30 19:57:21 -08:00
data2 = append(data2, a.groupKey(src)...)
2015-10-25 18:21:51 -07:00
data2 = append(data2, hashbuf[:]...)
hash2 := doubleSha256(data2)
return int(binary.BigEndian.Uint64(hash2) % newBucketCount)
}
// doublesha256( key + group +
// int64(doublesha256(key + addr))%buckets_per_group ) % num_old_buckets
2018-01-20 21:33:53 -08:00
func (a *addrBook) calcOldBucket(addr *p2p.NetAddress) int {
2015-10-25 18:21:51 -07:00
data1 := []byte{}
data1 = append(data1, []byte(a.key)...)
data1 = append(data1, []byte(addr.String())...)
hash1 := doubleSha256(data1)
hash64 := binary.BigEndian.Uint64(hash1)
hash64 %= oldBucketsPerGroup
var hashbuf [8]byte
binary.BigEndian.PutUint64(hashbuf[:], hash64)
data2 := []byte{}
data2 = append(data2, []byte(a.key)...)
2016-11-30 19:57:21 -08:00
data2 = append(data2, a.groupKey(addr)...)
2015-10-25 18:21:51 -07:00
data2 = append(data2, hashbuf[:]...)
hash2 := doubleSha256(data2)
return int(binary.BigEndian.Uint64(hash2) % oldBucketCount)
}
// Return a string representing the network group of this address.
2017-11-15 20:43:07 -08:00
// This is the /16 for IPv4, the /32 (/36 for he.net) for IPv6, the string
// "local" for a local address and the string "unroutable" for an unroutable
2015-10-25 18:21:51 -07:00
// address.
2018-01-20 21:33:53 -08:00
func (a *addrBook) groupKey(na *p2p.NetAddress) string {
2016-11-30 19:57:21 -08:00
if a.routabilityStrict && na.Local() {
2015-10-25 18:21:51 -07:00
return "local"
}
2016-11-30 19:57:21 -08:00
if a.routabilityStrict && !na.Routable() {
2015-10-25 18:21:51 -07:00
return "unroutable"
}
if ipv4 := na.IP.To4(); ipv4 != nil {
return (&net.IPNet{IP: na.IP, Mask: net.CIDRMask(16, 32)}).String()
}
if na.RFC6145() || na.RFC6052() {
// last four bytes are the ip address
ip := net.IP(na.IP[12:16])
return (&net.IPNet{IP: ip, Mask: net.CIDRMask(16, 32)}).String()
}
if na.RFC3964() {
ip := net.IP(na.IP[2:7])
return (&net.IPNet{IP: ip, Mask: net.CIDRMask(16, 32)}).String()
}
if na.RFC4380() {
// teredo tunnels have the last 4 bytes as the v4 address XOR
// 0xff.
ip := net.IP(make([]byte, 4))
for i, byte := range na.IP[12:16] {
ip[i] = byte ^ 0xff
}
return (&net.IPNet{IP: ip, Mask: net.CIDRMask(16, 32)}).String()
}
// OK, so now we know ourselves to be a IPv6 address.
// bitcoind uses /32 for everything, except for Hurricane Electric's
// (he.net) IP range, which it uses /36 for.
bits := 32
heNet := &net.IPNet{IP: net.ParseIP("2001:470::"),
Mask: net.CIDRMask(32, 128)}
if heNet.Contains(na.IP) {
bits = 36
}
return (&net.IPNet{IP: na.IP, Mask: net.CIDRMask(bits, 128)}).String()
}
2018-01-13 14:25:51 -08:00
// doubleSha256 calculates sha256(sha256(b)) and returns the resulting bytes.
func doubleSha256(b []byte) []byte {
hasher := sha256.New()
hasher.Write(b) // nolint: errcheck, gas
sum := hasher.Sum(nil)
hasher.Reset()
hasher.Write(sum) // nolint: errcheck, gas
return hasher.Sum(nil)
}