BTCPrivate
/
BitcoinPrivate-seeder
mirror of https://github.com/BTCPrivate/BitcoinPrivate-seeder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix per-flag-cache size tracking

This commit is contained in:
Pieter Wuille 2016-10-29 12:08:46 -07:00
parent 0ec6e14c17
commit d5764c9149
3 changed files with 42 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -5,9 +5,9 @@ dnsseed: dns.o bitcoin.o netbase.o protocol.o db.o main.o util.o
g++ -pthread $(LDFLAGS) -o dnsseed dns.o bitcoin.o netbase.o protocol.o db.o main.o util.o -lcrypto g++ -pthread $(LDFLAGS) -o dnsseed dns.o bitcoin.o netbase.o protocol.o db.o main.o util.o -lcrypto
%.o: %.cpp bitcoin.h netbase.h protocol.h db.h serialize.h uint256.h util.h %.o: %.cpp bitcoin.h netbase.h protocol.h db.h serialize.h uint256.h util.h
g++ -pthread $(CXXFLAGS) -Wno-invalid-offsetof -c -o $@ $< g++ -std=c++11 -pthread $(CXXFLAGS) -Wall -Wno-unused -Wno-sign-compare -Wno-reorder -Wno-comment -c -o $@ $<
dns.o: dns.c dns.o: dns.c
gcc -pthread -std=c99 $(CXXFLAGS) dns.c -c -o dns.o gcc -pthread -std=c99 $(CXXFLAGS) dns.c -Wall -c -o dns.o
%.o: %.cpp %.o: %.cpp

69
main.cpp
View File

@ -7,6 +7,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <getopt.h> #include <getopt.h>
#include <atomic>
#include "bitcoin.h" #include "bitcoin.h"
#include "db.h" #include "db.h"
@ -188,19 +189,25 @@ extern "C" void* ThreadCrawler(void* data) {
db.ResultMany(ips); db.ResultMany(ips);
db.Add(addr); db.Add(addr);
} while(1); } while(1);
return nullptr;
} }
extern "C" int GetIPList(void *thread, char *requestedHostname, addr_t *addr, int max, int ipv4, int ipv6); extern "C" int GetIPList(void *thread, char *requestedHostname, addr_t *addr, int max, int ipv4, int ipv6);
class CDnsThread { class CDnsThread {
public: public:
struct FlagSpecificData {
int nIPv4, nIPv6;
std::vector<addr_t> cache;
time_t cacheTime;
unsigned int cacheHits;
FlagSpecificData() : nIPv4(0), nIPv6(0), cacheTime(0), cacheHits(0) {}
};
dns_opt_t dns_opt; // must be first dns_opt_t dns_opt; // must be first
const int id; const int id;
std::map<uint64_t, vector<addr_t> > cache; std::map<uint64_t, FlagSpecificData> perflag;
int nIPv4, nIPv6; std::atomic<uint64_t> dbQueries;
std::map<uint64_t, time_t> cacheTime;
unsigned int cacheHits;
uint64_t dbQueries;
std::set<uint64_t> filterWhitelist; std::set<uint64_t> filterWhitelist;
void cacheHit(uint64_t requestedFlags, bool force = false) { void cacheHit(uint64_t requestedFlags, bool force = false) {
@ -210,15 +217,16 @@ public:
nets[NET_IPV6] = true; nets[NET_IPV6] = true;
} }
time_t now = time(NULL); time_t now = time(NULL);
cacheHits++; FlagSpecificData thisflag = perflag[requestedFlags];
if (force || cacheHits > (cache[requestedFlags].size()*cache[requestedFlags].size()/400) || (cacheHits*cacheHits > cache[requestedFlags].size() / 20 && (now - cacheTime[requestedFlags] > 5))) { thisflag.cacheHits++;
if (force || thisflag.cacheHits * 400 > (thisflag.cache.size()*thisflag.cache.size()) || (thisflag.cacheHits*thisflag.cacheHits * 20 > thisflag.cache.size() && (now - thisflag.cacheTime > 5))) {
set<CNetAddr> ips; set<CNetAddr> ips;
db.GetIPs(ips, requestedFlags, 1000, nets); db.GetIPs(ips, requestedFlags, 1000, nets);
dbQueries++; dbQueries++;
cache[requestedFlags].clear(); thisflag.cache.clear();
nIPv4 = 0; thisflag.nIPv4 = 0;
nIPv6 = 0; thisflag.nIPv6 = 0;
cache[requestedFlags].reserve(ips.size()); thisflag.cache.reserve(ips.size());
for (set<CNetAddr>::iterator it = ips.begin(); it != ips.end(); it++) { for (set<CNetAddr>::iterator it = ips.begin(); it != ips.end(); it++) {
struct in_addr addr; struct in_addr addr;
struct in6_addr addr6; struct in6_addr addr6;
@ -226,18 +234,18 @@ public:
addr_t a; addr_t a;
a.v = 4; a.v = 4;
memcpy(&a.data.v4, &addr, 4); memcpy(&a.data.v4, &addr, 4);
cache[requestedFlags].push_back(a); thisflag.cache.push_back(a);
nIPv4++; thisflag.nIPv4++;
} else if ((*it).GetIn6Addr(&addr6)) { } else if ((*it).GetIn6Addr(&addr6)) {
addr_t a; addr_t a;
a.v = 6; a.v = 6;
memcpy(&a.data.v6, &addr6, 16); memcpy(&a.data.v6, &addr6, 16);
cache[requestedFlags].push_back(a); thisflag.cache.push_back(a);
nIPv6++; thisflag.nIPv6++;
} }
} }
cacheHits = 0; thisflag.cacheHits = 0;
cacheTime[requestedFlags] = now; thisflag.cacheTime = now;
} }
} }
@ -250,12 +258,8 @@ public:
dns_opt.cb = GetIPList; dns_opt.cb = GetIPList;
dns_opt.port = opts->nPort; dns_opt.port = opts->nPort;
dns_opt.nRequests = 0; dns_opt.nRequests = 0;
cache.clear();
cacheTime.clear();
cacheHits = 0;
dbQueries = 0; dbQueries = 0;
nIPv4 = 0; perflag.clear();
nIPv6 = 0;
filterWhitelist = opts->filter_whitelist; filterWhitelist = opts->filter_whitelist;
} }
@ -280,8 +284,9 @@ extern "C" int GetIPList(void *data, char *requestedHostname, addr_t* addr, int
else if (strcasecmp(requestedHostname, thread->dns_opt.host)) else if (strcasecmp(requestedHostname, thread->dns_opt.host))
return 0; return 0;
thread->cacheHit(requestedFlags); thread->cacheHit(requestedFlags);
unsigned int size = thread->cache[requestedFlags].size(); auto& thisflag = thread->perflag[requestedFlags];
unsigned int maxmax = (ipv4 ? thread->nIPv4 : 0) + (ipv6 ? thread->nIPv6 : 0); unsigned int size = thisflag.cache.size();
unsigned int maxmax = (ipv4 ? thisflag.nIPv4 : 0) + (ipv6 ? thisflag.nIPv6 : 0);
if (max > size) if (max > size)
max = size; max = size;
if (max > maxmax) if (max > maxmax)
@ -290,16 +295,16 @@ extern "C" int GetIPList(void *data, char *requestedHostname, addr_t* addr, int
while (i<max) { while (i<max) {
int j = i + (rand() % (size - i)); int j = i + (rand() % (size - i));
do { do {
bool ok = (ipv4 && thread->cache[requestedFlags][j].v == 4) || bool ok = (ipv4 && thisflag.cache[j].v == 4) ||
(ipv6 && thread->cache[requestedFlags][j].v == 6); (ipv6 && thisflag.cache[j].v == 6);
if (ok) break; if (ok) break;
j++; j++;
if (j==size) if (j==size)
j=i; j=i;
} while(1); } while(1);
addr[i] = thread->cache[requestedFlags][j]; addr[i] = thisflag.cache[j];
thread->cache[requestedFlags][j] = thread->cache[requestedFlags][i]; thisflag.cache[j] = thisflag.cache[i];
thread->cache[requestedFlags][i] = addr[i]; thisflag.cache[i] = addr[i];
i++; i++;
} }
return max; return max;
@ -310,6 +315,7 @@ vector<CDnsThread*> dnsThread;
extern "C" void* ThreadDNS(void* arg) { extern "C" void* ThreadDNS(void* arg) {
CDnsThread *thread = (CDnsThread*)arg; CDnsThread *thread = (CDnsThread*)arg;
thread->run(); thread->run();
return nullptr;
} }
int StatCompare(const CAddrReport& a, const CAddrReport& b) { int StatCompare(const CAddrReport& a, const CAddrReport& b) {
@ -346,7 +352,7 @@ extern "C" void* ThreadDumper(void*) {
double stat[5]={0,0,0,0,0}; double stat[5]={0,0,0,0,0};
for (vector<CAddrReport>::const_iterator it = v.begin(); it < v.end(); it++) { for (vector<CAddrReport>::const_iterator it = v.begin(); it < v.end(); it++) {
CAddrReport rep = *it; CAddrReport rep = *it;
fprintf(d, "%-47s %4d %11"PRId64" %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6i %08"PRIx64" %5i \"%s\"\n", rep.ip.ToString().c_str(), (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str()); fprintf(d, "%-47s %4d %11" PRId64 " %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6i %08" PRIx64 " %5i \"%s\"\n", rep.ip.ToString().c_str(), (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str());
stat[0] += rep.uptime[0]; stat[0] += rep.uptime[0];
stat[1] += rep.uptime[1]; stat[1] += rep.uptime[1];
stat[2] += rep.uptime[2]; stat[2] += rep.uptime[2];
@ -359,6 +365,7 @@ extern "C" void* ThreadDumper(void*) {
fclose(ff); fclose(ff);
} }
} while(1); } while(1);
return nullptr;
} }
extern "C" void* ThreadStats(void*) { extern "C" void* ThreadStats(void*) {
@ -387,6 +394,7 @@ extern "C" void* ThreadStats(void*) {
printf("%s %i/%i available (%i tried in %is, %i new, %i active), %i banned; %llu DNS requests, %llu db queries", c, stats.nGood, stats.nAvail, stats.nTracked, stats.nAge, stats.nNew, stats.nAvail - stats.nTracked - stats.nNew, stats.nBanned, (unsigned long long)requests, (unsigned long long)queries); printf("%s %i/%i available (%i tried in %is, %i new, %i active), %i banned; %llu DNS requests, %llu db queries", c, stats.nGood, stats.nAvail, stats.nTracked, stats.nAge, stats.nNew, stats.nAvail - stats.nTracked - stats.nNew, stats.nBanned, (unsigned long long)requests, (unsigned long long)queries);
Sleep(1000); Sleep(1000);
} while(1); } while(1);
return nullptr;
} }
static const string mainnet_seeds[] = {"dnsseed.bluematt.me", "bitseed.xf2.org", "dnsseed.bitcoin.dashjr.org", "seed.bitcoin.sipa.be", ""}; static const string mainnet_seeds[] = {"dnsseed.bluematt.me", "bitseed.xf2.org", "dnsseed.bitcoin.dashjr.org", "seed.bitcoin.sipa.be", ""};
@ -411,6 +419,7 @@ extern "C" void* ThreadSeeder(void*) {
} }
Sleep(1800000); Sleep(1800000);
} while(1); } while(1);
return nullptr;
} }
int main(int argc, char **argv) { int main(int argc, char **argv) {

2
uint256.h
View File

@ -322,7 +322,7 @@ public:
// hex string to uint // hex string to uint
static char phexdigit[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, 0,0xa,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0xa,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0,0,0,0,0 }; static char phexdigit[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, 0,0xa,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0xa,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0,0,0,0,0 };
const char* pbegin = psz; const char* pbegin = psz;
while (phexdigit[*psz] || *psz == '0') while (phexdigit[(unsigned char)*psz] || *psz == '0')
psz++; psz++;
psz--; psz--;
unsigned char* p1 = (unsigned char*)pn; unsigned char* p1 = (unsigned char*)pn;