diff --git a/BitVector.cpp b/BitVector.cpp index f24b17e..f29ef52 100644 --- a/BitVector.cpp +++ b/BitVector.cpp @@ -375,6 +375,22 @@ void BitVector::pack(unsigned char* targ) const targ[bytes] = peekField(whole,rem) << (8-rem); } +string BitVector::packToString() const +{ + string result; + result.reserve((size()+7)/8); + // Tempting to call this->pack(result.c_str()) but technically c_str() is read-only. + unsigned bytes = size()/8; + for (unsigned i=0; i /** Pack into a char array. */ void pack(unsigned char*) const; + // Same as pack but return a string. + std::string packToString() const; /** Unpack from a char array. */ void unpack(const unsigned char*); diff --git a/Configuration.cpp b/Configuration.cpp index b567d81..2488a6b 100644 --- a/Configuration.cpp +++ b/Configuration.cpp @@ -676,6 +676,8 @@ const ConfigurationRecord& ConfigurationTable::lookup(const string& key) sqlite3_single_lookup(mDB,"CONFIG", "KEYSTRING",key.c_str(),"VALUESTRING",value); + // (pat 9-2014) If sqlite3_single_lookup returns false, the behavior below is incorrect. + // value found, cache the result if (value) { mCache[key] = ConfigurationRecord(key,value); @@ -688,7 +690,7 @@ const ConfigurationRecord& ConfigurationTable::lookup(const string& key) throw ConfigurationTableKeyNotFound(key); } - free(value); + if (value) free(value); // Leave mLock locked. The caller holds it still. return mCache[key]; diff --git a/Interthread.h b/Interthread.h index e56b360..b6057d2 100644 --- a/Interthread.h +++ b/Interthread.h @@ -131,7 +131,15 @@ template > class InterthreadQueue { mWriteSignalPointer = other.mWriteSignalPointer; } - Mutex &qGetLock() { return mLock; } + // (pat) This provides a client the ability to lock the InterthreadQueue and iterate it. + Mutex &qGetLock() const { return mLock; } + typedef typename Fifo::iterator iterator; + typedef typename Fifo::const_iterator const_iterator; + iterator begin() { assert(mLock.lockcnt()); return mQ.begin(); } + iterator end() { assert(mLock.lockcnt()); return mQ.end(); } + const_iterator begin() const { assert(mLock.lockcnt()); return mQ.begin(); } + const_iterator end() const { assert(mLock.lockcnt()); return mQ.end(); } + #if USE_SCOPED_ITERATORS // The Iterator locks the InterthreadQueue until the Iterator falls out of scope. // Semantics are different from normal C++ iterators - the begin,end,erase methods are in @@ -247,7 +255,7 @@ template > class InterthreadQueue { } /** Non-blocking peek at the first element; returns NULL if empty. */ - T* front() + T* front() const { ScopedLock lock(*mLockPointer); return (T*) (mQ.size() ? mQ.front() : NULL); diff --git a/InterthreadTest.cpp b/InterthreadTest.cpp index 6079593..ba7d172 100644 --- a/InterthreadTest.cpp +++ b/InterthreadTest.cpp @@ -91,13 +91,109 @@ void* mapReader(void*) return NULL; } +static const uint32_t Hyperframe = 1024; + +static int32_t FNDelta(int32_t v1, int32_t v2) +{ + static const int32_t halfModulus = Hyperframe/2; + int32_t delta = v1-v2; + if (delta>=halfModulus) delta -= Hyperframe; + else if (delta<-halfModulus) delta += Hyperframe; + return (int32_t) delta; +} + +static int FNCompare(int32_t v1, int32_t v2) +{ + int32_t delta = FNDelta(v1,v2); + if (delta>0) return 1; + if (delta<0) return -1; + return 0; +} + +struct TestTime { + int mFN; ///< frame number in the hyperframe + int mTN; ///< timeslot number + + public: + + TestTime(int wFN=0, int wTN=0) + :mFN(wFN),mTN(wTN) + { } + + bool operator<(const TestTime& other) const + { + if (mFN==other.mFN) return (mTN(const TestTime& other) const + { + if (mFN==other.mFN) return (mTN>other.mTN); + return FNCompare(mFN,other.mFN)>0; + } + + bool operator<=(const TestTime& other) const + { + if (mFN==other.mFN) return (mTN<=other.mTN); + return FNCompare(mFN,other.mFN)<=0; + } + + bool operator>=(const TestTime& other) const + { + if (mFN==other.mFN) return (mTN>=other.mTN); + return FNCompare(mFN,other.mFN)>=0; + } + + bool operator==(const TestTime& other) const + { + return (mFN == other.mFN) && (mTN==other.mTN); + } + +}; +// Welcome to wonderful C++. +struct CompareAdapter { + /** Compare the objects pointed to, not the pointers themselves. */ + // (pat) This is used when a RachInfo is placed in a priority_queue. + // Return true if rach1 should appear before rach2 in the priority_queue, + // meaning that rach1 will be serviced before rach2. + bool operator()(const TestTime *rach1, const TestTime *rach2) { + return *rach1 > *rach2; + } +}; + +void priority_queue_test() +{ + typedef InterthreadPriorityQueue,CompareAdapter> PQ_t; + PQ_t pq; + + pq.write(new TestTime(2,0)); + pq.write(new TestTime(1,0)); + pq.write(new TestTime(3,0)); + pq.write(new TestTime(2,3)); + pq.write(new TestTime(2,2)); + pq.write(new TestTime(2,1)); + pq.write(new TestTime(0,0)); + pq.write(new TestTime(1021,1)); + pq.write(new TestTime(1023,1)); + pq.write(new TestTime(1022,1)); + pq.write(new TestTime(1024,1)); + while (1) { + TestTime *peek = pq.peek(); + TestTime *ptime = pq.readNoBlock(); + if (!ptime) { break; } // Done. + assert(*peek == *ptime); + printf("TestTime(%d,%d)\n",ptime->mFN,ptime->mTN); + } +} int main(int argc, char *argv[]) { + priority_queue_test(); + Thread qReaderThread; qReaderThread.start(qReader,NULL); Thread mapReaderThread; diff --git a/LinkedLists.h b/LinkedLists.h index b97d37e..0e28c63 100644 --- a/LinkedLists.h +++ b/LinkedLists.h @@ -125,6 +125,7 @@ class SingleLinkList public: typedef void iterator; // Does not exist for this class, but needs to be defined. + typedef void const_iterator; // Does not exist for this class, but needs to be defined. SingleLinkList() : mHead(0), mTail(0), mSize(0), mTotalSize(0) {} unsigned size() const { return mSize; } unsigned totalSize() const { return mTotalSize; } @@ -161,8 +162,8 @@ class SingleLinkList mSize++; mTotalSize += item->size(); } - Node *front() { return mHead; } - Node *back() { return mTail; } + Node *front() const { return mHead; } + Node *back() const { return mTail; } // Interface to InterthreadQueue so it can used SingleLinkList as the Fifo. void put(void *val) { push_back((Node*)val); } diff --git a/LockTest.cpp b/LockTest.cpp new file mode 100644 index 0000000..db129e1 --- /dev/null +++ b/LockTest.cpp @@ -0,0 +1,125 @@ +#include +#include +#include +#include +using namespace std; + +#include "Configuration.h" +#include "Threads.h" + +ConfigurationTable gConfig; + +// LockTest starts three processes that lock and unlock three mutexes at random, to make sure no deadlock occurs. + +struct aprocess { + string id; + Thread t; + Mutex m; + void lockall(int lockOwnerBits); + void runtest(); + static void *pstart(void *v); + void start1(); + aprocess(string wid) : id(wid) {} +}; + +aprocess a("a"), b("b"), c("c"); + +void aprocess::lockall(int lockOwnerBits) { + if (id == "a") { + ScopedLockMultiple lock(lockOwnerBits,a.m,b.m,c.m,__FILE__,__LINE__); + } else if (id == "b") { + ScopedLockMultiple lock(lockOwnerBits,b.m,a.m,c.m,__FILE__,__LINE__); + } else if (id == "c") { + ScopedLockMultiple lock(lockOwnerBits,c.m,a.m,b.m,__FILE__,__LINE__); + } else { + assert(0); + } +} + +static void waitabit() { + // randomly return instantly. + if (random() & 1) return; + usleep(0xff&random()); +} + +void aprocess::runtest() { + for (int i = 0; i < 10000; i++) { + waitabit(); + //printf("%s %d here\n",id.c_str(),i); + lockall(0); + waitabit(); + //printf("%s1:a=%d b=%d c=%d\n",id.c_str(),a.m.lockcnt(), b.m.lockcnt(), c.m.lockcnt()); + // Add in some random locking behavior. + if (random() & 1) { + m.lock(__FILE__,__LINE__); + waitabit(); + m.unlock(); + } + //printf("%s2:a=%d b=%d c=%d\n",id.c_str(),a.m.lockcnt(), b.m.lockcnt(), c.m.lockcnt()); + waitabit(); + //printf("%s %d there\n",id.c_str(),i); + { ScopedLock lock(m); + lockall(1); + waitabit(); + //printf("%s3:a=%d b=%d c=%d\n",id.c_str(),a.m.lockcnt(), b.m.lockcnt(), c.m.lockcnt()); + } + //printf("%s4:a=%d b=%d c=%d\n",id.c_str(),a.m.lockcnt(), b.m.lockcnt(), c.m.lockcnt()); + } + printf("finished\n"); +} + +void *aprocess::pstart(void *v) { // This is the interface for the Thread.start() method. + aprocess *p = (aprocess*)v; + p->runtest(); + return 0; +} + +void aprocess::start1() { + this->t.start(&this->pstart,this); +} + +typedef void *(*task_t)(void*); + +int main(int argc, char **argv) +{ + // Start the three processes running. + a.start1(); + b.start1(); + c.start1(); + + // And let the main process fight for the locks as well. + // We also do a coverage check here: When we randomly sample the locks, all of them must have been locked at some point. + int fndA=0, fndB=0, fndC=0; + for (int n = 0; n < 1000; n++) { + waitabit(); + fndA += a.m.lockcnt(); + fndB += b.m.lockcnt(); + fndC += c.m.lockcnt(); + //printf("loop %d: a=%d b=%d c=%d\n",n, a.m.lockcnt(), b.m.lockcnt(), c.m.lockcnt()); + printf("loop %d: a=%s b=%s c=%s\n",n,a.m.mutext().c_str(), b.m.mutext().c_str(), c.m.mutext().c_str()); + + a.m.lock(__FILE__,__LINE__); + b.m.lock(__FILE__,__LINE__); + { ScopedLockMultiple tmp(3,a.m,b.m,c.m); waitabit(); } + c.m.lock(__FILE__,__LINE__); + + waitabit(); + a.m.unlock(); + a.m.lock(__FILE__,__LINE__); + + waitabit(); + b.m.unlock(); + b.m.lock(__FILE__,__LINE__); + + waitabit(); + a.m.unlock(); + b.m.unlock(); + c.m.unlock(); + } + //a.t.start(&a.pstart,&a); + //b.t.start((void*)&b); + //c.t.start((void*)&c); + a.t.join(); // Wait for it to finish. + b.t.join(); // Wait for it to finish. + printf("Test Finished. During random sampling, locks held were: A %d, B %d, C %d\n", fndA, fndB, fndC); +} diff --git a/Logger.cpp b/Logger.cpp index 8d8c340..3bc858d 100644 --- a/Logger.cpp +++ b/Logger.cpp @@ -225,8 +225,8 @@ Log::~Log() if (gLogToConsole) { // The COUT() macro prevents messages from stomping each other but adds uninteresting thread numbers, // so just use std::cout. - std::cout << mStream.str(); - if (neednl) std::cout<<"\n"; + std::cerr << mStream.str(); + if (neednl) std::cerr<<"\n"; } if (gLogToFile) { fputs(mStream.str().c_str(),gLogToFile); @@ -273,12 +273,10 @@ void gLogInitWithFile(const char* name, const char* level, int facility, char * if (gLogToFile==0 && LogFilePath != 0 && *LogFilePath != 0 && strlen(LogFilePath) > 0) { gLogToFile = fopen(LogFilePath,"w"); // New log file each time we start. if (gLogToFile) { - time_t now = time(NULL); - std::string result; - Timeval::isoTime(now, result); - fprintf(gLogToFile,"Starting at %s",result.c_str()); + string when = Timeval::isoTime(time(NULL),true); + fprintf(gLogToFile,"Starting at %s\n",when.c_str()); fflush(gLogToFile); - std::cout << name <<" logging to file: " << LogFilePath << "\n"; + std::cerr << name <<" logging to file: " << LogFilePath << "\n"; } } @@ -302,17 +300,15 @@ void gLogInit(const char* name, const char* level, int facility) // Pat added, tired of the syslog facility. // Both the transceiver and OpenBTS use this same Logger class, but only RMSC/OpenBTS/OpenNodeB may use this log file: string str = gConfig.getStr("Log.File"); - if (gLogToFile==0 && str.length() && (0==strncmp(gCmdName,"Open",4) || 0==strncmp(gCmdName,"RMSC",4))) { + if (gLogToFile==0 && str.length() && (0==strncmp(gCmdName,"Open",4) || 0==strncmp(gCmdName,"RMSC",4) || 0==strncmp(gCmdName,"RangeFinderGW",13))) { const char *fn = str.c_str(); if (fn && *fn && strlen(fn)>3) { // strlen because a garbage char is getting in sometimes. gLogToFile = fopen(fn,"w"); // New log file each time we start. if (gLogToFile) { - time_t now = time(NULL); - std::string result; - Timeval::isoTime(now, result); - fprintf(gLogToFile,"Starting at %s",result.c_str()); + string when = Timeval::isoTime(time(NULL),true); + fprintf(gLogToFile,"Starting at %s\n",when.c_str()); fflush(gLogToFile); - std::cout << name <<" logging to file: " << fn << "\n"; + std::cerr << name <<" logging to file: " << fn << "\n"; } } } diff --git a/Logger.h b/Logger.h index 12545e1..7d98a00 100644 --- a/Logger.h +++ b/Logger.h @@ -111,6 +111,7 @@ extern pid_t gPid; // The WATCHINFO macro prints an INFO level message that is also printed to the console if the log level is DEBUG. // Beware that the arguments are evaluated multiple times. #define WATCHF(...) { LOG(DEBUG)<= LOG_##level) syslog(LOG_##level,"%lu %s %s:%u:%s:lockid=%p " fmt,(unsigned long)pthread_self(),Utils::timestr().c_str(),__FILE__,__LINE__,__FUNCTION__,this,##__VA_ARGS__); + if (gMutexLogLevel >= LOG_##level) syslog(LOG_##level,"%lu %s %s:%u:%s:lockid=%p " fmt,gettid(),Utils::timestr().c_str(),__FILE__,__LINE__,__FUNCTION__,this,##__VA_ARGS__); + //if (gMutexLogLevel >= LOG_##level) syslog(LOG_##level,"%lu %s %s:%u:%s:lockid=%p " fmt,(unsigned long)pthread_self(),Utils::timestr().c_str(),__FILE__,__LINE__,__FUNCTION__,this,##__VA_ARGS__); //printf("%u %s %s:%u:%s:lockid=%u " fmt "\n",(unsigned)pthread_self(),Utils::timestr().c_str(),__FILE__,__LINE__,__FUNCTION__,(unsigned)this,##__VA_ARGS__); @@ -103,10 +107,12 @@ Mutex::~Mutex() assert(!res); } -bool Mutex::trylock() +bool Mutex::trylock(const char *file, unsigned line) { if (pthread_mutex_trylock(&mMutex)==0) { - if (mLockCnt < maxLocks) { mLockerFile[mLockCnt] = NULL; } + if (mLockCnt >= 0 && mLockCnt < maxLocks) { + mLockerFile[mLockCnt] = file; mLockerLine[mLockCnt] = line; // Now our thread has it locked from here. + } mLockCnt++; return true; } else { @@ -122,6 +128,9 @@ bool Mutex::timedlock(int msecs) // Wait this long in milli-seconds. return ETIMEDOUT != pthread_mutex_timedlock(&mMutex, &timeout); } +// There is a chance here that the mLockerFile&mLockerLine +// could change while we are printing it if multiple other threads are contending for the lock +// and swapping the lock around while we are in here. string Mutex::mutext() const { string result; @@ -138,46 +147,27 @@ string Mutex::mutext() const return result; } -void Mutex::lock() { - if (lockerFile()) LOCKLOG(DEBUG,"lock unchecked"); - _lock(); - mLockCnt++; -} +// Pat removed 10-1-2014. +//void Mutex::lock() { +// if (lockerFile()) LOCKLOG(DEBUG,"lock unchecked"); +// _lock(); +// mLockCnt++; +//} // WARNING: The LOG facility calls lock, so to avoid infinite recursion do not call LOG if file == NULL, // and the file argument should never be used from the Logger facility. void Mutex::lock(const char *file, unsigned line) { - // (pat 10-25-13) This is now going to be the default behavior so we can detect and report deadlocks at customer sites. - //if (file && gGetLoggingLevel(file)>=LOG_DEBUG) - //if (file) OBJLOG(DEBUG) <<"start at "<lock(_file,_line); + state[which] = true; +} +bool ScopedLockMultiple::_trylock(int which) { + if (state[which]) return true; + state[which] = mA[which]->trylock(_file,_line); + return state[which]; +} +void ScopedLockMultiple::_unlock(int which) { + if (state[which]) mA[which]->unlock(); + state[which] = false; +} +void ScopedLockMultiple::_saveState() { + // The caller may enter with mutex locked by the calling thread if the owner bit is set. + for (int i = 0; i <= 2; i++) { + // Test is deceptive because currently the owner bit is an assertion that owner has the bit locked. + // If we dont require that, then how would we know whether the lock was held by the current thread, or by some other thread? + // We would need to add some per-thread storage, and store it in the Mutex during Mutex::lock() or Mutex::trylock(). + state[i] = ownA[i]; // (ownA[i] && mA[i]->lockcnt()); + if (ownA[i]) { if (mA[i]->lockcnt() != 1) printf("mA[%d].lockcnt=%d\n",i,mA[i]->lockcnt()); assert(mA[i]->lockcnt() == 1); } + } +} +void ScopedLockMultiple::_restoreState() { + // Leave state of each lock the way we found it. + for (int i = 0; i <= 2; i++) { + // This is a little redundant because the _lock and _unlock now test state. + if (!ownA[i] && state[i]) _unlock(i); + else if (ownA[i] && ! state[i]) _lock(i); + } +} +void ScopedLockMultiple::_lockAll() { + // Do not return until we have locked all three mutexes. + for (int n=0; true; n++) { + // Attempt to lock in order 0,1,2. + _lock(0); // Wait on 0, unless it was locked by us on entry. + if (_trylock(1) && _trylock(2)) return; // Then try to acquire 1 and 2 + _unlock(1); // If failure, release all. + _unlock(0); + // Attempt to lock in order 1,0,2. + _lock(1); + if (_trylock(0) && _trylock(2)) return; + _unlock(0); + _unlock(1); + // Attempt to lock in order 2,1,0. + _lock(2); + if (_trylock(0) && _trylock(1)) return; + _unlock(0); + _unlock(2); + LOCKLOG(DEBUG,"Multiple lock attempt %d",n); // Seeing this message is a hint we are having contention issues. + } +} /** Block for the signal up to the cancellation timeout in msecs. */ diff --git a/Threads.h b/Threads.h index 1284fd5..6e004ba 100644 --- a/Threads.h +++ b/Threads.h @@ -27,6 +27,7 @@ #ifndef THREADS_H #define THREADS_H +#include #include #include #include @@ -69,6 +70,7 @@ void unlockCout(); ///< call after writing cout //@{ /** A class for recursive mutexes based on pthread_mutex. */ +// If at all possible, do not call lock/unlock from this class directly; use a ScopedLock instead. class Mutex { private: @@ -86,19 +88,17 @@ class Mutex { //unused: bool anyDebugging() { for (int i = 0; i < maxLocks; i++) { if (mLockerFile[i]) return true; return false; } } // pthread_mutex_trylock returns 0 and trylock returns true if the lock was acquired. - bool trylock(); - public: + bool trylock(const char *file=0, unsigned line=0); Mutex(); ~Mutex(); void _lock() { pthread_mutex_lock(&mMutex); } - void lock(); // (pat) Like the above but report blocking; to see report you must set both Log.Level to DEBUG for both Threads.cpp and the file. - void lock(const char *file, unsigned line); + void lock(const char *file=0, unsigned line=0); std::string mutext() const; @@ -160,9 +160,11 @@ class ScopedPointer { }; #endif +// Class to acquire a Mutex lock and release it automatically when this goes out of scope. +// ScopedLock should be used preferentially to Mutex::lock() and Mutex::unlock() in case a try-catch throw passes through +// the containing procedure while the lock is held; ScopedLock releases the lock in that case. +// "We dont use try-catch" you say? Yes we do - C++ string and many standard containers use throw to handle unexpected arguments. class ScopedLock { - - private: Mutex& mMutex; public: @@ -170,7 +172,54 @@ class ScopedLock { // Like the above but report blocking; to see report you must set both Log.Level to DEBUG for both Threads.cpp and the file. ScopedLock(Mutex& wMutex,const char *file, unsigned line):mMutex(wMutex) { mMutex.lock(file,line); } ~ScopedLock() { mMutex.unlock(); } +}; +// Lock multiple mutexes simultaneously. +class ScopedLockMultiple { + Mutex garbage; // Someplace to point mC if only two mutexes are specified. + Mutex *mA[3]; + bool ownA[3]; // If set, expect mA to be locked by this thread on entry. + bool state[3]; // Current state, true if our thread has locked the associated Mutex; doesnt say if Mutex is locked by other threads. + const char *_file; unsigned _line; + + void _lock(int which); + bool _trylock(int which); + void _unlock(int which); + void _saveState(); + void _restoreState(); + void _lockAll(); + void _init(int wOwner, Mutex& wA, Mutex&wB, Mutex&wCa); + + public: + + // Do not return until all three mutexes are locked. + // On entry, the caller may optionally already have locked mutexes, as specified by the wOwner flag bits. + // If owner&1, caller owns wA, if owner&2 caller owns wB, if owner&4 caller owns wC. + // There wouldnt be much point of this class if the caller already owned all three mutexes. + // Note that the mutexes may be temporarily surrendered during this call as the methodology to avoid deadlock, + // but in that case all will be re-acquired before this returns. + + ScopedLockMultiple(int wOwner, Mutex&wA, Mutex&wB, Mutex&wC) : _file(NULL), _line(0) { + _init(wOwner,wA,wB,wC); + _lockAll(); + } + // Like the above but report blocking; to see report you must set both Log.Level to DEBUG for both Threads.cpp and the file. + // Use like this: ScopedLockMultiple lock(bits,mutexa,mutexb,__FILE__,__LINE__); + ScopedLockMultiple(int wOwner, Mutex&wA, Mutex&wB, Mutex&wC, const char *wFile, int wLine) : _file(wFile), _line(wLine) { + _init(wOwner,wA,wB,wC); + _lockAll(); + } + + // Like the above but for two mutexes intead of three. + ScopedLockMultiple(int wOwner, Mutex& wA, Mutex&wB) : _file(NULL), _line(0) { + _init(wOwner,wA,wB,garbage); + _lockAll(); + } + ScopedLockMultiple(int wOwner, Mutex&wA, Mutex&wB, const char *wFile, int wLine) : _file(wFile), _line(wLine) { + _init(wOwner,wA,wB,garbage); + _lockAll(); + } + ~ScopedLockMultiple() { _restoreState(); } }; @@ -225,6 +274,8 @@ class Thread { public: + // (pat) This is the type of the function argument to pthread_create. + typedef void *(*Task_t)(void*); /** Create a thread in a non-running state. */ Thread(size_t wStackSize = (65536*4)):mThread((pthread_t)0) { @@ -241,8 +292,8 @@ class Thread { /** Start the thread on a task. */ - void start(void *(*task)(void*), void *arg); - void start2(void *(*task)(void*), void *arg, int stacksize); + void start(Task_t task, void *arg); + void start2(Task_t task, void *arg, int stacksize); /** Join a thread that will stop on its own. */ void join() { int s = pthread_join(mThread,NULL); assert(!s); mThread = 0; } diff --git a/Timeval.cpp b/Timeval.cpp index e6d7efb..e0ec49d 100644 --- a/Timeval.cpp +++ b/Timeval.cpp @@ -95,21 +95,25 @@ ostream& operator<<(ostream& os, const struct timespec& ts) return os; } +string Timeval::isoTime(time_t t, bool isLocal) +{ + char buf[100]; + struct tm tBuf; + if (isLocal) + localtime_r(&t, &tBuf); + else + gmtime_r(&t, &tBuf); + snprintf(buf, sizeof(buf)-1, "%04d-%02d-%02dT%02d:%02d:%02d%s", + tBuf.tm_year + 1900, tBuf.tm_mon + 1, tBuf.tm_mday, + tBuf.tm_hour, tBuf.tm_min, tBuf.tm_sec, + isLocal == false ? "Z" : ""); + return string(buf); +} + void Timeval::isoTime(time_t t, std::string &result, bool isLocal) { - char buf[BUFSIZ]; - struct tm tBuf; - if (isLocal) - localtime_r(&t, &tBuf); - else - gmtime_r(&t, &tBuf); - snprintf(buf, sizeof(buf)-1, "%04d-%02d-%02dT%02d:%02d:%02d%s", - tBuf.tm_year + 1900, tBuf.tm_mon + 1, tBuf.tm_mday, - tBuf.tm_hour, tBuf.tm_min, tBuf.tm_sec, - isLocal == false ? "Z" : ""); - result = buf; + result = isoTime(t,isLocal); } - // vim: ts=4 sw=4 diff --git a/Timeval.h b/Timeval.h index 265a531..5a5c609 100644 --- a/Timeval.h +++ b/Timeval.h @@ -99,7 +99,8 @@ class Timeval { * YYYY-MM-DDTHH:MM:SS[Z] format. If isLocal is true, use localtime, * otherwise, use gmtime. */ - static void isoTime(time_t t, std::string &result, bool isLocal = false); + static std::string isoTime(time_t t, bool isLocal); + static void isoTime(time_t t, std::string &result, bool isLocal); }; diff --git a/TimevalTest.cpp b/TimevalTest.cpp index b980f89..92a4c8e 100644 --- a/TimevalTest.cpp +++ b/TimevalTest.cpp @@ -48,6 +48,6 @@ int main(int argc, char *argv[]) std::string sLocal(""); std::string sGMT(""); Timeval::isoTime(t, sLocal, true); - Timeval::isoTime(t, sGMT); + Timeval::isoTime(t, sGMT, false); cout << "Localtime: " << sLocal << ", GMT: " << sGMT << std::endl; } diff --git a/Utils.cpp b/Utils.cpp index 92bcf06..aed42c7 100644 --- a/Utils.cpp +++ b/Utils.cpp @@ -21,17 +21,24 @@ #include // For ostringstream #include // For strcpy #include // For malloc -//#include "GSMCommon.h" +#include // For backtrace #include "Utils.h" #include "MemoryLeak.h" namespace Utils { using namespace std; +struct stringCaseInsensitive : public string { + bool operator==(string &other) { return 0==strcasecmp(this->c_str(),other.c_str()); } + bool operator!=(string &other) { return 0!=strcasecmp(this->c_str(),other.c_str()); } + stringCaseInsensitive(string &other) : string(other) {} + stringCaseInsensitive(const char* &other) : string(other) {} +}; + // (pat) This definition must be in the .cpp file to anchor the class vtable. RefCntBase::~RefCntBase() { LOG(DEBUG) << typeid(this).name(); } -int RefCntBase::decRefCnt() +int RefCntBase::decRefCnt() const { int saveRefCnt; // Passes the refcnt out of the locked block. { ScopedLock lock(mRefMutex); @@ -50,7 +57,7 @@ int RefCntBase::decRefCnt() } return saveRefCnt; } -void RefCntBase::incRefCnt() +void RefCntBase::incRefCnt() const { ScopedLock lock(mRefMutex); //printf("incRefCnt %p before=%d\n",this,mRefCnt); @@ -93,10 +100,12 @@ void MemStats::memChkDel(MemoryNames memIndex, const char *id) ScopedLock lock(memChkLock); /*cout << "del " #type "\n";*/ mMemNow[memIndex]--; - if (mMemNow[memIndex] < 0) { - LOG(ERR) << "Memory reference count underflow on type "<=1 //printf("firstlines return n=%d pos=%d\n",n,(int)pos); return msgstr.substr(0,pos); } -}; + +static string backtrace_failed = "backtrace failed"; + +string rn_backtrace() +{ + void *buffer[30]; + int nptrs = backtrace(buffer,30); + if (nptrs <= 0) { return backtrace_failed; } + char **strings = backtrace_symbols(buffer,nptrs); + if (strings == NULL) { return backtrace_failed; } + string result; + try { + result = "backtrace:"; + for (int j = 0; j < nptrs; j++) { + result = result + " " + strings[j]; + } + } catch(...) { + return backtrace_failed; + } + free(strings); + return result; +} + + +// Size is 65 instead of 64 so we can easily init from a char string, which is nul-terminated. +static unsigned char sMapBase64[65] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static unsigned char sMapBase64Reverse[256]; +static unsigned char sMapHexReverse[256]; + +// (pat) This class exists only to init the decoder maps above. +static struct BaseNDecoder { + BaseNDecoder() { + // Init the base64 decoder: + memset(sMapBase64Reverse,0xff,sizeof(sMapBase64Reverse)); // Set all illegal characters to 0xff. + for (unsigned i = 0; i < 64; i++) { sMapBase64Reverse[sMapBase64[i]] = i; } + sMapBase64Reverse['='] = 0; // The '=' char is used as the premature-end-of-string marker. + + // Init the hex decoder: + memset(sMapHexReverse,0xff,sizeof(sMapHexReverse)); // Set all illegal characters to 0xff. + for (unsigned ch = '0'; ch <= '9'; ch++) sMapHexReverse[ch] = ch - '0'; + for (unsigned ch = 'A'; ch <= 'Z'; ch++) sMapHexReverse[ch] = ch - 'A' + 10; + for (unsigned ch = 'a'; ch <= 'z'; ch++) sMapHexReverse[ch] = ch - 'a' + 10; + } +} sBaseNDecoderInit; + + +// (pat) Encode buffer of binary data to base64 encoded string. +static string base64EncodeToString(const unsigned char *ubuf, unsigned buflen) +{ + string result; + result.reserve((buflen*4+2)/3); + for (unsigned i = 0; i < buflen; i += 3) { + // Suck in 3 chars. + uint32_t accum = ubuf[i]; + accum <<= 8; if (i+1>18)]); + result.push_back((char)sMapBase64[0x3f&(accum>>12)]); + result.push_back(i+1 < buflen ? (char)sMapBase64[0x3f&(accum>>6)] : '='); + result.push_back(i+2 < buflen ? (char)sMapBase64[0x3f&accum] : '='); + } + return result; +} + + +// (pat) Decode base64 encoded buffer of characters to a string of binary data. Ignore interspersed spaces and newlines. +static string base64DecodeToString(const unsigned char *ubuf, unsigned buflen, string &errorMessage) +{ + string result; + result.reserve((buflen * 3 + 2) / 4); + for (unsigned i = 0; i < buflen;) { + uint32_t accum = 0; // Accumulator of up to 24 bits from 4 base64 chars. + unsigned j = 0; // Number of characters accumulated, 0-4. + // Suck in 4 chars. + while (j < 4 && i < buflen) { + unsigned ch = ubuf[i++]; + if (isspace(ch)) { continue; } // Ignore spaces and newlines. + if (ch == '=') { + // The '=' at the end of a base64 encoded string indicates size was not evenly divisible by 3 and pre-terminates. + // We will scan the rest of the string to detect errors. + for ( ; i < buflen; i++) { + if (! (isspace(ubuf[i]) || ubuf[i] == '=')) { errorMessage = "found data after '=' terminator in base64 data"; } + } + break; + } + unsigned decoded = sMapBase64Reverse[ch]; + if (decoded >= 64) { + errorMessage = "invalid characters in base64 data"; + continue; // But we'll keep going and hope for the best. + } + accum |= decoded << ((3-j)*6); + j++; + } + // Spit out 3 chars. + if (j >= 1) result.push_back(0xff&(accum >>16)); + if (j >= 3) result.push_back(0xff&(accum >>8)); + if (j == 4) result.push_back(0xff&accum); + } + return result; +} + + +// (pat) Decode hex encoded buffer of characters to a string of binary data. Ignore interspersed spaces and newlines. +// Undefined what happens if the number of input hex chars is odd. +static string hexDecodeToString(const unsigned char *ubuf, unsigned buflen, string &errorMessage) +{ + string result; + result.reserve((buflen + 1) / 2); + for (unsigned i = 0; i < buflen; ) { + unsigned j = 0; + unsigned char accum = 0; + // Suck in 2 chars. + while (j < 2 && i < buflen) { + unsigned ch = ubuf[i++]; + if (isspace(ch)) continue; + unsigned decoded = sMapHexReverse[ch]; + if (decoded >= 16) { + errorMessage = "unexpected character in hex data"; + } else { + accum = (accum << 4) | decoded; + } + j++; + } + // Spit out 1 char. + if (j) result.push_back(accum); + if (j == 1) { + errorMessage = "Unexpected odd length of hex encoded string"; + } + } + return result; +} + +static unsigned tohex1(unsigned data) +{ + data &= 0xf; + return data <= 9 ? data + '0' : data + 'A' - 10; +} + + +// Decode character data to binary data as per encodingArg which may be "binary", "hex", "base64" +// In the "binary" case, the data is just copied verbatim. +// On return the errorMessage will have a non-zero size if an error occurred. +string decodeToString(const char *buf, unsigned buflen, string encodingArg, string &errorMessage) +{ + stringCaseInsensitive encoding(encodingArg); + if (encoding == "binary") { + return string(buf,buflen); + } else if (encoding == "hex" || encoding == "base16") { + return hexDecodeToString((const unsigned char *)buf, buflen, errorMessage); + } else if (encoding == "base64") { + return base64DecodeToString((const unsigned char *)buf,buflen,errorMessage); + } else { + errorMessage = "Unexpected encoding specified:" + encodingArg; + return ""; + } +} + +// Encode binary data into a character string as per encodingArg which may be "binary", "hex", "base64". +// In the "binary" case, the data is just copied verbatim. +// On return the errorMessage will have a non-zero size if an error occurred. +string encodeToString(const char *data, unsigned datalen, string encodingArg, string &errorMessage) +{ + stringCaseInsensitive encoding(encodingArg); + if (encoding == "binary") { + return string(data,datalen); + } else if (encoding == "hex" || encoding == "base16") { + string result; + result.reserve(datalen*2); + for (const char *dp = data; dp < &data[datalen]; dp++) { + result.push_back(tohex1((*dp >> 4) & 0xf)); + result.push_back(tohex1(*dp & 0xf)); + } + return result; + } else if (encoding == "base64") { + return base64EncodeToString((const unsigned char *)data,datalen); + } else { + errorMessage = "Unexpected encoding specified:" + encodingArg; + return ""; + } +} + +}; // namespace diff --git a/Utils.h b/Utils.h index 67c2752..988235b 100644 --- a/Utils.h +++ b/Utils.h @@ -195,11 +195,11 @@ extern string uintToString(uint64_t h, uint64_t l); extern string uintToString(uint32_t x); //template class RefCntPointer; -// The class is created with a RefCnt of 0. The caller must assign the constructed result to a pointer -// of type RefCntPointer. When the last RefCntPointer is freed, this struct is too. +// The class is created with a RefCnt of 0. The caller may assign the constructed result to a pointer +// of type RefCntPointer. If so, then when the last RefCntPointer is freed, this struct is too. class RefCntBase { template friend class RefCntPointer; - Mutex mRefMutex; + mutable Mutex mRefMutex; mutable short mRefCnt; // signed, not unsigned! int setRefCnt(int val) { return mRefCnt = val; } // The semantics of reference counting mean you cannot copy an object that has reference counts; @@ -208,8 +208,10 @@ class RefCntBase { RefCntBase(RefCntBase &) { assert(0); } RefCntBase(const RefCntBase &) { assert(0); } RefCntBase operator=(RefCntBase &) {assert(0); mRefCnt = 0; return *this; } - int decRefCnt(); // Only RefCntPointer is permitted to use incRefCnt and decRefCnt. - void incRefCnt(); + // These are not 'const' but wonderful C++ requires us to declare them const so we can use a const pointer as the RefCntPointer target, + // then add 'mutable' to everything. What a botched up language. + int decRefCnt() const; // Only RefCntPointer is permitted to use incRefCnt and decRefCnt. + void incRefCnt() const; public: virtual ~RefCntBase(); RefCntBase() : mRefCnt(0) {} @@ -276,6 +278,10 @@ class RefCntPointer { extern string firstlines(string msgstr, int n=2); +extern std::string rn_backtrace(); +extern string decodeToString(const char *buf, unsigned buflen, string encodingArg, string &errorMessage); +extern string encodeToString(const char *data, unsigned datalen, string encodingArg, string &errorMessage); + }; // namespace using namespace Utils; diff --git a/UtilsTest.cpp b/UtilsTest.cpp index 2667944..be18776 100644 --- a/UtilsTest.cpp +++ b/UtilsTest.cpp @@ -62,10 +62,80 @@ void refcntpointertest() test3(); } +// Throw spaces into the string. +string messup(string in) +{ + string result; + for (string::iterator it = in.begin(); it != in.end(); it++) { + if (rand() & 1) result.push_back(' '); + result.push_back(*it); + } + return result; +} + +void asciiEncoderTest(bool verbose) +{ + char tbuf[102]; // binary data to try encoding and then decoding. + int numtests = 0; + printf("asciiEncoderTest...\n"); + for (unsigned len = 0; len < 100; len++) { // Length of binary data. + for (unsigned testno = 0; testno < 100; testno++) { + numtests++; + for (unsigned charno = 0; charno < len; charno++) { + tbuf[charno] = rand(); + } + string binaryinput = string(tbuf,len); + + for (unsigned type = 0; type < 2; type++) { + const char *method = type ? "base64" : "hex"; + string errorMessage; + string encoded = encodeToString(tbuf,len,method,errorMessage); + if (testno & 1) encoded = messup(encoded); + string decoded = decodeToString(encoded.data(),encoded.size(),method,errorMessage); + + // Check our results. + if (decoded.size() != len) { printf("FAIL %s len=%d size=%d encoded=%s\n",method,len,decoded.size(),encoded.c_str()); } + if (binaryinput != decoded) { printf("FAIL %s testno=%d len=%d encoded=%s\n",method,testno,len,encoded.c_str()); } + // Paranoid double check manually: + for (unsigned j = 0; j < len; j++) { + if (tbuf[j] != decoded[j]) { printf("FAIL %s len=%d j=%d\n",method,len,j); } + } + if (errorMessage.size()) { + printf("%s error message=%s\n",method,errorMessage.c_str()); + } + if (verbose) printf("TEST %s len=%d test=%d encoded=%s\n",method,len,testno,encoded.c_str()); + } + + /**** + { // Hex encoder/decoder tests. + string errorMessage = ""; + string hexencoded = encodeToString(tbuf,len,"hex",errorMessage); + if (hexencoded.size() != 2 * len) { + printf("FAIL hex encoded size=%d expected=%d\n",hexencoded.size(),2*len); + } + string hexdecoded = decodeToString(hexencoded.data(),hexencoded.size(),"hex",errorMessage); + //printf("input size=%d hexencoded size=%d hexdecoded size=%d %s\n",len,hexencoded.size(),hexdecoded.size(),hexencoded.c_str()); + string e2 = encodeToString(hexdecoded.data(),hexdecoded.size(),"hex",errorMessage); + //printf("in=%s out=%s\n",hexencoded.c_str(),e2.c_str()); + if (e2 != hexencoded) { + printf("FAIL hex here\n"); + } + if (binaryinput != hexdecoded) { printf("FAIL hex testno=%d len=%d\n",testno,len); } + if (errorMessage.size()) { + printf("base64 error message=%s\n",errorMessage.c_str()); + } + } + ****/ + } + } + printf("PASS %d tests\n",numtests); +} int main(int argc, char **argv) { + bool verbose = argc > 1; printf("Utils Test starting.\n"); refcntpointertest(); + asciiEncoderTest(verbose); } diff --git a/sqlite3util.cpp b/sqlite3util.cpp index f40b129..b7ebd43 100644 --- a/sqlite3util.cpp +++ b/sqlite3util.cpp @@ -60,6 +60,8 @@ int sqlite3_prepare_statement(sqlite3* DB, sqlite3_stmt **stmt, const char* quer if (src) { // (pat) You must not call LOG() from this file because it causes infinite recursion through gGetLoggingLevel and ConfigurationTable::lookup _LOG(ERR)<< format("sqlite3_prepare_v2 failed code=%u for \"%s\": %s\n",src,query,sqlite3_errmsg(DB)); + + // (pat) And I quote from sql documentation: "Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op." sqlite3_finalize(*stmt); } return src;