diff --git a/.gitignore b/.gitignore index 1a2cdbd..42412a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +._* .deps .libs *.o diff --git a/A51.cpp b/A51.cpp deleted file mode 100644 index 1e9098d..0000000 --- a/A51.cpp +++ /dev/null @@ -1,228 +0,0 @@ -/* - * A pedagogical implementation of A5/1. - * - * Copyright (C) 1998-1999: Marc Briceno, Ian Goldberg, and David Wagner - * - * The source code below is optimized for instructional value and clarity. - * Performance will be terrible, but that's not the point. - * The algorithm is written in the C programming language to avoid ambiguities - * inherent to the English language. Complain to the 9th Circuit of Appeals - * if you have a problem with that. - * - * This software may be export-controlled by US law. - * - * This software is free for commercial and non-commercial use as long as - * the following conditions are aheared to. - * Copyright remains the authors' and as such any Copyright notices in - * the code are not to be removed. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The license and distribution terms for any publicly available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution license - * [including the GNU Public License.] - * - * Background: The Global System for Mobile communications is the most widely - * deployed cellular telephony system in the world. GSM makes use of - * four core cryptographic algorithms, neither of which has been published by - * the GSM MOU. This failure to subject the algorithms to public review is all - * the more puzzling given that over 100 million GSM - * subscribers are expected to rely on the claimed security of the system. - * - * The four core GSM algorithms are: - * A3 authentication algorithm - * A5/1 "strong" over-the-air voice-privacy algorithm - * A5/2 "weak" over-the-air voice-privacy algorithm - * A8 voice-privacy key generation algorithm - * - * In April of 1998, our group showed that COMP128, the algorithm used by the - * overwhelming majority of GSM providers for both A3 and A8 - * functionality was fatally flawed and allowed for cloning of GSM mobile - * phones. - * Furthermore, we demonstrated that all A8 implementations we could locate, - * including the few that did not use COMP128 for key generation, had been - * deliberately weakened by reducing the keyspace from 64 bits to 54 bits. - * The remaining 10 bits are simply set to zero! - * - * See http://www.scard.org/gsm for additional information. - * - * The question so far unanswered is if A5/1, the "stronger" of the two - * widely deployed voice-privacy algorithm is at least as strong as the - * key. Meaning: "Does A5/1 have a work factor of at least 54 bits"? - * Absent a publicly available A5/1 reference implementation, this question - * could not be answered. We hope that our reference implementation below, - * which has been verified against official A5/1 test vectors, will provide - * the cryptographic community with the base on which to construct the - * answer to this important question. - * - * Initial indications about the strength of A5/1 are not encouraging. - * A variant of A5, while not A5/1 itself, has been estimated to have a - * work factor of well below 54 bits. See http://jya.com/crack-a5.htm for - * background information and references. - * - * With COMP128 broken and A5/1 published below, we will now turn our attention - * to A5/2. The latter has been acknowledged by the GSM community to have - * been specifically designed by intelligence agencies for lack of security. - * - */ - - -#include -#include -#include "A51.h" - -/* Masks for the three shift registers */ -#define R1MASK 0x07FFFF /* 19 bits, numbered 0..18 */ -#define R2MASK 0x3FFFFF /* 22 bits, numbered 0..21 */ -#define R3MASK 0x7FFFFF /* 23 bits, numbered 0..22 */ - -/* Middle bit of each of the three shift registers, for clock control */ -#define R1MID 0x000100 /* bit 8 */ -#define R2MID 0x000400 /* bit 10 */ -#define R3MID 0x000400 /* bit 10 */ - -/* The three shift registers. They're in global variables to make the code - * easier to understand. - * A better implementation would not use global variables. */ -word R1, R2, R3; - -/* Look at the middle bits of R1,R2,R3, take a vote, and - * return the majority value of those 3 bits. */ -bit majority() { - int sum; - sum = ((R1>>8)&1) + ((R2>>10)&1) + ((R3>>10)&1); - if (sum >= 2) - return 1; - else - return 0; -} - -/* Clock two or three of R1,R2,R3, with clock control - * according to their middle bits. - * Specifically, we clock Ri whenever Ri's middle bit - * agrees with the majority value of the three middle bits.*/ -void clock() { - bit maj = majority(); - if (((R1&R1MID)!=0) == maj) - R1 = ((R1<<1) & R1MASK) | (1 & (R1>>18 ^ R1>>17 ^ R1>>16 ^ R1>>13)); - if (((R2&R2MID)!=0) == maj) - R2 = ((R2<<1) & R2MASK) | (1 & (R2>>21 ^ R2>>20)); - if (((R3&R3MID)!=0) == maj) - R3 = ((R3<<1) & R3MASK) | (1 & (R3>>22 ^ R3>>21 ^ R3>>20 ^ R3>>7)); -} - -/* Clock all three of R1,R2,R3, ignoring their middle bits. - * This is only used for key setup. */ -void clockallthree() { - R1 = ((R1<<1) & R1MASK) | (1 & (R1>>18 ^ R1>>17 ^ R1>>16 ^ R1>>13)); - R2 = ((R2<<1) & R2MASK) | (1 & (R2>>21 ^ R2>>20)); - R3 = ((R3<<1) & R3MASK) | (1 & (R3>>22 ^ R3>>21 ^ R3>>20 ^ R3>>7)); -} - -/* Generate an output bit from the current state. - * You grab a bit from each register via the output generation taps; - * then you XOR the resulting three bits. */ -bit getbit() { - return ((R1>>18)^(R2>>21)^(R3>>22))&1; -} - -/* Do the A5/1 key setup. This routine accepts a 64-bit key and - * a 22-bit frame number. */ -void keysetup(byte key[8], word frame) { - int i; - bit keybit, framebit; - - /* Zero out the shift registers. */ - R1 = R2 = R3 = 0; - - /* Load the key into the shift registers, - * LSB of first byte of key array first, - * clocking each register once for every - * key bit loaded. (The usual clock - * control rule is temporarily disabled.) */ - for (i=0; i<64; i++) { - clockallthree(); /* always clock */ - keybit = (key[i/8] >> (i&7)) & 1; /* The i-th bit of the -key */ - R1 ^= keybit; R2 ^= keybit; R3 ^= keybit; - } - - /* Load the frame number into the shift - * registers, LSB first, - * clocking each register once for every - * key bit loaded. (The usual clock - * control rule is still disabled.) */ - for (i=0; i<22; i++) { - clockallthree(); /* always clock */ - framebit = (frame >> i) & 1; /* The i-th bit of the frame # -*/ - R1 ^= framebit; R2 ^= framebit; R3 ^= framebit; - } - - /* Run the shift registers for 100 clocks - * to mix the keying material and frame number - * together with output generation disabled, - * so that there is sufficient avalanche. - * We re-enable the majority-based clock control - * rule from now on. */ - for (i=0; i<100; i++) { - clock(); - } - - /* Now the key is properly set up. */ -} - -/* Generate output. We generate 228 bits of - * keystream output. The first 114 bits is for - * the A->B frame; the next 114 bits is for the - * B->A frame. You allocate a 15-byte buffer - * for each direction, and this function fills - * it in. */ -void run(byte AtoBkeystream[], byte BtoAkeystream[]) { - int i; - - /* Zero out the output buffers. */ - for (i=0; i<=113/8; i++) - AtoBkeystream[i] = BtoAkeystream[i] = 0; - - /* Generate 114 bits of keystream for the - * A->B direction. Store it, MSB first. */ - for (i=0; i<114; i++) { - clock(); - AtoBkeystream[i/8] |= getbit() << (7-(i&7)); - } - - /* Generate 114 bits of keystream for the - * B->A direction. Store it, MSB first. */ - for (i=0; i<114; i++) { - clock(); - BtoAkeystream[i/8] |= getbit() << (7-(i&7)); - } -} - -void A51_GSM( byte *key, int klen, int count, byte *block1, byte *block2 ) -{ - assert(klen == 64); - keysetup(key, count); // TODO - frame and count are not the same - run(block1, block2); -} diff --git a/A51.h b/A51.h deleted file mode 100644 index d1337de..0000000 --- a/A51.h +++ /dev/null @@ -1,11 +0,0 @@ - - -#include -#include - -typedef unsigned char byte; -typedef unsigned long word; -typedef word bit; - -void A51_GSM( byte *key, int klen, int count, byte *block1, byte *block2 ); - diff --git a/A51Test.cpp b/A51Test.cpp deleted file mode 100644 index 57ce467..0000000 --- a/A51Test.cpp +++ /dev/null @@ -1,162 +0,0 @@ -/* - * A pedagogical implementation of A5/1. - * - * Copyright (C) 1998-1999: Marc Briceno, Ian Goldberg, and David Wagner - * - * The source code below is optimized for instructional value and clarity. - * Performance will be terrible, but that's not the point. - * The algorithm is written in the C programming language to avoid ambiguities - * inherent to the English language. Complain to the 9th Circuit of Appeals - * if you have a problem with that. - * - * This software may be export-controlled by US law. - * - * This software is free for commercial and non-commercial use as long as - * the following conditions are aheared to. - * Copyright remains the authors' and as such any Copyright notices in - * the code are not to be removed. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The license and distribution terms for any publicly available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution license - * [including the GNU Public License.] - * - * Background: The Global System for Mobile communications is the most widely - * deployed cellular telephony system in the world. GSM makes use of - * four core cryptographic algorithms, neither of which has been published by - * the GSM MOU. This failure to subject the algorithms to public review is all - * the more puzzling given that over 100 million GSM - * subscribers are expected to rely on the claimed security of the system. - * - * The four core GSM algorithms are: - * A3 authentication algorithm - * A5/1 "strong" over-the-air voice-privacy algorithm - * A5/2 "weak" over-the-air voice-privacy algorithm - * A8 voice-privacy key generation algorithm - * - * In April of 1998, our group showed that COMP128, the algorithm used by the - * overwhelming majority of GSM providers for both A3 and A8 - * functionality was fatally flawed and allowed for cloning of GSM mobile - * phones. - * Furthermore, we demonstrated that all A8 implementations we could locate, - * including the few that did not use COMP128 for key generation, had been - * deliberately weakened by reducing the keyspace from 64 bits to 54 bits. - * The remaining 10 bits are simply set to zero! - * - * See http://www.scard.org/gsm for additional information. - * - * The question so far unanswered is if A5/1, the "stronger" of the two - * widely deployed voice-privacy algorithm is at least as strong as the - * key. Meaning: "Does A5/1 have a work factor of at least 54 bits"? - * Absent a publicly available A5/1 reference implementation, this question - * could not be answered. We hope that our reference implementation below, - * which has been verified against official A5/1 test vectors, will provide - * the cryptographic community with the base on which to construct the - * answer to this important question. - * - * Initial indications about the strength of A5/1 are not encouraging. - * A variant of A5, while not A5/1 itself, has been estimated to have a - * work factor of well below 54 bits. See http://jya.com/crack-a5.htm for - * background information and references. - * - * With COMP128 broken and A5/1 published below, we will now turn our attention - * to A5/2. The latter has been acknowledged by the GSM community to have - * been specifically designed by intelligence agencies for lack of security. - * - */ - - -#include -#include -#include -#include "./A51.h" - - -/* Test the code by comparing it against - * a known-good test vector. */ -void test() { - byte key[8] = {0x12, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}; - word frame = 0x134; - byte goodAtoB[15] = { 0x53, 0x4E, 0xAA, 0x58, 0x2F, 0xE8, 0x15, - 0x1A, 0xB6, 0xE1, 0x85, 0x5A, 0x72, 0x8C, 0x00 }; - byte goodBtoA[15] = { 0x24, 0xFD, 0x35, 0xA3, 0x5D, 0x5F, 0xB6, - 0x52, 0x6D, 0x32, 0xF9, 0x06, 0xDF, 0x1A, 0xC0 }; - byte AtoB[15], BtoA[15]; - int i, failed=0; - - A51_GSM(key, 64, frame, AtoB, BtoA); - - /* Compare against the test vector. */ - for (i=0; i<15; i++) - if (AtoB[i] != goodAtoB[i]) - failed = 1; - for (i=0; i<15; i++) - if (BtoA[i] != goodBtoA[i]) - failed = 1; - - /* Print some debugging output. */ - printf("key: 0x"); - for (i=0; i<8; i++) - printf("%02X", key[i]); - printf("\n"); - printf("frame number: 0x%06X\n", (unsigned int)frame); - printf("known good output:\n"); - printf(" A->B: 0x"); - for (i=0; i<15; i++) - printf("%02X", goodAtoB[i]); - printf(" B->A: 0x"); - for (i=0; i<15; i++) - printf("%02X", goodBtoA[i]); - printf("\n"); - printf("observed output:\n"); - printf(" A->B: 0x"); - for (i=0; i<15; i++) - printf("%02X", AtoB[i]); - printf(" B->A: 0x"); - for (i=0; i<15; i++) - printf("%02X", BtoA[i]); - printf("\n"); - - if (!failed) { - printf("Self-check succeeded: everything looks ok.\n"); - } else { - /* Problems! The test vectors didn't compare*/ - printf("\nI don't know why this broke; contact the authors.\n"); - exit(1); - } - - printf("time test\n"); - int n = 10000; - float t = clock(); - for (i = 0; i < n; i++) { - A51_GSM(key, 64, frame, AtoB, BtoA); - } - t = (clock() - t) / (CLOCKS_PER_SEC * (float)n); - printf("A51_GSM takes %g seconds per iteration\n", t); -} - -int main(void) { - test(); - return 0; -} diff --git a/BitVector.cpp b/BitVector.cpp index 7487834..a36ab7b 100644 --- a/BitVector.cpp +++ b/BitVector.cpp @@ -1,5 +1,5 @@ /* -* Copyright 2008, 2009 Free Software Foundation, Inc. +* Copyright 2008, 2009, 2014 Free Software Foundation, Inc. * * * This software is distributed under the terms of the GNU Affero Public License. @@ -30,38 +30,26 @@ #include #include #include +#include using namespace std; -/** - Apply a Galois polymonial to a binary seqeunce. - @param val The input sequence. - @param poly The polynomial. - @param order The order of the polynomial. - @return Single-bit result. -*/ -unsigned applyPoly(uint64_t val, uint64_t poly, unsigned order) -{ - uint64_t prod = val & poly; - unsigned sum = prod; - for (unsigned i=1; i>i; - return sum & 0x01; -} - - - - - BitVector::BitVector(const char *valString) - :Vector(strlen(valString)) { - uint32_t accum = 0; - for (size_t i=0; iiState) << 1; // input state for 0 - const uint32_t iState1 = iState0 | 0x01; // input state for 1 - const uint32_t oStateShifted = (sp->oState) << mIRate; // shifted output - const float cost = sp->cost; - sp++; - // 0 input extension - mCandidates[i].cost = cost; - mCandidates[i].oState = oStateShifted | mGeneratorTable[iState0 & mCMask]; - mCandidates[i].iState = iState0; - // 1 input extension - mCandidates[i+1].cost = cost; - mCandidates[i+1].oState = oStateShifted | mGeneratorTable[iState1 & mCMask]; - mCandidates[i+1].iState = iState1; - } -} - - -void ViterbiR2O4::getSoftCostMetrics(const uint32_t inSample, const float *matchCost, const float *mismatchCost) -{ - const float *cTab[2] = {matchCost,mismatchCost}; - for (unsigned i=0; i>1)&0x01][0]; - } -} - - -void ViterbiR2O4::pruneCandidates() -{ - const vCand* c1 = mCandidates; // 0-prefix - const vCand* c2 = mCandidates + mIStates; // 1-prefix - for (unsigned i=0; i=minCost) continue; - minCost = thisCost; - minIndex=i; - } - return mSurvivors[minIndex]; -} - - -const ViterbiR2O4::vCand& ViterbiR2O4::step(uint32_t inSample, const float *probs, const float *iprobs) -{ - branchCandidates(); - getSoftCostMetrics(inSample,probs,iprobs); - pruneCandidates(); - return minCost(); -} - - uint64_t Parity::syndrome(const BitVector& receivedCodeword) { return receivedCodeword.syndrome(*this); @@ -446,85 +294,6 @@ BitVector SoftVector::sliced() const -void SoftVector::decode(ViterbiR2O4 &decoder, BitVector& target) const -{ - const size_t sz = size(); - const unsigned deferral = decoder.deferral(); - const size_t ctsz = sz + deferral*decoder.iRate(); - assert(sz <= decoder.iRate()*target.size()); - - // Build a "history" array where each element contains the full history. - uint32_t history[ctsz]; - { - BitVector bits = sliced(); - uint32_t accum = 0; - for (size_t i=0; i0.5F) pVal = 1.0F-pVal; - float ipVal = 1.0F-pVal; - // This is a cheap approximation to an ideal cost function. - if (pVal<0.01F) pVal = 0.01; - if (ipVal<0.01F) ipVal = 0.01; - matchCostTable[i] = 0.25F/ipVal; - mismatchCostTable[i] = 0.25F/pVal; - } - - // pad end of table with unknowns - for (size_t i=sz; i=deferral) *op++ = (minCost.iState >> deferral)&0x01; - oCount++; - } - } -} - - - // (pat) Added 6-22-2012 float SoftVector::getEnergy(float *plow) const { @@ -541,6 +310,36 @@ float SoftVector::getEnergy(float *plow) const return avg; } +// (pat) Added 1-2014. Compute SNR of a soft vector. Very similar to above. +// Since we dont really know what the expected signal values are, we will assume that the signal is 0 or 1 +// and return the SNR on that basis. +// SNR is power(signal) / power(noise) where power can be calculated as (RMS(signal) / RMS(noise))**2 of the values. +// Since RMS is square-rooted, ie RMS = sqrt(1/n * (x1**2 + x2**2 ...)), we just add up the squares. +// To compute RMS of the signal we will remove any constant offset, so the signal values are either 0.5 or -0.5, +// so the RMS of the signal is just 0.5**2 * len; all we need to compute is the noise component. +float SoftVector::getSNR() const +{ + float sumSquaresNoise = 0; + const SoftVector &vec = *this; + int len = vec.size(); + if (len == 0) { return 0.0; } + for (int i = 0; i < len; i++) { + float bit = vec[i]; + if (bit < 0.5) { + // Assume signal is 0. + sumSquaresNoise += (bit - 0.0) * (bit - 0.0); + } else { + // Assume signal is 1. + sumSquaresNoise += (bit - 1.0) * (bit - 1.0); + } + } + float sumSquaresSignal = 0.5 * 0.5 * len; + // I really want log10 of this to convert to dB, but log is expensive, and Harvind seems to like absolute SNR. + // Clamp max to 999; it shouldnt get up there but be sure. This also avoids divide by zero. + if (sumSquaresNoise * 1000 < sumSquaresSignal) return 999; + return sumSquaresSignal / sumSquaresNoise; +} + ostream& operator<<(ostream& os, const SoftVector& sv) { @@ -622,4 +421,60 @@ bool BitVector::unhex(const char* src) return true; } +bool BitVector::operator==(const BitVector &other) const +{ + unsigned l = size(); + return l == other.size() && 0==memcmp(begin(),other.begin(),l); +} + +void BitVector::copyPunctured(BitVector &dst, const unsigned *puncture, const size_t plth) +{ + assert(size() - plth == dst.size()); + char *srcp = mStart; + char *dstp = dst.mStart; + const unsigned *pend = puncture + plth; + while (srcp < mEnd) { + if (puncture < pend) { + int n = (*puncture++) - (srcp - mStart); + assert(n >= 0); + for (int i = 0; i < n; i++) { + assert(srcp < mEnd && dstp < dst.mEnd); + *dstp++ = *srcp++; + } + srcp++; + } else { + while (srcp < mEnd) { + assert(dstp < dst.mEnd); + *dstp++ = *srcp++; + } + } + } + assert(dstp == dst.mEnd && puncture == pend); +} + +void SoftVector::copyUnPunctured(SoftVector &dst, const unsigned *puncture, const size_t plth) +{ + assert(size() + plth == dst.size()); + float *srcp = mStart; + float *dstp = dst.mStart; + const unsigned *pend = puncture + plth; + while (dstp < dst.mEnd) { + if (puncture < pend) { + int n = (*puncture++) - (dstp - dst.mStart); + assert(n >= 0); + for (int i = 0; i < n; i++) { + assert(srcp < mEnd && dstp < dst.mEnd); + *dstp++ = *srcp++; + } + *dstp++ = 0.5; + } else { + while (srcp < mEnd) { + assert(dstp < dst.mEnd); + *dstp++ = *srcp++; + } + } + } + assert(dstp == dst.mEnd && puncture == pend); +} + // vim: ts=4 sw=4 diff --git a/BitVector.h b/BitVector.h index 63c4e91..f1faf92 100644 --- a/BitVector.h +++ b/BitVector.h @@ -1,5 +1,5 @@ /* -* Copyright 2008, 2009 Free Software Foundation, Inc. +* Copyright 2008, 2009, 2014 Free Software Foundation, Inc. * * This software is distributed under the terms of the GNU Affero Public License. * See the COPYING file in the main directory for details. @@ -23,11 +23,12 @@ */ -#ifndef FECVECTORS_H -#define FECVECTORS_H +#ifndef BITVECTORS_H +#define BITVECTORS_H #include "Vector.h" #include +#include class BitVector; @@ -35,6 +36,7 @@ class SoftVector; + /** Shift-register (LFSR) generator. */ class Generator { @@ -112,171 +114,83 @@ class Parity : public Generator { }; - - -/** - Class to represent convolutional coders/decoders of rate 1/2, memory length 4. - This is the "workhorse" coder for most GSM channels. -*/ -class ViterbiR2O4 { - - private: - /**name Lots of precomputed elements so the compiler can optimize like hell. */ - //@{ - /**@name Core values. */ - //@{ - static const unsigned mIRate = 2; ///< reciprocal of rate - static const unsigned mOrder = 4; ///< memory length of generators - //@} - /**@name Derived values. */ - //@{ - static const unsigned mIStates = 0x01 << mOrder; ///< number of states, number of survivors - static const uint32_t mSMask = mIStates-1; ///< survivor mask - static const uint32_t mCMask = (mSMask<<1) | 0x01; ///< candidate mask - static const uint32_t mOMask = (0x01< +{ public: - - /** - A candidate sequence in a Viterbi decoder. - The 32-bit state register can support a deferral of 6 with a 4th-order coder. - */ - typedef struct candStruct { - uint32_t iState; ///< encoder input associated with this candidate - uint32_t oState; ///< encoder output associated with this candidate - float cost; ///< cost (metric value), float to support soft inputs - } vCand; - - /** Clear a structure. */ - void clear(vCand& v) - { - v.iState=0; - v.oState=0; - v.cost=0; - } - - - private: - - /**@name Survivors and candidates. */ - //@{ - vCand mSurvivors[mIStates]; ///< current survivor pool - vCand mCandidates[2*mIStates]; ///< current candidate pool - //@} - - public: - - unsigned iRate() const { return mIRate; } - uint32_t cMask() const { return mCMask; } - uint32_t stateTable(unsigned g, unsigned i) const { return mStateTable[g][i]; } - unsigned deferral() const { return mDeferral; } - - - ViterbiR2O4(); - - /** Set all cost metrics to zero. */ - void initializeStates(); - - /** - Full cycle of the Viterbi algorithm: branch, metrics, prune, select. - @return reference to minimum-cost candidate. - */ - const vCand& step(uint32_t inSample, const float *probs, const float *iprobs); - - private: - - /** Branch survivors into new candidates. */ - void branchCandidates(); - - /** Compute cost metrics for soft-inputs. */ - void getSoftCostMetrics(uint32_t inSample, const float *probs, const float *iprobs); - - /** Select survivors from the candidate set. */ - void pruneCandidates(); - - /** Find the minimum cost survivor. */ - const vCand& minCost() const; - - /** - Precompute the state tables. - @param g Generator index 0..((1/rate)-1) - */ - void computeStateTables(unsigned g); - - /** - Precompute the generator outputs. - mCoeffs must be defined first. - */ - void computeGeneratorTable(); - -}; - - - - -class BitVector : public Vector { - - - public: - /**@name Constructors. */ //@{ /**@name Casts of Vector constructors. */ - //@{ - BitVector(char* wData, char* wStart, char* wEnd) - :Vector(wData,wStart,wEnd) - { } - BitVector(size_t len=0):Vector(len) {} - BitVector(const Vector& source):Vector(source) {} - BitVector(Vector& source):Vector(source) {} - BitVector(const Vector& source1, const Vector source2):Vector(source1,source2) {} - //@} + BitVector(VectorDataType wData, char* wStart, char* wEnd) : VectorBase(wData, wStart, wEnd) {} + + // The one and only copy-constructor. + BitVector(const BitVector&other) : VectorBase() { + VECTORDEBUG("BitVector(%p)",(void*)&other); + if (other.getData()) { + this->clone(other); + } else { + this->makeAlias(other); + } + } + + // (pat) Removed default value for len and added 'explicit'. Please do not remove 'explicit'; + // it prevents auto-conversion of int to BitVector in constructors. + // Previous code was often ambiguous, especially for L3Frame and descendent constructors, leading to latent bugs. + explicit BitVector(size_t len) { this->vInit(len); } + BitVector() { this->vInit(0); } + + /** Build a BitVector by concatenation. */ + BitVector(const BitVector& other1, const BitVector& other2) : VectorBase() + { + assert(this->getData() == 0); + this->vConcat(other1,other2); + } /** Construct from a string of "0" and "1". */ + // (pat) Characters that are not '0' or '1' map to '0'. BitVector(const char* valString); //@} - /** Index a single bit. */ - bool bit(size_t index) const - { - // We put this code in .h for fast inlining. - const char *dp = mStart+index; - assert(dpbegin() + start; char* wEnd = wStart + span; - assert(wEnd<=mEnd); + assert(wEnd<=this->end()); +#if BITVECTOR_REFCNTS + return BitVector(mData,wStart,wEnd); +#else return BitVector(NULL,wStart,wEnd); +#endif } - BitVector alias() - { return segment(0,size()); } + // (pat) Historically the BitVector segment method had const and non-const versions with different behavior. + // I changed the name of the const version to cloneSegment and replaced all uses throughout OpenBTS. + const BitVector cloneSegment(size_t start, size_t span) const + { + BitVector seg = const_cast(this)->segment(start,span); + // (pat) We are depending on the Return Value Optimization not to invoke the copy-constructor on the result, + // which would result in its immediate destruction while we are still using it. + BitVector result; + result.clone(seg); + return result; + } - const BitVector segment(size_t start, size_t span) const - { return (BitVector)(Vector::segment(start,span)); } + BitVector alias() const { + return const_cast(this)->segment(0,size()); + } BitVector head(size_t span) { return segment(0,span); } - const BitVector head(size_t span) const { return segment(0,span); } BitVector tail(size_t start) { return segment(start,size()-start); } - const BitVector tail(size_t start) const { return segment(start,size()-start); } + + // (pat) Please do NOT put the const version of head and tail back in, because historically they were messed up. + // Use cloneSegment instead. + //const BitVector head(size_t span) const { return segment(0,span); } + //const BitVector tail(size_t start) const { return segment(start,size()-start); } //@} @@ -288,8 +202,6 @@ class BitVector : public Vector { uint64_t syndrome(Generator& gen) const; /** Calculate the parity word for the vector with the given Generator. */ uint64_t parity(Generator& gen) const; - /** Encode the signal with the GSM rate 1/2 convolutional encoder. */ - void encode(const ViterbiR2O4& encoder, BitVector& target); //@} @@ -342,25 +254,62 @@ class BitVector : public Vector { * @returns true on success, false on error. */ bool unhex(const char*); - void set(BitVector other) // That's right. No ampersand. + // For this method, 'other' should have been run through the copy-constructor already + // (unless it was newly created, ie foo.dup(L2Frame(...)), in which case we are screwed anyway) + // so the call to makeAlias is redundant. + // This only works if other is already an alias. + void dup(BitVector other) { assert(!this->getData()); makeAlias(other); assert(this->mStart == other.mStart); } + void dup(BitVector &other) { makeAlias(other); assert(this->mStart == other.mStart); } + +#if 0 + void operator=(const BitVector& other) { + printf("BitVector::operator=\n"); + assert(0); + //this->dup(other); + } +#endif + + bool operator==(const BitVector &other) const; + + /** Copy to dst, not including those indexed in puncture. */ + void copyPunctured(BitVector &dst, const unsigned *puncture, const size_t plth); + + /** Index a single bit. */ + // (pat) Cant have too many ways to do this, I guess. + bool bit(size_t index) const { - clear(); - mData=other.mData; - mStart=other.mStart; - mEnd=other.mEnd; - other.mData=NULL; + // We put this code in .h for fast inlining. + const char *dp = this->begin()+index; + assert(dpend()); + return (*dp) & 0x01; + } + + char& operator[](size_t index) + { + assert(this->mStart+indexmEnd); + return this->mStart[index]; + } + + const char& operator[](size_t index) const + { + assert(this->mStart+indexmEnd); + return this->mStart[index]; } /** Set a bit */ void settfb(size_t index, int value) { - char *dp = mStart+index; - assert(dpmStart+index; + assert(dpmEnd); *dp = value; } + typedef char* iterator; + typedef const char* const_iterator; }; +// (pat) BitVector2 was an intermediate step in fixing BitVector but is no longer needed. +#define BitVector2 BitVector std::ostream& operator<<(std::ostream&, const BitVector&); @@ -430,13 +379,11 @@ class SoftVector: public Vector { const SoftVector tail(size_t start) const { return segment(start,size()-start); } //@} - /** Decode soft symbols with the GSM rate-1/2 Viterbi decoder. */ - void decode(ViterbiR2O4 &decoder, BitVector& target) const; - // (pat) How good is the SoftVector in the sense of the bits being solid? // Result of 1 is perfect and 0 means all the bits were 0.5 // If plow is non-NULL, also return the lowest energy bit. float getEnergy(float *low=0) const; + float getSNR() const; /** Fill with "unknown" values. */ void unknown() { fill(0.5F); } @@ -452,6 +399,9 @@ class SoftVector: public Vector { /** Slice the whole signal into bits. */ BitVector sliced() const; + /** Copy to dst, adding in 0.5 for those indexed in puncture. */ + void copyUnPunctured(SoftVector &dst, const unsigned *puncture, const size_t plth); + /** Return a soft bit. */ float softbit(size_t index) const { @@ -467,7 +417,6 @@ class SoftVector: public Vector { assert(dp #include +#include using namespace std; +// We must have a gConfig now to include BitVector. +#include "Configuration.h" +ConfigurationTable gConfig; -int main(int argc, char *argv[]) + +void origTest() { - BitVector v1("0000111100111100101011110000"); - cout << v1 << endl; + BitVector v0("0000111100111100101011110000"); + cout << v0 << endl; + // (pat) The conversion from a string was inserting garbage into the result BitVector. + // Fixed now so only 0 or 1 are inserted, but lets check: + for (char *cp = v0.begin(); cp < v0.end(); cp++) cout << (int)*cp<<" "; + cout << endl; + + BitVector v1(v0); v1.LSB8MSB(); - cout << v1 << endl; - ViterbiR2O4 vCoder; - BitVector v2(v1.size()*2); - v1.encode(vCoder,v2); - cout << v2 << endl; - SoftVector sv2(v2); - cout << sv2 << endl; - for (unsigned i=0; i #include #include +#ifdef DEBUG_CONFIG +#define debugLogEarly gLogEarly +#else +#define debugLogEarly(...) +#endif using namespace std; @@ -47,12 +53,43 @@ static const char* createConfigTable = { ")" }; +// (pat) LOG() may not be initialized yet, so use syslog directly. +#define LOGNOW(fmt,...) syslog(LOG_ERR, "ERR %s:" fmt,Utils::timestr().c_str(),__VA_ARGS__) + + +long ConfigurationRecord::number() const +{ + if (mValue.size() == 0 && ! mCRWarned) { + LOGNOW("request for configuration key '%s' as number: configuration value is null", mCRKey.c_str()); + mCRWarned = true; + return 0; + } + char *endptr; + long result = strtol(mValue.c_str(),&endptr,0); + if (*endptr && ! mCRWarned) { + // (pat) Warn if the number is invalid. + LOGNOW("request for configuration key '%s' as number: value could not be fully converted: '%s'", mCRKey.c_str(),mValue.c_str()); + mCRWarned = true; + } + return result; +} float ConfigurationRecord::floatNumber() const { - float val; - sscanf(mValue.c_str(),"%f",&val); + if (mValue.size() == 0 && ! mCRWarned) { + LOGNOW("request for configuration key '%s' as float number: configuration value is null", mCRKey.c_str()); + mCRWarned = true; + return 0; + } + //sscanf(mValue.c_str(),"%f",&val); + char *endptr; + float val = strtof(mValue.c_str(),&endptr); + if (*endptr && ! mCRWarned) { + // (pat) Warn if the number is invalid. + LOGNOW("request for configuration key '%s' as float number: value could not be fully converted: '%s'", mCRKey.c_str(),mValue.c_str()); + mCRWarned = true; + } return val; } @@ -78,10 +115,12 @@ ConfigurationTable::ConfigurationTable(const char* filename, const char *wCmdNam if (!sqlite3_command(mDB,createConfigTable)) { gLogEarly(LOG_EMERG, "cannot create configuration table in database at %s, error message: %s", filename, sqlite3_errmsg(mDB)); } + + // Pat and David both do not want to use WAL mode on the config databases. // Set high-concurrency WAL mode. - if (!sqlite3_command(mDB,enableWAL)) { - gLogEarly(LOG_EMERG, "cannot enable WAL mode on database at %s, error message: %s", filename, sqlite3_errmsg(mDB)); - } + //if (!sqlite3_command(mDB,enableWAL)) { + // gLogEarly(LOG_EMERG, "cannot enable WAL mode on database at %s, error message: %s", filename, sqlite3_errmsg(mDB)); + //} // Build CommonLibs schema ConfigurationKey *tmp; @@ -144,6 +183,32 @@ ConfigurationTable::ConfigurationTable(const char* filename, const char *wCmdNam // Init the cross checking callback to something predictable mCrossCheck = NULL; + +#define DUMP_CONFIGURATION_TABLE 1 +#if DUMP_CONFIGURATION_TABLE + // (pat) Dump any non-default config variables... + try { + if (wCmdName == NULL) { wCmdName = ""; } + LOG(INFO) << wCmdName << ":" << " List of non-default config parameters:"; + string snippet(""); + ConfigurationKeyMap view = getSimilarKeys(snippet); + for (ConfigurationKeyMap::iterator it = view.begin(); it != view.end(); it++) { + string name = it->first; + ConfigurationKey key = it->second; + if (name != key.getName()) { + LOG(ALERT) << "SQL database is corrupt at name:"<getStr(name); + if (value != defaultValue) { + LOG(INFO) << "Config Variable"<second.getDescription(); + // Try to use a quote that will work: if the description contains ' quote with " else '. + // This is not perfect because these quotes could themselves be quoted. + const char * quote = string::npos != description.find('\'') ? "\"" : "'"; + ss << quote; if (mp->second.getType() == ConfigurationKey::BOOLEAN) { ss << "1=enabled, 0=disabled - "; } @@ -188,7 +257,7 @@ string ConfigurationTable::getDefaultSQL(const std::string& program, const std:: if (mp->second.isStatic()) { ss << " Static."; } - ss << "'"; + ss << quote; ss << ");" << endl; mp++; } @@ -208,15 +277,15 @@ string ConfigurationTable::getTeX(const std::string& program, const std::string& ss << "% -- these sections were generated using: " << program << " --gentex" << endl; ss << "% -- binary version: " << version << endl; - ss << "\\subsection{Customer Site Parameters}" << endl; + ss << "\\section{Customer Site Parameters}" << endl; ss << "These parameters must be changed to fit your site." << endl; ss << "\\begin{itemize}" << endl; mp = mSchema.begin(); while (mp != mSchema.end()) { if (mp->second.getVisibility() == ConfigurationKey::CUSTOMERSITE) { - ss << " \\item "; + ss << " \\item \\texttt{"; // name - ss << mp->first << " -- "; + ss << mp->first << "} -- "; // description ss << mp->second.getDescription(); ss << endl; @@ -226,7 +295,7 @@ string ConfigurationTable::getTeX(const std::string& program, const std::string& ss << "\\end{itemize}" << endl; ss << endl; - ss << "\\subsection{Customer Tuneable Parameters}" << endl; + ss << "\\section{Customer Tuneable Parameters}" << endl; ss << "These parameters can be changed to optimize your site." << endl; ss << "\\begin{itemize}" << endl; mp = mSchema.begin(); @@ -237,9 +306,9 @@ string ConfigurationTable::getTeX(const std::string& program, const std::string& mp->second.getVisibility() == ConfigurationKey::CUSTOMERTUNE || mp->second.getVisibility() == ConfigurationKey::CUSTOMERWARN )) { - ss << " \\item "; + ss << " \\item \\texttt{"; // name - ss << mp->first << " -- "; + ss << mp->first << "} -- "; // description ss << mp->second.getDescription(); ss << endl; @@ -249,16 +318,16 @@ string ConfigurationTable::getTeX(const std::string& program, const std::string& ss << "\\end{itemize}" << endl; ss << endl; - ss << "\\subsection{Developer/Factory Parameters}" << endl; + ss << "\\section{Developer/Factory Parameters}" << endl; ss << "These parameters should only be changed by when developing new code." << endl; ss << "\\begin{itemize}" << endl; mp = mSchema.begin(); while (mp != mSchema.end()) { if (mp->second.getVisibility() == ConfigurationKey::FACTORY || mp->second.getVisibility() == ConfigurationKey::DEVELOPER) { - ss << " \\item "; + ss << " \\item \\texttt{"; // name - ss << mp->first << " -- "; + ss << mp->first << "} -- "; // description ss << mp->second.getDescription(); ss << endl; @@ -377,6 +446,46 @@ bool ConfigurationTable::isValidValue(const std::string& name, const std::string break; } + case ConfigurationKey::HOSTANDPORT_OPT: { + if (val.length() == 0) { + ret = true; + break; + } + } + case ConfigurationKey::HOSTANDPORT: { + uint delimiter; + std::string host; + int port = -1; + bool validHost = false; + + delimiter = val.find(':'); + if (delimiter != std::string::npos) { + host = val.substr(0, delimiter); + std::stringstream(val.substr(delimiter+1)) >> port; + if (ConfigurationKey::isValidIP(host)) { + validHost = true; + } else { + regex_t r; + const char* expression = "^[a-zA-Z0-9_.-]+$"; + int result = regcomp(&r, expression, REG_EXTENDED); + if (result) { + char msg[256]; + regerror(result,&r,msg,255); + break;//abort(); + } + if (regexec(&r, host.c_str(), 0, NULL, 0)==0) { + validHost = true; + } + regfree(&r); + } + + if (validHost && 1 <= port && port <= 65535) { + ret = true; + } + } + break; + } + case ConfigurationKey::IPADDRESS_OPT: { if (val.length() == 0) { ret = true; @@ -553,13 +662,13 @@ const ConfigurationRecord& ConfigurationTable::lookup(const string& key) // value found, cache the result if (value) { - mCache[key] = ConfigurationRecord(value); + mCache[key] = ConfigurationRecord(key,value); // key definition found, cache the default } else if (keyDefinedInSchema(key)) { - mCache[key] = ConfigurationRecord(mSchema[key].getDefaultValue()); + mCache[key] = ConfigurationRecord(key,mSchema[key].getDefaultValue()); // total miss, cache the error } else { - mCache[key] = ConfigurationRecord(false); + mCache[key] = ConfigurationRecord(key,false); throw ConfigurationTableKeyNotFound(key); } @@ -749,9 +858,11 @@ ConfigurationRecordMap ConfigurationTable::getAllPairs() const const char* key = (const char*)sqlite3_column_text(stmt,0); const char* value = (const char*)sqlite3_column_text(stmt,1); if (key && value) { - tmp[string(key)] = ConfigurationRecord(value); + string skey(key); + tmp[skey] = ConfigurationRecord(skey,value); } else if (key && !value) { - tmp[string(key)] = ConfigurationRecord(false); + string skey(key); + tmp[skey] = ConfigurationRecord(skey,false); } src = sqlite3_run_query(mDB,stmt); } @@ -773,7 +884,7 @@ bool ConfigurationTable::set(const string& key, const string& value) bool success = sqlite3_command(mDB,cmd.c_str()); // Cache the result. - if (success) mCache[key] = ConfigurationRecord(value); + if (success) mCache[key] = ConfigurationRecord(key,value); return success; } @@ -944,55 +1055,6 @@ template bool ConfigurationKey::isInValRange(const ConfigurationKey &ke return ret; } -const std::string ConfigurationKey::getARFCNsString() { - stringstream ss; - int i; - float downlink; - float uplink; - - // 128:251 GSM850 - downlink = 869.2; - uplink = 824.2; - for (i = 128; i <= 251; i++) { - ss << i << "|GSM850 #" << i << " : " << downlink << " MHz downlink / " << uplink << " MHz uplink,"; - downlink += 0.2; - uplink += 0.2; - } - - // 0:124 EGSM900 - downlink = 935.0; - uplink = 890.0; - for (i = 0; i <= 124; i++) { - ss << i << "|PGSM900 #" << i << " : " << (downlink + (0.2*i)) << " MHz downlink / " << (uplink + (0.2*i)) << " MHz uplink,"; - } - // 975:1023 EGSM900 - for (i = 975; i <= 1023 ; i++) { - ss << i << "|PGSM900 #" << i << " : " << (downlink + (0.2*i)) << " MHz downlink / " << (uplink + (0.2*i)) << " MHz uplink,"; - } - - // 512:885 DCS1800 - downlink = 1805.2; - uplink = 1710.2; - for (i = 512; i <= 885; i++) { - ss << i << "|DCS1800 #" << i << " : " << downlink << " MHz downlink / " << uplink << " MHz uplink,"; - downlink += 0.2; - uplink += 0.2; - } - - // 512:810 PCS1900 - downlink = 1930.2; - uplink = 1850.2; - for (i = 512; i <= 810; i++) { - ss << i << "|PCS1900 #" << i << " : " << downlink << " MHz downlink / " << uplink << " MHz uplink,"; - downlink += 0.2; - uplink += 0.2; - } - - ss << endl; - - return ss.str(); -} - const std::string ConfigurationKey::visibilityLevelToString(const ConfigurationKey::VisibilityLevel& visibility) { std::string ret = "UNKNOWN ERROR"; @@ -1045,6 +1107,12 @@ const std::string ConfigurationKey::typeToString(const ConfigurationKey::Type& t case FILEPATH: ret = "file path"; break; + case HOSTANDPORT_OPT: + ret = "hostname or IP address and port (optional)"; + break; + case HOSTANDPORT: + ret = "hostname or IP address and port"; + break; case IPADDRESS_OPT: ret = "IP address (optional)"; break; @@ -1101,6 +1169,7 @@ void ConfigurationKey::printKey(const ConfigurationKey &key, const std::string& void ConfigurationKey::printDescription(const ConfigurationKey &key, ostream& os) { std::string tmp; + unsigned scope; os << " - description: " << key.getDescription() << std::endl; if (key.getUnits().length()) { @@ -1155,6 +1224,28 @@ void ConfigurationKey::printDescription(const ConfigurationKey &key, ostream& os } else if (key.getValidValues().length()) { os << " - raw valid values: " << tmp << std::endl; } + + scope = key.getScope(); + if (scope) { + if (scope & ConfigurationKey::GLOBALLYUNIQUE) { + os << " - scope: value must be unique across all nodes" << std::endl; + } + if (scope & ConfigurationKey::GLOBALLYSAME) { + os << " - scope: value must be the same across all nodes" << std::endl; + } + if (scope & ConfigurationKey::NEIGHBORSUNIQUE) { + os << " - scope: value must be unique across all neighbors" << std::endl; + } + if (scope & ConfigurationKey::NEIGHBORSSAME) { + os << " - scope: value must be the same across all neighbors" << std::endl; + } + if (scope & ConfigurationKey::NEIGHBORSSAME) { + os << " - scope: must be the same across all neighbors" << std::endl; + } + if (scope & ConfigurationKey::NODESPECIFIC) { + os << " - scope: specfic to each individual node" << std::endl; + } + } } diff --git a/Configuration.h b/Configuration.h index 959cdf8..3a51622 100644 --- a/Configuration.h +++ b/Configuration.h @@ -1,7 +1,7 @@ /* * Copyright 2009, 2010 Free Software Foundation, Inc. * Copyright 2010 Kestrel Signal Processing, Inc. -* Copyright 2011, 2012 Range Networks, Inc. +* Copyright 2011, 2012, 2014 Range Networks, Inc. * * This software is distributed under the terms of the GNU Affero Public License. * See the COPYING file in the main directory for details. @@ -69,35 +69,45 @@ class ConfigurationTableKeyNotFound : public ConfigurationTableError { }; +// (pat) 10-5-2013 Add the key to this record so we can print better error messages, +// and warn if number() of floatNumber() have non-numeric values. class ConfigurationRecord { private: + std::string mCRKey; std::string mValue; - long mNumber; bool mDefined; + mutable bool mCRWarned; public: - ConfigurationRecord(bool wDefined=true): - mDefined(wDefined) + ConfigurationRecord() : mDefined(false), mCRWarned(false) {} + ConfigurationRecord(const std::string &key,bool wDefined): + mCRKey(key), + mDefined(wDefined), + mCRWarned(false) { } - ConfigurationRecord(const std::string& wValue): + ConfigurationRecord(const std::string&key, const std::string& wValue): + mCRKey(key), mValue(wValue), - mNumber(strtol(wValue.c_str(),NULL,0)), - mDefined(true) + //mNumber(strtol(wValue.c_str(),&endptr,0)), + mDefined(true), + mCRWarned(false) { } - ConfigurationRecord(const char* wValue): + ConfigurationRecord(const std::string&key, const char* wValue): + mCRKey(key), mValue(std::string(wValue)), - mNumber(strtol(wValue,NULL,0)), - mDefined(true) + //mNumber(strtol(wValue.c_str(),&endptr,0)), + mDefined(true), + mCRWarned(false) { } const std::string& value() const { return mValue; } - long number() const { return mNumber; } + long number() const; bool defined() const { return mDefined; } float floatNumber() const; @@ -193,6 +203,7 @@ class ConfigurationTable { ConfigurationKeyMap mSchema;///< definition of configuration default values and validation logic + // (pat) filename is the sql file name, wCmdName is the name of the executable we are running, used for better error messages. ConfigurationTable(const char* filename = ":memory:", const char *wCmdName = 0, ConfigurationKeyMap wSchema = ConfigurationKeyMap()); /** Generate an up-to-date example sql file for new installs. */ @@ -350,6 +361,8 @@ class ConfigurationKey { CIDR, FILEPATH_OPT, FILEPATH, + HOSTANDPORT_OPT, + HOSTANDPORT, IPADDRESS_OPT, IPADDRESS, IPANDPORT, @@ -361,7 +374,17 @@ class ConfigurationKey { REGEX, STRING_OPT, STRING, - VALRANGE + VALRANGE // (pat) string format is: ':' [ '(' ')' ] + // step is not currently enforced. + }; + + enum Scope + { + GLOBALLYUNIQUE = 1, + GLOBALLYSAME = 2, + NEIGHBORSUNIQUE = 4, + NEIGHBORSSAME = 8, + NODESPECIFIC = 16 }; private: @@ -374,11 +397,12 @@ class ConfigurationKey { std::string mValidValues; bool mIsStatic; std::string mDescription; + unsigned mScope; public: - ConfigurationKey(const std::string& wName, const std::string& wDefaultValue, const std::string& wUnits, const VisibilityLevel wVisibility, const Type wType, const std::string& wValidValues, bool wIsStatic, const std::string& wDescription): + ConfigurationKey(const std::string& wName, const std::string& wDefaultValue, const std::string& wUnits, const VisibilityLevel wVisibility, const Type wType, const std::string& wValidValues, bool wIsStatic, const std::string& wDescription, unsigned wScope = 0): mName(wName), mDefaultValue(wDefaultValue), mUnits(wUnits), @@ -386,7 +410,8 @@ class ConfigurationKey { mType(wType), mValidValues(wValidValues), mIsStatic(wIsStatic), - mDescription(wDescription) + mDescription(wDescription), + mScope(wScope) { } ConfigurationKey() @@ -396,12 +421,14 @@ class ConfigurationKey { const std::string& getDefaultValue() const { return mDefaultValue; } void updateDefaultValue(const std::string& newValue) { mDefaultValue = newValue; } void updateDefaultValue(const int newValue) { std::stringstream ss; ss << newValue; updateDefaultValue(ss.str()); } + void updateValidValues(const std::string& newValue) { mValidValues = newValue; } const std::string& getUnits() const { return mUnits; } const VisibilityLevel& getVisibility() const { return mVisibility; } const Type& getType() const { return mType; } const std::string& getValidValues() const { return mValidValues; } bool isStatic() const { return mIsStatic; } const std::string& getDescription() const { return mDescription; } + unsigned getScope() const { return mScope; } static bool isValidIP(const std::string& ip); static void getMinMaxStepping(const ConfigurationKey &key, std::string &min, std::string &max, std::string &stepping); @@ -410,7 +437,6 @@ class ConfigurationKey { static const std::string typeToString(const ConfigurationKey::Type& type); static void printKey(const ConfigurationKey &key, const std::string& currentValue, std::ostream& os); static void printDescription(const ConfigurationKey &key, std::ostream& os); - static const std::string getARFCNsString(); }; diff --git a/ConfigurationTest.cpp b/ConfigurationTest.cpp index 290db64..3d8014b 100644 --- a/ConfigurationTest.cpp +++ b/ConfigurationTest.cpp @@ -47,7 +47,7 @@ int main(int argc, char *argv[]) gConfig.setUpdateHook(purgeConfig); - char *keys[5] = {"key1", "key2", "key3", "key4", "key5"}; + const char *keys[5] = {"key1", "key2", "key3", "key4", "key5"}; for (int i=0; i<5; i++) { gConfig.set(keys[i],i); diff --git a/Defines.h b/Defines.h new file mode 100644 index 0000000..5fb431d --- /dev/null +++ b/Defines.h @@ -0,0 +1,52 @@ +// Pat added this file. +// We need an include file that is included before any other include files. +// Might I suggest that Range Networks specific global #defines be prefixed with RN_ + +#ifndef DEFINES_H +#define DEFINES_H + +#define GPRS_ENCODER 1 // Use SharedL1Encoder and SharedL1Decoder +#define GPRS_TESTSI4 1 +#define GPRS_TEST 1 // Compile in other GPRS stuff. +#define GPRS_PAT 1 // Compile in GPRS code. Turn this off to get previous non-GRPS code, + // although I am not maintaining it so you may have to fix compile + // problems to use it. + + +// (pat) This removes the constness from a pointer, eg: const T *barfo; T *notsobarfo = Unconst(barfo); +template +T* Unconst(const T*foo) { return const_cast(foo); } + +// (pat) Like assert() but dont core dump unless we are testing. +// Note: to use this macro you must include the dependencies first. +#define devassert(code) {if (IS_LOG_LEVEL(DEBUG)) {assert(code);} else if (!(code)) {LOG(ERR)<<"assertion failed:"<< #code;}} + +// __GNUG__ is true for g++ and __GNUC__ for gcc. +#if __GNUC__&0==__GNUG__ + +#define RN_UNUSED __attribute__((unused)) + +#define RN_UNUSED_PARAM(var) RN_UNUSED var + +// Pack structs onto byte boundaries. +// Note that if structs are nested, this must appear on all of them. +#define RN_PACKED __attribute__((packed)) + +#else + +// Suppress warning message about a variable or function being unused. +// In C++ you can leave out the variable name to suppress the 'unused variable' warning. +#define RN_UNUSED_PARAM(var) /*nothing*/ +#define RN_UNUSED /*not defined*/ +#define RN_PACKED /*not defined*/ +#endif + +// Bound value between min and max values. +#define RN_BOUND(value,min,max) ( (value)<(min) ? (min) : (value)>(max) ? (max) : (value) ) + +#define RN_PRETTY_TEXT(name) (" " #name "=(") << name << ")" +#define RN_PRETTY_TEXT1(name) (" " #name "=") << name +#define RN_WRITE_TEXT(name) os << RN_PRETTY_TEXT(name) +#define RN_WRITE_OPT_TEXT(name,flag) if (flag) { os << RN_WRITE_TEXT(name); } + +#endif diff --git a/Interthread.h b/Interthread.h index 42e6f7f..35294a2 100644 --- a/Interthread.h +++ b/Interthread.h @@ -26,12 +26,14 @@ #ifndef INTERTHREAD_H #define INTERTHREAD_H +#include "Defines.h" #include "Timeval.h" #include "Threads.h" #include "LinkedLists.h" #include #include #include +#include @@ -49,27 +51,258 @@ // or SingleLinkedList, which implements the queue using an internal pointer in type T, // which must implement the functional interface of class SingleLinkListNode, // namely: functions T*next() and void setNext(T*). -template class InterthreadQueue { +#if UNUSED +//template class InterthreadQueue { +// +// protected: +// +// Fifo mQ; +// mutable Mutex mLock; +// mutable Signal mWriteSignal; +// +// public: +// +// /** Delete contents. */ +// void clear() +// { +// ScopedLock lock(mLock); +// while (mQ.size()>0) delete (T*)mQ.get(); +// } +// +// /** Empty the queue, but don't delete. */ +// void flushNoDelete() +// { +// ScopedLock lock(mLock); +// while (mQ.size()>0) mQ.get(); +// } +// +// +// ~InterthreadQueue() +// { clear(); } +// +// +// size_t size() const +// { +// ScopedLock lock(mLock); +// return mQ.size(); +// } +// +// size_t totalSize() const // pat added +// { +// ScopedLock lock(mLock); +// return mQ.totalSize(); +// } +// +// /** +// Blocking read. +// @return Pointer to object (will not be NULL). +// */ +// T* read() +// { +// ScopedLock lock(mLock); +// T* retVal = (T*)mQ.get(); +// while (retVal==NULL) { +// mWriteSignal.wait(mLock); +// retVal = (T*)mQ.get(); +// } +// return retVal; +// } +// +// /** Non-blocking peek at the first element; returns NULL if empty. */ +// T* front() +// { +// ScopedLock lock(mLock); +// return (T*) mQ.front(); +// } +// +// /** +// Blocking read with a timeout. +// @param timeout The read timeout in ms. +// @return Pointer to object or NULL on timeout. +// */ +// T* read(unsigned timeout) +// { +// if (timeout==0) return readNoBlock(); +// Timeval waitTime(timeout); +// ScopedLock lock(mLock); +// while ((mQ.size()==0) && (!waitTime.passed())) +// mWriteSignal.wait(mLock,waitTime.remaining()); +// T* retVal = (T*)mQ.get(); +// return retVal; +// } +// +// /** +// Non-blocking read. +// @return Pointer to object or NULL if FIFO is empty. +// */ +// T* readNoBlock() +// { +// ScopedLock lock(mLock); +// return (T*)mQ.get(); +// } +// +// /** Non-blocking write. */ +// void write(T* val) +// { +// ScopedLock lock(mLock); +// mQ.put(val); +// mWriteSignal.signal(); +// } +// +// /** Non-block write to the front of the queue. */ +// void write_front(T* val) // pat added +// { +// ScopedLock lock(mLock); +// mQ.push_front(val); +// mWriteSignal.signal(); +// } +//}; +#endif + +// A list designed for pointers so we can use get methods that return NULL on error. +template +class PtrList : public std::list { + //typedef typename std::list type_t; + public: + typedef typename std::list::iterator iter_t; + // Like pop_front but return the value, or NULL if none. + T* pop_frontr() { + if (this->empty()) { return NULL; } + T* result = this->front(); + this->pop_front(); + return result; + } + // Like pop_back but return the value, or NULL if none. + T* pop_backr() { + if (this->empty()) { return NULL; } + T* result = this->back(); + this->pop_back(); + return result; + } + + // These functions are solely for use by InterthreadQueue, necessitated by backward compatibility with PointerFIFO. + void put(T*v) { this->push_back(v); } + T* get() { return this->pop_frontr(); } +}; + + + +// (pat 8-2013) The scoped iterators are a great idea and worked fine, but it is not KISS, so I stopped using it. +// If you want to iterate through an InterthreadQueue or InterthreadMap, just get the lock using the appropropriate qGetLock method. +#if USE_SCOPED_ITERATORS +// This ScopedIterator locks the container for as long as the iterator exists. +// You have to create the ScopedIterator using the thread-safe container object, example: +// ThreadSafeMap mymap; +// ScopedIterator it(mymap); +// for (it = mymap.begin(); it != mymap.end(); it++) {} +// Then begin() and end() are passed through to the normal underlying iterator. +// An alternative implementation would have been to modify begin and end to return scoped iterators, +// but that is more complicated and would probably require multiple locks/unlocks on the Mutex. +template +class ScopedIteratorTemplate : public BaseType::iterator { + Mutex &mLockRef; + public: + ScopedIteratorTemplate(DerivedType &wOwner) : mLockRef(wOwner.qGetLock()) { mLockRef.lock(); } + ~ScopedIteratorTemplate() { mLockRef.unlock(); } + void operator=(typename BaseType::iterator it) { this->BaseType::iterator::operator=(it); } + void operator++() { this->BaseType::iterator::operator++(); } // ++prefix + void operator++(int) { this->BaseType::iterator::operator++(0); } // postfix++ + ValueType& operator*() { return this->BaseType::iterator::operator*(); } + ValueType* operator->() { return this->BaseType::iterator::operator->(); } +}; +#endif + + + +// (pat) There was a transition period from the old to the new InterthreadQueue when the new +// one was named InterthreadQueue2, and that still exists in some versions of the SGSN/GPRS code. +#define InterthreadQueue2 InterthreadQueue + +// (pat) The original InterthreadQueue had a complicated threading problem that this version fixed. +// I started out using this new version only in GPRS and SGSN, for fear of breaking something in GSM, +// but in release 4 I removed the old version above. +// 5-2013: Changed the ultimate base class to a PtrList and added ScopedIterator. +// 8-2013: Removed the ScopedIterator even though it is an elegant solution, because it is easy to +// just use the internal lock directly when one needs to iterate through one of these. +//template class InterthreadQueue { +template > class InterthreadQueue { + //protected: + + Fifo mQ; + // (pat) DO NOT USE mLock and mWriteSignal; instead use mLockPointer and mWriteSignalPointer. + // That allows us to connect two InterthreadQueue together such that a single thread can wait on either. + mutable Mutex mLock, *mLockPointer; + mutable Signal mWriteSignal, *mWriteSignalPointer; protected: - Fifo mQ; - mutable Mutex mLock; - mutable Signal mWriteSignal; - public: + InterthreadQueue() : mLockPointer(&mLock), mWriteSignalPointer(&mWriteSignal) {} + + // This connects the two InterthreadQueue permanently so they use the same lock and Signal. + // Subsequently you can use iqWaitForEither. + void iqConnect(InterthreadQueue &other) { + mLockPointer = other.mLockPointer; + mWriteSignalPointer = other.mWriteSignalPointer; + } + + Mutex &qGetLock() { return mLock; } +#if USE_SCOPED_ITERATORS + // The Iterator locks the InterthreadQueue until the Iterator falls out of scope. + // Semantics are different from normal C++ iterators - the begin,end,erase methods are in + // the Iterator, not the base type. + // Use like this: + // InterthreadQueue::ScopedIterator sit(someInterthreadQueue); + // for (T*val = sit.front(); val = *sit; sit++) ... + // if (something) val.erase(); + typedef typename Fifo::iterator iterator; + class ScopedIterator { + typedef InterthreadQueue BaseType_t; + typedef typename Fifo::iterator iterator_t; + BaseType_t &mParent; + iterator_t mit; + + public: + ScopedIterator(BaseType_t&wParent) : mParent(wParent) { mParent.mLockPointer->lock(); } + ~ScopedIterator() { mParent.mLockPointer->unlock(); } + + // Regular old iterators in case you want to use em. + iterator_t begin() { return mParent.mQ.begin(); } + iterator_t end() { return mParent.mQ.end(); } + + // Accessors and operators. Accessors move the iterator, eg, using front() rewinds iter to begin(). + T* current() { return mit != end() ? *mit : NULL; } + T* front() { mit = begin(); return current(); } + T* next() { if (mit != end()) { ++mit; } return current(); } + // Erase current element and advance the iterator forward. + void erase() { if (mit != end()) mit = mParent.mQ.erase(mit); } + T* operator++() { return next(); } // prefix ++ + T* operator++(int) { T*result = current(); next(); return result; } // postfix ++ + T* operator*() { return current(); } + + // And here is random access in case you want it. + // Note that this is inefficient, so dont use it unless you know the queue is small. + // This is inside ScopedIterator so that the entire InterthreadQueue is locked while you do whatever it is you are doing. + // Eg: { InterthreadQueue::ScopedIterator sit(myinterthreadqueue); for (unsigned i=0; i<10; i++) { T*foo = sit[i]; ... } } + T* operator[](unsigned ind) { + unsigned i = 0; + for (iterator_t itr = begin(); itr != end(); itr++) { if (i++ == ind) return *itr; } + return NULL; // Out of bounds. + } + }; +#endif /** Delete contents. */ void clear() { - ScopedLock lock(mLock); + ScopedLock lock(*mLockPointer); while (mQ.size()>0) delete (T*)mQ.get(); } /** Empty the queue, but don't delete. */ void flushNoDelete() { - ScopedLock lock(mLock); + ScopedLock lock(*mLockPointer); while (mQ.size()>0) mQ.get(); } @@ -80,26 +313,49 @@ template class InterthreadQueue { size_t size() const { - ScopedLock lock(mLock); + ScopedLock lock(*mLockPointer); return mQ.size(); } size_t totalSize() const // pat added { - ScopedLock lock(mLock); + ScopedLock lock(*mLockPointer); return mQ.totalSize(); } + // Wait for something on either of the two queues connected by iqConnect. Kind of hokey, but it works. Timeout is in msecs. + void iqWaitForEither(InterthreadQueue &other, unsigned timeout) { + ScopedLock lock(*mLockPointer); + if (timeout) { + Timeval waitTime(timeout); + while (mQ.size() == 0 && other.mQ.size() == 0) { + mWriteSignalPointer->wait(*mLockPointer,waitTime.remaining()); + } + } else { // Wait forever. + while (mQ.size() == 0 && other.mQ.size() == 0) { + mWriteSignalPointer->wait(*mLockPointer); + } + } + } + + // (pat 8-2013) Removed. Bad idea to use this name - conflicts with wait() in InterthreadQueueWithWait + //void wait() { // (pat 7-25-2013) Added. Wait for something to appear in the queue. + // ScopedLock lock(*mLockPointer); + // while (mQ.size() == 0) { + // mWriteSignalPointer->wait(*mLockPointer); + // } + //} + /** - Blocking read. + Blocking read from back of queue. @return Pointer to object (will not be NULL). */ T* read() { - ScopedLock lock(mLock); + ScopedLock lock(*mLockPointer); T* retVal = (T*)mQ.get(); while (retVal==NULL) { - mWriteSignal.wait(mLock); + mWriteSignalPointer->wait(*mLockPointer); retVal = (T*)mQ.get(); } return retVal; @@ -108,7 +364,7 @@ template class InterthreadQueue { /** Non-blocking peek at the first element; returns NULL if empty. */ T* front() { - ScopedLock lock(mLock); + ScopedLock lock(*mLockPointer); return (T*) mQ.front(); } @@ -121,131 +377,29 @@ template class InterthreadQueue { { if (timeout==0) return readNoBlock(); Timeval waitTime(timeout); - ScopedLock lock(mLock); - while ((mQ.size()==0) && (!waitTime.passed())) - mWriteSignal.wait(mLock,waitTime.remaining()); - T* retVal = (T*)mQ.get(); - return retVal; - } - - /** - Non-blocking read. - @return Pointer to object or NULL if FIFO is empty. - */ - T* readNoBlock() - { - ScopedLock lock(mLock); - return (T*)mQ.get(); - } - - /** Non-blocking write. */ - void write(T* val) - { - ScopedLock lock(mLock); - mQ.put(val); - mWriteSignal.signal(); - } - - /** Non-block write to the front of the queue. */ - void write_front(T* val) // pat added - { - ScopedLock lock(mLock); - mQ.push_front(val); - mWriteSignal.signal(); - } -}; - -// (pat) Identical to above but with the threading problem fixed. -template class InterthreadQueue2 { - - protected: - - Fifo mQ; - mutable Mutex mLock; - mutable Signal mWriteSignal; - - public: - - /** Delete contents. */ - void clear() - { - ScopedLock lock(mLock); - while (mQ.size()>0) delete (T*)mQ.get(); - } - - /** Empty the queue, but don't delete. */ - void flushNoDelete() - { - ScopedLock lock(mLock); - while (mQ.size()>0) mQ.get(); - } - - - ~InterthreadQueue2() - { clear(); } - - - size_t size() const - { - ScopedLock lock(mLock); - return mQ.size(); - } - - size_t totalSize() const // pat added - { - ScopedLock lock(mLock); - return mQ.totalSize(); - } - - /** - Blocking read. - @return Pointer to object (will not be NULL). - */ - T* read() - { - ScopedLock lock(mLock); - T* retVal = (T*)mQ.get(); - while (retVal==NULL) { - mWriteSignal.wait(mLock); - retVal = (T*)mQ.get(); + ScopedLock lock(*mLockPointer); + while (mQ.size()==0) { + long remaining = waitTime.remaining(); + // (pat) How high to we expect the precision here to be? I dont think they are used a precision timers, + // so dont try to wait if the remainder is just a few msecs. + if (remaining < 2) { return NULL; } + mWriteSignalPointer->wait(*mLockPointer,remaining); } - return retVal; - } - - /** Non-blocking peek at the first element; returns NULL if empty. */ - T* front() - { - ScopedLock lock(mLock); - return (T*) mQ.front(); - } - - /** - Blocking read with a timeout. - @param timeout The read timeout in ms. - @return Pointer to object or NULL on timeout. - */ - T* read(unsigned timeout) - { - if (timeout==0) return readNoBlock(); - Timeval waitTime(timeout); - ScopedLock lock(mLock); - while ((mQ.size()==0) && (!waitTime.passed())) - mWriteSignal.wait(mLock,waitTime.remaining()); T* retVal = (T*)mQ.get(); return retVal; } /** - Non-blocking read. + Non-blocking read. aka pop_front. @return Pointer to object or NULL if FIFO is empty. */ T* readNoBlock() { - ScopedLock lock(mLock); + ScopedLock lock(*mLockPointer); return (T*)mQ.get(); } - /** Non-blocking write. */ + /** Non-blocking write. aka push_back */ void write(T* val) { // (pat) The Mutex mLock must be released before signaling the mWriteSignal condition. @@ -257,26 +411,27 @@ template class InterthreadQueue2 { // until the read thread's accumulated temporary priority causes it to // get a second pre-emptive activation over the writing thread, // resulting in bursts of activity by the read thread. - { ScopedLock lock(mLock); + { ScopedLock lock(*mLockPointer); mQ.put(val); } - mWriteSignal.signal(); + mWriteSignalPointer->signal(); } - /** Non-block write to the front of the queue. */ + /** Non-block write to the front of the queue. aka push_front */ void write_front(T* val) // pat added { // (pat) See comments above. - { ScopedLock lock(mLock); + { ScopedLock lock(*mLockPointer); mQ.push_front(val); } - mWriteSignal.signal(); + mWriteSignalPointer->signal(); } }; /** Pointer FIFO for interthread operations. */ +// Pat thinks this should be combined with InterthreadQueue by simply moving the wait method there. template class InterthreadQueueWithWait { protected: @@ -336,8 +491,16 @@ template class InterthreadQueueWithWait { if (timeout==0) return readNoBlock(); Timeval waitTime(timeout); ScopedLock lock(mLock); - while ((mQ.size()==0) && (!waitTime.passed())) - mWriteSignal.wait(mLock,waitTime.remaining()); + // (pat 8-2013) This commented out code has a deadlock problem. + //while ((mQ.size()==0) && (!waitTime.passed())) + // mWriteSignal.wait(mLock,waitTime.remaining()); + while (mQ.size()==0) { + long remaining = waitTime.remaining(); + // (pat) How high to we expect the precision here to be? I dont think they are used a precision timers, + // so dont try to wait if the remainder is just a few msecs. + if (remaining < 2) { return NULL; } + mWriteSignal.wait(mLock,remaining); + } T* retVal = (T*)mQ.get(); if (retVal!=NULL) mReadSignal.signal(); return retVal; @@ -382,13 +545,16 @@ template class InterthreadQueueWithWait { /** Thread-safe map of pointers to class D, keyed by class K. */ -template class InterthreadMap { +template class InterthreadMap +{ +public: + typedef std::map Map; // (pat) It would be more generally useful if this were D instead of D* protected: - typedef std::map Map; Map mMap; mutable Mutex mLock; + //mutable Mutex mModificationLock; // (pat) Lock prevents deletion from the map while an iterator is active. Signal mWriteSignal; public: @@ -396,6 +562,7 @@ public: void clear() { // Delete everything in the map. + //ScopedLock lock1(mModificationLock); // Lock this first! ScopedLock lock(mLock); typename Map::iterator iter = mMap.begin(); while (iter != mMap.end()) { @@ -408,12 +575,13 @@ public: ~InterthreadMap() { clear(); } /** - Non-blocking write. + Non-blocking write. WARNING: This deletes any pre-existing element! @param key The index to write to. @param wData Pointer to data, not to be deleted until removed from the map. */ void write(const K &key, D * wData) { + //ScopedLock lock1(mModificationLock); // Lock this first! ScopedLock lock(mLock); typename Map::iterator iter = mMap.find(key); if (iter!=mMap.end()) { @@ -426,17 +594,17 @@ public: } /** - Non-blocking read with element removal. + Identical to readNoBlock but with element removal. @param key Key to read from. @return Pointer at key or NULL if key not found, to be deleted by caller. */ - D* getNoBlock(const K& key) + D* getNoBlock(const K& key, bool bRemove = true) { ScopedLock lock(mLock); typename Map::iterator iter = mMap.find(key); if (iter==mMap.end()) return NULL; D* retVal = iter->second; - mMap.erase(iter); + if (bRemove) { mMap.erase(iter); } return retVal; } @@ -446,20 +614,32 @@ public: @param timeout The blocking timeout in ms. @return Pointer at key or NULL on timeout, to be deleted by caller. */ - D* get(const K &key, unsigned timeout) + D* get(const K &key, unsigned timeout, bool bRemove = true) { - if (timeout==0) return getNoBlock(key); - Timeval waitTime(timeout); + if (timeout==0) return getNoBlock(key,bRemove); ScopedLock lock(mLock); - typename Map::iterator iter = mMap.find(key); - while ((iter==mMap.end()) && (!waitTime.passed())) { - mWriteSignal.wait(mLock,waitTime.remaining()); - iter = mMap.find(key); + Timeval waitTime(timeout); + // (pat 8-2013) This commented out code suffered from a deadlock problem. + //typename Map::iterator iter = mMap.find(key); + //while ((iter==mMap.end()) && (!waitTime.passed())) { + // mWriteSignal.wait(mLock,waitTime.remaining()); + // iter = mMap.find(key); + //} + //if (iter==mMap.end()) return NULL; + //D* retVal = iter->second; + //mMap.erase(iter); + //return retVal; + while (1) { + typename Map::iterator iter = mMap.find(key); + if (iter!=mMap.end()) { + D* retVal = iter->second; + if (bRemove) { mMap.erase(iter); } + return retVal; + } + long remaining = waitTime.remaining(); + if (remaining < 2) { return NULL; } + mWriteSignal.wait(mLock,remaining); } - if (iter==mMap.end()) return NULL; - D* retVal = iter->second; - mMap.erase(iter); - return retVal; } /** @@ -467,7 +647,7 @@ public: @param key The key to read from. @return Pointer at key, to be deleted by caller. */ - D* get(const K &key) + D* get(const K &key, bool bRemove = true) { ScopedLock lock(mLock); typename Map::iterator iter = mMap.find(key); @@ -476,7 +656,7 @@ public: iter = mMap.find(key); } D* retVal = iter->second; - mMap.erase(iter); + if (bRemove) { mMap.erase(iter); } return retVal; } @@ -485,9 +665,11 @@ public: Remove an entry and delete it. @param key The key of the entry to delete. @return True if it was actually found and deleted. + (pat) If you just want remove without deleting, see getNoBlock. */ bool remove(const K &key ) { + //ScopedLock lock(mModificationLock); D* val = getNoBlock(key); if (!val) return false; delete val; @@ -496,17 +678,19 @@ public: /** - Non-blocking read. + Non-blocking read. (pat) Actually, it blocks until the map is available. @param key Key to read from. @return Pointer at key or NULL if key not found. */ D* readNoBlock(const K& key) const { - D* retVal=NULL; - ScopedLock lock(mLock); - typename Map::const_iterator iter = mMap.find(key); - if (iter!=mMap.end()) retVal = iter->second; - return retVal; + return Unconst(this)->getNoBlock(key,false); + // (pat 8-2013) Lets use common code. + //D* retVal=NULL; + //ScopedLock lock(mLock); + //typename Map::const_iterator iter = mMap.find(key); + //if (iter!=mMap.end()) retVal = iter->second; + //return retVal; } /** @@ -517,36 +701,56 @@ public: */ D* read(const K &key, unsigned timeout) const { - if (timeout==0) return readNoBlock(key); - ScopedLock lock(mLock); - Timeval waitTime(timeout); - typename Map::const_iterator iter = mMap.find(key); - while ((iter==mMap.end()) && (!waitTime.passed())) { - mWriteSignal.wait(mLock,waitTime.remaining()); - iter = mMap.find(key); - } - if (iter==mMap.end()) return NULL; - D* retVal = iter->second; - return retVal; + return Unconst(this)->get(key,timeout,false); + // (pat 8-2013) This commented out code suffered from a deadlock problem. + //if (timeout==0) return readNoBlock(key); + //ScopedLock lock(mLock); + //Timeval waitTime(timeout); + //typename Map::const_iterator iter = mMap.find(key); + //while ((iter==mMap.end()) && (!waitTime.passed())) { + // mWriteSignal.wait(mLock,waitTime.remaining()); + // iter = mMap.find(key); + //} + //if (iter==mMap.end()) return NULL; + //D* retVal = iter->second; + //return retVal; } /** - Blocking read. + Blocking read. Blocks until the key exists. @param key The key to read from. @return Pointer at key. */ D* read(const K &key) const { - ScopedLock lock(mLock); - typename Map::const_iterator iter = mMap.find(key); - while (iter==mMap.end()) { - mWriteSignal.wait(mLock); - iter = mMap.find(key); - } - D* retVal = iter->second; - return retVal; + return Unconst(this)->get(key,false); + // (pat 8-2013) Lets use common code. + //ScopedLock lock(mLock); + //typename Map::const_iterator iter = mMap.find(key); + //while (iter==mMap.end()) { + // mWriteSignal.wait(mLock); + // iter = mMap.find(key); + //} + //D* retVal = iter->second; + //return retVal; } + // pat added. + unsigned size() const { return mMap.size(); } + + //Mutex &getModificationLock() { return mModificationLock; } + //void iterLock() { mModificationLock.lock(); } + //void iterUnlock() { mModificationLock.unlock(); } + +#if USE_SCOPED_ITERATORS + typedef ScopedIteratorTemplate,std::pair > ScopedIterator; +#endif + // WARNING: These iterators are not intrinsically thread safe. + // Caller must use ScopiedIterator or the modification lock or enclose the entire iteration in some higher level lock. + Mutex &qGetLock() { return mLock; } + typedef typename Map::iterator iterator; + typename Map::iterator begin() { return mMap.begin(); } + typename Map::iterator end() { return mMap.end(); } }; diff --git a/InterthreadTest.cpp b/InterthreadTest.cpp index 03445d9..552c1ee 100644 --- a/InterthreadTest.cpp +++ b/InterthreadTest.cpp @@ -28,6 +28,8 @@ #include "Threads.h" #include "Interthread.h" #include +#include "Configuration.h" +ConfigurationTable gConfig; using namespace std; diff --git a/LinkedLists.h b/LinkedLists.h index 31fb9c5..95fdc27 100644 --- a/LinkedLists.h +++ b/LinkedLists.h @@ -74,13 +74,13 @@ class PointerFIFO { unsigned size() const { return mSize; } unsigned totalSize() const { return 0; } // Not used in this version. - /** Put an item into the FIFO at the back of the queue. */ + /** Put an item into the FIFO at the back of the queue. aka push_back */ void put(void* val); /** Push an item on the front of the FIFO. */ void push_front(void*val); // pat added. /** - Take an item from the FIFO. + Take an item from the FIFO. aka pop_front, but returns NULL Returns NULL for empty list. */ void* get(); @@ -123,6 +123,7 @@ class SingleLinkList unsigned mTotalSize; // Total of size() method of elements in list. public: + typedef void iterator; // Does not exist for this class, but needs to be defined. SingleLinkList() : mHead(0), mTail(0), mSize(0), mTotalSize(0) {} unsigned size() const { return mSize; } unsigned totalSize() const { return mTotalSize; } diff --git a/Logger.cpp b/Logger.cpp index 06e91f6..fc95ce0 100644 --- a/Logger.cpp +++ b/Logger.cpp @@ -1,7 +1,7 @@ /* * Copyright 2009, 2010 Free Software Foundation, Inc. * Copyright 2010 Kestrel Signal Processing, Inc. -* Copyright 2011, 2012 Range Networks, Inc. +* Copyright 2011, 2012, 2014 Range Networks, Inc. * * * This software is distributed under the terms of the GNU Affero Public License. @@ -70,6 +70,7 @@ int numLevels = 8; bool gLogToConsole = 0; FILE *gLogToFile = NULL; Mutex gLogToLock; +LogGroup gLogGroup; int levelStringToInt(const string& name) @@ -118,6 +119,13 @@ int getLoggingLevel(const char* filename) return lookupLevel("Log.Level"); } +//bool gCheckGroupLogLevel(const char *groupname, int loglevel) +//{ +// // Gag me +// string keyName = string("Log.Group.") + groupname; +// return gConfig.defines(keyName) ? (lookupLevel(gConfig.getStr(keyName)) >= loglevel) : false; +//} + int gGetLoggingLevel(const char* filename) @@ -220,8 +228,17 @@ Log::~Log() } +// (pat) This is the log initialization function. +// It is invoked by this line in OpenBTS.cpp, and similar lines in other programs like the TransceiverRAD1: +// Log dummy("openbts",gConfig.getStr("Log.Level").c_str(),LOG_LOCAL7); +// The LOCAL7 corresponds to the "local7" line in the file /etc/rsyslog.d/OpenBTS.log. Log::Log(const char* name, const char* level, int facility) { + // (pat) This 'constructor' has nothing to do with the regular use of the Log class, so we have + // to set this special flag to prevent the destructor from generating a syslog message. + // This is really goofy, but there is a reason - this is the way whoever wrote this got the Logger initialized early during + // static class initialization since OpenBTS has so many static classes whose constructors do work (a really bad idea) + // and may generate log messages. mDummyInit = true; gLogInit(name, level, facility); } @@ -236,6 +253,34 @@ ostringstream& Log::get() +// Allow applications to also pass in a filename. Filename should come from the database +void gLogInitWithFile(const char* name, const char* level, int facility, char * LogFilePath) +{ + // Set the level if one has been specified. + if (level) { + gConfig.set("Log.Level",level); + } + + if (gLogToFile==0 && LogFilePath != 0 && *LogFilePath != 0 && strlen(LogFilePath) > 0) { + gLogToFile = fopen(LogFilePath,"w"); // New log file each time we start. + if (gLogToFile) { + time_t now; + time(&now); + fprintf(gLogToFile,"Starting at %s",ctime(&now)); + fflush(gLogToFile); + std::cout << name <<" logging to file: " << LogFilePath << "\n"; + } + } + + // Open the log connection. + openlog(name,0,facility); + + // We cant call this from the Mutex itself because the Logger uses Mutex. + gMutexLogLevel = gGetLoggingLevel("Mutex.cpp"); +} + + + void gLogInit(const char* name, const char* level, int facility) { // Set the level if one has been specified. @@ -255,13 +300,16 @@ void gLogInit(const char* name, const char* level, int facility) time(&now); fprintf(gLogToFile,"Starting at %s",ctime(&now)); fflush(gLogToFile); - std::cout << "Logging to file: " << fn << "\n"; + std::cout << name <<" logging to file: " << fn << "\n"; } } } // Open the log connection. openlog(name,0,facility); + + // We cant call this from the Mutex itself because the Logger uses Mutex. + gMutexLogLevel = gGetLoggingLevel("Mutex.cpp"); } @@ -274,4 +322,104 @@ void gLogEarly(int level, const char *fmt, ...) va_end(args); } +// Return _NumberOfLogGroups if invalid. +LogGroup::Group LogGroup::groupNameToIndex(const char *groupName) const +{ + for (unsigned g = (Group)0; g < _NumberOfLogGroups; g++) { + if (0 == strcasecmp(mGroupNames[g],groupName)) { return (Group) g; } // happiness + } + return _NumberOfLogGroups; // failed +} + +LogGroup::LogGroup() { LogGroupInit(); } + +// These must match LogGroup::Group. +const char *LogGroup::mGroupNames[] = { "Control", "SIP", "GSM", "GPRS", "Layer2", NULL }; + +void LogGroup::LogGroupInit() +{ + // Error check some more. + assert(0==strcmp(mGroupNames[Control],"Control")); + assert(0==strcmp(mGroupNames[SIP],"SIP")); + assert(0==strcmp(mGroupNames[GSM],"GSM")); + assert(0==strcmp(mGroupNames[GPRS],"GPRS")); + assert(0==strcmp(mGroupNames[Layer2],"Layer2")); + + // Error check mGroupNames is the correct length; + unsigned g; + for (g = 0; mGroupNames[g]; g++) { continue; } + assert(g == _NumberOfLogGroups); // If you get this, go fix mGroupNames to match enum LogGroup::Group. + + for (unsigned g = 0; g < _NumberOfLogGroups; g++) { + mDebugLevel[g] = 0; + } + +#if 0 + if (mGroupNameToIndex.size()) { return; } // inited previously. + mGroupNameToIndex[string("Control")] = Control; + mGroupNameToIndex[string("SIP")] = SIP; + mGroupNameToIndex[string("GSM")] = GSM; + mGroupNameToIndex[string("GPRS")] = GPRS; + mGroupNameToIndex[string("Layer2")] = Layer2; +#endif +} + +static const char *LogGroupPrefix = "Log.Group."; + + +#if UNUSED +// Return true if this was a LogGroup config parameter. +// These dont have to be fast. +bool LogGroup::setGroup(const string groupName, const string levelName) +{ + const int len = strlen(LogGroupPrefix); + if (0 != strncasecmp(groupName.c_str(),LogGroupPrefix,len)) { return false; } + Group g = groupNameToIndex(groupName.c_str() + len); + if (g >= _NumberOfLogGroups) { + LOG(ALERT) << "Unrecognized Log.Group config parameter:"<second] = lookupLevel(levelName); + //} + return true; +} + +bool LogGroup::unsetGroup(const string groupName) +{ + const int len = strlen(LogGroupPrefix); + if (0 != strncasecmp(groupName.c_str(),LogGroupPrefix,len)) { return false; } + Group g = groupNameToIndex(groupName.c_str() + len); + if (g >= _NumberOfLogGroups) { + LOG(ALERT) << "Unrecognized Log.Group config parameter:"<second] = lookupLevel(levelName); + //} + return true; +} +#endif + +void LogGroup::setAll() +{ + LOG(DEBUG); + string prefix = string(LogGroupPrefix); + for (unsigned g = 0; g < _NumberOfLogGroups; g++) { + string param = prefix + mGroupNames[g]; + if (gConfig.defines(param)) { + string levelName = gConfig.getStr(param); + LOG(DEBUG) << "Setting "< #include #include +#include +// We cannot include Utils.h because it includes Logger.h, so just declare timestr() here. +// If timestr decl is changed G++ will whine when Utils.h is included. +namespace Utils { const std::string timestr(); }; #define _LOG(level) \ Log(LOG_##level).get() << pthread_self() \ - << timestr() << " " __FILE__ ":" << __LINE__ << ":" << __FUNCTION__ << ": " + << Utils::timestr() << " " __FILE__ ":" << __LINE__ << ":" << __FUNCTION__ << ": " +// (pat) If you '#define LOG_GROUP groupname' before including Logger.h, then you can set Log.Level.groupname as well as Log.Level.filename. +#ifdef LOG_GROUP +//#define CHECK_GROUP_LOG_LEVEL(groupname,loglevel) gCheckGroupLogLevel(#groupname,loglevel) +//#define IS_LOG_LEVEL(wLevel) (CHECK_GROUP_LOG_LEVEL(LOG_GROUP,LOG_##wLevel) || gGetLoggingLevel(__FILE__)>=LOG_##wLevel) +#define IS_LOG_LEVEL(wLevel) (gCheckGroupLogLevel(LOG_GROUP,LOG_##wLevel) || gGetLoggingLevel(__FILE__)>=LOG_##wLevel) +#else #define IS_LOG_LEVEL(wLevel) (gGetLoggingLevel(__FILE__)>=LOG_##wLevel) +#endif #ifdef NDEBUG #define LOG(wLevel) \ @@ -68,6 +82,8 @@ // Use like this: int descriptive_name; LOG(INFO)< GroupMapType; + //GroupMapType mGroupNameToIndex; +}; +extern LogGroup gLogGroup; + +// We inline this: +static __inline__ bool gCheckGroupLogLevel(LogGroup::Group group, unsigned level) { + assert(group < LogGroup::_NumberOfLogGroups); + //_LOG(DEBUG) << LOGVAR(group)<= level; +} + + std::list gGetLoggerAlarms(); ///< Get a copy of the recent alarm list. /**@ Global control and initialization of the logging system. */ //@{ + + +/** Initialize the global logging system with filename test 10*/ +void gLogInitWithFile(const char* name, const char* level, int facility, char* LogFilePath=NULL); + /** Initialize the global logging system. */ void gLogInit(const char* name, const char* level=NULL, int facility=LOG_USER); /** Get the logging level associated with a given file. */ @@ -133,6 +202,9 @@ int gGetLoggingLevel(const char *filename=NULL); void gLogEarly(int level, const char *fmt, ...) __attribute__((format(printf, 2, 3))); //@} +// (pat) This is historical, some files include Logger.h and expect to get these too. These should be removed. +#include "Threads.h" // must be after defines above, if these files are to be allowed to use LOG() +#include "Utils.h" #endif diff --git a/Makefile.am b/Makefile.am index 3c4e5bf..f253cd7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,8 +41,7 @@ libcommon_la_SOURCES = \ Configuration.cpp \ sqlite3util.cpp \ URLEncode.cpp \ - Utils.cpp \ - A51.cpp + Utils.cpp noinst_PROGRAMS = \ BitVectorTest \ @@ -54,12 +53,12 @@ noinst_PROGRAMS = \ ConfigurationTest \ LogTest \ URLEncodeTest \ - F16Test \ - A51Test + F16Test # ReportingTest noinst_HEADERS = \ + Defines.h \ BitVector.h \ Interthread.h \ LinkedLists.h \ @@ -74,8 +73,7 @@ noinst_HEADERS = \ URLEncode.h \ Utils.h \ Logger.h \ - sqlite3util.h \ - A51.h + sqlite3util.h URLEncodeTest_SOURCES = URLEncodeTest.cpp URLEncodeTest_LDADD = libcommon.la @@ -84,11 +82,11 @@ BitVectorTest_SOURCES = BitVectorTest.cpp BitVectorTest_LDADD = libcommon.la $(SQLITE_LA) InterthreadTest_SOURCES = InterthreadTest.cpp -InterthreadTest_LDADD = libcommon.la +InterthreadTest_LDADD = libcommon.la $(SQLITE_LA) InterthreadTest_LDFLAGS = -lpthread SocketsTest_SOURCES = SocketsTest.cpp -SocketsTest_LDADD = libcommon.la +SocketsTest_LDADD = libcommon.la $(SQLITE_LA) SocketsTest_LDFLAGS = -lpthread TimevalTest_SOURCES = TimevalTest.cpp @@ -111,9 +109,6 @@ LogTest_LDADD = libcommon.la $(SQLITE_LA) F16Test_SOURCES = F16Test.cpp -A51Test_SOURCES = A51Test.cpp -A51Test_LDADD = libcommon.la - MOSTLYCLEANFILES += testSource testDestination diff --git a/MemoryLeak.h b/MemoryLeak.h index 4948534..1f7bcd6 100644 --- a/MemoryLeak.h +++ b/MemoryLeak.h @@ -48,6 +48,14 @@ struct MemStats { mScramblingCode, mURlcDownSdu, mURlcPdu, + mSipBase, + mSipDialog, + mSipMessage, + mSipTransaction, + mMMContext, + mMMUser, + mTranEntry, + mMachineBase, // Must be last: mMax, }; diff --git a/ScalarTypes.h b/ScalarTypes.h index 077d889..21fa66d 100644 --- a/ScalarTypes.h +++ b/ScalarTypes.h @@ -84,6 +84,12 @@ struct Double_z { _INITIALIZED_SCALAR_FUNCS(Double_z,double,0) }; +template +struct Enum_z { + basetype value; + _INITIALIZED_SCALAR_BASE_FUNCS(Enum_z,basetype,((basetype)0)) +}; + class ItemWithValueAndWidth { public: diff --git a/Sockets.cpp b/Sockets.cpp index dd7527c..dca483e 100644 --- a/Sockets.cpp +++ b/Sockets.cpp @@ -1,5 +1,5 @@ /* -* Copyright 2008, 2010 Free Software Foundation, Inc. +* Copyright 2008, 2010, 2014 Free Software Foundation, Inc. * * * This software is distributed under the terms of the GNU Affero Public License. @@ -33,6 +33,7 @@ #include "Threads.h" #include "Sockets.h" +#include "Logger.h" #include #include #include @@ -42,6 +43,9 @@ +SocketError::SocketError() { + LOG(DEBUG) << "SocketError"; +} @@ -51,7 +55,10 @@ bool resolveAddress(struct sockaddr_in *address, const char *hostAndPort) assert(hostAndPort); char *copy = strdup(hostAndPort); char *colon = strchr(copy,':'); - if (!colon) return false; + if (!colon) { + LOG(WARNING) << "missing port number in:"<h_addrtype != AF_INET) { - CERR("WARNING -- gethostbyname() resolved " << host << " to something other then AF_INET"); + LOG(WARNING) << "gethostbyname() resolved " << host << " to something other then AF_INET"; return false; } - address->sin_family = hp->h_addrtype; + address->sin_family = hp->h_addrtype; // Above guarantees it is AF_INET assert(sizeof(address->sin_addr) == hp->h_length); memcpy(&(address->sin_addr), hp->h_addr_list[0], hp->h_length); address->sin_port = htons(port); @@ -141,7 +149,7 @@ DatagramSocket::~DatagramSocket() int DatagramSocket::write( const char * message, size_t length ) { - assert(length<=MAX_UDP_LENGTH); + //assert(length<=MAX_UDP_LENGTH); // (pat 8-2013) Removed on David's orders. int retVal = sendto(mSocketFD, message, length, 0, (struct sockaddr *)mDestination, addressSize()); if (retVal == -1 ) perror("DatagramSocket::write() failed"); @@ -150,7 +158,7 @@ int DatagramSocket::write( const char * message, size_t length ) int DatagramSocket::writeBack( const char * message, size_t length ) { - assert(length<=MAX_UDP_LENGTH); + //assert(length<=MAX_UDP_LENGTH); // (pat 8-2013) Removed on David's orders. int retVal = sendto(mSocketFD, message, length, 0, (struct sockaddr *)mSource, addressSize()); if (retVal == -1 ) perror("DatagramSocket::write() failed"); @@ -175,7 +183,9 @@ int DatagramSocket::writeBack( const char * message) int DatagramSocket::send(const struct sockaddr* dest, const char * message, size_t length ) { - assert(length<=MAX_UDP_LENGTH); + // (pat 8-2013) Dont assert! + // assert(length<=MAX_UDP_LENGTH); + // sendto is supposed to return an error if the packet is too long. int retVal = sendto(mSocketFD, message, length, 0, dest, addressSize()); if (retVal == -1 ) perror("DatagramSocket::send() failed"); return retVal; @@ -272,7 +282,9 @@ void UDPSocket::open(unsigned short localPort) address.sin_addr.s_addr = INADDR_ANY; address.sin_port = htons(localPort); if (bind(mSocketFD,(struct sockaddr*)&address,length)<0) { - perror("bind() failed"); + char buf[100]; + sprintf(buf,"bind(port %d) failed",localPort); + perror(buf); throw SocketError(); } } @@ -318,7 +330,9 @@ void UDDSocket::open(const char* localPath) strcpy(address.sun_path,localPath); unlink(localPath); if (bind(mSocketFD,(struct sockaddr*)&address,length)<0) { - perror("bind() failed"); + char buf[1100]; + sprintf(buf,"bind(path %s) failed",localPath); + perror(buf); throw SocketError(); } } diff --git a/Sockets.h b/Sockets.h index c79f79a..f607f7c 100644 --- a/Sockets.h +++ b/Sockets.h @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -50,7 +49,9 @@ bool resolveAddress(struct sockaddr_in *address, const char *host, unsigned shor bool resolveAddress(struct sockaddr_in *address, const char *hostAndPort); /** An exception to throw when a critical socket operation fails. */ -class SocketError {}; +struct SocketError { + SocketError(); +}; #define SOCKET_ERROR {throw SocketError(); } /** Abstract class for connectionless sockets. */ @@ -142,6 +143,11 @@ public: class UDPSocket : public DatagramSocket { public: + // (pat) If you want to set the local port using some sql option, you MUST NOT do it in a constructor + // unless that constructor is called from OpenBTS.cpp, because there is a constructor race between + // class ConfigurationTable (needed by the call to gConfig) and the class containing the UDPSocket and calling gConfig. + // Alternatively, since we cannot add an empty constructor (because it is ambiguous with the following) + // you must use a UDPSocket* and allocate it with 'new'. /** Open a USP socket with an OS-assigned port and no default destination. */ UDPSocket( unsigned short localPort=0); diff --git a/SocketsTest.cpp b/SocketsTest.cpp index 9a4997b..e8e8082 100644 --- a/SocketsTest.cpp +++ b/SocketsTest.cpp @@ -31,6 +31,8 @@ #include #include +#include "Configuration.h" +ConfigurationTable gConfig; static const int gNumToSend = 10; diff --git a/Threads.cpp b/Threads.cpp index de6520b..853ac45 100644 --- a/Threads.cpp +++ b/Threads.cpp @@ -1,5 +1,5 @@ /* -* Copyright 2008 Free Software Foundation, Inc. +* Copyright 2008, 2014 Free Software Foundation, Inc. * * * This software is distributed under the terms of the GNU Affero Public License. @@ -29,10 +29,19 @@ #include "Threads.h" #include "Timeval.h" +#include "Logger.h" +#include using namespace std; +int gMutexLogLevel = LOG_INFO; // The mutexes cannot call gConfig or gGetLoggingLevel so we have to get the log level indirectly. + + +#define LOCKLOG(level,fmt,...) \ + if (gMutexLogLevel >= LOG_##level) syslog(LOG_##level,"%lu %s %s:%u:%s:lockid=%p " fmt,(unsigned long)pthread_self(),Utils::timestr().c_str(),__FILE__,__LINE__,__FUNCTION__,this,##__VA_ARGS__); + //printf("%u %s %s:%u:%s:lockid=%u " fmt "\n",(unsigned)pthread_self(),Utils::timestr().c_str(),__FILE__,__LINE__,__FUNCTION__,(unsigned)this,##__VA_ARGS__); + @@ -72,8 +81,10 @@ void unlockCerr() -Mutex::Mutex() +Mutex::Mutex() : mLockCnt(0) //, mLockerFile(0), mLockerLine(0) { + memset(mLockerFile,0,sizeof(mLockerFile)); + // Must use getLoggingLevel, not gGetLoggingLevel, to avoid infinite recursion. bool res; res = pthread_mutexattr_init(&mAttribs); assert(!res); @@ -91,12 +102,126 @@ Mutex::~Mutex() assert(!res); } - - - -/** Block for the signal up to the cancellation timeout. */ -void Signal::wait(Mutex& wMutex, unsigned timeout) const +bool Mutex::trylock() { + if (pthread_mutex_trylock(&mMutex)==0) { + if (mLockCnt < maxLocks) { mLockerFile[mLockCnt] = NULL; } + mLockCnt++; + return true; + } else { + return false; + } +} + +// Returns true if the lock was acquired within the timeout, or false if it timed out. +bool Mutex::timedlock(int msecs) // Wait this long in milli-seconds. +{ + Timeval future(msecs); + struct timespec timeout = future.timespec(); + return ETIMEDOUT != pthread_mutex_timedlock(&mMutex, &timeout); +} + +string Mutex::mutext() const +{ + string result; + result.reserve(100); + //result += format("lockid=%u lockcnt=%d",(unsigned)this,mLockCnt); + result += format("lockcnt=%d",mLockCnt); + for (int i = 0; i < mLockCnt && i < maxLocks; i++) { + if (mLockerFile[i]) { + result += format(" %s:%u",mLockerFile[i],mLockerLine[i]); + } else { + result += " ?"; + } + } + return result; +} + +void Mutex::lock() { + if (lockerFile()) LOCKLOG(DEBUG,"lock unchecked"); + _lock(); + mLockCnt++; +} + +// WARNING: The LOG facility calls lock, so to avoid infinite recursion do not call LOG if file == NULL, +// and the file argument should never be used from the Logger facility. +void Mutex::lock(const char *file, unsigned line) +{ + // (pat 10-25-13) This is now going to be the default behavior so we can detect and report deadlocks at customer sites. + //if (file && gGetLoggingLevel(file)>=LOG_DEBUG) + //if (file) OBJLOG(DEBUG) <<"start at "<= 0 && mLockCnt < maxLocks) { + mLockerFile[mLockCnt] = file; mLockerLine[mLockCnt] = line; // Now our thread has it locked from here. + } + mLockCnt++; + if (file) { LOCKLOG(DEBUG,"lock by %s",mutext().c_str()); } + //else { LOCKLOG(DEBUG,"lock no file"); } +} + +void Mutex::unlock() +{ + if (lockerFile()) { LOCKLOG(DEBUG,"unlock at %s",mutext().c_str()); } + //else { LOCKLOG(DEBUG,"unlock unchecked"); } + mLockCnt--; + pthread_mutex_unlock(&mMutex); +} + +RWLock::RWLock() +{ + bool res; + res = pthread_rwlockattr_init(&mAttribs); + assert(!res); + res = pthread_rwlock_init(&mRWLock,&mAttribs); + assert(!res); +} + + +RWLock::~RWLock() +{ + pthread_rwlock_destroy(&mRWLock); + bool res = pthread_rwlockattr_destroy(&mAttribs); + assert(!res); +} + + + +/** Block for the signal up to the cancellation timeout in msecs. */ +// (pat 8-2013) Our code had places (InterthreadQueue) that passed in negative timeouts which create deadlock. +// To prevent that, use signed, not unsigned timeout. +void Signal::wait(Mutex& wMutex, long timeout) const +{ + if (timeout <= 0) { return; } // (pat) Timeout passed already Timeval then(timeout); struct timespec waitTime = then.timespec(); pthread_cond_timedwait(&mSignal,&wMutex.mMutex,&waitTime); diff --git a/Threads.h b/Threads.h index a38a73a..ccee199 100644 --- a/Threads.h +++ b/Threads.h @@ -1,5 +1,5 @@ /* -* Copyright 2008, 2011 Free Software Foundation, Inc. +* Copyright 2008, 2011, 2014 Free Software Foundation, Inc. * * This software is distributed under the terms of the GNU Affero Public License. * See the COPYING file in the main directory for details. @@ -37,6 +37,8 @@ class Mutex; /**@name Multithreaded access for standard streams. */ //@{ +extern int gMutexLogLevel; // The mutexes cannot call gConfig or gGetLoggingLevel so we have to get the log level indirectly. + /**@name Functions for gStreamLock. */ //@{ extern Mutex gStreamLock; ///< global lock for cout and cerr @@ -72,6 +74,18 @@ class Mutex { pthread_mutex_t mMutex; pthread_mutexattr_t mAttribs; + int mLockCnt; + int mMutexLogLevel; // We cant use LOG inside the Mutex because LOG itself uses mutexes, so get the LOG level at mutex creation time + // and use it for this mutex from then on. + + static const int maxLocks = 5; // Just the maximum number of recursive locks we report during debugging, not the max possible. + const char *mLockerFile[maxLocks]; + unsigned mLockerLine[maxLocks]; + const char *lockerFile() { int i = mLockCnt-1; return (i >= 0 && i < maxLocks) ? mLockerFile[i] : NULL; } + //unused: bool anyDebugging() { for (int i = 0; i < maxLocks; i++) { if (mLockerFile[i]) return true; return false; } } + + // pthread_mutex_trylock returns 0 and trylock returns true if the lock was acquired. + bool trylock(); public: @@ -79,16 +93,71 @@ class Mutex { ~Mutex(); - void lock() { pthread_mutex_lock(&mMutex); } + void _lock() { pthread_mutex_lock(&mMutex); } + void lock(); - bool trylock() { return pthread_mutex_trylock(&mMutex)==0; } + // (pat) Like the above but report blocking; to see report you must set both Log.Level to DEBUG for both Threads.cpp and the file. + void lock(const char *file, unsigned line); - void unlock() { pthread_mutex_unlock(&mMutex); } + std::string mutext() const; + + // Returns true if the lock was acquired, or false if it timed out. + bool timedlock(int msecs); + + void unlock(); + + // (pat) I use this to assert that the Mutex is locked on entry to some method that requres it, but only in debug mode. + int lockcnt() { return mLockCnt; } friend class Signal; }; +/** A class for reader/writer based on pthread_rwlock. */ +class RWLock { + + private: + + pthread_rwlock_t mRWLock; + pthread_rwlockattr_t mAttribs; + + public: + + RWLock(); + + ~RWLock(); + + void wlock() { pthread_rwlock_wrlock(&mRWLock); } + void rlock() { pthread_rwlock_rdlock(&mRWLock); } + + bool trywlock() { return pthread_rwlock_trywrlock(&mRWLock)==0; } + bool tryrlock() { return pthread_rwlock_tryrdlock(&mRWLock)==0; } + + void unlock() { pthread_rwlock_unlock(&mRWLock); } + +}; + + +#if 0 +// (pat) NOT FINISHED OR TESTED. A pointer that releases a specified mutex when it goes out of scope. +template +class ScopedPointer { + Mutex &mControllingMutex; // A pointer to the mutex for the object being protected. + PointsTo *mPtr; + + public: + ScopedPointer(Mutex& wMutex) :mControllingMutex(wMutex) { mControllingMutex.lock(); } + // Requisite Copy Constructor: The mutex is already locked, but we need to lock it again because the + // other ScopedPointer is about to go out of scope and will call unlock. + ScopedPointer(ScopedPointer &other) :mControllingMutex(other.mControllingMutex) { mControllingMutex.lock(); } + ~ScopedPointer() { mControllingMutex.unlock(); } + + // You are allowed to assign and derference the underlying pointer - it still holds the Mutex locked. + PointsTo *operator->() const { return mPtr; } + PointsTo * operator=(PointsTo *other) { mPtr = other; } + PointsTo& operator*() { return *mPtr; } +}; +#endif class ScopedLock { @@ -97,6 +166,8 @@ class ScopedLock { public: ScopedLock(Mutex& wMutex) :mMutex(wMutex) { mMutex.lock(); } + // Like the above but report blocking; to see report you must set both Log.Level to DEBUG for both Threads.cpp and the file. + ScopedLock(Mutex& wMutex,const char *file, unsigned line):mMutex(wMutex) { mMutex.lock(file,line); } ~ScopedLock() { mMutex.unlock(); } }; @@ -121,7 +192,7 @@ class Signal { Block for the signal up to the cancellation timeout. Under Linux, spurious returns are possible. */ - void wait(Mutex& wMutex, unsigned timeout) const; + void wait(Mutex& wMutex, long timeout) const; /** Block for the signal. diff --git a/Timeval.cpp b/Timeval.cpp index 50ce05d..642c05a 100644 --- a/Timeval.cpp +++ b/Timeval.cpp @@ -29,7 +29,7 @@ using namespace std; -void Timeval::future(unsigned offset) +void Timeval::future(unsigned offset) // In msecs { now(); unsigned sec = offset/1000; diff --git a/Utils.cpp b/Utils.cpp index f06420d..8c4fe9b 100644 --- a/Utils.cpp +++ b/Utils.cpp @@ -1,5 +1,5 @@ /* -* Copyright 2011 Range Networks, Inc. +* Copyright 2011, 2014 Range Networks, Inc. * All Rights Reserved. * * This software is distributed under multiple licenses; @@ -20,14 +20,46 @@ #include // For ostream #include // For ostringstream #include // For strcpy +#include // For malloc //#include "GSMCommon.h" #include "Utils.h" #include "MemoryLeak.h" namespace Utils { +using namespace std; + +// (pat) This definition must be in the .cpp file to anchor the class vtable. +RefCntBase::~RefCntBase() { LOG(DEBUG) << typeid(this).name(); } + +int RefCntBase::decRefCnt() +{ + int saveRefCnt; // Passes the refcnt out of the locked block. + { ScopedLock lock(mRefMutex); + assert(mRefCnt >= 0); + mRefCnt = mRefCnt - 1; + saveRefCnt = mRefCnt; + } // Must not keep locked during the delete, since the Mutex itself will be deleted. + // The typeid(this).name() doesnt add anything because it is just the name of the class here, not the derived class. + LOG(DEBUG) <<" "<<(void*)this <<" " <= 0); + mRefCnt++; +} + MemStats gMemStats; int gMemLeakDebug = 0; +static Mutex memChkLock; MemStats::MemStats() { @@ -36,7 +68,7 @@ MemStats::MemStats() memset(mMemName,0,sizeof(mMemName)); } -void MemStats::text(std::ostream &os) +void MemStats::text(ostream &os) { os << "Structs current total:\n"; for (int i = 0; i < mMax; i++) { @@ -46,7 +78,8 @@ void MemStats::text(std::ostream &os) void MemStats::memChkNew(MemoryNames memIndex, const char *id) { - /*std::cout << "new " #type "\n";*/ + /*cout << "new " #type "\n";*/ + ScopedLock lock(memChkLock); mMemNow[memIndex]++; mMemTotal[memIndex]++; mMemName[memIndex] = id; @@ -54,21 +87,22 @@ void MemStats::memChkNew(MemoryNames memIndex, const char *id) void MemStats::memChkDel(MemoryNames memIndex, const char *id) { - /*std::cout << "del " #type "\n";*/ + ScopedLock lock(memChkLock); + /*cout << "del " #type "\n";*/ mMemNow[memIndex]--; if (mMemNow[memIndex] < 0) { - LOG(ERR) << "Memory underflow on type "<= (300-4)) { strcpy(&buf[(300-4)],"..."); } - return std::string(buf); + string result; + if (n <= 199) { + result = string(buf); + } else { + if (n > 5000) { LOG(ERR) << "oversized string in format"; n = 5000; } + // We could use vasprintf but we already computed the length... + // We are not using alloca because it might overflow the small stacks used for our threads. + char *buffer = (char*)malloc(n+2); // add 1 extra superstitiously. + va_start(ap,fmt); + vsnprintf(buffer,n+1,fmt,ap); + va_end(ap); + //if (n >= (2000-4)) { strcpy(&buf[(2000-4)],"..."); } + result = string(buffer); + free(buffer); + } + return result; +#if 0 // Maybe ok, but not recommended. data() is const char* + string result; + va_list ap; + va_start(ap,fmt); + result.reserve(200); + int n = vsnprintf(result.data(),198,fmt,ap); + va_end(ap); + if (n > 198) { + if (n > 5000) { LOG(ERR) << "oversized string in format"; n = 5000; } + result.reserve(n+2); // add 1 extra superstitiously. + va_start(ap,fmt); + vsnprintf(result.data(),n+1,fmt,ap); + va_end(ap); + } + result.resize(n); + return result; +#endif } +// Absolutely identical to format above. This sucks... +string format1(const char *fmt, ...) +{ + va_list ap; + char buf[200]; + va_start(ap,fmt); + int n = vsnprintf(buf,199,fmt,ap); + va_end(ap); + string result; + if (n <= 199) { + result = string(buf); + } else { + if (n > 5000) { LOG(ERR) << "oversized string in format"; n = 5000; } + // We could use vasprintf but we already computed the length... + // We are not using alloca because it might overflow the small stacks used for our threads. + char *buffer = (char*)malloc(n+2); // add 1 extra superstitiously. + va_start(ap,fmt); + vsnprintf(buffer,n+1,fmt,ap); + va_end(ap); + //if (n >= (2000-4)) { strcpy(&buf[(2000-4)],"..."); } + result = string(buffer); + free(buffer); + } + return result; +} + +int myscanf(const char *str, const char *fmt, string *s1) +{ + int maxlen = strlen(str)+1; + char *a1 = (char*)alloca(maxlen); + int n = sscanf(str,fmt,a1); + s1->assign(a1); + return n; +} +int myscanf(const char *str, const char *fmt, string *s1, string *s2) +{ + int maxlen = strlen(str)+1; + char *a1 = (char*)alloca(maxlen); + char *a2 = (char*)alloca(maxlen); + int n = sscanf(str,fmt,a1,a2); + switch (n) { + case 2: s2->assign(a2); + case 1: s1->assign(a1); + } + return n; +} +int myscanf(const char *str, const char *fmt, string *s1, string *s2, string *s3) +{ + int maxlen = strlen(str)+1; + char *a1 = (char*)alloca(maxlen); + char *a2 = (char*)alloca(maxlen); + char *a3 = (char*)alloca(maxlen); + int n = sscanf(str,fmt,a1,a2,a3); + switch (n) { + case 3: s3->assign(a3); + case 2: s2->assign(a2); + case 1: s1->assign(a1); + } + return n; +} +int myscanf(const char *str, const char *fmt, string *s1, string *s2, string *s3, string *s4) +{ + int maxlen = strlen(str)+1; + char *a1 = (char*)alloca(maxlen); + char *a2 = (char*)alloca(maxlen); + char *a3 = (char*)alloca(maxlen); + char *a4 = (char*)alloca(maxlen); + int n = sscanf(str,fmt,a1,a2,a3,a4); + switch (n) { + case 4: s4->assign(a4); + case 3: s3->assign(a3); + case 2: s2->assign(a2); + case 1: s1->assign(a1); + } + return n; +} + +#if 0 +string format(const char *fmt, string s1) { + return format(fmt,s1.c_str()); +} +string format(const char *fmt, string s1, string s2) { + return format(fmt,s1.c_str(),s2.c_str()); +} +string format(const char *fmt, string s1, string s2, string s3) { + return format(fmt,s1.c_str(),s2.c_str(),s3.c_str()); +} +string format(const char *fmt, string s1, int i1) { + return format(fmt,s1.c_str(),i1); +} +string format(const char *fmt, int i1, string s1) { + return format(fmt,i1,s1.c_str()); +} +string format(const char *fmt, string s1, string s2, int i1) { + return format(fmt,s1.c_str(),s2.c_str(),i1); +} +string format(const char *fmt, string s1, string s2, int i1, int i2) { + return format(fmt,s1.c_str(),s2.c_str(),i1,i2); +} +#endif + // Return time in seconds with high resolution. // Note: In the past I found this to be a surprisingly expensive system call in linux. double timef() @@ -100,16 +266,25 @@ double timef() return tv.tv_usec / 1000000.0 + tv.tv_sec; } -const std::string timestr() +const string timestr(unsigned fieldwidth) // Use to pick the number of chars in the output. { struct timeval tv; struct tm tm; gettimeofday(&tv,NULL); localtime_r(&tv.tv_sec,&tm); unsigned tenths = tv.tv_usec / 100000; // Rounding down is ok. - return format(" %02d:%02d:%02d.%1d",tm.tm_hour,tm.tm_min,tm.tm_sec,tenths); + string result = format(" %02d:%02d:%02d.%1d",tm.tm_hour,tm.tm_min,tm.tm_sec,tenths); + return result.substr(fieldwidth >= result.size() ? 0 : result.size() - fieldwidth); + //switch (maxfield) { + //case 'h': case 'H': return format("%02d:%02d:%02d.%1d",tm.tm_hour,tm.tm_min,tm.tm_sec,tenths); + //case 'm': case 'M': return format("%02d:%02d.%1d",tm.tm_min,tm.tm_sec,tenths); + //case 's': case 'S': return format("%02d.%1d",tm.tm_sec,tenths); + //default: return format(" %02d:%02d:%02d.%1d",tm.tm_hour,tm.tm_min,tm.tm_sec,tenths); + //} } +const string timestr() { return timestr(12); } + // High resolution sleep for the specified time. // Return FALSE if time is already past. void sleepf(double howlong) @@ -127,16 +302,16 @@ void sleepf(double howlong) //sleepf(sleeptime); //} -std::string Text2Str::str() const +string Text2Str::str() const { - std::ostringstream ss; + ostringstream ss; text(ss); return ss.str(); } -std::ostream& operator<<(std::ostream& os, const Text2Str *val) +ostream& operator<<(std::ostream& os, const Text2Str *val) { - std::ostringstream ss; + ostringstream ss; if (val) { val->text(ss); os << ss.str(); @@ -185,27 +360,192 @@ int cstrSplit(char *in, char **pargv,int maxargc, const char *splitchars) return argc; } -std::ostream& operator<<(std::ostream& os, const Statistic &stat) { stat.text(os); return os; } -std::ostream& operator<<(std::ostream& os, const Statistic &stat) { stat.text(os); return os; } -std::ostream& operator<<(std::ostream& os, const Statistic &stat) { stat.text(os); return os; } -std::ostream& operator<<(std::ostream& os, const Statistic &stat) { stat.text(os); return os; } -std::string replaceAll(const std::string input, const std::string search, const std::string replace) +// Return pointer to the nth (1 for first) space-separated non-empty argument from this string, and length in plength. +// Note that strtok is not thread safe. +char *cstrGetArg(const char *in, int nth, unsigned *length) { - std::string output = input; - unsigned index = 0; + const char *result, *cp = in; + while (*cp && nth-- > 0) { + while (*cp && isspace(*cp)) { cp++; } + result = cp; + while (*cp && !isspace(*cp)) { cp++; } + if (nth == 0) { + *length = cp - result; + // remove the ever-to-be-hated const for the convenience of our callers. + return *length ? const_cast(result) : NULL; + } + } + return NULL; +} - while (true) { - index = output.find(search, index); - if (index == std::string::npos) { + +vector& stringSplit(vector &result,const char *input) +{ + char *argv[40]; + //char buf[202]; + //strncpy(buf,input,200); buf[200] = 0; + char *buf = strdup(input); + int cnt = cstrSplit(buf,argv,40,NULL); + for (int i = 0; i < cnt; i++) { + result.push_back(string(argv[i])); + } + free(buf); + return result; +} + +// Print a table formatted as a vector of vector of strings. +// The columns will be aligned. +// Column size is determined from the columns. +// An entry of "_" is suppressed. + +void printPrettyTable(prettyTable_t &tab, ostream&os, bool tabSeparated) +{ + LOG(DEBUG); + const unsigned maxcols = 30; + // Determine the maximum width of each column. + int width[maxcols]; memset(width,0,sizeof(width)); + if (!tabSeparated) { + for (prettyTable_t::iterator it = tab.begin(); it != tab.end(); ++it) { + std::vector &row = *it; + for (unsigned col = 0; col 100) colwidth = 100; + width[col] = std::max(width[col],colwidth); + } + } + } + // Now print it. + for (unsigned nrow = 0; nrow < tab.size(); nrow++) { + vector &row = tab[nrow]; + + // DEBUG: print the column widths. + if (0 && IS_LOG_LEVEL(DEBUG) && nrow == 0) { + for (unsigned col = 0; col &stat) { stat.text(os); return os; } +ostream& operator<<(std::ostream& os, const Statistic &stat) { stat.text(os); return os; } +ostream& operator<<(std::ostream& os, const Statistic &stat) { stat.text(os); return os; } +ostream& operator<<(std::ostream& os, const Statistic &stat) { stat.text(os); return os; } + +string replaceAll(const std::string input, const std::string search, const std::string replace) +{ + string output = input; + unsigned index1 = 0; + + while (index1 < output.size()) { + try { + index1 = output.find(search, index1); + if (index1 == string::npos) { break; } - output.replace(index, replace.length(), replace); - index += replace.length(); + output.replace(index1, search.length(), replace); + // We want to scan past the piece we just replaced. + index1 += replace.length(); + } catch (...) { + LOG(ERR) << "string replaceAll error"<> *hRAND; + stringstream ssl; + ssl << hex << strlRAND; + ssl >> *lRAND; +} + +string uintToString(uint64_t h, uint64_t l) +{ + ostringstream os1; + os1.width(16); + os1.fill('0'); + os1 << hex << h; + ostringstream os2; + os2.width(16); + os2.fill('0'); + os2 << hex << l; + ostringstream os3; + os3 << os1.str() << os2.str(); + return os3.str(); +} + +string uintToString(uint32_t x) +{ + ostringstream os; + os.width(8); + os.fill('0'); + os << hex << x; + return os.str(); +} }; diff --git a/Utils.h b/Utils.h index 0bc738e..a0bd6c6 100644 --- a/Utils.h +++ b/Utils.h @@ -1,5 +1,5 @@ /* -* Copyright 2011 Range Networks, Inc. +* Copyright 2011, 2014 Range Networks, Inc. * All Rights Reserved. * * This software is distributed under multiple licenses; @@ -14,28 +14,65 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#ifndef GPRSUTILS_H -#define GPRSUTILS_H +#ifndef _UTILS_H +#define _UTILS_H #include #include #include +#include #include #include // for sqrtf +#include #include "Logger.h" namespace Utils { +using namespace std; extern double timef(); // high resolution time +// We dont use a default arg here timestr(unsigned fieldwidth=12) because g++ complains about duplicate decl in Logger.h extern const std::string timestr(); // A timestamp to print in messages. +extern const std::string timestr(unsigned fieldwidth); // A timestamp to print in messages. extern void sleepf(double howlong); // high resolution sleep extern int gcd(int x, int y); -// It is irritating to create a string just to interface to the brain-damaged -// C++ stream class, but this is only used for debug messages. -std::string format(const char *fmt, ...) __attribute__((format (printf,1,2))); +string format(const char *fmt, ...) __attribute__((format (printf,1,2))); + +// format1 used to prevent C++ confusion over what function to call here. +string format1(const char *fmt, ...) __attribute__((format (printf,1,2))); +// We have to enumerate the cross product of argument types here. This is fixed in C++11. +inline string format(const char *fmt, string s1) { + return format1(fmt,s1.c_str()); +} +inline string format(const char *fmt, string s1, string s2) { + return format1(fmt,s1.c_str(),s2.c_str()); +} +inline string format(const char *fmt, string s1, string s2, string s3) { + return format1(fmt,s1.c_str(),s2.c_str(),s3.c_str()); +} +inline string format(const char *fmt, string s1, string s2, string s3, string s4) { + return format1(fmt,s1.c_str(),s2.c_str(),s3.c_str(),s4.c_str()); +} +inline string format(const char *fmt, string s1, int i1) { + return format1(fmt,s1.c_str(),i1); +} +inline string format(const char *fmt, int i1, string s1) { + return format1(fmt,i1,s1.c_str()); +} +inline string format(const char *fmt, string s1, string s2, int i1) { + return format1(fmt,s1.c_str(),s2.c_str(),i1); +} +inline string format(const char *fmt, string s1, string s2, int i1, int i2) { + return format1(fmt,s1.c_str(),s2.c_str(),i1,i2); +} + +int myscanf(const char *str, const char *fmt, string *s1); +int myscanf(const char *str, const char *fmt, string *s1, string *s2); +int myscanf(const char *str, const char *fmt, string *s1, string *s2, string *s3); +int myscanf(const char *str, const char *fmt, string *s1, string *s2, string *s3, string *s4); int cstrSplit(char *in, char **pargv,int maxargc, const char *splitchars=NULL); +char *cstrGetArg(const char *in, int nth, unsigned *length); // For classes with a text() function, provide a function to return a String, // and also a standard << stream function that takes a pointer to the object. @@ -106,7 +143,7 @@ template struct Statistic { void text(std::ostream &os) const { // Print everything in parens. os << "("<& stringSplit(vector &result,const char *input); +typedef vector > prettyTable_t; +void printPrettyTable(prettyTable_t &tab, ostream&os, bool tabSeparated = false); + +// The need for this is eliminated in C++11. +string stringcat(string a, string b); +string stringcat(string a, string b, string c); +string stringcat(string a, string b, string c, string d); +string stringcat(string a, string b, string c, string d, string e); +string stringcat(string a, string b, string c, string d, string e, string f); +string stringcat(string a, string b, string c, string d, string e, string f, string g); + +extern void stringToUint(string strRAND, uint64_t *hRAND, uint64_t *lRAND); +extern string uintToString(uint64_t h, uint64_t l); +extern string uintToString(uint32_t x); + +// The class is created with a RefCnt of 0. The caller must assign the constructed result to a pointer +// of type RefCntPointer. When the last RefCntPointer is freed, this struct is too. +class RefCntBase { + Mutex mRefMutex; + mutable short mRefCnt; // signed, not unsigned! + int setRefCnt(int val) { return mRefCnt = val; } + // The semantics of reference counting mean you cannot copy an object that has reference counts; + // they are only manipulated by pointers. This constructor is private, which makes C++ whine if you do this. + // Actually, you could copy it, if the refcnt in the new object were 0 and the pointer was assigned to a RefCntPointer. + RefCntBase(RefCntBase &) { assert(0); } + RefCntBase(const RefCntBase &) { assert(0); } + RefCntBase operator=(RefCntBase &) {assert(0); mRefCnt = 0; return *this; } + public: + virtual ~RefCntBase(); + RefCntBase() : mRefCnt(0) {} + int getRefCnt() const { return mRefCnt; } + int decRefCnt(); + void incRefCnt(); +}; + +// Semantics: +// SomeDescendentOfRefCntBase foo; +// RefCntPointer ptrToFoo; +// ptrToFoo = &foo; // increment the reference count of foo. +// ... ptrToFoo->... // Normal code using ptrToFoo as a (SomeDescendentOfRefCntBase*) +// ptrToFoo = 0; // release pointer and decrement reference count of foo. +// ptrToFoo.free(); // A better way to release it. +template // Type must be a descendent of RefCntBase. +class RefCntPointer { + Type *rcPointer; + void rcInc() { if (rcPointer) rcPointer->incRefCnt(); } + void rcDec() { + if (rcPointer) if (rcPointer->decRefCnt() <= 0) { rcPointer = NULL; }; + } + public: + RefCntPointer() : rcPointer(NULL) { } + RefCntPointer(const RefCntPointer &other) { rcPointer = other.rcPointer; rcInc(); } + RefCntPointer(RefCntPointer &other) { rcPointer = other.rcPointer; rcInc(); } + RefCntPointer(Type*other) { rcPointer = other; rcInc(); } + // (pat) Making this virtual ~RefCntPointer makes ~MMContext crash, dont know why. + // (pat) Update: lets try making it virtual again; it should work, but no one derives off RefCntPointer so I dont know why it crashes.. + ~RefCntPointer() { + if (rcPointer) { LOG(DEBUG) <<" rcPointer="<<((void*)rcPointer) <<" refcnt="<<(rcPointer?rcPointer->getRefCnt():0); } + rcDec(); + rcPointer = NULL; // should be redundant + } + Type* self() const { return rcPointer; } + // The operator=(Type*) is called if the argument is NULL, so we dont need int. + //Type* operator=(int value) { assert(value == 0); rcDec(); rcPointer = 0; return rcPointer; } + // We increment before decrement if possible so that a = a; does not crash. + RefCntPointer& operator=(RefCntPointer &other) { other.rcInc(); rcDec(); rcPointer = other.rcPointer; return *this; } + RefCntPointer& operator=(Type* other) { Type *old = rcPointer; rcPointer = other; rcInc(); if (old) {old->decRefCnt();} return *this; } + bool operator==(const RefCntPointer &other) const { return rcPointer == other.rcPointer; } + bool operator==(const Type* other) const { return rcPointer == other; } + bool operator!=(const RefCntPointer &other) const { return rcPointer != other.rcPointer; } + bool operator!=(const Type* other) const { return rcPointer != other; } + Type* operator->() const { return rcPointer; } + Type& operator*() const { return *rcPointer; } + // This auto-conversion causes gcc warning messages, which are of no import, but to avoid them I now use self() everywhere instead. + //operator Type*() const { return rcPointer; } + // free is a synonym for *this = NULL; but it is a better comment in the code what is happening. RefCntBase is only freed if this was the last pointer to it. + void free() { + int refcnt = rcPointer ? rcPointer->getRefCnt() : 0; + rcDec(); + if (refcnt > 1) { LOG(DEBUG)<<"RefCntPointer "<<(void*)rcPointer<<" refcnt before="< class RCData : public RefCntBase { + public: + T* mPointer; +}; +#endif /** @@ -43,35 +59,117 @@ extern int gVectorDebug; Unlike std::vector, this class does not support dynamic resizing. Unlike std::vector, this class does support "aliases" and subvectors. */ -template class Vector { - - // TODO -- Replace memcpy calls with for-loops. - - public: - - /**@name Iterator types. */ - //@{ - typedef T* iterator; - typedef const T* const_iterator; - //@} +// (pat) Nov 2013: Vector and the derived classes BitVector and SoftVector were originally written with behavior +// that differed for const and non-const cases, making them very difficult to use and resulting in many extremely +// difficult to find bugs in the code base. +// Ultimately these classes should all be converted to reference counted methodologies, but as an interim measure +// I am rationalizing their behavior until we flush out all places in the code base that inadvertently depended +// on the original behavior. This is done with assert statements in BitVector methods. +// ==== +// What the behavior was probably supposed to be: +// Vectors can 'own' the data they point to or not. Only one Vector 'owns' the memory at a time, +// so that automatic destruction can be used. So whenever there is an operation that yields one +// vector from another the options were: clone (allocate a new vector from memory), alias (make the +// new vector point into the memory of the original vector) or shift (the new Vector steals the +// memory ownership from the original vector.) +// The const copy-constructor did a clone, the non-const copy constructor did a shiftMem, and the segment and +// related methods (head, tail, etc) returned aliases. +// Since a copy-constructor is inserted transparently in sometimes surprising places, this made the +// class very difficult to use. Moreover, since the C++ standard specifies that a copy-constructor is used +// to copy the return value from functions, it makes it literally impossible for a function to fully control +// the return value. Our code has relied on the "Return Value Optimization" which says that the C++ compiler +// may omit the copy-construction of the return value even if the copy-constructor has side-effects, which ours does. +// This methodology is fundamentally incompatible with C++. +// What the original behavior actually was: +// class Vector: +// The copy-constructor and assignment operators did a clone for the const case and a shift for the non-const case. +// This is really horrible. +// The segment methods were identical for const and non-const cases, always returning an alias. +// This also resulted in zillions of redundant mallocs and copies throughout the code base. +// class BitVector: +// Copy-constructor: +// BitVector did not have any copy-constructors, and I think the intent was that it would have the same behavior +// as Vector, but that is not how C++ works: with no copy-constructor the default copy-constructor +// uses only the const case, so only the const Vector copy-constructor was used. Therefore it always cloned, +// and the code base relied heavily on the "Return Value Optimization" to work at all. +// Assignment operator: +// BitVector did not have one, so C++ makes a default one that calls Vector::operator=() as a side effect, +// which did a clone; not sure if there was a non-const version and no longer care. +// segment methods: +// The non-const segment() returned an alias, and the const segment() returned a clone. +// I think the intent was that the behavior should be the same as Vector, but there was a conversion +// of the result of the const segment() method from Vector to BitVector which caused the Vector copy-constructor +// to be (inadvertently) invoked, resulting in the const version of the segment method returning a clone. +// What the behavior is now: +// VectorBase: +// There is a new VectorBase class that has only the common methods and extremely basic constructors. +// The VectorBase class MUST NOT CONTAIN: copy constructors, non-trivial constructors called from derived classes, +// or any method that returns a VectorBase type object. Why? Because any of the above when used in derived classes +// can cause copy-constructor invocation, often surprisingly, obfuscating the code. +// Each derived class must provide its own: copy-constructors and segment() and related methods, since we do not +// want to inadvertently invoke a copy-constructor to convert the segment() result from VectorBase to the derived type. +// BitVector: +// The BitVector copy-constructor and assignment operator (inherited from VectorBase) paradigm is: +// if the copied Vector owned memory, perform a clone so the new vector owns memory also, +// otherwise just do a simple copy, which is another alias. This isnt perfect but works every place +// in our code base and easier to use than the previous paradigm. +// The segment method always returns an alias. +// If you want a clone of a segment, use cloneSegment(), which replaces the previous: const segment(...) const method. +// Note that the semantics of cloneSegment still rely on the Return Value Optimization. Oh well, we should use refcnts. +// Vector: +// I left Vector alone (except for rearrangement to separate out VectorBase.) Vector should just not be used. +// SoftVector: +// SoftVector and signalVector should be updated similar to BitVector, but I did not want to disturb them. +// What the behavior should be: +// All these should be reference-counted, similar to ByteVector. +template class VectorBase +{ + // TODO -- Replace memcpy calls with for-loops. (pat) in case class T is not POD [Plain Old Data] protected: - - T* mData; ///< allocated data block, if any +#if BITVECTOR_REFCNTS + typedef RefCntPointer > VectorDataType; +#else + typedef T* VectorDataType; +#endif + VectorDataType mData; ///< allocated data block. T* mStart; ///< start of useful data T* mEnd; ///< end of useful data + 1 + // Init vector with specified size. Previous contents are completely discarded. This is only used for initialization. + void vInit(size_t elements) + { + mData = elements ? new T[elements] : NULL; + mStart = mData; + mEnd = mStart + elements; + } + + /** Assign from another Vector, shifting ownership. */ + // (pat) This should be eliminated, but it is used by Vector and descendents. + void shiftMem(VectorBase&other) + { + VECTORDEBUG("VectorBase::shiftMem(%p)",(void*)&other); + this->clear(); + this->mData=other.mData; + this->mStart=other.mStart; + this->mEnd=other.mEnd; + other.mData=NULL; + } + + // Assign from another Vector, making this an alias to other. + void makeAlias(const VectorBase &other) + { + if (this->getData()) { + assert(this->getData() != other.getData()); // Not possible by the semantics of Vector. + this->clear(); + } + this->mStart=const_cast(other.mStart); + this->mEnd=const_cast(other.mEnd); + } + public: - /**** - char *inspect() { - static char buf[100]; - sprintf(buf," mData=%p mStart=%p mEnd=%p ",mData,mStart,mEnd); - return buf; - } - ***/ - - /** Return the size of the Vector. */ + /** Return the size of the Vector in units, ie, the number of T elements. */ size_t size() const { assert(mStart>=mData); @@ -80,114 +178,59 @@ template class Vector { } /** Return size in bytes. */ - size_t bytes() const { return size()*sizeof(T); } + size_t bytes() const { return this->size()*sizeof(T); } - /** Change the size of the Vector, discarding content. */ - void resize(size_t newSize) - { + /** Change the size of the Vector in items (not bytes), discarding content. */ + void resize(size_t newElements) { + //VECTORDEBUG("VectorBase::resize("<<(void*)this<<","<resize(0); } /** Copy data from another vector. */ - void clone(const Vector& other) - { - resize(other.size()); + void clone(const VectorBase& other) { + this->resize(other.size()); memcpy(mData,other.mStart,other.bytes()); } + void vConcat(const VectorBase&other1, const VectorBase&other2) { + this->resize(other1.size()+other2.size()); + memcpy(this->mStart, other1.mStart, other1.bytes()); + memcpy(this->mStart+other1.size(), other2.mStart, other2.bytes()); + } + protected: - - //@{ - - /** Build an empty Vector of a given size. */ - Vector(size_t wSize=0):mData(NULL) { resize(wSize); } - - /** Build a Vector by shifting the data block. */ - Vector(Vector& other) - :mData(other.mData),mStart(other.mStart),mEnd(other.mEnd) - { other.mData=NULL; } - - /** Build a Vector by copying another. */ - Vector(const Vector& other):mData(NULL) { clone(other); } + VectorBase() : mData(0), mStart(0), mEnd(0) {} /** Build a Vector with explicit values. */ - Vector(T* wData, T* wStart, T* wEnd) - :mData(wData),mStart(wStart),mEnd(wEnd) - { } - - /** Build a vector from an existing block, NOT to be deleted upon destruction. */ - Vector(T* wStart, size_t span) - :mData(NULL),mStart(wStart),mEnd(wStart+span) - { } - - /** Build a Vector by concatenation. */ - Vector(const Vector& other1, const Vector& other2) - :mData(NULL) - { - resize(other1.size()+other2.size()); - memcpy(mStart, other1.mStart, other1.bytes()); - memcpy(mStart+other1.size(), other2.mStart, other2.bytes()); + VectorBase(VectorDataType wData, T* wStart, T* wEnd) :mData(wData),mStart(wStart),mEnd(wEnd) { + //VECTORDEBUG("VectorBase("<<(void*)wData); + VECTORDEBUG("VectorBase(%p,%p,%p)",this->getData(),wStart,wEnd); } - //@} + public: /** Destroy a Vector, deleting held memory. */ - ~Vector() { clear(); } - - - - - //@{ - - /** Assign from another Vector, shifting ownership. */ - void operator=(Vector& other) - { - clear(); - mData=other.mData; - mStart=other.mStart; - mEnd=other.mEnd; - other.mData=NULL; + ~VectorBase() { + //VECTORDEBUG("~VectorBase("<<(void*)this<<")"); + VECTORDEBUG("~VectorBase(%p)",this); + this->clear(); } - /** Assign from another Vector, copying. */ - void operator=(const Vector& other) { clone(other); } + bool isOwner() { return !!this->mData; } // Do we own any memory ourselves? - //@} - - - //@{ - - /** Return an alias to a segment of this Vector. */ - Vector segment(size_t start, size_t span) - { - T* wStart = mStart + start; - T* wEnd = wStart + span; - assert(wEnd<=mEnd); - return Vector(NULL,wStart,wEnd); + std::string inspect() const { + char buf[100]; + snprintf(buf,100," mData=%p mStart=%p mEnd=%p ",(void*)mData,mStart,mEnd); + return std::string(buf); } - /** Return an alias to a segment of this Vector. */ - const Vector segment(size_t start, size_t span) const - { - T* wStart = mStart + start; - T* wEnd = wStart + span; - assert(wEnd<=mEnd); - return Vector(NULL,wStart,wEnd); - } - - Vector head(size_t span) { return segment(0,span); } - const Vector head(size_t span) const { return segment(0,span); } - Vector tail(size_t start) { return segment(start,size()-start); } - const Vector tail(size_t start) const { return segment(start,size()-start); } /** Copy part of this Vector to a segment of another Vector. @@ -195,7 +238,7 @@ template class Vector { @param start The start point in the other vector. @param span The number of elements to copy. */ - void copyToSegment(Vector& other, size_t start, size_t span) const + void copyToSegment(VectorBase& other, size_t start, size_t span) const { T* base = other.mStart + start; assert(base+span<=other.mEnd); @@ -204,17 +247,18 @@ template class Vector { } /** Copy all of this Vector to a segment of another Vector. */ - void copyToSegment(Vector& other, size_t start=0) const { copyToSegment(other,start,size()); } + void copyToSegment(VectorBase& other, size_t start=0) const { copyToSegment(other,start,size()); } - void copyTo(Vector& other) const { copyToSegment(other,0,size()); } + void copyTo(VectorBase& other) const { copyToSegment(other,0,size()); } /** Copy a segment of this vector into another. @param other The other vector (to copt into starting at 0.) @param start The start point in this vector. @param span The number of elements to copy. + WARNING: This function does NOT resize the result - you must set the result size before entering. */ - void segmentCopyTo(Vector& other, size_t start, size_t span) const + void segmentCopyTo(VectorBase& other, size_t start, size_t span) const { const T* base = mStart + start; assert(base+span<=mEnd); @@ -236,11 +280,19 @@ template class Vector { while (dp& other) { + //std::cout << "Vector=(this="<inspect()<<",other="<clone(other); + } else { + this->makeAlias(other); + } + //std::cout << "Vector= after(this="<inspect()<<")"< class Vector { return mStart[index]; } - const T* begin() const { return mStart; } - T* begin() { return mStart; } - const T* end() const { return mEnd; } - T* end() { return mEnd; } - bool isOwner() { return !!mData; } // Do we own any memory ourselves? - //@} - - + const T* begin() const { return this->mStart; } + T* begin() { return this->mStart; } + const T* end() const { return this->mEnd; } + T* end() { return this->mEnd; } +#if BITVECTOR_REFCNTS + const T*getData() const { return this->mData.isNULL() ? 0 : this->mData->mPointer; } +#else + const T*getData() const { return this->mData; } +#endif }; +// (pat) Nov 2013. This class retains the original poor behavior. See comments at VectorBase +template class Vector : public VectorBase +{ + public: + + /** Build an empty Vector of a given size. */ + Vector(size_t wSize=0) { this->resize(wSize); } + + /** Build a Vector by shifting the data block. */ + Vector(Vector& other) : VectorBase(other.mData,other.mStart,other.mEnd) { other.mData=NULL; } + + /** Build a Vector by copying another. */ + Vector(const Vector& other):VectorBase() { this->clone(other); } + + /** Build a Vector with explicit values. */ + Vector(T* wData, T* wStart, T* wEnd) : VectorBase(wData,wStart,wEnd) { } + + /** Build a vector from an existing block, NOT to be deleted upon destruction. */ + Vector(T* wStart, size_t span) : VectorBase(NULL,wStart,wStart+span) { } + + /** Build a Vector by concatenation. */ + Vector(const Vector& other1, const Vector& other2):VectorBase() { + assert(this->mData == 0); + vConcat(other1,other2); + } + + //@{ + + /** Assign from another Vector, shifting ownership. */ + void operator=(Vector& other) { this->shiftMem(other); } + + /** Assign from another Vector, copying. */ + void operator=(const Vector& other) { this->clone(other); } + + /** Return an alias to a segment of this Vector. */ + Vector segment(size_t start, size_t span) + { + T* wStart = this->mStart + start; + T* wEnd = wStart + span; + assert(wEnd<=this->mEnd); + return Vector(NULL,wStart,wEnd); + } + + /** Return an alias to a segment of this Vector. */ + const Vector segment(size_t start, size_t span) const + { + T* wStart = this->mStart + start; + T* wEnd = wStart + span; + assert(wEnd<=this->mEnd); + return Vector(NULL,wStart,wEnd); + } + + Vector head(size_t span) { return segment(0,span); } + const Vector head(size_t span) const { return segment(0,span); } + Vector tail(size_t start) { return segment(start,this->size()-start); } + const Vector tail(size_t start) const { return segment(start,this->size()-start); } + + /**@name Iterator types. */ + //@{ + typedef T* iterator; + typedef const T* const_iterator; + //@} + + //@} +}; + + diff --git a/VectorTest.cpp b/VectorTest.cpp index ad5c473..4065968 100644 --- a/VectorTest.cpp +++ b/VectorTest.cpp @@ -23,7 +23,7 @@ */ - +#define ENABLE_VECTORDEBUG #include "Vector.h" #include @@ -35,9 +35,28 @@ ConfigurationTable gConfig; using namespace std; typedef Vector TestVector; +int barfo; +void foo(TestVector a) +{ + barfo = a.size(); // Do something so foo wont be optimized out. +} +void anotherTest() +{ + cout << "START Vector anotherTest" << endl; + TestVector v0(10); + TestVector atest = v0.head(3); + cout << atest << endl; + cout << "calling head" << endl; + cout << v0.head(3) << endl; + cout << "Passing Vector" << endl; + // This calls the Vector non-const copy constructor + foo(v0); + cout << "FINISH anotherTest" << endl; +} int main(int argc, char *argv[]) { + anotherTest(); TestVector test1(5); for (int i=0; i<5; i++) test1[i]=i; TestVector test2(5); @@ -49,7 +68,9 @@ int main(int argc, char *argv[]) { TestVector testC(test1,test2); cout << testC << endl; - cout << testC.head(3) << endl; + + TestVector foo = testC.head(3); + //cout << testC.head(3) << endl; cout << testC.tail(3) << endl; testC.fill(8); cout << testC << endl; diff --git a/sqlite3util.cpp b/sqlite3util.cpp index 2500c51..ddffe69 100644 --- a/sqlite3util.cpp +++ b/sqlite3util.cpp @@ -6,11 +6,16 @@ #include "sqlite3.h" #include "sqlite3util.h" +#include "Logger.h" #include #include #include +#include +#include +using namespace std; + // Wrappers to sqlite operations. // These will eventually get moved to commonlibs. @@ -31,7 +36,7 @@ int sqlite3_prepare_statement(sqlite3* DB, sqlite3_stmt **stmt, const char* quer usleep(200); } if (src) { - fprintf(stderr,"sqlite3_prepare_v2 failed for \"%s\": %s\n",query,sqlite3_errmsg(DB)); + LOG(ERR)<< format("sqlite3_prepare_v2 failed code=%u for \"%s\": %s\n",src,query,sqlite3_errmsg(DB)); sqlite3_finalize(*stmt); } return src; @@ -49,11 +54,88 @@ int sqlite3_run_query(sqlite3* DB, sqlite3_stmt *stmt, unsigned retries) usleep(200); } if ((src!=SQLITE_DONE) && (src!=SQLITE_ROW)) { - fprintf(stderr,"sqlite3_run_query failed: %s: %s\n", sqlite3_sql(stmt), sqlite3_errmsg(DB)); + LOG(ERR) << format("sqlite3_run_query failed code=%u for: %s: %s\n", src, sqlite3_sql(stmt), sqlite3_errmsg(DB)); } return src; } +// condition buffer is size 100 minimum. +static const unsigned sqlBufferSize = 100; +static const char *condition_c(char *resultBuffer,const char *keyName, const char*keyValue) +{ + snprintf(resultBuffer,sqlBufferSize,"WHERE %s == '%s'",keyName,keyValue); + return resultBuffer; +} + +static const char *condition_u(char *resultBuffer,const char *keyName, unsigned keyValue) +{ + snprintf(resultBuffer,sqlBufferSize,"WHERE %s == %u",keyName,keyValue); + return resultBuffer; +} + +void sqlQuery::queryStart(sqlite3*db, const char *tableName,const char *resultColumns, const char*condition) +{ + int retries = 5; + mdb = db; + mQueryRC = SQLITE_ERROR; // Until we know better. + //size_t stringSize = sqlBufferSize + strlen(resultColumns) + strlen(tableName) + strlen(condition); + //char query[stringSize]; + //snprintf(query,stringSize,"SELECT %s FROM %s %s",resultColumns,tableName,condition); + // We save the query in a string so the caller can print it out in error messages if the query fails. + mQueryString = format("SELECT %s FROM %s %s",resultColumns,tableName,condition); + // Prepare the statement. + if (sqlite3_prepare_statement(mdb,&mStmt,mQueryString.c_str(),retries)) { mStmt = NULL; return; } + // Read the result. + mQueryRC = sqlite3_run_query(mdb,mStmt,retries); +} + +// Load the next row. Return true if there is another row, false if finished or error. +bool sqlQuery::sqlStep() +{ + if (mQueryRC == SQLITE_ROW) { + // Get the next row. + mQueryRC = sqlite3_run_query(mdb,mStmt,5); + } + return mQueryRC == SQLITE_ROW; +} + +sqlQuery::sqlQuery(sqlite3*db, const char *tableName,const char *resultColumns,const char *condition) +{ + queryStart(db,tableName,resultColumns,condition); +} + +sqlQuery::sqlQuery(sqlite3*db, const char *tableName,const char *resultColumns,const char *keyName, unsigned keyData) +{ + char conditionBuffer[sqlBufferSize]; + queryStart(db,tableName,resultColumns,condition_u(conditionBuffer,keyName,keyData)); +} + +sqlQuery::sqlQuery(sqlite3*db, const char *tableName,const char *resultColumns,const char *keyName, const char *keyData) +{ + char conditionBuffer[sqlBufferSize]; + queryStart(db,tableName,resultColumns,condition_c(conditionBuffer,keyName,keyData)); +} + +sqlQuery::~sqlQuery() +{ + if (mStmt) sqlite3_finalize(mStmt); +} + +string sqlQuery::getResultText(int colNum) +{ + if (sqlSuccess()) { + const char* ptr = (const char*)sqlite3_column_text(mStmt,colNum); + return ptr ? string(ptr,sqlite3_column_bytes(mStmt,colNum)) : string(""); + } + return string(""); +} + +sqlite3_int64 sqlQuery::getResultInt(int colNum) +{ + return sqlSuccess() ? sqlite3_column_int64(mStmt,colNum) : 0; +} + + bool sqlite3_exists(sqlite3* DB, const char *tableName, const char* keyName, const char* keyData, unsigned retries) @@ -94,6 +176,114 @@ bool sqlite3_single_lookup(sqlite3* DB, const char *tableName, return retVal; } +#if 0 // This code works fine, but sqlQuery is a better way. +// If result is a row return the sqlite3_stmt, else NULL. +// Pass a comma-separated list of column names to return, or if you want all the columns in the result, pass "*" as the resultColumns. +// Almost all the other functions below could use this. +sqlite3_stmt *sqlite_lookup_row(sqlite3*db, const char *tableName, const char* condition, const char *resultColumns) +{ + int retries = 5; + size_t stringSize = sqlBufferSize + strlen(resultColumns) + strlen(tableName) + strlen(condition); + char query[stringSize]; + snprintf(query,stringSize,"SELECT %s FROM %s %s",resultColumns,tableName,condition); + // Prepare the statement. + sqlite3_stmt *stmt; + if (sqlite3_prepare_statement(db,&stmt,query,retries)) return NULL; + // Read the result. + int src = sqlite3_run_query(db,stmt,retries); + if (src == SQLITE_ROW) { + return stmt; // Caller must sqlite3_finalize(); + } + sqlite3_finalize(stmt); + return NULL; +} +sqlite3_stmt *sqlite_lookup_row_c(sqlite3*db, const char *tableName, const char* keyName, const char *keyData, const char *resultColumns) +{ + char conditionBuffer[sqlBufferSize]; + return sqlite_lookup_row(db,tableName,condition_c(conditionBuffer,keyName,keyData),resultColumns); +} +sqlite3_stmt *sqlite_lookup_row_u(sqlite3*db, const char *tableName, const char* keyName, unsigned keyValue, const char* resultColumns) +{ + char conditionBuffer[sqlBufferSize]; + return sqlite_lookup_row(db,tableName,condition_u(conditionBuffer,keyName,keyValue),resultColumns); +} +// Pass a comma-separated list of column names to return, or if you want all the columns in the result, pass "*" as the resultColumns. +vector sqlite_multi_lookup_vector(sqlite3* db, const char* tableName, const char* keyName, const char* keyData, const char *resultColumns) +{ + vector result; + if (sqlite3_stmt *stmt = sqlite_lookup_row_c(db,tableName,keyName,keyData,resultColumns)) { + int n = sqlite3_column_count(stmt); + if (n < 0 || n > 100) { goto done; } // Would like to LOG an error but afraid to use LOG in here. + result.reserve(n+1); + for (int i = 0; i < n; i++) { + const char* ptr = (const char*)sqlite3_column_text(stmt,i); + result.push_back(ptr ? string(ptr,sqlite3_column_bytes(stmt,i)) : string("")); + } + done: + sqlite3_finalize(stmt); + } + return result; +} +#endif + + +bool sqlite_single_lookup(sqlite3* db, const char* tableName, + const char* keyName, const char* keyData, + const char* resultName, string &resultData) +{ + sqlQuery query(db,tableName,resultName,keyName,keyData); + if (query.sqlSuccess()) { + resultData = query.getResultText(); + return true; + } + return false; +#if 0 + if (sqlite3_stmt *stmt = sqlite_lookup_row_c(db,tableName,keyName,keyData,valueName)) { + if (const char* ptr = (const char*)sqlite3_column_text(stmt,0)) { + valueData = string(ptr,sqlite3_column_bytes(stmt,0)); + } + sqlite3_finalize(stmt); + return true; + } + return false; +#endif +} + +bool sqlite_single_lookup(sqlite3* db, const char* tableName, + const char* keyName, unsigned keyValue, + const char* resultName, string &resultData) +{ + sqlQuery query(db,tableName,resultName,keyName,keyValue); + if (query.sqlSuccess()) { + resultData = query.getResultText(); + return true; + } + return false; +#if 0 + if (sqlite3_stmt *stmt = sqlite_lookup_row_u(db,tableName,keyName,keyValue,valueName)) { + if (const char* ptr = (const char*)sqlite3_column_text(stmt,0)) { + valueData = string(ptr,sqlite3_column_bytes(stmt,0)); + } + sqlite3_finalize(stmt); + return true; + } + return false; +#endif +} + +// Do the lookup and just return the string. +// For this function an empty value is indistinguishable from failure - both return an empty string. +string sqlite_single_lookup_string(sqlite3* db, const char* tableName, + const char* keyName, const char* keyData, const char* resultName) +{ + return sqlQuery(db,tableName,resultName,keyName,keyData).getResultText(); +#if 0 + string result; + (void) sqlite_single_lookup(db,tableName,keyName,keyData,valueName,result); + return result; +#endif +} + // This function returns an allocated string that must be free'd by the caller. bool sqlite3_single_lookup(sqlite3* DB, const char* tableName, @@ -103,7 +293,7 @@ bool sqlite3_single_lookup(sqlite3* DB, const char* tableName, valueData=NULL; size_t stringSize = 100 + strlen(valueName) + strlen(tableName) + strlen(keyName) + strlen(keyData); char query[stringSize]; - sprintf(query,"SELECT %s FROM %s WHERE %s == \"%s\"",valueName,tableName,keyName,keyData); + snprintf(query,stringSize,"SELECT %s FROM %s WHERE %s == \"%s\"",valueName,tableName,keyName,keyData); // Prepare the statement. sqlite3_stmt *stmt; if (sqlite3_prepare_statement(DB,&stmt,query,retries)) return false; @@ -144,16 +334,42 @@ bool sqlite3_single_lookup(sqlite3* DB, const char* tableName, return retVal; } +bool sqlite_set_attr(sqlite3*db,const char *attr_name,const char*attr_value) +{ + if (! sqlite3_command(db,"CREATE TABLE IF NOT EXISTS ATTR_TABLE (ATTR_NAME TEXT PRIMARY KEY, ATTR_VALUE TEXT)")) { + const char *fn = sqlite3_db_filename(db,"main"); + LOG(WARNING) << "Could not create ATTR_TABLE in database file " <<(fn?fn:""); + return false; + } + char query[100]; + snprintf(query,100,"REPLACE INTO ATTR_TABLE (ATTR_NAME,ATTR_VALUE) VALUES('%s','%s')",attr_name,attr_value); + if (! sqlite3_command(db,query)) { + const char *fn = sqlite3_db_filename(db,"main"); + LOG(WARNING) << "Could not set attribute: "< +#include +#include // (pat) Dont put statics in .h files - they generate a zillion g++ error messages. extern const char *enableWAL; @@ -9,6 +11,35 @@ extern const char *enableWAL; // "PRAGMA journal_mode=WAL" //}; +// Pat added. +class sqlQuery { + sqlite3 *mdb; + sqlite3_stmt *mStmt; + int mQueryRC; + void queryStart(sqlite3*db, const char *tableName,const char *condition, const char*resultCols); + + public: + std::string mQueryString; + // Query for row(s) matching this condition. Can request one or more result columns, or "*" for all columns. + sqlQuery(sqlite3*db, const char *tableName,const char*resultColumns,const char *condition); + // Query for a row where keyName == keyData. + sqlQuery(sqlite3*db, const char *tableName,const char*resultColumns,const char *keyName, const char*keyData); + // Query for a row where keyName == keyData. + sqlQuery(sqlite3*db, const char *tableName,const char*resultColumns,const char *keyName, unsigned keyData); + // Did the query succeed and find a result row? + bool sqlSuccess() { return mStmt && mQueryRC == SQLITE_ROW; } + // Return the results as text or integer. + std::string getResultText(int colNum=0); + sqlite3_int64 getResultInt(int colNum=0); + // Return the number of columns in the result, or 0 if the result did not contain any data. + // Note: If the table is completely empty, sqlite3_column_count returns non-0, so check mQueryRC first. + unsigned sqlResultSize() { return mStmt && mQueryRC == SQLITE_ROW ? sqlite3_column_count(mStmt) : 0; } + // Step to the next row. Return false if there are no more rows. + bool sqlStep(); + ~sqlQuery(); +}; + +// (pat) These functions should probably not begin with "sqlite3_" since that is reserved for sqlite3 itself... int sqlite3_prepare_statement(sqlite3* DB, sqlite3_stmt **stmt, const char* query, unsigned retries = 5); int sqlite3_run_query(sqlite3* DB, sqlite3_stmt *stmt, unsigned retries = 5); @@ -17,6 +48,7 @@ bool sqlite3_single_lookup(sqlite3* DB, const char *tableName, const char* keyName, const char* keyData, const char* valueName, unsigned &valueData, unsigned retries = 5); +// This function returns an allocated string that must be free'd by the caller. bool sqlite3_single_lookup(sqlite3* DB, const char* tableName, const char* keyName, const char* keyData, const char* valueName, char* &valueData, unsigned retries = 5); @@ -26,10 +58,25 @@ bool sqlite3_single_lookup(sqlite3* DB, const char* tableName, const char* keyName, unsigned keyData, const char* valueName, char* &valueData, unsigned retries = 5); +bool sqlite_single_lookup(sqlite3* DB, const char* tableName, + const char* keyName, const char* keyData, + const char* valueName, std::string &valueData); +bool sqlite_single_lookup(sqlite3* DB, const char* tableName, + const char* keyName, unsigned keyData, + const char* valueName, std::string &valueData); + +//std::vector sqlite_multi_lookup_vector(sqlite3* DB, const char* tableName, const char* keyName, const char* keyData, const char *resultColumns); +std::string sqlite_single_lookup_string(sqlite3* DB, const char* tableName, const char* keyName, unsigned keyData, const char* valueName); + +// Get and set attributes on an sqlite database. Works by creating an ATTR_TABLE in the database. +bool sqlite_set_attr(sqlite3*db,const char *attr_name,const char *attr_value); +std::string sqlite_get_attr(sqlite3*db,const char *attr_name); + bool sqlite3_exists(sqlite3* DB, const char* tableName, const char* keyName, const char* keyData, unsigned retries = 5); /** Run a query, ignoring the result; return true on success. */ -bool sqlite3_command(sqlite3* DB, const char* query, unsigned retries = 5); +bool sqlite_command(sqlite3* DB, const char* query, int *pResultCode = NULL, unsigned retries=5); +bool inline sqlite3_command(sqlite3* DB, const char* query, unsigned retries = 5) { return sqlite_command(DB,query,NULL,retries); } #endif