Merge branch 'feature-tpc-compression'

Finishing development and testing in branch

HLT/BASE/AliHLTDataDeflater.cxx

@@ -327,26 +327,28 @@ int AliHLTDataDeflater::FillStatistics(int id, AliHLTUInt64_t value, unsigned le
       fHistograms->GetEntriesFast()<=id ||
       id<0) return 0;
 
+  if (value<(~(AliHLTUInt64_t)0)) {
   TObject* o=fHistograms->At(id);
   if (o) {
     TH1* h=dynamic_cast<TH1*>(o);
     if (h) {
       h->Fill(value);
     }
   }
+  }
 
   if (!fParameterCompression) {
     int bins=fHistograms->GetEntriesFast();
     fParameterCompression=new TH2D("ParameterCompression", "ParameterCompression", bins, 0, bins, 1000, 0., 5.0);
   }
-  if (fParameterCompression) {
+  if (fParameterCompression && codingWeight>=.0) {
     fParameterCompression->Fill(id, codingWeight);
   }
   if (!fParameterSize) {
     int bins=fHistograms->GetEntriesFast();
     fParameterSize=new TH2D("ParameterSize", "ParameterSize", bins, 0, bins, 1000, 0., 64.0);
   }
-  if (fParameterSize) {
+  if (fParameterSize && length>0) {
     fParameterSize->Fill(id, length);
   }
 

HLT/BASE/AliHLTDataDeflater.h

@@ -151,6 +151,12 @@ class AliHLTDataDeflater : public AliHLTLogging
   /// return unique version of the deflater, base class has version 0
   virtual int GetDeflaterVersion() const {return 0;}
 
+  /// start the encoding of a new sequence
+  virtual int StartEncoder() { return 0; };
+
+  /// stop the encoding current sequence
+  virtual int StopEncoder() { return 0; };
+
 protected:
 
 private:

HLT/BASE/AliHLTDataDeflaterHuffman.cxx

@@ -29,6 +29,7 @@
 #include "TFile.h"
 #include <memory>
 #include <algorithm>
+#include <cmath>
 #include <iostream>
 
 /** ROOT macro for the implementation of ROOT specific class methods */
@@ -40,6 +41,8 @@ AliHLTDataDeflaterHuffman::AliHLTDataDeflaterHuffman(bool bTrainingMode)
   , fHuffmanCoders()
   , fHuffmanCoderList(NULL)
   , fTrainingMode(bTrainingMode)
+  , fParameterClusterCount()
+  , fBitCount()
 {
   // see header file for class documentation
   // or
@@ -84,8 +87,10 @@ int AliHLTDataDeflaterHuffman::AddParameterDefinition(const char* name, unsigned
     return -EPERM;
   }
 
-  fReferenceLength.push_back(refLength);
+  fReferenceLength.push_back(refLength>0?refLength:bitLength);
   fHuffmanCoders.push_back(pHuffman);
+  fParameterClusterCount.push_back(0);
+  fBitCount.push_back(0);
 
   int memberId=fHuffmanCoders.size()-1;
   if (DoStatistics()) {
@@ -134,6 +139,8 @@ bool AliHLTDataDeflaterHuffman::OutputParameterBits( int memberId, AliHLTUInt64_
     return false;
   }
 
+  fParameterClusterCount[memberId]++;
+
   AliHLTUInt64_t length = 0;
   const std::bitset<64>& v=fHuffmanCoders[memberId]->Encode((value>fHuffmanCoders[memberId]->GetMaxValue())?fHuffmanCoders[memberId]->GetMaxValue():value, length);
   //cout << fHuffmanCoders[memberId]->GetName() << " value " << value << ": code lenght " << length << " " << v << endl;
@@ -146,7 +153,8 @@ bool AliHLTDataDeflaterHuffman::OutputParameterBits( int memberId, AliHLTUInt64_
       weight=length;
       weight/=parameterLength;
     }
-    FillStatistics(memberId, value, length, weight);
+    FillStatistics(memberId, value, 0, -1.0);
+    fBitCount[memberId]+=length;
   }
 
   if (length>0) {
@@ -304,6 +312,32 @@ void AliHLTDataDeflaterHuffman::SaveAs(const char *filename, Option_t *option) c
   return AliHLTDataDeflater::SaveAs(filename, remainingOptions);
 }
 
+int AliHLTDataDeflaterHuffman::StartEncoder()
+{
+  int memberId=0;
+  for (vector<unsigned>::iterator it = fBitCount.begin(); it!=fBitCount.end(); it++, memberId++) {
+    *it=0;
+    fParameterClusterCount[memberId]=0;
+  }
+  return 0;
+}
+
+int AliHLTDataDeflaterHuffman::StopEncoder()
+{
+  int memberId=0;
+  for (vector<unsigned>::iterator it = fBitCount.begin(); it!=fBitCount.end(); it++, memberId++) {
+    if (fParameterClusterCount[memberId]==0) continue;
+    UInt_t outputByteCount = (*it+7)/8;
+
+    float weight=outputByteCount*8.0;
+    weight/=fParameterClusterCount[memberId];
+    unsigned parameterSize=(unsigned)ceil(weight);
+    weight/=fReferenceLength[memberId];
+    FillStatistics(memberId, ~(AliHLTUInt64_t)0, parameterSize, weight);
+  }
+  return GetBitDataOutputSizeBytes();
+}
+
 ostream& operator<<(ostream &out, const AliHLTDataDeflaterHuffman& me)
 {
   me.Print(out);

HLT/BASE/AliHLTDataDeflaterHuffman.h

@@ -37,6 +37,12 @@ class AliHLTDataDeflaterHuffman : public AliHLTDataDeflater
   /// init list of decoders
   int InitDecoders(TList* decoderlist);
 
+  /// inherited from AliHLTDataDeflater: start the encoding for a new event
+  virtual int StartEncoder();
+
+  /// inherited from AliHLTDataDeflater: stop the encoding for current event
+  virtual int StopEncoder();
+
   /// inherited from AliHLTDataDeflater: write bit pattern according to configuration
   virtual bool OutputParameterBits( int parameterId, AliHLTUInt64_t const & value );
 
@@ -89,6 +95,9 @@ class AliHLTDataDeflaterHuffman : public AliHLTDataDeflater
 
   bool fTrainingMode; //! indicate training mode
 
+  vector<unsigned> fParameterClusterCount; // cluster count for every parameter
+  vector<unsigned> fBitCount; // bit count for every parameter
+
   ClassDef(AliHLTDataDeflaterHuffman, 0)
 };
 

HLT/BASE/AliHLTDataInflater.h

@@ -136,7 +136,7 @@ class AliHLTDataInflater : public AliHLTLogging
   /** function to determine input bit
    * @return boolean (if bit is 1 or 0)
    */
-  bool InputBit( AliHLTUInt8_t & value );
+  virtual bool InputBit( AliHLTUInt8_t & value );
 
   /** function to read bits from bitstream
    * @param value
@@ -152,7 +152,7 @@ class AliHLTDataInflater : public AliHLTLogging
   bool RewindBitPosition(UInt_t const & bitCount);
 
   /** function pad 8 bits */
-  void Pad8Bits();
+  virtual void Pad8Bits();
 
   /** function to determine input bytes
    * @param data       AliHLTUInt8_t* pointer to input data

HLT/BASE/AliHLTDataInflaterHuffman.cxx

@@ -153,6 +153,31 @@ bool AliHLTDataInflaterHuffman::NextValue(AliHLTUInt64_t& value, AliHLTUInt32_t&
   return true;
 }
 
+bool AliHLTDataInflaterHuffman::InputBit( AliHLTUInt8_t & value )
+{
+  /// special overload of InputBit method to consider the
+  /// internal register
+  if (fInputLength > 0) {
+    const int shiftval=sizeof(fInput)*8 - 1;
+    value = (fInput>>shiftval) & 0x1;
+    fInput<<=1;
+    fInputLength-=1;
+    return true;
+  }
+
+  return AliHLTDataInflater::InputBit(value);
+}
+
+void AliHLTDataInflaterHuffman::Pad8Bits()
+{
+  /// special overload of Pad8Bits method to clear the
+  /// internal register and rewind the read pointer
+  RewindBitPosition(fInputLength);
+  fInputLength = 0;
+  fInput = 0;
+  AliHLTDataInflater::Pad8Bits();
+}
+
 void AliHLTDataInflaterHuffman::Print(Option_t* option) const
 {
   /// Print info

HLT/BASE/AliHLTDataInflaterHuffman.h

@@ -31,8 +31,18 @@ class AliHLTDataInflaterHuffman : public AliHLTDataInflater
   /// init list of decoders
   int InitDecoders(TList* decoderlist);
 
-  /// overloaded from AliHLTDataInflater
+  /**
+   * Retrieve next value from data stream
+   * The next variable-length code is read from the stream and decoded
+   * using the initialized Huffman instance. The maximum number of input
+   * bits is always required for the Huffman decoder. Internal buffering
+   * is implemented to avoid repetitive backward seek in the input stream
+   * after decoding of a symbol when the length is known.
+   *
+   * overloaded from AliHLTDataInflater
+   */
   virtual bool NextValue(AliHLTUInt64_t& value, AliHLTUInt32_t& length);
+
   /// switch to next parameter
   virtual int NextParameter() {
     if (fHuffmanCoders.size()==0) return -1;
@@ -42,6 +52,20 @@ class AliHLTDataInflaterHuffman : public AliHLTDataInflater
     return fCurrentParameter;
   }
 
+  /**
+   * Read next bit from the input.
+   * This overload of AliHLTDataInflater::InputBit handles the internal
+   * buffer and forwards to the base class method if the buffer is empty.
+   */
+  bool InputBit( AliHLTUInt8_t & value );
+
+  /**
+   * Pad read pointer to next 8 bit boundary
+   * This overload first clears the internal register and rewinds the read
+   * pointer appropriately, then calls Pad8Bits of the base class
+   */
+  void Pad8Bits();
+
   /// Print info
   void Print(Option_t* option = "") const;
   /// clear the object and reset pointer references

HLT/TPCLib/AliHLTTPCHWCFSpacePointContainer.cxx

@@ -678,6 +678,7 @@ int AliHLTTPCHWCFSpacePointContainer::WriteSorted(AliHLTUInt8_t* outputPtr,
   unsigned lastPadRow=0;
   AliHLTUInt64_t lastPad64=0;
   AliHLTUInt64_t lastTime64=0;
+  pDeflater->StartEncoder();
   AliHLTSpacePointPropertyGrid::iterator clusterID=pGrid->begin();
   if (clusterID!=pGrid->end()) {
     for (; clusterID!=pGrid->end(); clusterID++) {
@@ -791,6 +792,7 @@ int AliHLTTPCHWCFSpacePointContainer::WriteSorted(AliHLTUInt8_t* outputPtr,
       blockout->fCount++;
     }
   }
+  pDeflater->StopEncoder();
   AliHLTComponent_BlockData bd;
   AliHLTComponent::FillBlockData(bd);
   bd.fOffset        = size+offset;

HLT/TPCLib/AliHLTTPCHWCFSpacePointContainer.h

@@ -55,6 +55,7 @@ class AliHLTTPCHWCFSpacePointContainer : public AliHLTSpacePointContainer
   virtual float GetZWidth(AliHLTUInt32_t clusterID) const;
   virtual float GetCharge(AliHLTUInt32_t clusterID) const;
   virtual float GetQMax(AliHLTUInt32_t clusterID) const;
+  virtual float GetMaxSignal(AliHLTUInt32_t clusterID) const {return GetQMax(clusterID);}
   virtual float GetPhi(AliHLTUInt32_t clusterID) const;
 
   /// add input block to the collection

HLT/TPCLib/AliHLTTPCRawSpacePointContainer.cxx

@@ -654,6 +654,7 @@ int AliHLTTPCRawSpacePointContainer::WriteSorted(AliHLTUInt8_t* outputPtr,
   unsigned lastPadRow=0;
   AliHLTUInt64_t lastPad64=0;
   AliHLTUInt64_t lastTime64=0;
+  pDeflater->StartEncoder();
   AliHLTSpacePointPropertyGrid::iterator clusterID=pGrid->begin();
 
   AliHLTUInt32_t filledBytes = 0;
@@ -777,8 +778,16 @@ int AliHLTTPCRawSpacePointContainer::WriteSorted(AliHLTUInt8_t* outputPtr,
       }
       AliHLTUInt64_t sigmaY264=0;
       if (!isnan(sigmaY2)) sigmaY264=(AliHLTUInt64_t)round(sigmaY2*AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaY2].fScale);
+      // we can safely use the upper limit as this is an unphysical cluster, no impact to physics
+      if (sigmaY264 >= 1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaY2].fBitLength) {
+        sigmaY264 = (1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaY2].fBitLength)-1;
+      }
       AliHLTUInt64_t sigmaZ264=0;
       if (!isnan(sigmaZ2)) sigmaZ264=(AliHLTUInt64_t)round(sigmaZ2*AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaZ2].fScale);
+      // we can safely use the upper limit as this is an unphysical cluster, no impact to physics
+      if (sigmaZ264 >= 1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaZ2].fBitLength) {
+        sigmaZ264 = (1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaZ2].fBitLength)-1;
+      }
       pDeflater->OutputParameterBits(AliHLTTPCDefinitions::kPadRow , padrow64);
       pDeflater->OutputParameterBits(AliHLTTPCDefinitions::kPad    , pad64);
       if (fMode&kModeDifferentialPadTime) pDeflater->OutputBit(padType);
@@ -794,8 +803,7 @@ int AliHLTTPCRawSpacePointContainer::WriteSorted(AliHLTUInt8_t* outputPtr,
     }
     blockout->fCount++;
   }
-
-
+  pDeflater->StopEncoder();
   AliHLTComponent_BlockData bd;
   AliHLTComponent::FillBlockData(bd);
   bd.fOffset        = offset;

HLT/TPCLib/AliHLTTPCRawSpacePointContainer.h

@@ -55,6 +55,7 @@ class AliHLTTPCRawSpacePointContainer : public AliHLTSpacePointContainer
   virtual float GetZWidth(AliHLTUInt32_t clusterID) const;
   virtual float GetCharge(AliHLTUInt32_t clusterID) const;
   virtual float GetQMax(AliHLTUInt32_t clusterID) const;
+  virtual float GetMaxSignal(AliHLTUInt32_t clusterID) const {return GetQMax(clusterID);}
   virtual float GetPhi(AliHLTUInt32_t clusterID) const;
 
   /// add input block to the collection

HLT/TPCLib/AliHLTTPCTrackGeometry.cxx

@@ -27,7 +27,7 @@
 #include "AliHLTTPCSpacePointData.h"
 #include "AliHLTTPCClusterDataFormat.h"
 #include "AliHLTTPCSpacePointContainer.h"
-#include "AliHLTTPCHWCFSpacePointContainer.h"
+#include "AliHLTTPCRawSpacePointContainer.h"
 #include "AliHLTTPCDefinitions.h"
 #include "AliHLTComponent.h"
 #include "AliHLTGlobalBarrelTrack.h"
@@ -469,8 +469,11 @@ int AliHLTTPCTrackGeometry::WriteAssociatedClusters(AliHLTSpacePointContainer* p
 {
   // write associated clusters to buffer via deflater
   if (!pDeflater || !pSpacePoints) return -EINVAL;
-  AliHLTTPCHWCFSpacePointContainer* pTPCRawSpacePoints=dynamic_cast<AliHLTTPCHWCFSpacePointContainer*>(pSpacePoints);
-  if (!pTPCRawSpacePoints) return -EINVAL;
+  AliHLTTPCRawSpacePointContainer* pTPCRawSpacePoints=dynamic_cast<AliHLTTPCRawSpacePointContainer*>(pSpacePoints);
+  if (!pTPCRawSpacePoints) {
+    HLTError("invalid type of SpacePointContainer \"%s\", required AliHLTTPCRawSpacePointContainer", pSpacePoints->ClassName());
+    return -EINVAL;
+  }
   bool bReverse=true;
   bool bWriteSuccess=true;
   int writtenClusters=0;
@@ -589,8 +592,16 @@ int AliHLTTPCTrackGeometry::WriteAssociatedClusters(AliHLTSpacePointContainer* p
 	    }
 	    AliHLTUInt64_t sigmaY264=0;
 	    if (!isnan(sigmaY2)) sigmaY264=(AliHLTUInt64_t)round(sigmaY2*AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaY2].fScale);
+	    // we can safely use the upper limit as this is an unphysical cluster, no impact to physics
+	    if (sigmaY264 >= 1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaY2].fBitLength) {
+	      sigmaY264 = (1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaY2].fBitLength)-1;
+	    }
 	    AliHLTUInt64_t sigmaZ264=0;
 	    if (!isnan(sigmaZ2)) sigmaZ264=(AliHLTUInt64_t)round(sigmaZ2*AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaZ2].fScale);
+	    // we can safely use the upper limit as this is an unphysical cluster, no impact to physics
+	    if (sigmaZ264 >= 1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaZ2].fBitLength) {
+	      sigmaZ264 = (1<<AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kSigmaZ2].fBitLength)-1;
+	    }
 	    bWriteSuccess=bWriteSuccess && pDeflater->OutputParameterBits(AliHLTTPCDefinitions::kResidualPad    , deltapad64);  
 	    bWriteSuccess=bWriteSuccess && pDeflater->OutputBit(signDeltaPad);
 	    bWriteSuccess=bWriteSuccess && pDeflater->OutputParameterBits(AliHLTTPCDefinitions::kResidualTime   , deltatime64);
@@ -641,7 +652,13 @@ int AliHLTTPCTrackGeometry::WriteAssociatedClusters(AliHLTSpacePointContainer* p
     bWriteSuccess=bWriteSuccess && pDeflater->OutputBit(haveClusters);
   }
 
-  if (!bWriteSuccess) return -ENOSPC;
+  if (!bWriteSuccess) {
+    // TODO: code review 2015-02-10 [email protected]
+    // misleading error code, there are two reasons for failed write operation
+    // 1) target buffer overflow -> -ENOSPC
+    // 2) value range excess -> -ERANGE
+    return -ENOSPC;
+  }
 
   int allClusters=0;
   for (clrow=fRawTrackPoints.begin(); clrow!=fRawTrackPoints.end(); clrow++) {

HLT/TPCLib/comp/AliHLTTPCDataCompressionComponent.cxx

@@ -753,7 +753,10 @@ int AliHLTTPCDataCompressionComponent::WriteTrackClusters(const vector<AliHLTGlo
     }
 
     int result=pTrackPoints->Write(*track, pSpacePoints, pDeflater, outputPtr+size, capacity-size, fpWrittenAssociatedClusterIds);
-    if (result<0) return result;
+    if (result<0) {
+      HLTError("failed to write track points for track %d with error code %d, aborting", track->GetID(), result);
+      return result;
+    }
     size+=result;
 
     UInt_t nofTrackPoints=track->GetNumberOfPoints();

HLT/TPCLib/comp/AliHLTTPCDataCompressionDecoder.h

@@ -483,6 +483,10 @@ int AliHLTTPCDataCompressionDecoder::ReadTrackClustersCompressed(T& c, AliHLTDat
 	  trackpad*=AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kResidualPad].fScale;
 	  if (currentTrackPoint->GetU()>0.) trackpad64=(AliHLTUInt64_t)round(trackpad);
 	  if (sign) {
+	    if (trackpad64<value) {
+	      bReadSuccess=false;
+	      break;
+	    }
 	    value=trackpad64-value;
 	  } else {
 	    value+=trackpad64;
@@ -500,6 +504,10 @@ int AliHLTTPCDataCompressionDecoder::ReadTrackClustersCompressed(T& c, AliHLTDat
 	  tracktime*=AliHLTTPCDefinitions::fgkClusterParameterDefinitions[AliHLTTPCDefinitions::kResidualTime].fScale;
 	  if (currentTrackPoint->GetV()>0.) tracktime64=(AliHLTUInt64_t)round(tracktime);
 	  if (sign) {
+	    if (tracktime64<value) {
+	      bReadSuccess=false;
+	      break;
+	    }
 	    value=tracktime64-value;
 	  } else {
 	    value+=tracktime64;