Skip to content

Commit 14ff7db

Browse files
tubagundemtubagundem
authored
TPC: Processing of common mode values in O2 (#15137)
* TPC: Processing of common mode values in O2 * Added CMVContainer.cxx, fixed missing links and includes * Fix formatting * Removed unused includes, directly write the TTree object to CCDB without TMem file, small fixes * Changed the decoding and encoding of CMVs, removed grouping per side * Update the dataformat of CMV * Updated the CMVContainer, corrected the timestamp range for CCDB * Fix formatting * Removed factorize workflow, updated the distribute workflow accordingly * Fix formatting * Extend error tracking in CMVToVectorSpec.cxx * Replace CMVPerInterval with per TF TTree accumulation and raw uint16_t storage, fix CCDB timestamp for partial intervals, fix TMemFile padding * Added delta+zigzag+varint compression, added drawCMV.C macro for visualization * Added small value zeroing, added `--use-compression` and `--cmv-zero-threshold` flags to TPCDistributeCMVSpec, updated drawCMV.C macro to auto detect branch format (compressed or not) * Added sparse and HUffman encoding to the CMVContainer, updatet the drawCMV.C macro and TPCDistributeCMVSpec.h accordingly * Added CMVPerTFCombined to CMVContainer to combine sparse encoding with varint/Huffman compression, updated drawing macro and workflow options accordingly * Added gaussian dynamic precision * Refactored CMVContainer, unified CMV compression in a flag based container * Added CMV workflow documentation to README and fixed CMV packet size mismatch handling
1 parent c9acd57 commit 14ff7db

File tree

16 files changed

+2854
-3
lines changed

16 files changed

+2854
-3
lines changed

‎DataFormats/Detectors/TPC/include/DataFormatsTPC/CMV.h‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
/// @file CMV.h
13+
/// @author Tuba Gündem, tuba.gundem@cern.ch
14+
/// @brief Common mode values data format definition
15+
16+
/// The data is sent by the CRU as 256+16 bit words. The CMV data layout is as follows:
17+
/// - 256-bit Header: [version:8][packetID:8][errorCode:8][magicWord:8][heartbeatOrbit:32][heartbeatBC:16][padding:176]
18+
/// - 16-bit CMV value: [sign:1][I8F7:15] where bit 15 is the sign (1=positive, 0=negative) and the lower 15 bits are a fixed point I8F7 value (8 integer bits, 7 fractional bits)
19+
/// Float conversion: sign ? (value & 0x7FFF) / 128.0 : -(value & 0x7FFF) / 128.0
20+
21+
#ifndef ALICEO2_DATAFORMATSTPC_CMV_H
22+
#define ALICEO2_DATAFORMATSTPC_CMV_H
23+
24+
#include <cstdint>
25+
#include <cmath>
26+
27+
namespace o2::tpc::cmv
28+
{
29+
30+
static constexpr uint32_t NTimeBinsPerPacket = 3564; ///< number of time bins (covering 8 heartbeats)
31+
static constexpr uint32_t NPacketsPerTFPerCRU = 4; ///< 4 packets per timeframe
32+
static constexpr uint32_t NTimeBinsPerTF = NTimeBinsPerPacket * NPacketsPerTFPerCRU; ///< maximum number of timebins per timeframe (14256)
33+
34+
/// Data padding: NTimeBinsPerPacket * sizeof(Data) = 3564 * 2 = 7128 bytes
35+
static constexpr uint32_t DataSizeBytes = NTimeBinsPerPacket * sizeof(uint16_t); ///< 7128 bytes
36+
static constexpr uint32_t DataPaddingBytes = (32 - (DataSizeBytes % 32)) % 32; ///< 8 bytes
37+
38+
/// Header definition of the CMVs
39+
struct Header {
40+
static constexpr uint8_t MagicWord = 0xDC;
41+
union {
42+
uint64_t word0 = 0; ///< bits 0 - 63
43+
struct {
44+
uint8_t version : 8; ///< version
45+
uint8_t packetID : 8; ///< packet id
46+
uint8_t errorCode : 8; ///< errors
47+
uint8_t magicWord : 8; ///< magic word
48+
uint32_t heartbeatOrbit : 32; ///< first heart beat timing of the package
49+
};
50+
};
51+
union {
52+
uint64_t word1 = 0; ///< bits 64 - 127
53+
struct {
54+
uint16_t heartbeatBC : 16; ///< first BC id of the package
55+
uint16_t unused1 : 16; ///< reserved
56+
uint32_t unused2 : 32; ///< reserved
57+
};
58+
};
59+
union {
60+
uint64_t word3 = 0; ///< bits 128 - 191
61+
struct {
62+
uint64_t unused3 : 64; ///< reserved
63+
};
64+
};
65+
union {
66+
uint64_t word4 = 0; ///< bits 192 - 255
67+
struct {
68+
uint64_t unused4 : 64; ///< reserved
69+
};
70+
};
71+
};
72+
73+
/// CMV single data container
74+
struct Data {
75+
uint16_t cmv{0}; ///< 16-bit signed fixed point value: bit 15 = sign (1=positive, 0=negative), bits 14-0 = I8F7 magnitude
76+
77+
uint16_t getCMV() const { return cmv; } ///< raw 16-bit integer representation
78+
void setCMV(uint16_t value) { cmv = value; } ///< set raw 16-bit integer representation
79+
80+
// Decode to float: sign-magnitude with 7 fractional bits, range ±255.992
81+
float getCMVFloat() const
82+
{
83+
const bool positive = (cmv >> 15) & 1; // bit 15: sign (1=positive, 0=negative)
84+
const float magnitude = (cmv & 0x7FFF) / 128.f; // lower 15 bits, shift right by 7 (divide by 2^7)
85+
return positive ? magnitude : -magnitude;
86+
}
87+
88+
// Encode from float: clamps magnitude to 15 bits, range ±255.992
89+
void setCMVFloat(float value)
90+
{
91+
const bool positive = (value >= 0.f);
92+
const uint16_t magnitude = static_cast<uint16_t>(std::abs(value) * 128.f + 0.5f) & 0x7FFF;
93+
cmv = (positive ? 0x8000 : 0x0000) | magnitude;
94+
}
95+
};
96+
97+
/// CMV full data container: one packet carries NTimeBinsPerPacket CMV values followed by padding
98+
/// Layout: Header (32 bytes) + Data[NTimeBinsPerPacket] (7128 bytes) + padding (8 bytes) = 7168 bytes total (224 * 32 = 7168)
99+
/// The padding bytes at the end of the data array are rubbish/unused and must not be interpreted as CMV values
100+
struct Container {
101+
Header header; ///< CMV data header
102+
Data data[NTimeBinsPerPacket]; ///< data values
103+
uint8_t padding[DataPaddingBytes]{}; ///< trailing padding to align data to 32-byte boundary
104+
105+
// Header and data accessors
106+
const Header& getHeader() const { return header; }
107+
Header& getHeader() { return header; }
108+
109+
const Data* getData() const { return data; }
110+
Data* getData() { return data; }
111+
112+
// Per timebin CMV accessors
113+
uint16_t getCMV(uint32_t timeBin) const { return data[timeBin].getCMV(); }
114+
void setCMV(uint32_t timeBin, uint16_t value) { data[timeBin].setCMV(value); }
115+
116+
float getCMVFloat(uint32_t timeBin) const { return data[timeBin].getCMVFloat(); }
117+
void setCMVFloat(uint32_t timeBin, float value) { data[timeBin].setCMVFloat(value); }
118+
};
119+
120+
} // namespace o2::tpc::cmv
121+
122+
#endif

‎Detectors/TPC/base/include/TPCBase/RDHUtils.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
#define AliceO2_TPC_RDHUtils_H
1414

1515
#include "DetectorsRaw/RDHUtils.h"
16-
//#include "Headers/RAWDataHeader.h"
16+
// #include "Headers/RAWDataHeader.h"
1717

1818
namespace o2
1919
{
@@ -28,6 +28,7 @@ static constexpr FEEIDType UserLogicLinkID = 15; ///< virtual link ID for ZS dat
2828
static constexpr FEEIDType IDCLinkID = 20; ///< Identifier for integrated digital currents
2929
static constexpr FEEIDType ILBZSLinkID = 21; ///< Identifier for improved link-based ZS
3030
static constexpr FEEIDType DLBZSLinkID = 22; ///< Identifier for dense link-based ZS
31+
static constexpr FEEIDType CMVLinkID = 23; ///< Identifier for common mode values
3132
static constexpr FEEIDType SACLinkID = 25; ///< Identifier for sampled analog currents
3233

3334
/// compose feeid from cru, endpoint and link

‎Detectors/TPC/calibration/CMakeLists.txt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ o2_add_library(TPCCalibration
5858
src/DigitAdd.cxx
5959
src/CorrectdEdxDistortions.cxx
6060
src/PressureTemperatureHelper.cxx
61+
src/CMVContainer.cxx
6162
PUBLIC_LINK_LIBRARIES O2::DataFormatsTPC O2::TPCBaseRecSim
6263
O2::TPCReconstruction ROOT::Minuit
6364
Microsoft.GSL::GSL
@@ -115,7 +116,8 @@ o2_target_root_dictionary(TPCCalibration
115116
include/TPCCalibration/TPCMShapeCorrection.h
116117
include/TPCCalibration/DigitAdd.h
117118
include/TPCCalibration/CorrectdEdxDistortions.h
118-
include/TPCCalibration/PressureTemperatureHelper.h)
119+
include/TPCCalibration/PressureTemperatureHelper.h
120+
include/TPCCalibration/CMVContainer.h)
119121

120122
o2_add_test_root_macro(macro/comparePedestalsAndNoise.C
121123
PUBLIC_LINK_LIBRARIES O2::TPCBaseRecSim
@@ -153,6 +155,10 @@ o2_add_test_root_macro(macro/prepareITFiles.C
153155
COMPILE_ONLY
154156
PUBLIC_LINK_LIBRARIES O2::TPCCalibration
155157
LABELS tpc)
158+
o2_add_test_root_macro(macro/drawCMV.C
159+
COMPILE_ONLY
160+
PUBLIC_LINK_LIBRARIES O2::TPCCalibration O2::TPCBase
161+
LABELS tpc)
156162

157163
o2_add_test(IDCFourierTransform
158164
COMPONENT_NAME calibration

‎Detectors/TPC/calibration/include/TPCCalibration/CMVContainer.h‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
/// @file CMVContainer.h
13+
/// @author Tuba Gündem, tuba.gundem@cern.ch
14+
/// @brief Structs for storing CMVs to the CCDB
15+
16+
#ifndef ALICEO2_TPC_CMVCONTAINER_H_
17+
#define ALICEO2_TPC_CMVCONTAINER_H_
18+
19+
#include <string>
20+
#include <memory>
21+
#include <vector>
22+
#include <cstdint>
23+
24+
#include "TTree.h"
25+
#include "TPCBase/CRU.h"
26+
#include "DataFormatsTPC/CMV.h"
27+
28+
namespace o2::tpc
29+
{
30+
31+
struct CMVPerTF; // forward declaration
32+
struct CMVPerTFCompressed; // forward declaration
33+
34+
/// Bitmask flags describing which encoding stages are applied in CMVPerTFCompressed
35+
struct CMVEncoding {
36+
static constexpr uint8_t kNone = 0x00; ///< No compression — raw uint16 values stored flat
37+
static constexpr uint8_t kSparse = 0x01; ///< Non-zero positions stored sparsely (varint-encoded deltas)
38+
static constexpr uint8_t kDelta = 0x02; ///< Delta coding between consecutive values (dense only)
39+
static constexpr uint8_t kZigzag = 0x04; ///< Zigzag encoding of deltas or signed values
40+
static constexpr uint8_t kVarint = 0x08; ///< Varint compression of the value stream
41+
static constexpr uint8_t kHuffman = 0x10; ///< Canonical Huffman compression of the value stream
42+
};
43+
44+
/// Single compressed representation for one TF across all CRUs, stored in a TTree
45+
/// mFlags is a bitmask of CMVEncoding values that fully describes the encoding pipeline
46+
/// mData holds the encoded payload whose binary layout depends on mFlags:
47+
///
48+
/// Dense path (!kSparse):
49+
/// kZigzag absent → N × uint16_t LE (raw values, CRU-major order)
50+
/// kZigzag + kVarint → N × varint(zigzag(delta(signed(raw))))
51+
/// kZigzag + kHuffman → [Huffman table] + [bitstream] of zigzag(delta(signed(raw)))
52+
///
53+
/// Sparse path (kSparse):
54+
/// 4 bytes LE uint32_t : posStreamSize
55+
/// posStream: for each CRU: varint(N), N × varint(tb_delta)
56+
/// valStream (one entry per non-zero):
57+
/// default → uint16_t LE raw value
58+
/// kZigzag + kVarint → varint(zigzag(signed(raw)))
59+
/// kZigzag + kHuffman → [Huffman table] + [bitstream] of zigzag(signed(raw))
60+
struct CMVPerTFCompressed {
61+
uint32_t firstOrbit{0}; ///< First orbit of this TF
62+
uint16_t firstBC{0}; ///< First bunch crossing of this TF
63+
uint8_t mFlags{0}; ///< Bitmask of CMVEncoding values
64+
65+
std::vector<uint8_t> mData; ///< Encoded payload
66+
67+
/// Restore a CMVPerTF from this compressed object into *cmv (must not be null)
68+
void decompress(CMVPerTF* cmv) const;
69+
70+
/// Serialise into a TTree; each Fill() call appends one entry (one TF)
71+
std::unique_ptr<TTree> toTTree() const;
72+
73+
private:
74+
/// Decode the sparse position stream; advances ptr past the position block
75+
/// Returns (cru, timeBin) pairs for every non-zero entry, in CRU-major order
76+
static std::vector<std::pair<int, uint32_t>> decodeSparsePositions(const uint8_t*& ptr, const uint8_t* end);
77+
78+
/// Decode the value stream into raw uint32_t symbols
79+
/// Dispatches to Huffman, varint, or raw uint16 based on flags
80+
static std::vector<uint32_t> decodeValueStream(const uint8_t*& ptr, const uint8_t* end, uint32_t N, uint8_t flags);
81+
82+
/// Apply inverse zigzag and scatter decoded values into the sparse positions of *cmv
83+
static void decodeSparseValues(const std::vector<uint32_t>& symbols,
84+
const std::vector<std::pair<int, uint32_t>>& positions,
85+
uint8_t flags, CMVPerTF* cmv);
86+
87+
/// Apply inverse zigzag and inverse delta, then fill the full dense CMV array in *cmv
88+
static void decodeDenseValues(const std::vector<uint32_t>& symbols, uint8_t flags, CMVPerTF* cmv);
89+
90+
public:
91+
ClassDefNV(CMVPerTFCompressed, 1)
92+
};
93+
94+
/// CMV data for one TF across all CRUs
95+
/// Raw 16-bit CMV values are stored in a flat C array indexed as [cru * NTimeBinsPerTF + timeBin]
96+
struct CMVPerTF {
97+
uint32_t firstOrbit{0}; ///< First orbit of this TF, from heartbeatOrbit of the first CMV packet
98+
uint16_t firstBC{0}; ///< First bunch crossing of this TF, from heartbeatBC of the first CMV packet
99+
100+
// Raw 16-bit CMV values, flat array indexed as [cru * NTimeBinsPerTF + timeBin]
101+
uint16_t mDataPerTF[CRU::MaxCRU * cmv::NTimeBinsPerTF]{};
102+
103+
/// Return the raw 16-bit CMV value for a given CRU and timebin within this TF
104+
uint16_t getCMV(const int cru, const int timeBin) const;
105+
106+
/// Return the float CMV value for a given CRU and timebin within this TF
107+
float getCMVFloat(const int cru, const int timeBin) const;
108+
109+
/// Zero out raw CMV values whose float magnitude is below threshold
110+
void zeroSmallValues(float threshold = 1.0f);
111+
112+
/// Round values to the nearest integer ADC for all values whose rounded magnitude is <= threshold
113+
void roundToIntegers(uint16_t threshold);
114+
115+
/// Quantise |v| with a Gaussian-CDF recovery profile:
116+
/// Coarse decimal-style precision below and around mean, then a smooth return to the full native I8F7 precision as the magnitude increases with width sigma
117+
void trimGaussianPrecision(float mean, float sigma);
118+
119+
/// Compress this object into a CMVPerTFCompressed using the encoding pipeline described by flags
120+
/// Quantisation (trimGaussianPrecision / roundToIntegers / zeroSmallValues) should be applied to this object before calling compress(); it is not part of the flags pipeline
121+
CMVPerTFCompressed compress(uint8_t flags) const;
122+
123+
/// Serialise into a TTree; each Fill() call appends one entry (one TF)
124+
std::unique_ptr<TTree> toTTree() const;
125+
126+
/// Write the TTree to a ROOT file
127+
static void writeToFile(const std::string& filename, const std::unique_ptr<TTree>& tree);
128+
129+
private:
130+
static int32_t cmvToSigned(uint16_t raw); ///< Sign-magnitude uint16_t → signed integer
131+
static uint16_t quantizeBelowThreshold(uint16_t raw, float quantizationMean, float quantizationSigma); ///< Quantise sub-threshold values with a Gaussian-shaped recovery to full precision
132+
static uint32_t zigzagEncode(int32_t value); ///< Zigzag encode
133+
static void encodeVarintInto(uint32_t value, std::vector<uint8_t>& out); ///< Varint encode
134+
135+
public:
136+
ClassDefNV(CMVPerTF, 1)
137+
};
138+
139+
} // namespace o2::tpc
140+
141+
#endif // ALICEO2_TPC_CMVCONTAINER_H_

0 commit comments

Comments
 (0)