xrootd
XrdPfcInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDPFC_INFO_HH__
2 #define __XRDPFC_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include <cstdio>
22 #include <ctime>
23 #include <assert.h>
24 #include <vector>
25 
26 #include "XrdSys/XrdSysPthread.hh"
27 #include "XrdCl/XrdClConstants.hh"
28 #include "XrdCl/XrdClDefaultEnv.hh"
29 
30 #include "XrdPfcTypes.hh"
31 
32 class XrdOssDF;
33 class XrdCksCalc;
34 class XrdSysTrace;
35 
36 namespace XrdPfc
37 {
38 class Stats;
39 
40 //----------------------------------------------------------------------------
42 //----------------------------------------------------------------------------
43 
44 class Info
45 {
46 public:
47  struct Status {
48  union {
49  struct {
50  int f_cksum_check : 3;
51 
52  int _free_bits_ : 29;
53  };
54  unsigned int _raw_;
55  };
56  Status() : _raw_(0) {}
57  };
58 
60  struct AStat
61  {
62  time_t AttachTime;
63  time_t DetachTime;
64  int NumIos;
65  int Duration;
66  int NumMerged;
67  int Reserved;
68  long long BytesHit;
69  long long BytesMissed;
70  long long BytesBypassed;
71 
72  AStat() :
73  AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0), Reserved(0),
75  {}
76 
77  void MergeWith(const AStat &a);
78  };
79 
80  struct Store
81  {
82  long long m_buffer_size;
83  long long m_file_size;
84  time_t m_creationTime;
85  time_t m_noCkSumTime;
86  size_t m_accessCnt;
89 
90  Store () :
93  {}
94  };
95 
96 
97  //------------------------------------------------------------------------
99  //------------------------------------------------------------------------
100  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
101 
102  //------------------------------------------------------------------------
104  //------------------------------------------------------------------------
105  ~Info();
106 
107  //---------------------------------------------------------------------
109  //---------------------------------------------------------------------
110  void SetBitWritten(int i);
111 
112  //---------------------------------------------------------------------
114  //---------------------------------------------------------------------
115  bool TestBitWritten(int i) const;
116 
117  //---------------------------------------------------------------------
119  //---------------------------------------------------------------------
120  bool TestBitPrefetch(int i) const;
121 
122  //---------------------------------------------------------------------
124  //---------------------------------------------------------------------
125  void SetBitPrefetch(int i);
126 
127  //---------------------------------------------------------------------
129  //---------------------------------------------------------------------
130  void SetBitSynced(int i);
131 
132  //---------------------------------------------------------------------
134  //---------------------------------------------------------------------
136 
137  void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs);
138 
139  //---------------------------------------------------------------------
141  //---------------------------------------------------------------------
142  void ResizeBits();
143 
144  //---------------------------------------------------------------------
150  //---------------------------------------------------------------------
151  bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0);
152 
153  //---------------------------------------------------------------------
159  //---------------------------------------------------------------------
160  bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0);
161 
162  //---------------------------------------------------------------------
164  //---------------------------------------------------------------------
166 
167  //---------------------------------------------------------------------
169  //---------------------------------------------------------------------
171 
172  //---------------------------------------------------------------------
174  //---------------------------------------------------------------------
176 
177  //---------------------------------------------------------------------
179  //---------------------------------------------------------------------
180  void WriteIOStat(Stats& s);
181 
182  //---------------------------------------------------------------------
184  //---------------------------------------------------------------------
186 
187  //---------------------------------------------------------------------
189  //---------------------------------------------------------------------
190  void WriteIOStatSingle(long long bytes_disk);
191 
192  //---------------------------------------------------------------------
194  //---------------------------------------------------------------------
195  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
196 
197  //---------------------------------------------------------------------
199  //---------------------------------------------------------------------
200  int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const;
201 
202  //---------------------------------------------------------------------
204  //---------------------------------------------------------------------
205  int GetBitvecSizeInBytes() const;
206 
207  //---------------------------------------------------------------------
209  //---------------------------------------------------------------------
210  int GetNBlocks() const;
211 
212  //---------------------------------------------------------------------
214  //---------------------------------------------------------------------
215  long long GetFileSize() const;
216 
217  //---------------------------------------------------------------------
219  //---------------------------------------------------------------------
220  bool GetLatestDetachTime(time_t& t) const;
221 
222  //---------------------------------------------------------------------
224  //---------------------------------------------------------------------
225  const AStat* GetLastAccessStats() const;
226 
227  //---------------------------------------------------------------------
229  //---------------------------------------------------------------------
230  long long GetBufferSize() const;
231 
232  //---------------------------------------------------------------------
234  //---------------------------------------------------------------------
235  bool IsComplete() const;
236 
237  //---------------------------------------------------------------------
239  //---------------------------------------------------------------------
240  int GetNDownloadedBlocks() const;
241 
242  //---------------------------------------------------------------------
244  //---------------------------------------------------------------------
245  long long GetNDownloadedBytes() const;
246 
247  //---------------------------------------------------------------------
249  //---------------------------------------------------------------------
250  int GetLastDownloadedBlock() const;
251 
252  //---------------------------------------------------------------------
254  //---------------------------------------------------------------------
255  long long GetExpectedDataFileSize() const;
256 
257  //---------------------------------------------------------------------
259  //---------------------------------------------------------------------
261 
262  //---------------------------------------------------------------------
264  //---------------------------------------------------------------------
265  size_t GetAccessCnt() const { return m_store.m_accessCnt; }
266 
267  //---------------------------------------------------------------------
269  //---------------------------------------------------------------------
270  int GetVersion() { return m_version; }
271 
272  //---------------------------------------------------------------------
274  //---------------------------------------------------------------------
275  const Store& RefStoredData() const { return m_store; }
276  const std::vector<AStat>& RefAStats() const { return m_astats; }
277 
278  //---------------------------------------------------------------------
280  //---------------------------------------------------------------------
281  time_t GetCreationTime() const { return m_store.m_creationTime; }
282 
283  //---------------------------------------------------------------------
285  //---------------------------------------------------------------------
286  uint32_t CalcCksumStore();
288  void CalcCksumMd5(unsigned char* buff, char* digest);
289 
291  const char* GetCkSumStateAsText() const;
292 
294  bool IsCkSumNet() const { return m_store.m_status.f_cksum_check & CSChk_Net; }
297 
302 
303  bool HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; }
304  time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; }
307 
308 #ifdef XRDPFC_CKSUM_TEST
309  static void TestCksumStuff();
310 #endif
311 
312  static const char* m_traceID; // has to be m_ (convention in TRACE macros)
313  static const char* s_infoExtension;
314  static const size_t s_infoExtensionLen;
315  static size_t s_maxNumAccess; // can be set from configuration
316  static const int s_defaultVersion;
317 
318  XrdSysTrace* GetTrace() const {return m_trace; }
319 
320 protected:
322 
324  unsigned char *m_buff_synced;
325  unsigned char *m_buff_written;
326  unsigned char *m_buff_prefetch;
327  std::vector<AStat> m_astats;
328 
332  bool m_complete;
334 
335 private:
336  inline unsigned char cfiBIT(int n) const { return 1 << n; }
337 
338  // Reading functions for older cinfo file formats
339  bool ReadV2(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
340  bool ReadV3(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
341 
343 };
344 
345 //------------------------------------------------------------------------------
346 
347 inline bool Info::TestBitWritten(int i) const
348 {
349  const int cn = i/8;
350  assert(cn < GetBitvecSizeInBytes());
351 
352  const int off = i - cn*8;
353  return (m_buff_written[cn] & cfiBIT(off)) != 0;
354 }
355 
356 inline void Info::SetBitWritten(int i)
357 {
358  const int cn = i/8;
359  assert(cn < GetBitvecSizeInBytes());
360 
361  const int off = i - cn*8;
362 
363  m_buff_written[cn] |= cfiBIT(off);
364 
365  if (--m_missingBlocks == 0)
366  m_complete = true;
367 }
368 
369 inline void Info::SetBitPrefetch(int i)
370 {
371  if (!m_buff_prefetch) return;
372 
373  const int cn = i/8;
374  assert(cn < GetBitvecSizeInBytes());
375 
376  const int off = i - cn*8;
377  m_buff_prefetch[cn] |= cfiBIT(off);
378 }
379 
380 inline bool Info::TestBitPrefetch(int i) const
381 {
382  if (!m_buff_prefetch) return false;
383 
384  const int cn = i/8;
385  assert(cn < GetBitvecSizeInBytes());
386 
387  const int off = i - cn*8;
388  return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
389 }
390 
391 inline void Info::SetBitSynced(int i)
392 {
393  const int cn = i/8;
394  assert(cn < GetBitvecSizeInBytes());
395 
396  const int off = i - cn*8;
397  m_buff_synced[cn] |= cfiBIT(off);
398 }
399 
400 //------------------------------------------------------------------------------
401 
402 inline int Info::GetNDownloadedBlocks() const
403 {
404  int cntd = 0;
405  for (int i = 0; i < m_bitvecSizeInBits; ++i)
406  if (TestBitWritten(i)) cntd++;
407 
408  return cntd;
409 }
410 
411 inline long long Info::GetNDownloadedBytes() const
412 {
414 }
415 
417 {
418  for (int i = m_bitvecSizeInBits - 1; i >= 0; --i)
419  if (TestBitWritten(i)) return i;
420 
421  return -1;
422 }
423 
424 inline long long Info::GetExpectedDataFileSize() const
425 {
426  int last_block = GetLastDownloadedBlock();
427  if (last_block == m_bitvecSizeInBits - 1)
428  return m_store.m_file_size;
429  else
430  return (last_block + 1) * m_store.m_buffer_size;
431 }
432 
433 inline int Info::GetBitvecSizeInBytes() const
434 {
435  if (m_bitvecSizeInBits)
436  return ((m_bitvecSizeInBits - 1)/8 + 1);
437  else
438  return 0;
439 }
440 
441 inline int Info::GetNBlocks() const
442 {
443  return m_bitvecSizeInBits;
444 }
445 
446 inline long long Info::GetFileSize() const
447 {
448  return m_store.m_file_size;
449 }
450 
451 inline bool Info::IsComplete() const
452 {
453  return m_complete;
454 }
455 
456 inline int Info::CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
457 {
458  // TODO rewrite to use full byte comparisons outside of edges ?
459  // Also, it seems to be always called with firstIdx = 0, lastIdx = m_bitvecSizeInBits.
460  int cnt = 0;
461  for (int i = firstIdx; i < lastIdx; ++i)
462  if (! TestBitWritten(i)) ++cnt;
463 
464  return cnt;
465 }
466 
468 {
470  m_complete = (m_missingBlocks == 0);
471 }
472 
473 inline long long Info::GetBufferSize() const
474 {
475  return m_store.m_buffer_size;
476 }
477 
478 }
479 #endif
Definition: XrdCksCalc.hh:40
Definition: XrdOss.hh:63
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:45
bool ReadV2(XrdOssDF *fp, off_t off, const char *dname, const char *fname)
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
Definition: XrdPfcInfo.hh:369
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdPfcInfo.hh:467
bool IsCkSumAny() const
Definition: XrdPfcInfo.hh:295
~Info()
Destructor.
void SetBitSynced(int i)
Mark block as synced to disk.
Definition: XrdPfcInfo.hh:391
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
void SetAllBitsSynced()
Mark all blocks as synced to disk.
int GetBitvecSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdPfcInfo.hh:433
time_t GetNoCkSumTimeForUVKeep() const
Definition: XrdPfcInfo.hh:305
void ResetCkSumNet()
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdPfcInfo.hh:326
void CompactifyAccessRecords()
Compactify access records to the configured maximum.
uint32_t CalcCksumSyncedAndAStats()
CkSumCheck_e GetCkSumState() const
Definition: XrdPfcInfo.hh:290
const char * GetCkSumStateAsText() const
int m_version
Definition: XrdPfcInfo.hh:329
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
static const int s_defaultVersion
Definition: XrdPfcInfo.hh:316
void DowngradeCkSumState(CkSumCheck_e css_ref)
Definition: XrdPfcInfo.hh:299
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdPfcInfo.hh:411
XrdCksCalc * m_cksCalcMd5
Definition: XrdPfcInfo.hh:342
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
Definition: XrdPfcInfo.hh:416
time_t GetCreationTime() const
Get file size.
Definition: XrdPfcInfo.hh:281
unsigned char * m_buff_written
download state vector
Definition: XrdPfcInfo.hh:325
bool IsCkSumNet() const
Definition: XrdPfcInfo.hh:294
unsigned char * m_buff_synced
disk written state vector
Definition: XrdPfcInfo.hh:324
bool m_complete
cached; if false, set to true when missingBlocks hit zero
Definition: XrdPfcInfo.hh:332
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
Definition: XrdPfcInfo.hh:380
bool IsComplete() const
Get complete status.
Definition: XrdPfcInfo.hh:451
const std::vector< AStat > & RefAStats() const
Definition: XrdPfcInfo.hh:276
void ResetAllAccessStats()
Reset IO Stats.
bool IsCkSumCache() const
Definition: XrdPfcInfo.hh:293
unsigned char cfiBIT(int n) const
Definition: XrdPfcInfo.hh:336
const Store & RefStoredData() const
Get stored data.
Definition: XrdPfcInfo.hh:275
void SetBitWritten(int i)
Mark block as written to disk.
Definition: XrdPfcInfo.hh:356
time_t GetNoCkSumTime() const
Definition: XrdPfcInfo.hh:304
int m_missingBlocks
cached, updated in SetBitWritten()
Definition: XrdPfcInfo.hh:331
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdPfcInfo.hh:473
long long GetExpectedDataFileSize() const
Get expected data file size.
Definition: XrdPfcInfo.hh:424
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
Definition: XrdPfcInfo.hh:347
int m_bitvecSizeInBits
cached
Definition: XrdPfcInfo.hh:330
long long GetFileSize() const
Get file size.
Definition: XrdPfcInfo.hh:446
XrdSysTrace * m_trace
Definition: XrdPfcInfo.hh:321
std::vector< AStat > m_astats
access records
Definition: XrdPfcInfo.hh:327
static const char * s_infoExtension
Definition: XrdPfcInfo.hh:313
void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs)
Store m_store
Definition: XrdPfcInfo.hh:323
const AStat * GetLastAccessStats() const
Get latest access stats.
bool Write(XrdOssDF *fp, const char *dname, const char *fname=0)
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
bool ReadV3(XrdOssDF *fp, off_t off, const char *dname, const char *fname)
XrdSysTrace * GetTrace() const
Definition: XrdPfcInfo.hh:318
void SetCkSumState(CkSumCheck_e css)
Definition: XrdPfcInfo.hh:298
void ResetNoCkSumTime()
Definition: XrdPfcInfo.hh:306
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
static const char * m_traceID
Definition: XrdPfcInfo.hh:312
bool IsCkSumBoth() const
Definition: XrdPfcInfo.hh:296
void ResizeBits()
Reserve bit vectors for file_size / buffer_size bytes.
void ResetCkSumCache()
static size_t s_maxNumAccess
Definition: XrdPfcInfo.hh:315
int GetVersion()
Get version.
Definition: XrdPfcInfo.hh:270
uint32_t CalcCksumStore()
Get cksum, MD5 is for backward compatibility with V2 and V3.
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdPfcInfo.hh:402
size_t GetAccessCnt() const
Get number of accesses.
Definition: XrdPfcInfo.hh:265
int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdPfcInfo.hh:456
void CalcCksumMd5(unsigned char *buff, char *digest)
static const size_t s_infoExtensionLen
Definition: XrdPfcInfo.hh:314
void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc)
Write open/close with given time and bytes read from disk.
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
int GetNBlocks() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdPfcInfo.hh:441
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdPfcInfo.hh:333
bool Read(XrdOssDF *fp, const char *dname, const char *fname=0)
Read content of cinfo file into this object.
bool HasNoCkSumTime() const
Definition: XrdPfcInfo.hh:303
Statistics of cache utilisation by a File object.
Definition: XrdPfcStats.hh:31
Definition: XrdSysTrace.hh:49
Definition: XrdPfc.hh:41
CkSumCheck_e
Definition: XrdPfcTypes.hh:23
@ CSChk_Both
Definition: XrdPfcTypes.hh:23
@ CSChk_Net
Definition: XrdPfcTypes.hh:23
@ CSChk_Cache
Definition: XrdPfcTypes.hh:23
Access statistics.
Definition: XrdPfcInfo.hh:61
long long BytesHit
read from cache
Definition: XrdPfcInfo.hh:68
void MergeWith(const AStat &a)
long long BytesBypassed
read from remote and dropped
Definition: XrdPfcInfo.hh:70
AStat()
Definition: XrdPfcInfo.hh:72
int Duration
total duration of all IOs attached
Definition: XrdPfcInfo.hh:65
int Reserved
reserved / alignment
Definition: XrdPfcInfo.hh:67
int NumIos
number of IO objects attached during this access
Definition: XrdPfcInfo.hh:64
time_t DetachTime
close time
Definition: XrdPfcInfo.hh:63
long long BytesMissed
read from remote and cached
Definition: XrdPfcInfo.hh:69
time_t AttachTime
open time
Definition: XrdPfcInfo.hh:62
int NumMerged
number of times the record has been merged
Definition: XrdPfcInfo.hh:66
Definition: XrdPfcInfo.hh:47
int f_cksum_check
as in enum CkSumCheck_e
Definition: XrdPfcInfo.hh:50
unsigned int _raw_
Definition: XrdPfcInfo.hh:54
int _free_bits_
Definition: XrdPfcInfo.hh:52
Status()
Definition: XrdPfcInfo.hh:56
Definition: XrdPfcInfo.hh:81
Store()
Definition: XrdPfcInfo.hh:90
int m_astatSize
size of AStat vector
Definition: XrdPfcInfo.hh:88
Status m_status
status information
Definition: XrdPfcInfo.hh:87
long long m_buffer_size
buffer / block size
Definition: XrdPfcInfo.hh:82
time_t m_noCkSumTime
time when first non-cksummed block was detected
Definition: XrdPfcInfo.hh:85
size_t m_accessCnt
total access count for the file
Definition: XrdPfcInfo.hh:86
long long m_file_size
size of file in bytes
Definition: XrdPfcInfo.hh:83
time_t m_creationTime
time the info file was created
Definition: XrdPfcInfo.hh:84