44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
71 template <
typename Data_T>
76 template <
typename Data_T>
81 namespace SparseFile {
94 template <
class Data_T>
101 typedef std::vector<Sparse::SparseBlock<Data_T>*>
BlockPtrs;
214 template <
class Data_T>
219 template <
class Data_T>
223 template <
class Data_T>
321 template <
class Data_T>
378 template <
class Data_T>
385 template <
class Data_T>
391 template <
class Data_T>
397 template <
class Data_T>
402 template <
class Data_T>
403 int getNextId(
const std::string filename,
const std::string layerPath);
405 template <
class Data_T>
425 template <
class Data_T>
429 template <
class Data_T>
470 namespace SparseFile {
474 template <
class Data_T>
476 const std::string a_layerPath)
477 : filename(a_filename), layerPath(a_layerPath),
478 valuesPerBlock(-1), occupiedBlocks(-1),
479 blockMutex(NULL), m_fileHandle(-1), m_reader(NULL) {
485 template <
class Data_T>
492 delete [] blockMutex;
497 template <
class Data_T>
507 template <
class Data_T>
524 blockMutex =
new boost::mutex[blocks.size()];
529 if (m_fileHandle >= 0) {
530 m_layerGroup.open(m_fileHandle, layerPath.c_str());
542 template <
class Data_T>
545 return m_fileHandle >= 0;
550 template <
class Data_T>
553 boost::mutex::scoped_lock lock(m_mutex);
555 fileBlockIndices.resize(numBlocks);
556 blockLoaded.resize(numBlocks, 0);
557 blocks.resize(numBlocks, 0);
558 blockUsed.resize(numBlocks,
false);
559 loadCounts.resize(numBlocks, 0);
560 refCounts.resize(numBlocks, 0);
563 blockMutex =
new boost::mutex[numBlocks];
568 template <
class Data_T>
572 using namespace Hdf5Util;
574 boost::mutex::scoped_lock lock_A(m_mutex);
581 m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
582 if (m_fileHandle < 0)
583 throw NoSuchFileException(filename);
585 m_layerGroup.open(m_fileHandle, layerPath.c_str());
586 if (m_layerGroup.id() < 0) {
588 "Couldn't find layer group " + layerPath +
590 throw FileIntegrityException(filename);
599 template <
class Data_T>
602 boost::mutex::scoped_lock lock(m_mutex);
605 blocks[blockIdx]->resize(valuesPerBlock);
606 assert(blocks[blockIdx]->data.size() > 0);
609 m_reader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->dataRef());
611 blockLoaded[blockIdx] = 1;
616 template <
class Data_T>
620 blocks[blockIdx]->clear();
623 blockLoaded[blockIdx] = 0;
628 template <
class Data_T>
631 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
632 ++refCounts[blockIdx];
637 template <
class Data_T>
640 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
641 --refCounts[blockIdx];
646 template <
class Data_T>
649 return valuesPerBlock *
sizeof(Data_T);
654 template <
class Data_T>
657 std::vector<int>::const_iterator i = loadCounts.begin();
658 std::vector<int>::const_iterator end = loadCounts.end();
660 for (; i != end; ++i)
668 template <
class Data_T>
671 std::vector<int>::const_iterator i = blockLoaded.begin();
672 std::vector<int>::const_iterator end = blockLoaded.end();
674 for (; i != end; ++i)
683 template <
class Data_T>
686 std::vector<int>::const_iterator i = loadCounts.begin();
687 std::vector<int>::const_iterator li = blockLoaded.begin();
688 std::vector<int>::const_iterator end = loadCounts.end();
691 if (blockLoaded.size() == 0) {
692 for (; i != end; ++i)
696 assert(loadCounts.size() == blockLoaded.size());
698 for (; i != end; ++i, ++li)
708 template <
class Data_T>
711 std::vector<int>::const_iterator i = loadCounts.begin();
712 std::vector<int>::const_iterator end = loadCounts.end();
713 int numLoads = 0, numBlocks = 0;
714 for (; i != end; ++i) {
721 return (
float)numLoads / std::max(1, numBlocks);
726 template <
class Data_T>
729 std::vector<int>::iterator li = loadCounts.begin();
730 std::vector<int>::iterator lend = loadCounts.end();
731 for (; li != lend; ++li)
743 namespace SparseFile {
748 inline Reference<half>&
760 return m_vhRefs[idx];
778 return m_vfRefs[idx];
796 return m_vdRefs[idx];
804 m_hRefs.push_back(ref);
805 return m_hRefs.size() - 1;
813 m_vhRefs.push_back(ref);
814 return m_vhRefs.size() - 1;
822 m_fRefs.push_back(ref);
823 return m_fRefs.size() - 1;
831 m_vfRefs.push_back(ref);
832 return m_vfRefs.size() - 1;
840 m_dRefs.push_back(ref);
841 return m_dRefs.size() - 1;
849 m_vdRefs.push_back(ref);
850 return m_vdRefs.size() - 1;
856 inline int FileReferences::numRefs<half>()
const
858 return m_hRefs.size();
864 inline int FileReferences::numRefs<V3h>()
const
866 return m_vhRefs.size();
872 inline int FileReferences::numRefs<float>()
const
874 return m_fRefs.size();
880 inline int FileReferences::numRefs<V3f>()
const
882 return m_vfRefs.size();
888 inline int FileReferences::numRefs<double>()
const
890 return m_dRefs.size();
896 inline int FileReferences::numRefs<V3d>()
const
898 return m_vdRefs.size();
905 template <
class Data_T>
908 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
909 "simple scalars or vectors!");
911 "FileReferences::ref(): Do not use memory limiting on sparse "
912 "fields that aren't simple scalars or vectors!");
919 template <
class Data_T>
922 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
923 "simple scalars or vectors!");
925 "FileReferences::append(): Do not use memory limiting on sparse "
926 "fields that aren't simple scalars or vectors!");
932 template <
class Data_T>
935 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
936 "simple scalars or vectors!");
938 "FileReferences::numRefs(): "
939 "Do not use memory limiting on sparse "
940 "fields that aren't "
941 "simple scalars or vectors!");
953 template <
class Data_T>
956 const std::string layerPath)
958 using namespace SparseFile;
966 template <
class Data_T>
970 boost::mutex::scoped_lock lock(m_mutex);
975 CacheList::iterator it = m_blockCacheList.begin();
976 CacheList::iterator end = m_blockCacheList.end();
977 CacheList::iterator next;
982 if (it->blockType == blockType && it->refIdx == refIdx) {
983 if (it == m_nextBlock) {
988 bytesFreed += reference.
blockSize(it->blockIdx);
989 m_blockCacheList.erase(it);
995 m_memUse -= bytesFreed;
1002 reference.
blocks.clear();
1009 template <
class Data_T>
1013 return m_fileData.ref<Data_T>(index);
1018 template <
class Data_T>
1026 int blockSize = reference.
blockSize(blockIdx);
1027 if (m_limitMemUse) {
1030 deallocateBlocks(blockSize);
1037 boost::mutex::scoped_lock lock_A(m_mutex);
1038 boost::mutex::scoped_lock lock_B(reference.
blockMutex[blockIdx]);
1045 m_memUse += blockSize;
1054 template <
class Data_T>
1067 template <
class Data_T>