Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use cache during OBF section reading #799

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Use cache during OBF section reading #799

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/Data/ObfMapSectionInfo_P.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "QtExtensions.h"
#include "ignore_warnings_on_external_includes.h"
#include <QMutex>
#include <QReadWriteLock>
#include <QSet>
#include <QHash>
#include <QMap>
Expand Down Expand Up @@ -44,6 +45,8 @@ namespace OsmAnd

bool hasChildrenDataBoxes;
uint32_t firstDataBoxInnerOffset;
mutable std::shared_ptr<const QList<ObfMapSectionDataBlockId>> subNodeIds;
mutable QMutex _subNodeIdsMutex;

friend class OsmAnd::ObfMapSectionReader_P;
};
Expand All @@ -57,6 +60,8 @@ namespace OsmAnd
mutable std::shared_ptr< const QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> > > _rootNodes;
mutable QAtomicInt _rootNodesLoaded;
mutable QMutex _rootNodesLoadMutex;
mutable QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>> _nodeCache;
mutable QReadWriteLock _nodeCacheAccessMutex;
public:
virtual ~ObfMapSectionLevel_P();

Expand Down
234 changes: 183 additions & 51 deletions src/Data/ObfMapSectionReader_P.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
#include "Logging.h"
#include "Utilities.h"

const int MAX_TREE_DEPTH_IN_CACHE = 7;

using google::protobuf::internal::WireFormatLite;

OsmAnd::ObfMapSectionReader_P::ObfMapSectionReader_P()
Expand Down Expand Up @@ -396,40 +398,155 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode,
QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>>& nodeCache,
QReadWriteLock& nodeCacheAccessMutex,
int treeDepth,
MapSurfaceType& outChildrenSurfaceType,
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
const std::shared_ptr<const IQueryController>& queryController,
ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric)
{
const auto cis = reader.getCodedInputStream().get();
QList<std::shared_ptr<const ObfMapSectionLevelTreeNode>> childNodes;

outChildrenSurfaceType = MapSurfaceType::Undefined;
for (;;)
bool fromCache = false;
if (!treeNode->subNodeIds)
{
const auto tag = cis->ReadTag();
switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
{
case 0:
if (!ObfReaderUtilities::reachedDataEnd(cis))
return;
QMutexLocker scopedLocker(&treeNode->_subNodeIdsMutex);

return;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
if (!treeNode->subNodeIds)
{
// Read all child nodes and make a lookup table
const std::shared_ptr<QList<ObfMapSectionDataBlockId>> subNodeIds(new QList<ObfMapSectionDataBlockId>);
const auto cis = reader.getCodedInputStream().get();
cis->Seek(treeNode->offset);
const auto oldLimit = cis->PushLimit(treeNode->length);
cis->Skip(treeNode->firstDataBoxInnerOffset);
outChildrenSurfaceType = MapSurfaceType::Undefined;
bool keepReading = true;
while (keepReading)
{
const auto length = ObfReaderUtilities::readBigEndianInt(cis);
const auto offset = cis->CurrentPosition();
const auto oldLimit = cis->PushLimit(length);
const auto tag = cis->ReadTag();
switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
{
case 0:
keepReading = false;
if (!ObfReaderUtilities::reachedDataEnd(cis))
break;

const std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->surfaceType = treeNode->surfaceType;
childNode->offset = offset;
childNode->length = length;
readTreeNode(reader, section, treeNode->area31, childNode);
break;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
{
const auto length = ObfReaderUtilities::readBigEndianInt(cis);
const auto offset = cis->CurrentPosition();
const auto prevLimit = cis->PushLimit(length);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(
new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->surfaceType = treeNode->surfaceType;
childNode->offset = offset;
childNode->length = length;
readTreeNode(reader, section, treeNode->area31, childNode);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(prevLimit);

// Update metric
if (metric)
metric->visitedNodes++;

ObfMapSectionDataBlockId subNodeId;
subNodeId.sectionRuntimeGeneratedId = section->runtimeGeneratedId;
subNodeId.offset = childNode->offset;

// Register node in lookup table
subNodeIds->append(subNodeId);

// Store node in cache
if (treeDepth < MAX_TREE_DEPTH_IN_CACHE)
{
/*
if (!nodeCacheAccessMutex.tryLockForWrite(0))
LogPrintf(OsmAnd::LogSeverityLevel::Debug, "OBF cache write is locked!");
else
nodeCacheAccessMutex.unlock();
*/
QWriteLocker scopedLocker(&nodeCacheAccessMutex);

nodeCache.insert(subNodeId, childNode);
}

if (bbox31)
{
const auto shouldSkip =
!bbox31->contains(childNode->area31) &&
!childNode->area31.contains(*bbox31) &&
!bbox31->intersects(childNode->area31);
if (shouldSkip)
break;
}

// Update metric
if (metric)
metric->acceptedNodes++;

if (nodesWithData && childNode->dataOffset > 0)
nodesWithData->push_back(childNode);

if (childNode->hasChildrenDataBoxes)
childNodes.append(childNode);

if (childNode->surfaceType != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = childNode->surfaceType;
else if (outChildrenSurfaceType != childNode->surfaceType)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}

break;
}
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);

// Store lookup table
if (treeDepth < MAX_TREE_DEPTH_IN_CACHE)
{
treeNode->subNodeIds = subNodeIds;
}
}
else
fromCache = true;
}
else
fromCache = true;

if (fromCache)
{
// Use lookup table to access child nodes stored in cache
for (const auto& subNodeId : constOf(*treeNode->subNodeIds))
{
std::shared_ptr<const ObfMapSectionLevelTreeNode> childNode;
{
/*
if (!nodeCacheAccessMutex.tryLockForRead(0))
LogPrintf(OsmAnd::LogSeverityLevel::Debug, "OBF cache read is locked!");
else
nodeCacheAccessMutex.unlock();
*/
QReadLocker scopedLocker(&nodeCacheAccessMutex);

const auto& citChildNode = nodeCache.constFind(subNodeId);
if (citChildNode != nodeCache.cend())
childNode = citChildNode.value();
}
if (childNode)
{
// Update metric
if (metric)
metric->visitedNodes++;
Expand All @@ -441,7 +558,7 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
!childNode->area31.contains(*bbox31) &&
!bbox31->intersects(childNode->area31);
if (shouldSkip)
break;
continue;
}

// Update metric
Expand All @@ -451,33 +568,43 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
if (nodesWithData && childNode->dataOffset > 0)
nodesWithData->push_back(childNode);

auto subchildrenSurfaceType = MapSurfaceType::Undefined;
if (childNode->hasChildrenDataBoxes)
{
cis->Seek(childNode->offset);
const auto oldLimit = cis->PushLimit(childNode->length);
childNodes.append(childNode);

cis->Skip(childNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, childNode, subchildrenSurfaceType, nodesWithData, bbox31, queryController, metric);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
}

const auto surfaceTypeToMerge = (subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
if (childNode->surfaceType != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = surfaceTypeToMerge;
else if (outChildrenSurfaceType != surfaceTypeToMerge)
outChildrenSurfaceType = childNode->surfaceType;
else if (outChildrenSurfaceType != childNode->surfaceType)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}

break;
}
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}

for (const auto& childNode : constOf(childNodes))
{
auto subchildrenSurfaceType = MapSurfaceType::Undefined;
readTreeNodeChildren(
reader,
section,
childNode,
nodeCache,
nodeCacheAccessMutex,
treeDepth + 1,
subchildrenSurfaceType,
nodesWithData,
bbox31,
queryController,
metric);
const auto surfaceTypeToMerge =
(subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = surfaceTypeToMerge;
else if (outChildrenSurfaceType != surfaceTypeToMerge)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}
}
}
Expand Down Expand Up @@ -1102,17 +1229,22 @@ void OsmAnd::ObfMapSectionReader_P::loadMapObjects(
auto rootSubnodesSurfaceType = MapSurfaceType::Undefined;
if (rootNode->hasChildrenDataBoxes)
{
cis->Seek(rootNode->offset);
auto oldLimit = cis->PushLimit(rootNode->length);

cis->Skip(rootNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, rootNode, rootSubnodesSurfaceType, &treeNodesWithData, bbox31, queryController, metric);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
readTreeNodeChildren(
reader,
section,
rootNode,
mapLevel->_p->_nodeCache,
mapLevel->_p->_nodeCacheAccessMutex,
0,
rootSubnodesSurfaceType,
&treeNodesWithData,
bbox31,
queryController,
metric);
}

const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined) ? rootSubnodesSurfaceType : rootNode->surfaceType;
const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined)
? rootSubnodesSurfaceType : rootNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (bboxOrSectionSurfaceType == MapSurfaceType::Undefined)
Expand Down
4 changes: 4 additions & 0 deletions src/Data/ObfMapSectionReader_P.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <QHash>
#include <QMap>
#include <QSet>
#include <QReadWriteLock>
#include "restore_internal_warnings.h"

#include "OsmAndCore.h"
Expand Down Expand Up @@ -81,6 +82,9 @@ namespace OsmAnd
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode,
QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>>& nodeCache,
QReadWriteLock& nodeCacheAccessMutex,
int treeDepth,
MapSurfaceType& outChildrenSurfaceType,
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
Expand Down