309 lines
12 KiB
C++
309 lines
12 KiB
C++
#include "headcache.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <cstring>
|
|
#include <istream>
|
|
#include <limits>
|
|
#include <stdexcept>
|
|
|
|
#include <components/files/streamwithbuffer.hpp>
|
|
#include <components/vfs/manager.hpp>
|
|
|
|
namespace MWSound
|
|
{
|
|
namespace
|
|
{
|
|
// Both buffers implement only the block-read and seek subset of the
|
|
// std::streambuf protocol that FFmpegDecoder uses (read, gcount,
|
|
// seekg, tellg); single-character access is unsupported by design.
|
|
|
|
// Per-file ceiling. Measured stream init consumption tops out under
|
|
// 100KB across vanilla and heavy audio mods once embedded pictures are
|
|
// out of the way; files whose init reads more than this are not cached.
|
|
constexpr std::streamoff sMaxHeadBytes = 256 * 1024;
|
|
|
|
// Covers vanilla's 658KB maximum.
|
|
constexpr std::streamoff sMaxWholeFileBytes = 1024 * 1024;
|
|
|
|
// Reads in the lower half of the file extend the prefix; reads in the
|
|
// upper half accumulate a suffix range, so both regions can be cached
|
|
// and served next time.
|
|
class RecordingBuf final : public std::streambuf
|
|
{
|
|
public:
|
|
explicit RecordingBuf(Files::IStreamPtr inner)
|
|
: mInner(std::move(inner))
|
|
{
|
|
}
|
|
|
|
std::streamoff prefixEnd() const { return mPrefixEnd; }
|
|
std::streamoff suffixStart() const { return mSuffixStart; }
|
|
|
|
protected:
|
|
std::streamsize xsgetn(char* s, std::streamsize n) override
|
|
{
|
|
mInner->clear();
|
|
mInner->read(s, n);
|
|
const std::streamsize got = mInner->gcount();
|
|
const std::streamoff begin = mPos;
|
|
mPos += got;
|
|
if (mFileSize > 0 && begin >= mFileSize / 2)
|
|
mSuffixStart = std::min(mSuffixStart, begin);
|
|
else
|
|
mPrefixEnd = std::max(mPrefixEnd, mPos);
|
|
return got;
|
|
}
|
|
|
|
std::streampos seekoff(std::streamoff off, std::ios_base::seekdir dir, std::ios_base::openmode) override
|
|
{
|
|
mInner->clear();
|
|
mInner->seekg(off, dir);
|
|
if (!*mInner)
|
|
return std::streampos(std::streamoff(-1));
|
|
mPos = mInner->tellg();
|
|
if (dir == std::ios_base::end)
|
|
mFileSize = std::max(mFileSize, mPos - off);
|
|
setg(nullptr, nullptr, nullptr);
|
|
return mPos;
|
|
}
|
|
|
|
std::streampos seekpos(std::streampos pos, std::ios_base::openmode mode) override
|
|
{
|
|
return seekoff(static_cast<std::streamoff>(pos), std::ios_base::beg, mode);
|
|
}
|
|
|
|
private:
|
|
Files::IStreamPtr mInner;
|
|
std::streamoff mPos = 0;
|
|
std::streamoff mPrefixEnd = 0;
|
|
std::streamoff mSuffixStart = std::numeric_limits<std::streamoff>::max();
|
|
std::streamoff mFileSize = 0;
|
|
};
|
|
|
|
class HeadBuf final : public std::streambuf
|
|
{
|
|
public:
|
|
HeadBuf(std::shared_ptr<const HeadBuffer>&& buffer, const VFS::Manager& vfs)
|
|
: mBuffer(std::move(buffer))
|
|
, mVfs(vfs)
|
|
{
|
|
}
|
|
|
|
protected:
|
|
std::streamsize xsgetn(char* s, std::streamsize n) override
|
|
{
|
|
std::streamsize done = 0;
|
|
while (done < n && mPos < mBuffer->mFileSize)
|
|
{
|
|
const std::streamoff headSize = static_cast<std::streamoff>(mBuffer->mHead.size());
|
|
if (mPos < headSize)
|
|
{
|
|
const std::streamsize take = std::min<std::streamsize>(n - done, headSize - mPos);
|
|
std::memcpy(s + done, mBuffer->mHead.data() + mPos, take);
|
|
done += take;
|
|
mPos += take;
|
|
continue;
|
|
}
|
|
if (!mBuffer->mSuffix.empty() && mPos >= mBuffer->mSuffixStart)
|
|
{
|
|
const std::streamoff inSuffix = mPos - mBuffer->mSuffixStart;
|
|
const std::streamsize take = std::min<std::streamsize>(
|
|
n - done, static_cast<std::streamoff>(mBuffer->mSuffix.size()) - inSuffix);
|
|
if (take <= 0)
|
|
break;
|
|
std::memcpy(s + done, mBuffer->mSuffix.data() + inSuffix, take);
|
|
done += take;
|
|
mPos += take;
|
|
continue;
|
|
}
|
|
if (mInner == nullptr)
|
|
mInner = mVfs.get(mBuffer->mName);
|
|
mInner->clear();
|
|
mInner->seekg(mPos);
|
|
const std::streamsize want = !mBuffer->mSuffix.empty() && mPos < mBuffer->mSuffixStart
|
|
? std::min<std::streamsize>(n - done, mBuffer->mSuffixStart - mPos)
|
|
: n - done;
|
|
mInner->read(s + done, want);
|
|
const std::streamsize got = mInner->gcount();
|
|
if (got <= 0)
|
|
break;
|
|
done += got;
|
|
mPos += got;
|
|
}
|
|
return done;
|
|
}
|
|
|
|
std::streampos seekoff(std::streamoff off, std::ios_base::seekdir dir, std::ios_base::openmode) override
|
|
{
|
|
std::streamoff base = 0;
|
|
if (dir == std::ios_base::cur)
|
|
base = mPos;
|
|
else if (dir == std::ios_base::end)
|
|
base = mBuffer->mFileSize;
|
|
const std::streamoff target = base + off;
|
|
// Reject seeks past the end like the constrained streams the
|
|
// file is served from on a miss, so replays behave the same.
|
|
if (target < 0 || target > mBuffer->mFileSize)
|
|
return std::streampos(std::streamoff(-1));
|
|
mPos = target;
|
|
setg(nullptr, nullptr, nullptr);
|
|
return mPos;
|
|
}
|
|
|
|
std::streampos seekpos(std::streampos pos, std::ios_base::openmode mode) override
|
|
{
|
|
return seekoff(static_cast<std::streamoff>(pos), std::ios_base::beg, mode);
|
|
}
|
|
|
|
private:
|
|
std::shared_ptr<const HeadBuffer> mBuffer;
|
|
const VFS::Manager& mVfs;
|
|
Files::IStreamPtr mInner;
|
|
std::streamoff mPos = 0;
|
|
};
|
|
}
|
|
|
|
Files::IStreamPtr makeHeadStream(std::shared_ptr<const HeadBuffer>&& buffer, const VFS::Manager& vfs)
|
|
{
|
|
return std::make_unique<Files::StreamWithBuffer<HeadBuf>>(std::make_unique<HeadBuf>(std::move(buffer), vfs));
|
|
}
|
|
|
|
Files::IStreamPtr makeRecordingStream(Files::IStreamPtr&& impl)
|
|
{
|
|
return std::make_unique<Files::StreamWithBuffer<RecordingBuf>>(std::make_unique<RecordingBuf>(std::move(impl)));
|
|
}
|
|
|
|
HeadCache::HeadCache(const VFS::Manager& vfs, std::size_t maxBytes)
|
|
: mVfs(vfs)
|
|
, mMaxBytes(maxBytes)
|
|
{
|
|
}
|
|
|
|
std::shared_ptr<const HeadBuffer> HeadCache::lookup(VFS::Path::NormalizedView name)
|
|
{
|
|
const std::lock_guard lock(mMutex);
|
|
const auto it = mEntries.find(name);
|
|
if (it == mEntries.end())
|
|
return nullptr;
|
|
mLru.splice(mLru.begin(), mLru, it->second);
|
|
return *it->second;
|
|
}
|
|
|
|
void HeadCache::insert(VFS::Path::NormalizedView name, const std::istream& stream)
|
|
{
|
|
const auto* const recording = dynamic_cast<const RecordingBuf*>(stream.rdbuf());
|
|
if (recording == nullptr)
|
|
throw std::invalid_argument("HeadCache::insert: stream is not from makeRecordingStream");
|
|
const std::streamoff prefixEnd = recording->prefixEnd();
|
|
if (prefixEnd <= 0 || prefixEnd > sMaxHeadBytes)
|
|
return;
|
|
|
|
// The file was just read by stream init, so this re-open and these
|
|
// copies are served from the OS caches; storage is not touched again.
|
|
Files::IStreamPtr fresh = mVfs.get(name);
|
|
fresh->seekg(0, std::ios_base::end);
|
|
const std::streamoff fileSize = fresh->tellg();
|
|
if (fileSize <= 0)
|
|
return;
|
|
std::vector<char> head(static_cast<std::size_t>(std::min(prefixEnd, fileSize)));
|
|
fresh->seekg(0);
|
|
fresh->read(head.data(), head.size());
|
|
if (fresh->gcount() != static_cast<std::streamsize>(head.size()))
|
|
return;
|
|
|
|
std::vector<char> suffix;
|
|
std::streamoff suffixStart = fileSize;
|
|
if (recording->suffixStart() < fileSize)
|
|
{
|
|
suffixStart = recording->suffixStart();
|
|
if (fileSize - suffixStart > sMaxHeadBytes)
|
|
return;
|
|
suffix.resize(static_cast<std::size_t>(fileSize - suffixStart));
|
|
fresh->clear();
|
|
fresh->seekg(suffixStart);
|
|
fresh->read(suffix.data(), suffix.size());
|
|
if (fresh->gcount() != static_cast<std::streamsize>(suffix.size()))
|
|
return;
|
|
}
|
|
|
|
insert(name, std::move(head), std::move(suffix), suffixStart, fileSize);
|
|
}
|
|
|
|
bool HeadCache::contains(VFS::Path::NormalizedView name) const
|
|
{
|
|
const std::lock_guard lock(mMutex);
|
|
return mEntries.contains(name);
|
|
}
|
|
|
|
bool HeadCache::full() const
|
|
{
|
|
const std::lock_guard lock(mMutex);
|
|
return mMaxBytes - mBytes < static_cast<std::size_t>(sMaxWholeFileBytes);
|
|
}
|
|
|
|
bool HeadCache::warmWholeFile(VFS::Path::NormalizedView name)
|
|
{
|
|
{
|
|
// Permit head-only entries to upgrade.
|
|
const std::lock_guard lock(mMutex);
|
|
const auto it = mEntries.find(name);
|
|
if (it != mEntries.end())
|
|
{
|
|
const HeadBuffer& buffer = **it->second;
|
|
if (static_cast<std::streamoff>(buffer.mHead.size()) == buffer.mFileSize)
|
|
return true;
|
|
if (buffer.mFileSize > sMaxWholeFileBytes)
|
|
return false;
|
|
}
|
|
}
|
|
Files::IStreamPtr stream = mVfs.get(name);
|
|
stream->seekg(0, std::ios_base::end);
|
|
const std::streamoff fileSize = stream->tellg();
|
|
if (fileSize <= 0 || fileSize > sMaxWholeFileBytes)
|
|
return false;
|
|
std::vector<char> head(static_cast<std::size_t>(fileSize));
|
|
stream->seekg(0);
|
|
stream->read(head.data(), head.size());
|
|
if (stream->gcount() != static_cast<std::streamsize>(head.size()))
|
|
return false;
|
|
insert(name, std::move(head), std::vector<char>(), fileSize, fileSize);
|
|
return true;
|
|
}
|
|
|
|
void HeadCache::insert(VFS::Path::NormalizedView name, std::vector<char>&& head, std::vector<char>&& suffix,
|
|
std::streamoff suffixStart, std::streamoff fileSize)
|
|
{
|
|
const std::size_t bytes = head.size() + suffix.size();
|
|
// Nothing is allocated until every way out has been taken: an entry
|
|
// bigger than the whole budget can never fit, and evicting for it
|
|
// would empty the cache for no purpose.
|
|
if (bytes > mMaxBytes)
|
|
return;
|
|
const std::lock_guard lock(mMutex);
|
|
const auto existing = mEntries.find(name);
|
|
if (existing != mEntries.end())
|
|
{
|
|
// Preserve streams using the old entry.
|
|
const HeadBuffer& old = **existing->second;
|
|
const std::size_t oldBytes = old.mHead.size() + old.mSuffix.size();
|
|
if (bytes <= oldBytes)
|
|
return;
|
|
mBytes -= oldBytes;
|
|
mLru.erase(existing->second);
|
|
mEntries.erase(existing);
|
|
}
|
|
while (!mLru.empty() && mBytes + bytes > mMaxBytes)
|
|
{
|
|
const HeadBuffer& evicted = *mLru.back();
|
|
mBytes -= evicted.mHead.size() + evicted.mSuffix.size();
|
|
mEntries.erase(evicted.mName);
|
|
mLru.pop_back();
|
|
}
|
|
const LruIt lruIt = mLru.insert(mLru.begin(),
|
|
std::make_shared<const HeadBuffer>(
|
|
VFS::Path::Normalized(name), std::move(head), std::move(suffix), suffixStart, fileSize));
|
|
mEntries.emplace((*lruIt)->mName, lruIt);
|
|
mBytes += bytes;
|
|
}
|
|
}
|