MemoryProgramCache.cpp

Show log
Commit

Hash : 82826be0
Author :
Date : 2023-01-06T11:20:14

Reland "Make ANGLE program version only dependent on data that matters"

This is a reland of commit 61728827d2e5ecce685578bc54bb2c744b65fc9a

Changes made on top of previous commit
1. In src/program_serialize_data_version.py,
fix the script error on Windows: anglebug.com/7918.
2. In BUILD.gn, replace the 'write_file' with
'response_file_content'. See
https://gn.googlesource.com/gn/+/main/docs/reference.md#var_response_file_contents
3. In scripts/generate_android.bp.py, add code to
handle the special GN build flag {{response_file_name}}.
The code writes the list defined in srcs
(identified by $(in) in Android blueprint) into a
temp file named 'gn_response_file', and replaces the
{{response_file_name}} with 'gn_response_file'.

Original change's description

> Changes made on top of original commit
> 1. Enable execution permission on python script
> program_serialize_data_version.py
> 2. Remove unused list in libGLESv2.gni
> 3. In angle/BUILD.gn, change file path from
> "relative to angle_root", to "relative to root_build_dir",
> so that inside the script program_serialize_data_version.py,
> we don't have to find the absolute path of the code files for
> hashing.

> Original change's description

> > This change introduces a new variable ANGLE_PROGRAM_VERSION
> > to track the version of ANGLE source files that affect shader
> > program serialization/deserialization. This change include more
> > source files than necessary, to serve the purpose of a conservative
> > jumping off point. We will narrow down the list of files for
> > ANGLE_PROGRAM_VERSION hash generation in the future.

> > Add a new script program_serialize_data_version.py that will
> > be triggered during the build when the related source files changed.
> > The script will generate a hash and the hash size from the related
> > source files. In program serialization/deserialization and cache
> > key generation, we will use this hash value instead of the entire
> > ANGLE git hash. When the hash value changed, we know that the
> > related source files changed, and we should invalidate the program
> > cache and re-generate the blob cache / program binary.

> > Bug: angleproject:4981
> > Change-Id: I2fb609416738d459d3289190c232c2d797ba58e3
> > Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/4072215
> > Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>
> > Reviewed-by: Cody Northrop <cnorthrop@google.com>
> > Reviewed-by: Jamie Madill <jmadill@chromium.org>
> > Commit-Queue: Yuxin Hu <yuxinhu@google.com>

> Bug: angleproject:4981
> Change-Id: Iaa9eb0ab33439197bc30d03064fc245ea7ef1ea8
> Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/4113445
> Reviewed-by: Amirali Abdolrashidi <abdolrashidi@google.com>
> Reviewed-by: Cody Northrop <cnorthrop@google.com>
> Commit-Queue: Yuxin Hu <yuxinhu@google.com>

Bug: angleproject:4981
Change-Id: Ib5bba199be6d08a1e19807026df0e7b747dbc8a9
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/4144078
Reviewed-by: Roman Lavrov <romanl@google.com>
Commit-Queue: Yuxin Hu <yuxinhu@google.com>
Reviewed-by: Cody Northrop <cnorthrop@google.com>

Download

src/libANGLE/MemoryProgramCache.cpp

//
// Copyright 2017 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// MemoryProgramCache: Stores compiled and linked programs in memory so they don't
//   always have to be re-compiled. Can be used in conjunction with the platform
//   layer to warm up the cache from disk.

// Include zlib first, otherwise FAR gets defined elsewhere.
#define USE_SYSTEM_ZLIB
#include "compression_utils_portable.h"

#include "libANGLE/MemoryProgramCache.h"

#include <GLSLANG/ShaderVars.h>
#include <anglebase/sha1.h>

#include "common/BinaryStream.h"
#include "common/angle_version_info.h"
#include "common/utilities.h"
#include "libANGLE/Context.h"
#include "libANGLE/Debug.h"
#include "libANGLE/Uniform.h"
#include "libANGLE/capture/FrameCapture.h"
#include "libANGLE/histogram_macros.h"
#include "libANGLE/renderer/ProgramImpl.h"
#include "platform/PlatformMethods.h"

namespace gl
{

namespace
{
class HashStream final : angle::NonCopyable
{
  public:
    std::string str() { return mStringStream.str(); }

    template <typename T>
    HashStream &operator<<(T value)
    {
        mStringStream << value << kSeparator;
        return *this;
    }

  private:
    static constexpr char kSeparator = ':';
    std::ostringstream mStringStream;
};

HashStream &operator<<(HashStream &stream, Shader *shader)
{
    if (shader)
    {
        stream << shader->getSourceString().c_str() << shader->getSourceString().length()
               << shader->getCompilerResourcesString().c_str();
    }
    return stream;
}

HashStream &operator<<(HashStream &stream, const ProgramBindings &bindings)
{
    for (const auto &binding : bindings.getStableIterationMap())
    {
        stream << binding.first << binding.second;
    }
    return stream;
}

HashStream &operator<<(HashStream &stream, const ProgramAliasedBindings &bindings)
{
    for (const auto &binding : bindings.getStableIterationMap())
    {
        stream << binding.first << binding.second.location;
    }
    return stream;
}

HashStream &operator<<(HashStream &stream, const std::vector<std::string> &strings)
{
    for (const auto &str : strings)
    {
        stream << str;
    }
    return stream;
}

HashStream &operator<<(HashStream &stream, const std::vector<gl::VariableLocation> &locations)
{
    for (const auto &loc : locations)
    {
        stream << loc.index << loc.arrayIndex << loc.ignored;
    }
    return stream;
}

}  // anonymous namespace

MemoryProgramCache::MemoryProgramCache(egl::BlobCache &blobCache) : mBlobCache(blobCache) {}

MemoryProgramCache::~MemoryProgramCache() {}

void MemoryProgramCache::ComputeHash(const Context *context,
                                     const Program *program,
                                     egl::BlobCache::Key *hashOut)
{
    // Compute the program hash. Start with the shader hashes and resource strings.
    HashStream hashStream;
    for (ShaderType shaderType : AllShaderTypes())
    {
        hashStream << program->getAttachedShader(shaderType);
    }

    // Add some ANGLE metadata and Context properties, such as version and back-end.
    hashStream << angle::GetANGLEShaderProgramVersion() << angle::GetANGLESHVersion()
               << context->getClientMajorVersion() << context->getClientMinorVersion()
               << context->getString(GL_RENDERER);

    // Hash pre-link program properties.
    hashStream << program->getAttributeBindings() << program->getUniformLocationBindings()
               << program->getFragmentOutputLocations() << program->getFragmentOutputIndexes()
               << program->getState().getTransformFeedbackVaryingNames()
               << program->getState().getTransformFeedbackBufferMode()
               << program->getState().getOutputLocations()
               << program->getState().getSecondaryOutputLocations();

    // Include the status of FrameCapture, which adds source strings to the binary
    hashStream << context->getShareGroup()->getFrameCaptureShared()->enabled();

    // Call the secure SHA hashing function.
    const std::string &programKey = hashStream.str();
    angle::base::SHA1HashBytes(reinterpret_cast<const unsigned char *>(programKey.c_str()),
                               programKey.length(), hashOut->data());
}

angle::Result MemoryProgramCache::getProgram(const Context *context,
                                             Program *program,
                                             egl::BlobCache::Key *hashOut)
{
    // If caching is effectively disabled, don't bother calculating the hash.
    if (!mBlobCache.isCachingEnabled())
    {
        return angle::Result::Incomplete;
    }

    ComputeHash(context, program, hashOut);

    angle::MemoryBuffer uncompressedData;
    switch (mBlobCache.getAndDecompress(context->getScratchBuffer(), *hashOut, &uncompressedData))
    {
        case egl::BlobCache::GetAndDecompressResult::NotFound:
            return angle::Result::Incomplete;

        case egl::BlobCache::GetAndDecompressResult::DecompressFailure:
            ANGLE_PERF_WARNING(context->getState().getDebug(), GL_DEBUG_SEVERITY_LOW,
                               "Error decompressing program binary data fetched from cache.");
            return angle::Result::Incomplete;

        case egl::BlobCache::GetAndDecompressResult::GetSuccess:
            angle::Result result =
                program->loadBinary(context, GL_PROGRAM_BINARY_ANGLE, uncompressedData.data(),
                                    static_cast<int>(uncompressedData.size()));
            ANGLE_TRY(result);

            if (result == angle::Result::Continue)
                return angle::Result::Continue;

            // Cache load failed, evict
            ANGLE_PERF_WARNING(context->getState().getDebug(), GL_DEBUG_SEVERITY_LOW,
                               "Failed to load program binary from cache.");
            remove(*hashOut);

            return angle::Result::Incomplete;
    }

    UNREACHABLE();
    return angle::Result::Incomplete;
}

bool MemoryProgramCache::getAt(size_t index,
                               const egl::BlobCache::Key **hashOut,
                               egl::BlobCache::Value *programOut)
{
    return mBlobCache.getAt(index, hashOut, programOut);
}

void MemoryProgramCache::remove(const egl::BlobCache::Key &programHash)
{
    mBlobCache.remove(programHash);
}

angle::Result MemoryProgramCache::putProgram(const egl::BlobCache::Key &programHash,
                                             const Context *context,
                                             const Program *program)
{
    // If caching is effectively disabled, don't bother serializing the program.
    if (!mBlobCache.isCachingEnabled())
    {
        return angle::Result::Incomplete;
    }

    angle::MemoryBuffer serializedProgram;
    ANGLE_TRY(program->serialize(context, &serializedProgram));

    angle::MemoryBuffer compressedData;
    if (!egl::CompressBlobCacheData(serializedProgram.size(), serializedProgram.data(),
                                    &compressedData))
    {
        ANGLE_PERF_WARNING(context->getState().getDebug(), GL_DEBUG_SEVERITY_LOW,
                           "Error compressing binary data.");
        return angle::Result::Incomplete;
    }

    {
        std::scoped_lock<std::mutex> lock(mBlobCache.getMutex());
        // TODO: http://anglebug.com/7568
        // This was a workaround for Chrome until it added support for EGL_ANDROID_blob_cache,
        // tracked by http://anglebug.com/2516. This issue has since been closed, but removing this
        // still causes a test failure.
        auto *platform = ANGLEPlatformCurrent();
        platform->cacheProgram(platform, programHash, compressedData.size(), compressedData.data());
    }

    mBlobCache.put(programHash, std::move(compressedData));
    return angle::Result::Continue;
}

angle::Result MemoryProgramCache::updateProgram(const Context *context, const Program *program)
{
    egl::BlobCache::Key programHash;
    ComputeHash(context, program, &programHash);
    return putProgram(programHash, context, program);
}

bool MemoryProgramCache::putBinary(const egl::BlobCache::Key &programHash,
                                   const uint8_t *binary,
                                   size_t length)
{
    // Copy the binary.
    angle::MemoryBuffer newEntry;
    if (!newEntry.resize(length))
    {
        return false;
    }
    memcpy(newEntry.data(), binary, length);

    // Store the binary.
    mBlobCache.populate(programHash, std::move(newEntry));

    return true;
}

void MemoryProgramCache::clear()
{
    mBlobCache.clear();
}

void MemoryProgramCache::resize(size_t maxCacheSizeBytes)
{
    mBlobCache.resize(maxCacheSizeBytes);
}

size_t MemoryProgramCache::entryCount() const
{
    return mBlobCache.entryCount();
}

size_t MemoryProgramCache::trim(size_t limit)
{
    return mBlobCache.trim(limit);
}

size_t MemoryProgramCache::size() const
{
    return mBlobCache.size();
}

size_t MemoryProgramCache::maxSize() const
{
    return mBlobCache.maxSize();
}

}  // namespace gl

Download