pbf_reader.hpp

Go to the documentation of this file.

#ifndef PROTOZERO_PBF_READER_HPP
#define PROTOZERO_PBF_READER_HPP

/*****************************************************************************

protozero - Minimalistic protocol buffer decoder and encoder in C++.

This file is from https://github.com/mapbox/protozero where you can find more
documentation.

*****************************************************************************/

#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <iterator>
#include <string>
#include <utility>

#include <protozero/pbf_types.hpp>
#include <protozero/exception.hpp>
#include <protozero/varint.hpp>

#if __BYTE_ORDER != __LITTLE_ENDIAN
# include <protozero/byteswap.hpp>
#endif

#ifndef protozero_assert
# define protozero_assert(x) assert(x)
#endif

namespace protozero {

class pbf_reader {

    // A pointer to the next unread data.
    const char *m_data = nullptr;

    // A pointer to one past the end of data.
    const char *m_end = nullptr;

    // The wire type of the current field.
    pbf_wire_type m_wire_type = pbf_wire_type::unknown;

    // The tag of the current field.
    pbf_tag_type m_tag = 0;

    template <typename T>
    inline T get_fixed() {
        T result;
        skip_bytes(sizeof(T));
#if __BYTE_ORDER == __LITTLE_ENDIAN
        memcpy(&result, m_data - sizeof(T), sizeof(T));
#else
        byteswap<sizeof(T)>(m_data - sizeof(T), reinterpret_cast<char*>(&result));
#endif
        return result;
    }

#if __BYTE_ORDER == __LITTLE_ENDIAN
    template <typename T>
    inline std::pair<const T*, const T*> packed_fixed() {
        protozero_assert(tag() != 0 && "call next() before accessing field value");
        auto len = get_len_and_skip();
        protozero_assert(len % sizeof(T) == 0);
        return std::make_pair(reinterpret_cast<const T*>(m_data-len), reinterpret_cast<const T*>(m_data));
    }

#else

    template <typename T>
    class const_fixed_iterator : public std::iterator<std::forward_iterator_tag, T> {

        const char* m_data;
        const char* m_end;

    public:

        const_fixed_iterator() noexcept :
            m_data(nullptr),
            m_end(nullptr) {
        }

        const_fixed_iterator(const char *data, const char* end) noexcept :
            m_data(data),
            m_end(end) {
        }

        const_fixed_iterator(const const_fixed_iterator&) noexcept = default;
        const_fixed_iterator(const_fixed_iterator&&) noexcept = default;

        const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default;
        const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default;

        ~const_fixed_iterator() noexcept = default;

        T operator*() {
            T result;
            byteswap<sizeof(T)>(m_data, reinterpret_cast<char*>(&result));
            return result;
        }

        const_fixed_iterator& operator++() {
            m_data += sizeof(T);
            return *this;
        }

        const_fixed_iterator operator++(int) {
            const const_fixed_iterator tmp(*this);
            ++(*this);
            return tmp;
        }

        bool operator==(const const_fixed_iterator& rhs) const noexcept {
            return m_data == rhs.m_data && m_end == rhs.m_end;
        }

        bool operator!=(const const_fixed_iterator& rhs) const noexcept {
            return !(*this == rhs);
        }

    }; // class const_fixed_iterator

    template <typename T>
    inline std::pair<const_fixed_iterator<T>, const_fixed_iterator<T>> packed_fixed() {
        protozero_assert(tag() != 0 && "call next() before accessing field value");
        auto len = get_len_and_skip();
        protozero_assert(len % sizeof(T) == 0);
        return std::make_pair(const_fixed_iterator<T>(m_data-len, m_data),
                              const_fixed_iterator<T>(m_data, m_data));
    }
#endif

    template <typename T> inline T get_varint();
    template <typename T> inline T get_svarint();

    inline pbf_length_type get_length() { return get_varint<pbf_length_type>(); }

    inline void skip_bytes(pbf_length_type len);

    inline pbf_length_type get_len_and_skip();

public:

    inline pbf_reader(const char *data, size_t length) noexcept;

    inline pbf_reader(std::pair<const char *, size_t> data) noexcept;

    inline pbf_reader(const std::string& data) noexcept;

    inline pbf_reader() noexcept = default;

    inline pbf_reader(const pbf_reader&) noexcept = default;

    inline pbf_reader(pbf_reader&&) noexcept = default;

    inline pbf_reader& operator=(const pbf_reader& other) noexcept = default;

    inline pbf_reader& operator=(pbf_reader&& other) noexcept = default;

    inline ~pbf_reader() = default;

    inline operator bool() const noexcept;

    size_t length() const noexcept {
        return size_t(m_end - m_data);
    }

    inline bool next();

    inline bool next(pbf_tag_type tag);

    inline pbf_tag_type tag() const noexcept;

    inline pbf_wire_type wire_type() const noexcept;

    inline bool has_wire_type(pbf_wire_type type) const noexcept;

    inline void skip();


    inline bool get_bool();

    inline int32_t get_enum() {
        protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
        return get_varint<int32_t>();
    }

    inline int32_t get_int32() {
        protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
        return get_varint<int32_t>();
    }

    inline int32_t get_sint32() {
        protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
        return get_svarint<int32_t>();
    }

    inline uint32_t get_uint32() {
        protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
        return get_varint<uint32_t>();
    }

    inline int64_t get_int64() {
        protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
        return get_varint<int64_t>();
    }

    inline int64_t get_sint64() {
        protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
        return get_svarint<int64_t>();
    }

    inline uint64_t get_uint64() {
        protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
        return get_varint<uint64_t>();
    }

    inline uint32_t get_fixed32();

    inline int32_t get_sfixed32();

    inline uint64_t get_fixed64();

    inline int64_t get_sfixed64();

    inline float get_float();

    inline double get_double();

    inline std::pair<const char*, pbf_length_type> get_data();

    inline std::string get_bytes();

    inline std::string get_string();

    inline pbf_reader get_message() {
        return pbf_reader(get_data());
    }


private:

    template <typename T>
    class const_varint_iterator : public std::iterator<std::forward_iterator_tag, T> {

    protected:

        const char* m_data;
        const char* m_end;

    public:

        const_varint_iterator() noexcept :
            m_data(nullptr),
            m_end(nullptr) {
        }

        const_varint_iterator(const char *data, const char* end) noexcept :
            m_data(data),
            m_end(end) {
        }

        const_varint_iterator(const const_varint_iterator&) noexcept = default;
        const_varint_iterator(const_varint_iterator&&) noexcept = default;

        const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default;
        const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default;

        ~const_varint_iterator() noexcept = default;

        T operator*() {
            const char* d = m_data; // will be thrown away
            return static_cast<T>(decode_varint(&d, m_end));
        }

        const_varint_iterator& operator++() {
            // Ignore the result, we call decode_varint() just for the
            // side-effect of updating m_data.
            decode_varint(&m_data, m_end);
            return *this;
        }

        const_varint_iterator operator++(int) {
            const const_varint_iterator tmp(*this);
            ++(*this);
            return tmp;
        }

        bool operator==(const const_varint_iterator& rhs) const noexcept {
            return m_data == rhs.m_data && m_end == rhs.m_end;
        }

        bool operator!=(const const_varint_iterator& rhs) const noexcept {
            return !(*this == rhs);
        }

    }; // class const_varint_iterator

    template <typename T>
    class const_svarint_iterator : public const_varint_iterator<T> {

    public:

        const_svarint_iterator() noexcept :
            const_varint_iterator<T>() {
        }

        const_svarint_iterator(const char *data, const char* end) noexcept :
            const_varint_iterator<T>(data, end) {
        }

        const_svarint_iterator(const const_svarint_iterator&) = default;
        const_svarint_iterator(const_svarint_iterator&&) = default;

        const_svarint_iterator& operator=(const const_svarint_iterator&) = default;
        const_svarint_iterator& operator=(const_svarint_iterator&&) = default;

        ~const_svarint_iterator() = default;

        T operator*() {
            const char* d = this->m_data; // will be thrown away
            return static_cast<T>(decode_zigzag64(decode_varint(&d, this->m_end)));
        }

        const_svarint_iterator& operator++() {
            // Ignore the result, we call decode_varint() just for the
            // side-effect of updating m_data.
            decode_varint(&this->m_data, this->m_end);
            return *this;
        }

        const_svarint_iterator operator++(int) {
            const const_svarint_iterator tmp(*this);
            ++(*this);
            return tmp;
        }

    }; // class const_svarint_iterator

public:

    typedef const_varint_iterator< int32_t> const_bool_iterator;

    typedef const_varint_iterator< int32_t> const_enum_iterator;

    typedef const_varint_iterator< int32_t> const_int32_iterator;

    typedef const_svarint_iterator<int32_t> const_sint32_iterator;

    typedef const_varint_iterator<uint32_t> const_uint32_iterator;

    typedef const_varint_iterator< int64_t> const_int64_iterator;

    typedef const_svarint_iterator<int64_t> const_sint64_iterator;

    typedef const_varint_iterator<uint64_t> const_uint64_iterator;


    inline std::pair<pbf_reader::const_bool_iterator, pbf_reader::const_bool_iterator> get_packed_bool();

    inline std::pair<pbf_reader::const_enum_iterator, pbf_reader::const_enum_iterator> get_packed_enum();

    inline std::pair<pbf_reader::const_int32_iterator, pbf_reader::const_int32_iterator> get_packed_int32();

    inline std::pair<pbf_reader::const_sint32_iterator, pbf_reader::const_sint32_iterator> get_packed_sint32();

    inline std::pair<pbf_reader::const_uint32_iterator, pbf_reader::const_uint32_iterator> get_packed_uint32();

    inline std::pair<pbf_reader::const_int64_iterator, pbf_reader::const_int64_iterator> get_packed_int64();

    inline std::pair<pbf_reader::const_sint64_iterator, pbf_reader::const_sint64_iterator> get_packed_sint64();

    inline std::pair<pbf_reader::const_uint64_iterator, pbf_reader::const_uint64_iterator> get_packed_uint64();

    inline auto get_packed_fixed32() -> decltype(packed_fixed<uint32_t>()) {
        return packed_fixed<uint32_t>();
    }

    inline auto get_packed_sfixed32() -> decltype(packed_fixed<int32_t>()) {
        return packed_fixed<int32_t>();
    }

    inline auto get_packed_fixed64() -> decltype(packed_fixed<uint64_t>()) {
        return packed_fixed<uint64_t>();
    }

    inline auto get_packed_sfixed64() -> decltype(packed_fixed<int64_t>()) {
        return packed_fixed<int64_t>();
    }

    inline auto get_packed_float() -> decltype(packed_fixed<float>()) {
        return packed_fixed<float>();
    }

    inline auto get_packed_double() -> decltype(packed_fixed<double>()) {
        return packed_fixed<double>();
    }


}; // class pbf_reader

pbf_reader::pbf_reader(const char *data, size_t length) noexcept
    : m_data(data),
      m_end(data + length),
      m_wire_type(pbf_wire_type::unknown),
      m_tag(0) {
}

pbf_reader::pbf_reader(std::pair<const char *, size_t> data) noexcept
    : m_data(data.first),
      m_end(data.first + data.second),
      m_wire_type(pbf_wire_type::unknown),
      m_tag(0) {
}

pbf_reader::pbf_reader(const std::string& data) noexcept
    : m_data(data.data()),
      m_end(data.data() + data.size()),
      m_wire_type(pbf_wire_type::unknown),
      m_tag(0) {
}

pbf_reader::operator bool() const noexcept {
    return m_data < m_end;
}

bool pbf_reader::next() {
    if (m_data == m_end) {
        return false;
    }

    auto value = get_varint<uint32_t>();
    m_tag = value >> 3;

    // tags 0 and 19000 to 19999 are not allowed as per
    // https://developers.google.com/protocol-buffers/docs/proto
    protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range");

    m_wire_type = pbf_wire_type(value & 0x07);
// XXX do we want this check? or should it throw an exception?
//        protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type");
    return true;
}

bool pbf_reader::next(pbf_tag_type requested_tag) {
    while (next()) {
        if (m_tag == requested_tag) {
            return true;
        } else {
            skip();
        }
    }
    return false;
}

pbf_tag_type pbf_reader::tag() const noexcept {
    return m_tag;
}

pbf_wire_type pbf_reader::wire_type() const noexcept {
    return m_wire_type;
}

bool pbf_reader::has_wire_type(pbf_wire_type type) const noexcept {
    return wire_type() == type;
}

void pbf_reader::skip_bytes(pbf_length_type len) {
    if (m_data + len > m_end) {
        throw end_of_buffer_exception();
    }
    m_data += len;

// In debug builds reset the tag to zero so that we can detect (some)
// wrong code.
#ifndef NDEBUG
    m_tag = 0;
#endif
}

void pbf_reader::skip() {
    protozero_assert(tag() != 0 && "call next() before calling skip()");
    switch (wire_type()) {
        case pbf_wire_type::varint:
            (void)get_uint32(); // called for the side-effect of skipping value
            break;
        case pbf_wire_type::fixed64:
            skip_bytes(8);
            break;
        case pbf_wire_type::length_delimited:
            skip_bytes(get_length());
            break;
        case pbf_wire_type::fixed32:
            skip_bytes(4);
            break;
        default:
            throw unknown_pbf_wire_type_exception();
    }
}

pbf_length_type pbf_reader::get_len_and_skip() {
    auto len = get_length();
    skip_bytes(len);
    return len;
}

template <typename T>
T pbf_reader::get_varint() {
    return static_cast<T>(decode_varint(&m_data, m_end));
}

template <typename T>
T pbf_reader::get_svarint() {
    protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint");
    return static_cast<T>(decode_zigzag64(decode_varint(&m_data, m_end)));
}

uint32_t pbf_reader::get_fixed32() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
    return get_fixed<uint32_t>();
}

int32_t pbf_reader::get_sfixed32() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
    return get_fixed<int32_t>();
}

uint64_t pbf_reader::get_fixed64() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
    return get_fixed<uint64_t>();
}

int64_t pbf_reader::get_sfixed64() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
    return get_fixed<int64_t>();
}

float pbf_reader::get_float() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
    return get_fixed<float>();
}

double pbf_reader::get_double() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
    return get_fixed<double>();
}

bool pbf_reader::get_bool() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
    protozero_assert((*m_data & 0x80) == 0 && "not a 1 byte varint");
    skip_bytes(1);
    return m_data[-1] != 0; // -1 okay because we incremented m_data the line before
}

std::pair<const char*, pbf_length_type> pbf_reader::get_data() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message");
    auto len = get_len_and_skip();
    return std::make_pair(m_data-len, len);
}

std::string pbf_reader::get_bytes() {
    auto d = get_data();
    return std::string(d.first, d.second);
}

std::string pbf_reader::get_string() {
    return get_bytes();
}

std::pair<pbf_reader::const_bool_iterator, pbf_reader::const_bool_iterator> pbf_reader::get_packed_bool() {
    return get_packed_int32();
}

std::pair<pbf_reader::const_enum_iterator, pbf_reader::const_enum_iterator> pbf_reader::get_packed_enum() {
    return get_packed_int32();
}

std::pair<pbf_reader::const_int32_iterator, pbf_reader::const_int32_iterator> pbf_reader::get_packed_int32() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    auto len = get_len_and_skip();
    return std::make_pair(pbf_reader::const_int32_iterator(m_data-len, m_data),
                          pbf_reader::const_int32_iterator(m_data, m_data));
}

std::pair<pbf_reader::const_uint32_iterator, pbf_reader::const_uint32_iterator> pbf_reader::get_packed_uint32() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    auto len = get_len_and_skip();
    return std::make_pair(pbf_reader::const_uint32_iterator(m_data-len, m_data),
                          pbf_reader::const_uint32_iterator(m_data, m_data));
}

std::pair<pbf_reader::const_sint32_iterator, pbf_reader::const_sint32_iterator> pbf_reader::get_packed_sint32() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    auto len = get_len_and_skip();
    return std::make_pair(pbf_reader::const_sint32_iterator(m_data-len, m_data),
                          pbf_reader::const_sint32_iterator(m_data, m_data));
}

std::pair<pbf_reader::const_int64_iterator, pbf_reader::const_int64_iterator> pbf_reader::get_packed_int64() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    auto len = get_len_and_skip();
    return std::make_pair(pbf_reader::const_int64_iterator(m_data-len, m_data),
                          pbf_reader::const_int64_iterator(m_data, m_data));
}

std::pair<pbf_reader::const_uint64_iterator, pbf_reader::const_uint64_iterator> pbf_reader::get_packed_uint64() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    auto len = get_len_and_skip();
    return std::make_pair(pbf_reader::const_uint64_iterator(m_data-len, m_data),
                          pbf_reader::const_uint64_iterator(m_data, m_data));
}

std::pair<pbf_reader::const_sint64_iterator, pbf_reader::const_sint64_iterator> pbf_reader::get_packed_sint64() {
    protozero_assert(tag() != 0 && "call next() before accessing field value");
    auto len = get_len_and_skip();
    return std::make_pair(pbf_reader::const_sint64_iterator(m_data-len, m_data),
                          pbf_reader::const_sint64_iterator(m_data, m_data));
}

} // end namespace protozero

#endif // PROTOZERO_PBF_READER_HPP