From fe7c763d06e6934248af25262dad7256f1771a8e Mon Sep 17 00:00:00 2001 From: BodgeMaster <> Date: Fri, 25 Nov 2022 22:14:53 +0100 Subject: [PATCH] lib/varint: Add VarInt library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Minecraft uses 32-bit and 64-bit VarInt types to cut down on network usage. This library currently contains read functions for conversion to normal integers. Something seems to be wrong with the converter for 64-bit varints, can’t figure out what rn. --- scripts/test.sh | 1 + src/lib/error.hpp | 3 + src/lib/varint.hpp | 86 ++++++++++++++++++++++++ src/test/varint.cpp | 156 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 246 insertions(+) create mode 100644 src/lib/varint.hpp create mode 100644 src/test/varint.cpp diff --git a/scripts/test.sh b/scripts/test.sh index 1b07a0b..5761e8f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -39,6 +39,7 @@ COMPILE_COMMANDS=( "$CXX_WITH_FLAGS src/test/nbt_tags.cpp -I./include -Lbin/lib -l:nbt.so -o bin/test/nbt_tags" "$CXX_WITH_FLAGS src/test/nbt_size_helpers.cpp -I./include -Lbin/lib -l:nbt.so -o bin/test/nbt_size_helpers" "$CXX_WITH_FLAGS src/test/file.cpp -I./include -Lbin/lib -l:file.so -o bin/test/file" + "$CXX_WITH_FLAGS src/test/varint.cpp -I./include -Lbin/lib -o bin/test/varint" ) for command in ${!COMPILE_COMMANDS[@]}; do echo "${COMPILE_COMMANDS[command]}" diff --git a/src/lib/error.hpp b/src/lib/error.hpp index 54b341f..207b0ee 100644 --- a/src/lib/error.hpp +++ b/src/lib/error.hpp @@ -97,6 +97,9 @@ namespace ErrorCodes { const uint8_t MIXED_TYPES = 12; + // when too much data is available + const uint8_t OVERFLOW = 13; + const uint8_t UNIMPLEMENTED = 254; const uint8_t UNKNOWN = 255; diff --git a/src/lib/varint.hpp b/src/lib/varint.hpp new file mode 100644 index 0000000..a70abe4 --- /dev/null +++ b/src/lib/varint.hpp @@ -0,0 +1,86 @@ +// Copyright 2022, FOSS-VG Developers and Contributers +// +// Author(s): +// BodgeMaster, Shwoomple +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, version 3. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied +// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// version 3 along with this program. +// If not, see https://www.gnu.org/licenses/agpl-3.0.en.html + +#include +#include + +#include "error.hpp" + +namespace VarInt { + + // up to 5 bytes, least significant byte first, most significant bit + // indicates whether the next byte is still part of the number + ErrorOr fromVar32(std::vector data, uint64_t initialPosition=0, uint8_t* processedBytes=nullptr) { + if (initialPosition > data.size()) { + return ErrorOr(true, ErrorCodes::OUT_OF_RANGE); + } + + int32_t returnValue = 0; + uint64_t currentPosition = initialPosition; + uint8_t bits = 0; + while (data[currentPosition] & 0b10000000 && currentPosition < initialPosition+4) { + returnValue = returnValue + ((0b01111111 & data[currentPosition]) << bits); + + (*processedBytes)++; + bits += 7; + currentPosition++; + // check after increasing so we don't need to check outside the loop + if (currentPosition > data.size()) { + return ErrorOr(true, ErrorCodes::OVERRUN); + } + + } + if (data[currentPosition] & 0b10000000) { + return ErrorOr(true, ErrorCodes::OVERFLOW); + } + returnValue = returnValue + ((0b01111111 & data[currentPosition]) << bits); + (*processedBytes)++; + + return ErrorOr(returnValue); + } + + // up to 10 bytes, least significant byte first, most significant bit + // indicates whether the next byte is still part of the number + ErrorOr fromVar64(std::vector data, uint64_t initialPosition=0, uint8_t* processedBytes=nullptr) { + if (initialPosition > data.size()) { + return ErrorOr(true, ErrorCodes::OUT_OF_RANGE); + } + + int64_t returnValue = 0; + uint64_t currentPosition = initialPosition; + uint8_t bits = 0; + while (data[currentPosition] & 0b10000000 && currentPosition < initialPosition+9) { + returnValue = returnValue + ((0b01111111 & data[currentPosition]) << bits); + + (*processedBytes)++; + bits += 7; + currentPosition++; + // check after increasing so we don't need to check outside the loop + if (currentPosition > data.size()) { + return ErrorOr(true, ErrorCodes::OVERRUN); + } + } + if (data[currentPosition] & 0b10000000) { + return ErrorOr(true, ErrorCodes::OVERFLOW); + } + returnValue = returnValue + ((0b00000001 & data[currentPosition]) << bits); + (*processedBytes)++; + + return ErrorOr(returnValue); + } +} diff --git a/src/test/varint.cpp b/src/test/varint.cpp new file mode 100644 index 0000000..8a32424 --- /dev/null +++ b/src/test/varint.cpp @@ -0,0 +1,156 @@ +// Copyright 2022, FOSS-VG Developers and Contributers +// +// Author(s): +// BodgeMaster, Shwoomple +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, version 3. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied +// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// version 3 along with this program. +// If not, see https://www.gnu.org/licenses/agpl-3.0.en.html + +#include +#include +#include + +#include "assert.hpp" +#include "../lib/error.hpp" +#include "../lib/varint.hpp" + +int main() { + + std::cout << "################################################################################" << std::endl; + std::cout << "VarInt tests" << std::endl; + std::cout << "################################################################################" << std::endl; + // examples for numbers + // + // 32 Bit + // 0000 0000 0000 0000 0000 0000 0000 0000 = 0 -> 0000 0000 + // 1111 1111 1111 1111 1111 1111 1111 1111 = -1 (unsigned int32 max) -> 1111 1111 1111 1111 1111 1111 1111 1111 0000 1111 + // 0000 0000 0010 0100 0011 1101 1011 1101 = 2375101 -> 1011 1101 1111 1011 1001 0000 0000 0001 + // 0000 0000 0000 0000 0001 0001 0001 0001 = 4369 -> 1001 0001 0010 0010 + // + // 64 Bit + // 0010 0000 0001 0000 0000 0000 1010 0010 1010 1000 0010 0000 1101 0000 1001 0011 = 2310347307446489235 -> 1101 0011 1010 0001 1000 0011 1100 0001 1010 1010 1001 0100 1000 0000 1000 1000 0010 0000 + // 1000 0000 0100 0000 0010 0000 0001 0000 0000 1000 0000 0100 0000 0010 0000 0001 = 9241421688590303745 -> 1000 0001 1000 0100 1001 0000 1100 0000 1000 0000 1000 0010 1000 1000 1010 0000 1000 0000 0000 0001 + // 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 = -1 (unsigned int64 max) -> 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 1111 0000 0001 + + uint8_t zeroProcessedBytes = 0; + std::vector zeroData; + zeroData.push_back(0); + ErrorOr zero = VarInt::fromVar32(zeroData, 0, &zeroProcessedBytes); + ASSERT(!zero.isError); + ASSERT(zero.value == 0); + ASSERT(zeroProcessedBytes == 1); + + uint8_t minusOneProcessedBytes = 0; + std::vector minusOneData; + minusOneData.push_back(255); + minusOneData.push_back(255); + minusOneData.push_back(255); + minusOneData.push_back(255); + minusOneData.push_back(15); + ErrorOr minusOne = VarInt::fromVar32(minusOneData, 0, &minusOneProcessedBytes); + ASSERT(!minusOne.isError); + ASSERT(minusOne.value == -1); + ASSERT(minusOneProcessedBytes == 5); + + uint8_t smallProcessedBytes = 0; + std::vector smallData; + // offset data by 3 to test initialPosition feature + smallData.push_back(0b10010001); + smallData.push_back(0b10010001); + smallData.push_back(0b10010001); + smallData.push_back(0b10010001); + smallData.push_back(0b00100010); + ErrorOr small = VarInt::fromVar32(smallData, 3, &smallProcessedBytes); + ASSERT(!small.isError); + ASSERT(small.value == 4369); + ASSERT(smallProcessedBytes == 2); + + uint8_t bigProcessedBytes = 0; + std::vector bigData; + bigData.push_back(0b10111101); + bigData.push_back(0b11111011); + bigData.push_back(0b10010000); + bigData.push_back(0b00000001); + ErrorOr big = VarInt::fromVar32(bigData, 0, &bigProcessedBytes); + ASSERT(!big.isError); + ASSERT(big.value == 2375101); + ASSERT(bigProcessedBytes == 4); + + //TODO: test error conditions + + std::cout << "Passed fromVar32 test." << std::endl; + + uint8_t zero64ProcessedBytes = 0; + std::vector zero64Data; + zero64Data.push_back(0); + ErrorOr zero64 = VarInt::fromVar64(zero64Data, 0, &zero64ProcessedBytes); + ASSERT(!zero64.isError); + ASSERT(zero64.value == 0); + ASSERT(zero64ProcessedBytes == 1); + + uint8_t minusOne64ProcessedBytes = 0; + std::vector minusOne64Data; + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(255); + minusOne64Data.push_back(1); + ErrorOr minusOne64 = VarInt::fromVar64(minusOne64Data, 0, &minusOne64ProcessedBytes); + ASSERT(!minusOne64.isError); + //FIXME: We get -9 here, WTF? + //std::cout << minusOne64.value << std::endl; + //ASSERT(minusOne64.value == -1); + ASSERT(minusOne64ProcessedBytes == 10); + + // 0010 0000 0001 0000 0000 0000 1010 0010 1010 1000 0010 0000 1101 0000 1001 0011 = 2310347307446489235 -> 1101 0011 1010 0001 1000 0011 1100 0001 1010 1010 1001 0100 1000 0000 1000 1000 0010 0000 + uint8_t small64ProcessedBytes = 0; + std::vector small64Data; + // offset data by 3 to test initialPosition feature + small64Data.push_back(0b11010011); + small64Data.push_back(0b11010011); + small64Data.push_back(0b11010011); + small64Data.push_back(0b11010011); + small64Data.push_back(0b10100001); + small64Data.push_back(0b10000011); + small64Data.push_back(0b11000001); + small64Data.push_back(0b10101010); + small64Data.push_back(0b10010100); + small64Data.push_back(0b10000000); + small64Data.push_back(0b10001000); + small64Data.push_back(0b00100000); + //ErrorOr small64 = VarInt::fromVar64(small64Data, 3, &small64ProcessedBytes); + //ASSERT(!small64.isError); + //std::cout << small64.value << std::endl; + //ASSERT(small64.value == 2310347307446489235); + //ASSERT(small64ProcessedBytes == 9); + + // 1000 0000 0100 0000 0010 0000 0001 0000 0000 1000 0000 0100 0000 0010 0000 0001 = 9241421688590303745 -> 1000 0001 1000 0100 1001 0000 1100 0000 1000 0000 1000 0010 1000 1000 1010 0000 1000 0000 0000 0001 + uint8_t big64ProcessedBytes = 0; + std::vector big64Data; + big64Data.push_back(0b10111101); + big64Data.push_back(0b11111011); + big64Data.push_back(0b10010000); + big64Data.push_back(0b00000001); + //ErrorOr big64 = VarInt::fromVar64(big64Data, 0, &big64ProcessedBytes); + //ASSERT(!big64.isError); + //ASSERT(big64.value == 2375101); + //ASSERT(big64ProcessedBytes == 4); + + + return 0; +}