Compare commits

...

8 Commits

Author SHA1 Message Date
BodgeMaster e0cb220d37 resources/region_files: add a zlib compressed chunk 2024-03-24 18:58:26 +01:00
BodgeMaster b5bb2f45c1 Build system: start moving the download handler to its own file 2024-03-24 18:11:28 +01:00
BodgeMaster f35211689e scripts/lib: Fix a bug in the `remove` function 2024-03-24 14:53:57 +01:00
BodgeMaster 248175e0e1 test/hexnet: remove useless (unimplemented) test 2024-03-24 13:27:56 +01:00
BodgeMaster f563f96e87 Code style guidelines and resources/README: Minor documentation changes 2024-03-24 13:27:56 +01:00
BodgeMaster f20ed8b123 lib/net/*: Move stuff around
I did this months ago with the intention to finally start implementing networking....
Yeah, that definitely panned out.

Also, Git is drunk with how it says files have been renamed, they’re just empty apart from the license notice.
2024-03-24 13:27:56 +01:00
BodgeMaster 487d8f1636 tools: Add zlibutil
Splitting one of Joca’s commits by topic - Bodge

Add zlibutil.cpp and zlib to the project's dependencies. - Joca
2024-03-24 13:27:16 +01:00
BodgeMaster 34e19486d6 gitignore: Ignore Sublime Text’s files
Splitting Joca’s commit by topic
2024-03-24 12:05:20 +01:00
19 changed files with 376 additions and 90 deletions

3
.gitignore vendored
View File

@ -24,6 +24,9 @@
# ignore nano's temp files
*.swp
# Ignore sublime text project files.
*.sublime-*
#vscode
.vscode

View File

@ -22,10 +22,8 @@ each file.
Don't use excessive comments, use descriptive names instead.
There is no such thing as too long names (within reason, of course).
Don't have comments that go beyond 80 characters per line, preferably
slightly less.
Put comments on their own lines.
Format comments to improve readability, don't have long single-line comments.
Indent by four spaces.

View File

@ -59,3 +59,9 @@ Usage: `check_endianness > header_file`
Note that, while this tool should in theory be able to detect
PDP and Honeywell-316-style endianness, the FOSS-VG project itself
does not support these.
## region_files
Exactly what the name says. At this point just a random region file I grabbed
from Minecraft, will probably contain synthetic files for test cases in the
future as well.

Binary file not shown.

View File

@ -5,9 +5,18 @@ else
VERBOSE=false
fi
if [ "$1" = "-n" ]; then
DECOMPRESS=false
shift
else
DECOMPRESS=true
fi
if [ ! -f "$1" ]; then
echo "Usage: $0 [-v] FILE"
echo "Usage: $0 [-v] [-n] FILE"
echo " -v display raw content and NBT dump"
echo " -n don't decompress"
exit 1
fi
@ -62,13 +71,20 @@ sys.stdout.buffer.flush()
esac
echo " format: $FORMAT ($FORMAT_HEX)"
if $DECOMPRESS; then
dd if="$1" bs=4096 count=$LENGTH skip=$OFFSET 2>/dev/null | dd bs=1 skip=5 count=$(($COMPRESSED_LENGTH-1)) 2>/dev/null | UNCOMPRESS > /tmp/chunk_uncompressed_nbt
else
dd if="$1" bs=4096 count=$LENGTH skip=$OFFSET 2>/dev/null | dd bs=1 skip=5 count=$(($COMPRESSED_LENGTH-1)) 2>/dev/null > /tmp/chunk_compressed_nbt
fi
if $VERBOSE; then
echo "Raw chunk data:"
echo "$DATA"
echo "NBT dump:"
#TODO: use pipes instead of a file
dd if="$1" bs=4096 count=$LENGTH skip=$OFFSET 2>/dev/null | dd bs=1 skip=5 count=$(($COMPRESSED_LENGTH-1)) 2>/dev/null | UNCOMPRESS > /tmp/chunk_uncompressed_nbt
#TODO: fix this up to work with both compressed and uncompressed NBT
dumpnbt /tmp/chunk_uncompressed_nbt
rm /tmp/chunk_uncompressed_nbt
fi

View File

@ -62,6 +62,7 @@ COMPILE_COMMANDS=(
"$CXX_WITH_FLAGS src/tools/dumpnbt.cpp -I./include -Lbin/lib -l:nbt.so -l:cli.so -o bin/tools/dumpnbt"
"$CXX_WITH_FLAGS src/tools/arraydump.cpp -I./include -Lbin/lib -l:file.so -l:cli.so -o bin/tools/arraydump"
"$CXX_WITH_FLAGS src/tools/baseconvert.cpp -I./include -Lbin/lib -l:cli.so -o bin/tools/baseconvert"
"$CXX_WITH_FLAGS src/tools/zlibutil.cpp -I./include -Lbin/lib -l:cli.so -l:libz.so -l:file.so -o bin/tools/zlibutil"
"$CXX_WITH_FLAGS -pthread src/tools/hexnet.cpp -I./include -Lbin/lib -l:cli.so -l:libsockpp.so -o bin/tools/hexnet"
"$CXX_WITH_FLAGS -DFOSSVG_DEBUG src/fossvg.cpp -I./include -Lbin/lib -l:file.so -l:cli.so -lglfw -lvulkan -o bin/fossvg"
"$CXX_WITH_FLAGS src/fossvgd.cpp -I./include -Lbin/lib -l:cli.so -o bin/fossvgd"

View File

@ -39,6 +39,7 @@ fi
ln -vs ../dependencies/sockpp-0.8.1/include/sockpp/ ./include/
ln -vs ../dependencies/tiny-utf8-4.4.3/include/tinyutf8/ ./include/
ln -vs ../dependencies/zlib-1.3.1/ ./include/zlib
create_file ./bin/.placeholder
create_file ./include/.placeholder

View File

@ -17,6 +17,7 @@
source scripts/lib.sh
#TODO: move this to download.sh
echo ">>> Checking download cache for unneeded or corrupted files..."
for file in $(ls .download_cache); do
#TODO: remove if unknown shasum

61
scripts/download.sh Executable file
View File

@ -0,0 +1,61 @@
if [ "$#" -lt 3 -o "$#" -gt 3 ]; then
echo "Usage: $0 URL DESTINATION SHA256SUM"
exit 1
fi
source scripts/lib.sh
URL="$1"
DESTINATION="$2"
SHA256SUM="$3"
if command -v wget >/dev/null 2>&1; then
USE_WGET=yes
else
if command -v curl >/dev/null 2>&1; then
USE_WGET=no
else
echo "Found neither wget nor curl. Aborting."
exit 1
fi
fi
if [ ! -d .download_cache ]; then
echo "Cache directory missing."
create_directory .download_cache
fi
#TODO: keep track of which cached file is downloaded to which destination
#TODO: when downloading a file with a new hash to the same destination, mark the old one as no longer in use
#TODO: if there is an even older one for the same download path, delete it
#TODO: remove cache maintenance from clean_dependencies
if [ -f ".download_cache/$SHA256SUM" ]; then
if check_sha256 ".download_cache/$SHA256SUM" "$SHA256SUM"; then
echo "Using locally cached file for $DESTINATION"
cp ".download_cache/$SHA256SUM" "$DESTINATION"
exit 0
else
echo "Locally cached file for $DESTINATION is corrupted."
rm ".download_cache/$SHA256SUM"
$0 "$URL" "$DESTINATION" "$SHA256SUM"
exit $?
fi
else
echo -n "Downloading $URL to $DESTINATION... "
if [ $USE_WGET = yes ]; then
wget -O ".download_cache/$SHA256SUM" "$URL" >/dev/null 2>&1
else
curl -L "$URL" --output ".download_cache/$SHA256SUM" >/dev/null 2>&1
fi
if check_sha256 ".download_cache/$SHA256SUM" "$SHA256SUM"; then
cp ".download_cache/$SHA256SUM" "$DESTINATION"
echo "done."
exit 0
else
echo "error."
echo "Checksum verification failed. Your download is either corrupted or the file has been altered. Removing file."
rm ".download_cache/$SHA256SUM"
exit 1
fi
fi

8
scripts/lib.sh Normal file → Executable file
View File

@ -59,16 +59,16 @@ fi
function remove {
local USE_FORCE=""
if [ "$1" = "-f" ]; then
USE_FORCE="-f"
USE_FORCE="f"
shift
echo "Forcefully removing $1..."
else
echo "Removing $1..."
fi;
shift
if [ -d "$1" ]; then
rm "$USE_FORCE" -rv "$1"
rm -"$USE_FORCE"rv "$1"
else
rm "$USE_FORCE" -v "$1"
rm -"$USE_FORCE"v "$1"
fi
}

View File

@ -17,73 +17,19 @@
source scripts/lib.sh
echo -n "Wget or curl? "
if command -v wget >/dev/null 2>&1; then
USE_WGET=yes
echo "wget"
else
if command -v curl >/dev/null 2>&1; then
USE_WGET=no
echo "curl"
else
echo "Found neither wget nor curl. Aborting."
exit 1
fi
fi
function download {
URL="$1"
DESTINATION="$2"
SHA256SUM="$3"
if [ ! -d .download_cache ]; then
echo "Cache directory missing."
create_directory .download_cache
fi
if [ -f ".download_cache/$SHA256SUM" ]; then
if check_sha256 ".download_cache/$SHA256SUM" "$SHA256SUM"; then
echo "Using locally cached file for $DESTINATION"
cp ".download_cache/$SHA256SUM" "$DESTINATION"
return 0
else
echo "Locally cached file for $DESTINATION is corrupted."
rm ".download_cache/$SHA256SUM"
download "$URL" "$DESTINATION" "$SHA256SUM"
return $?
fi
else
echo -n "Downloading $URL to $DESTINATION... "
if [ $USE_WGET = yes ]; then
wget -O ".download_cache/$SHA256SUM" "$URL" >/dev/null 2>&1
else
curl -L "$URL" --output ".download_cache/$SHA256SUM" >/dev/null 2>&1
fi
if check_sha256 ".download_cache/$SHA256SUM" "$SHA256SUM"; then
cp ".download_cache/$SHA256SUM" "$DESTINATION"
echo "done."
return 0
else
echo "error."
echo "Checksum verification failed. Your download is either corrupted or the file has been altered. Removing file."
rm ".download_cache/$SHA256SUM"
return 1
fi
fi
}
scripts/clean.sh
scripts/clean_dependencies.sh
set -e # failures are not acceptable here
create_directory dependencies/tmp
download https://github.com/DuffsDevice/tiny-utf8/archive/refs/tags/v4.4.3.tar.gz dependencies/tmp/tiny-utf8.tar.gz 8e3f61651909c9f3105d3501932a96aa65733127fb6e7cf94cb1b0a2dff42c8f
download https://github.com/fpagliughi/sockpp/archive/refs/tags/v0.8.1.tar.gz dependencies/tmp/sockpp.tar.gz a8aedff8bd8c1da530b91be650352008fddabc9f1df0d19701d76cbc359c8651
scripts/download.sh https://github.com/DuffsDevice/tiny-utf8/archive/refs/tags/v4.4.3.tar.gz dependencies/tmp/tiny-utf8.tar.gz 8e3f61651909c9f3105d3501932a96aa65733127fb6e7cf94cb1b0a2dff42c8f
scripts/download.sh https://github.com/fpagliughi/sockpp/archive/refs/tags/v0.8.1.tar.gz dependencies/tmp/sockpp.tar.gz a8aedff8bd8c1da530b91be650352008fddabc9f1df0d19701d76cbc359c8651
scripts/download.sh https://www.zlib.net/zlib-1.3.1.tar.xz dependencies/tmp/zlib.tar.xz 38ef96b8dfe510d42707d9c781877914792541133e1870841463bfa73f883e32
#TODO: figure out how to cache shaderc
#also TODO: target a specific commit
git clone --branch known-good https://github.com/google/shaderc.git dependencies/tmp/shaderc
#download https://github.com/google/shaderc/archive/refs/tags/v2023.7.tar.gz dependencies/tmp/shaderc.tar.gz 681e1340726a0bf46bea7e31f10cbfe78e01e4446a35d90fedc2b78d400fcdeb
#scripts/download.sh https://github.com/google/shaderc/archive/refs/tags/v2023.7.tar.gz dependencies/tmp/shaderc.tar.gz 681e1340726a0bf46bea7e31f10cbfe78e01e4446a35d90fedc2b78d400fcdeb
echo -n ">>> Extracting tiny-utf8... "
gzip -d dependencies/tmp/tiny-utf8.tar.gz
@ -95,6 +41,11 @@ gzip -d dependencies/tmp/sockpp.tar.gz
tar -xf dependencies/tmp/sockpp.tar -C dependencies
echo "done"
echo -n ">>> Extracting zlib... "
xz -d dependencies/tmp/zlib.tar.xz
tar -xf dependencies/tmp/zlib.tar -C dependencies
echo "done"
echo ">>> Building sockpp..."
pushd dependencies/sockpp-0.8.1/ >/dev/null 2>&1
if uname -s | tr [:upper:] [:lower:] | grep cygwin >/dev/null; then
@ -138,5 +89,12 @@ mkdir -vp dependencies/shaderc/bin
cp -v "dependencies/tmp/shaderc/$SHADERC_BUILD/glslc/glslc" dependencies/shaderc/bin
echo ">>> done"
echo ">>> Building zlib..."
pushd dependencies/zlib-1.3.1/
./configure
make
popd
echo "done"
echo ">>> Cleaning up..."
remove -f dependencies/tmp

View File

@ -1,23 +0,0 @@
#!/usr/bin/env bash
# Copyright 2022, FOSS-VG Developers and Contributers
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# version 3 along with this program.
# If not, see https://www.gnu.org/licenses/agpl-3.0.en.html
echo "################################################################################
Testing hexnet
################################################################################"
echo "Test not yet implemented."
#TODO: implement unit test after merging back with master

View File

@ -0,0 +1,42 @@
// Copyright 2022, FOSS-VG Developers and Contributers
//
// Author(s):
// BodgeMaster
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, version 3.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// version 3 along with this program.
// If not, see https://www.gnu.org/licenses/agpl-3.0.en.html
#pragma once
#include <string>
#include "../error.hpp"
namespace network {
struct Connection {
}
// host could be a hostname or address
ErrorOr<Connection> connect(std::string host);
void disconnect(Connection);
// start listening for incoming connections
ErrorOrVoid startListening();
// stop listening for new incoming connections but keep existing
// connections open
void stopListening();
// stop listening for incoming connections and disconnect all
void stopCommunication();
ErrorOr<Connection> acceptNextIncoming();
}

16
src/lib/net/tcp_server.h Normal file
View File

@ -0,0 +1,16 @@
// Copyright 2022, FOSS-VG Developers and Contributers
//
// Author(s):
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, version 3.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// version 3 along with this program.
// If not, see https://www.gnu.org/licenses/agpl-3.0.en.html

206
src/tools/zlibutil.cpp Normal file
View File

@ -0,0 +1,206 @@
// Copyright 2022, FOSS-VG Developers and Contributers
//
// Author(s):
// Jocadbz
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, version 3.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// version 3 along with this program.
// If not, see https://www.gnu.org/licenses/agpl-3.0.en.html
#include <bitset>
#include <iomanip>
#include <iostream>
#include <vector>
#include <tinyutf8/tinyutf8.h>
#include <zlib/zlib.h>
#include <cstring>
#include "../lib/cli.hpp"
#include "../lib/file.hpp"
#define EXIT_SUCCESS 0
#define EXIT_RUNTIME 1
#define EXIT_USAGE 2
#include <iostream>
#include <fstream>
#include <vector>
#include <zlib.h>
#define CHUNK_SIZE 16384 // Chunk size
std::vector<char> compressData(const char* data, int size) {
z_stream zs;
memset(&zs, 0, sizeof(zs));
if (deflateInit(&zs, Z_DEFAULT_COMPRESSION) != Z_OK)
throw(std::runtime_error("deflateInit failed while compressing."));
zs.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data));
zs.avail_in = size;
int ret;
char outbuffer[CHUNK_SIZE];
std::vector<char> compressedData;
do {
zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
zs.avail_out = CHUNK_SIZE;
ret = deflate(&zs, Z_FINISH);
if (compressedData.size() < zs.total_out)
compressedData.insert(compressedData.end(), outbuffer, outbuffer + CHUNK_SIZE - zs.avail_out);
} while (ret == Z_OK);
deflateEnd(&zs);
if (ret != Z_STREAM_END)
throw(std::runtime_error("Error while compressing: " + std::to_string(ret)));
return compressedData;
}
std::vector<char> decompressData(const char* data, int size) {
z_stream zs;
memset(&zs, 0, sizeof(zs));
if (inflateInit(&zs) != Z_OK)
throw(std::runtime_error("inflateInit failed while decompressing."));
zs.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data));
zs.avail_in = size;
int ret;
char outbuffer[CHUNK_SIZE];
std::vector<char> decompressedData;
do {
zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
zs.avail_out = CHUNK_SIZE;
ret = inflate(&zs, 0);
if (decompressedData.size() < zs.total_out)
decompressedData.insert(decompressedData.end(), outbuffer, outbuffer + CHUNK_SIZE - zs.avail_out);
} while (ret == Z_OK);
inflateEnd(&zs);
if (ret != Z_STREAM_END)
throw(std::runtime_error("Error while decompressing: " + std::to_string(ret)));
return decompressedData;
}
/*
Finnaly, the main file
*/
int main(int argc, char* argv[]) {
std::vector<CLI::Flag> flags;
flags.push_back(CLI::Flag('c', "compress", "compress a file"));
flags.push_back(CLI::Flag('d', "decompress", "descompress a file"));
std::vector<CLI::Option> options;
std::vector<CLI::Argument> arguments;
arguments.push_back(CLI::Argument("FILE", "path of the file to compress/decompress"));
CLI::ArgumentsParser cliParser = CLI::ArgumentsParser(argc, argv, flags, options, arguments, "Compress or decompress files using zlib");
if (cliParser.getFlag("help").value) {
std::cout << cliParser.getUsage() << std::endl;
return EXIT_SUCCESS;
}
if (cliParser.getFlag("license").value){
std::cout
<< "Copyright 2022, FOSS-VG Developers and Contributers\n"
<< "\n"
<< "ZlibUtil is part of the FOSS-VG development tool suite.\n"
<< "\n"
<< "This program is free software: you can redistribute it and/or modify it\n"
<< "under the terms of the GNU Affero General Public License as published\n"
<< "by the Free Software Foundation, version 3.\n"
<< "\n"
<< "This program is distributed in the hope that it will be useful,\n"
<< "but WITHOUT ANY WARRANTY; without even the implied\n"
<< "warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n"
<< "See the GNU Affero General Public License for more details.\n"
<< "\n"
<< "You should have received a copy of the GNU Affero General Public License\n"
<< "version 3 along with this program.\n"
<< "If not, see https://www.gnu.org/licenses/agpl-3.0.en.html"
<< std::endl;
return EXIT_SUCCESS;
}
if (cliParser.wrongUsage) {
std::cout << cliParser.getUsage() << std::endl;
return EXIT_USAGE;
}
ErrorOr<File*> filePointer = File::open(cliParser.getArgument(0).value, 'r');
if (filePointer.isError) {
std::cout << "Failed to open file: " << cliParser.getArgument(0).value << std::endl;
return EXIT_RUNTIME;
}
File* file = filePointer.value;
File *writeFile;
if (cliParser.getFlag("compress").value) {
std::vector<uint8_t> bytes = file->read(file->size.value).value; // this is what you get from lib/file IIRC
std::vector<char> differentBytes = std::vector<char>(bytes.begin(), bytes.end());
std::vector<char> compressed = compressData(differentBytes.data(), file->size.value);
std::vector<unsigned char> unsigneddata = std::vector<unsigned char>(compressed.begin(), compressed.end());
writeFile = File::open(cliParser.getArgument(0).value + ".compressed", 'w').value;
writeFile->write(unsigneddata);
writeFile->close();
} else if (cliParser.getFlag("decompress").value) {
std::vector<uint8_t> bytes = file->read(file->size.value).value; // this is what you get from lib/file IIRC
std::vector<char> differentBytes = std::vector<char>(bytes.begin(), bytes.end());
std::vector<char> compressed = decompressData(differentBytes.data(), file->size.value);
std::vector<unsigned char> unsigneddata = std::vector<unsigned char>(compressed.begin(), compressed.end());
writeFile = File::open(cliParser.getArgument(0).value + ".uncompressed", 'w').value;
writeFile->write(unsigneddata);
writeFile->close();
}
}