File ZArchive-0.1.2+git20220908.d2c7177.obscpio of Package ZArchive

Overview Repositories Revisions Requests Users Attributes Meta

File ZArchive-0.1.2+git20220908.d2c7177.obscpio of Package ZArchive (Revision 16fffacd1da41e7d0e6f481e9dd3de12)

Currently displaying revision 16fffacd1da41e7d0e6f481e9dd3de12 , Show latest

07070100000000000081A40000000000000000000000016319452600000042000000000000000000000000000000000000002E00000000ZArchive-0.1.2+git20220908.d2c7177/.gitignorecmake-build-*/
.idea

# Visual Studio
.vs/
out/
CMakeSettings.json07070100000001000081A4000000000000000000000001631945260000085D000000000000000000000000000000000000003200000000ZArchive-0.1.2+git20220908.d2c7177/CMakeLists.txtcmake_minimum_required (VERSION 3.15)

if (WIN32)
    set(VCPKG_TARGET_TRIPLET "x64-windows-static" CACHE STRING "VCPKG Target Triplet to use")
endif()

project("ZArchive"
    VERSION "0.1.2"
    DESCRIPTION "Library for creating and reading zstd-compressed file archives"
    HOMEPAGE_URL "https://github.com/Exzap/ZArchive"
)

list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")

include(GNUInstallDirs)

set(SOURCE_FILES_LIB
    src/zarchivewriter.cpp
    src/zarchivereader.cpp
    src/sha_256.c
)

# build static library
add_library (zarchive ${SOURCE_FILES_LIB})
add_library (ZArchive::zarchive ALIAS zarchive)
target_compile_features(zarchive PUBLIC cxx_std_20)
set_target_properties(zarchive PROPERTIES
    MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>"
    VERSION "${PROJECT_VERSION}"
    SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
)

target_include_directories(zarchive
    PUBLIC
        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
)

find_package(zstd MODULE REQUIRED) # MODULE because zstd::zstd is not defined upstream
target_link_libraries(zarchive PRIVATE zstd::zstd)

# standalone executable
add_executable (zarchiveTool src/main.cpp)
set_property(TARGET zarchiveTool PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
set_target_properties(zarchiveTool PROPERTIES OUTPUT_NAME "zarchive")
target_link_libraries(zarchiveTool PRIVATE zarchive)

# install
install(DIRECTORY include/zarchive/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/zarchive" FILES_MATCHING PATTERN "zarchive*.h")
install(TARGETS zarchive)
install(TARGETS zarchiveTool)

# pkg-config
include(JoinPaths) # can be replaced by cmake_path(APPEND) in CMake 3.20
join_paths(PKGCONFIG_INCLUDEDIR "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
join_paths(PKGCONFIG_LIBDIR "\${prefix}" "${CMAKE_INSTALL_LIBDIR}")

configure_file("zarchive.pc.in" "zarchive.pc" @ONLY)
install(
    FILES "${CMAKE_CURRENT_BINARY_DIR}/zarchive.pc"
    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
)
07070100000002000081A40000000000000000000000016319452600000383000000000000000000000000000000000000002B00000000ZArchive-0.1.2+git20220908.d2c7177/LICENSEMIT No Attribution

Permission is hereby granted, free of charge, to any person obtaining a copy of this
software and associated documentation files (the "Software"), to deal in the Software
without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.07070100000003000081A400000000000000000000000163194526000009ED000000000000000000000000000000000000002D00000000ZArchive-0.1.2+git20220908.d2c7177/README.md## Overview
ZArchive is yet another file archive format. Think of zip, tar, 7z, etc. but with the requirement of allowing random-access reads and supporting compression.

## Features / Specifications
- Supports random-access reads within stored files
- Uses zstd compression (64KiB blocks)
- Scales reasonably well up to multiple terabytes with millions of files
- The theoretical size limit per-file is 2^48-1 (256 Terabyte)
- The encoding for paths within the archive is Windows-1252 (case-insensitive)
- Contains a SHA256 hash of the whole archive for integrity checks
- Endian-independent. The format always uses big-endian internally
- Stateless file and directory iterator handles which don't require memory allocation

## Example - Basic read file

```c++
#include "zarchive/zarchivereader.h"

int main()
{
	ZArchiveReader* reader = ZArchiveReader::OpenFromFile("archive.zar");
	if (!reader)
	 	return -1;
	ZArchiveNodeHandle fileHandle = reader->LookUp("myfolder/example.bin");
	if (reader->IsFile(fileHandle))
	{
		uint8_t buffer[1000];
		uint64_t n = reader->ReadFromFile(fileHandle, 0, 1000, buffer);
		// buffer now contains the first n (up to 1000) bytes of example.bin
	}
	delete reader;
	return 0;
}
```

For a more detailed example see [main.cpp](/src/main.cpp)

## Limitations
- Not designed for adding, removing or modifying files after the archive has been created

## No-seek creation
When creating new archives only byte append operations are used. No file seeking is necessary. This makes it possible to create archives on storage which is write-once. It also simplifies streaming ZArchive creation over network.

## UTF8 paths
UTF8 for file and folder paths is theoretically supported as paths are just binary blobs. But the case-insensitive comparison only applies to latin letters (a-z).

## Wii U specifics
Originally this format was created to store Wii U games dumps. These use the file extension .wua (Wii U Archive) but are otherwise regular ZArchive files. To allow multiple Wii U titles to be stored inside a single archive, each title must be placed in a subfolder following the naming scheme: 16-digit titleId followed by \_v and then the version as decimal. For example: 0005000e10102000_v32

## License
The ZArchive library is licensed under [MIT No Attribution](https://github.com/Exzap/ZArchive/blob/master/LICENSE), with the exception of [sha_256.c](/src/sha_256.c) and [sha_256.h](/src/sha_256.h) which are public domain, see: [ https://github.com/amosnier/sha-2]( https://github.com/amosnier/sha-2).
07070100000004000041ED0000000000000000000000016319452600000000000000000000000000000000000000000000002900000000ZArchive-0.1.2+git20220908.d2c7177/cmake07070100000005000081A40000000000000000000000016319452600000385000000000000000000000000000000000000003800000000ZArchive-0.1.2+git20220908.d2c7177/cmake/Findzstd.cmake# SPDX-FileCopyrightText: 2022 Andrea Pappacoda <andrea@pappacoda.it>
# SPDX-License-Identifier: ISC

include(FindPackageHandleStandardArgs)

find_package(zstd CONFIG QUIET)
if (zstd_FOUND)
	# Use upstream zstdConfig.cmake if possible
	if (NOT TARGET zstd::zstd)
		if (TARGET zstd::libzstd_static)
			add_library(zstd::zstd ALIAS zstd::libzstd_static)
		elseif (TARGET zstd::libzstd_shared)
			add_library(zstd::zstd ALIAS zstd::libzstd_shared)
		endif()
	endif()
	find_package_handle_standard_args(zstd CONFIG_MODE)
else()
	# Fallback to pkg-config otherwise
	find_package(PkgConfig)
	if (PKG_CONFIG_FOUND)
		pkg_search_module(libzstd IMPORTED_TARGET GLOBAL libzstd)
		if (libzstd_FOUND)
			add_library(zstd::zstd ALIAS PkgConfig::libzstd)
		endif()
	endif()

find_package_handle_standard_args(zstd
		REQUIRED_VARS
			libzstd_LINK_LIBRARIES
			libzstd_FOUND
		VERSION_VAR libzstd_VERSION
	)
endif()
07070100000006000081A40000000000000000000000016319452600000331000000000000000000000000000000000000003900000000ZArchive-0.1.2+git20220908.d2c7177/cmake/JoinPaths.cmake# This module provides function for joining paths
# known from most languages
#
# SPDX-License-Identifier: (MIT OR CC0-1.0)
# Copyright 2020 Jan Tojnar
# https://github.com/jtojnar/cmake-snips
#
# Modelled after Python’s os.path.join
# https://docs.python.org/3.7/library/os.path.html#os.path.join
# Windows not supported
function(join_paths joined_path first_path_segment)
    set(temp_path "${first_path_segment}")
    foreach(current_segment IN LISTS ARGN)
        if(NOT ("${current_segment}" STREQUAL ""))
            if(IS_ABSOLUTE "${current_segment}")
                set(temp_path "${current_segment}")
            else()
                set(temp_path "${temp_path}/${current_segment}")
            endif()
        endif()
    endforeach()
    set(${joined_path} "${temp_path}" PARENT_SCOPE)
endfunction()
07070100000007000041ED0000000000000000000000016319452600000000000000000000000000000000000000000000002B00000000ZArchive-0.1.2+git20220908.d2c7177/include07070100000008000041ED0000000000000000000000016319452600000000000000000000000000000000000000000000003400000000ZArchive-0.1.2+git20220908.d2c7177/include/zarchive07070100000009000081A40000000000000000000000016319452600002E19000000000000000000000000000000000000004500000000ZArchive-0.1.2+git20220908.d2c7177/include/zarchive/zarchivecommon.h#pragma once

#include <string>
#include <string_view>
#include <cstring>

/* Determine endianness */
/* Original code by https://github.com/rofl0r */
#if (defined __BYTE_ORDER__) && (defined __ORDER_LITTLE_ENDIAN__)
# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#  define ENDIANNESS_LE 1
#  define ENDIANNESS_BE 0
# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#  define ENDIANNESS_LE 0
#  define ENDIANNESS_BE 1
# endif
/* Try to derive from arch/compiler-specific macros */
#elif defined(_X86_) || defined(__x86_64__) || defined(__i386__) || \
      defined(__i486__) || defined(__i586__) || defined(__i686__) || \
      defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) || \
      defined(__ARMEL__) || \
      defined(__MSP430__) || \
      (defined(__LITTLE_ENDIAN__) && __LITTLE_ENDIAN__ == 1) || \
      (defined(_LITTLE_ENDIAN) && _LITTLE_ENDIAN == 1) || \
      defined(_M_ARM) || defined(_M_ARM64) || \
      defined(_M_IX86) || defined(_M_AMD64) /* MSVC */
# define ENDIANNESS_LE 1
# define ENDIANNESS_BE 0
#elif defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) || \
      defined(__MICROBLAZEEB__) || defined(__ARMEB__) || \
      (defined(__BIG_ENDIAN__) && __BIG_ENDIAN__ == 1) || \
      (defined(_BIG_ENDIAN) && _BIG_ENDIAN == 1)
# define ENDIANNESS_LE 0
# define ENDIANNESS_BE 1
/* Try to get it from a header */
#else
# if defined(__linux) || defined(__HAIKU__)
#  include <endian.h>
# elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
       defined(__DragonFly__)
#  include <sys/endian.h>
# elif defined(__APPLE__)
#  include <machine/endian.h>
# endif
#endif

#ifndef ENDIANNESS_LE
# undef ENDIANNESS_BE
# if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN)
#  if __BYTE_ORDER == __LITTLE_ENDIAN
#   define ENDIANNESS_LE 1
#   define ENDIANNESS_BE 0
#  elif __BYTE_ORDER == __BIG_ENDIAN
#   define ENDIANNESS_LE 0
#   define ENDIANNESS_BE 1
#  endif
# elif defined(BYTE_ORDER) && defined(LITTLE_ENDIAN)
#  if BYTE_ORDER == LITTLE_ENDIAN
#   define ENDIANNESS_LE 1
#   define ENDIANNESS_BE 0
#  elif BYTE_ORDER == BIG_ENDIAN
#   define ENDIANNESS_LE 0
#   define ENDIANNESS_BE 1
#  endif
# endif
#endif

namespace _ZARCHIVE
{
	inline constexpr size_t COMPRESSED_BLOCK_SIZE = 64 * 1024; // 64KiB
	inline constexpr size_t ENTRIES_PER_OFFSETRECORD = 16; // must be aligned to two

template<class T, std::size_t... N>
	constexpr T bswap_impl(T i, std::index_sequence<N...>)
	{
		return ((((i >> (N * 8)) & (T)(uint8_t)(-1)) << ((sizeof(T) - 1 - N) * 8)) | ...);
	}

template<class T, class U = std::make_unsigned_t<T>>
	constexpr T bswap(T i)
	{
		return (T)bswap_impl<U>((U)i, std::make_index_sequence<sizeof(T)>{});
	}

template<class T>
	inline T _store(const T src)
	{
#if ENDIANNESS_BE != 0
		return src;
#else
		return bswap<T>(src);
#endif
	}

template<class T>
	inline T _load(const T src)
	{
#if ENDIANNESS_BE != 0
		return src;
#else
		return bswap<T>(src);
#endif
	}

struct CompressionOffsetRecord
	{
		// for every Nth entry we store the full 64bit offset, the blocks in between calculate the offset from the size array
		uint64_t baseOffset;
		uint16_t size[ENTRIES_PER_OFFSETRECORD]; // compressed size - 1

static void Serialize(const CompressionOffsetRecord* input, size_t count, CompressionOffsetRecord* output)
		{
			while (count)
			{
				output->baseOffset = _store(input->baseOffset);
				for (size_t i = 0; i < ENTRIES_PER_OFFSETRECORD; i++)
					output->size[i] = _store(input->size[i]);
				input++;
				output++;
				count--;
			}
		}

static void Deserialize(CompressionOffsetRecord* input, size_t count, CompressionOffsetRecord* output)
		{
			Serialize(input, count, output);
		}
	};

static_assert(std::is_standard_layout<CompressionOffsetRecord>::value);
	static_assert(sizeof(CompressionOffsetRecord) == (8 + 2 * 16));

struct FileDirectoryEntry
	{
		uint32_t nameOffsetAndTypeFlag; // MSB is type. 0 -> directory, 1 -> file. Lower 31 bit are the offset into the node name table
		union
		{
			// note: Current serializer/deserializer code assumes both record types have the same data layout (three uint32_t) which allows for skipping a type check
			struct
			{
				uint32_t fileOffsetLow;
				uint32_t fileSizeLow;
				uint32_t fileOffsetAndSizeHigh; // upper 16 bits -> file size extension, lower 16 bits -> file offset extension
			}fileRecord;
			struct
			{
				uint32_t nodeStartIndex;
				uint32_t count;
				uint32_t _reserved;
			}directoryRecord;
		};

void SetTypeAndNameOffset(bool isFile, uint32_t nameOffset)
		{
			nameOffsetAndTypeFlag = 0;
			if (isFile)
				nameOffsetAndTypeFlag |= 0x80000000;
			else
				nameOffsetAndTypeFlag &= ~0x80000000;
			nameOffsetAndTypeFlag |= (nameOffset & 0x7FFFFFFF);
		}

uint32_t GetNameOffset() const
		{
			return nameOffsetAndTypeFlag & 0x7FFFFFFF;
		}

uint64_t GetFileOffset() const
		{
			uint64_t fileOffset = fileRecord.fileOffsetLow;
			fileOffset |= ((uint64_t)(fileRecord.fileOffsetAndSizeHigh & 0xFFFF) << 32);
			return fileOffset;
		}

uint64_t GetFileSize() const
		{
			uint64_t fileSize = fileRecord.fileSizeLow;
			fileSize |= ((uint64_t)(fileRecord.fileOffsetAndSizeHigh & 0xFFFF0000) << 16);
			return fileSize;
		}

void SetFileOffset(uint64_t fileOffset)
		{
			fileRecord.fileOffsetLow = (uint32_t)fileOffset;
			fileRecord.fileOffsetAndSizeHigh &= 0xFFFF0000;
			fileRecord.fileOffsetAndSizeHigh |= ((uint32_t)(fileOffset >> 32) & 0xFFFF);
		}

void SetFileSize(uint64_t fileSize)
		{
			fileRecord.fileSizeLow = (uint32_t)fileSize;
			fileRecord.fileOffsetAndSizeHigh &= 0x0000FFFF;
			fileRecord.fileOffsetAndSizeHigh |= ((uint32_t)(fileSize >> 16) & 0xFFFF0000);
		}

bool IsFile() const
		{
			return (nameOffsetAndTypeFlag & 0x80000000) != 0;
		}

static void Serialize(const FileDirectoryEntry* input, size_t count, FileDirectoryEntry* output)
		{
			// Optimized method where we exploit the fact that fileRecord and dirRecord have the same layout:
			while (count)
			{
				output->nameOffsetAndTypeFlag = _store(input->nameOffsetAndTypeFlag);
				output->fileRecord.fileOffsetLow = _store(input->fileRecord.fileOffsetLow);
				output->fileRecord.fileSizeLow = _store(input->fileRecord.fileSizeLow);
				output->fileRecord.fileOffsetAndSizeHigh = _store(input->fileRecord.fileOffsetAndSizeHigh);
				input++;
				output++;
				count--;
			}

/* Generic implementation:
			while (count)
			{
				if (input->IsFile())
				{
					output->fileRecord.fileOffsetLow = _store(input->fileRecord.fileOffsetLow);
					output->fileRecord.fileSizeLow = _store(input->fileRecord.fileSizeLow);
					output->fileRecord.fileOffsetAndSizeHigh = _store(input->fileRecord.fileOffsetAndSizeHigh);
				}
				else
				{
					output->directoryRecord.nodeStartIndex = _store(input->directoryRecord.nodeStartIndex);
					output->directoryRecord.count = _store(input->directoryRecord.count);
					output->directoryRecord._reserved = _store(input->directoryRecord._reserved);
				}
				output->nameOffsetAndTypeFlag = _store(input->nameOffsetAndTypeFlag);
				input++;
				output++;
				count--;
			}
			*/
		}

static void Deserialize(FileDirectoryEntry* input, size_t count, FileDirectoryEntry* output)
		{
			Serialize(input, count, output);
		}
	};

static_assert(std::is_standard_layout<FileDirectoryEntry>::value);
	static_assert(sizeof(FileDirectoryEntry) == 16);

struct Footer
	{
		static inline uint32_t kMagic = 0x169f52d6;
		static inline uint32_t kVersion1 = 0x61bf3a01; // also acts as an extended magic

struct OffsetInfo
		{
			uint64_t offset;
			uint64_t size;

bool IsWithinValidRange(uint64_t fileSize) const
			{
				return (offset + size) <= fileSize;
			}
		};

OffsetInfo sectionCompressedData;
		OffsetInfo sectionOffsetRecords;
		OffsetInfo sectionNames;
		OffsetInfo sectionFileTree;
		OffsetInfo sectionMetaDirectory;
		OffsetInfo sectionMetaData;
		uint8_t integrityHash[32];
		uint64_t totalSize;
		uint32_t version;
		uint32_t magic;

static void Serialize(const Footer* input, Footer* output)
		{
			output->magic = _store(input->magic);
			output->version = _store(input->version);
			output->totalSize = _store(input->totalSize);
			output->sectionCompressedData.offset = _store(input->sectionCompressedData.offset);
			output->sectionCompressedData.size = _store(input->sectionCompressedData.size);
			output->sectionOffsetRecords.offset = _store(input->sectionOffsetRecords.offset);
			output->sectionOffsetRecords.size = _store(input->sectionOffsetRecords.size);
			output->sectionNames.offset = _store(input->sectionNames.offset);
			output->sectionNames.size = _store(input->sectionNames.size);
			output->sectionFileTree.offset = _store(input->sectionFileTree.offset);
			output->sectionFileTree.size = _store(input->sectionFileTree.size);
			output->sectionMetaDirectory.offset = _store(input->sectionMetaDirectory.offset);
			output->sectionMetaDirectory.size = _store(input->sectionMetaDirectory.size);
			output->sectionMetaData.offset = _store(input->sectionMetaData.offset);
			output->sectionMetaData.size = _store(input->sectionMetaData.size);
			memcpy(output->integrityHash, input->integrityHash, 32);
		}

static void Deserialize(Footer* input, Footer* output)
		{
			Serialize(input, output);
		}
	};

static_assert(sizeof(Footer) == (16 * 6 + 32 + 8 + 4 + 4));

static bool GetNextPathNode(std::string_view& pathParser, std::string_view& node)
	{
		// skip leading slashes
		while (!pathParser.empty() && (pathParser.front() == '/' || pathParser.front() == '\\'))
			pathParser.remove_prefix(1);
		if (pathParser.empty())
			return false;
		// the next slash is the delimiter
		size_t index = 0;
		for (index = 0; index < pathParser.size(); index++)
		{
			if (pathParser[index] == '/' || pathParser[index] == '\\')
				break;
		}
		node = std::basic_string_view<char>(pathParser.data(), index);
		pathParser.remove_prefix(index);
		return true;
	}

static void SplitFilenameFromPath(std::string_view& pathInOut, std::string_view& filename)
	{
		if (pathInOut.empty())
		{
			filename = pathInOut;
			return;
		}
		// scan backwards until the first slash, this is where the filename starts
		// if there is no slash then stop at index zero
		size_t index = pathInOut.size() - 1;
		while (true)
		{
			if (pathInOut[index] == '/' || pathInOut[index] == '\\')
			{
				index++; // slash isn't part of the filename
				break;
			}
			if (index == 0)
				break;
			index--;
		}
		filename = std::basic_string_view<char>(pathInOut.data() + index, pathInOut.size() - index);
		pathInOut.remove_suffix(pathInOut.size() - index);
	}

static bool CompareNodeNameBool(std::string_view n1, std::string_view n2)
	{
		if (n1.size() != n2.size())
			return false;
		for (size_t i = 0; i < n1.size(); i++)
		{
			char c1 = n1[i];
			char c2 = n2[i];
			if (c1 >= 'A' && c1 <= 'Z')
				c1 -= ('A' - 'a');
			if (c2 >= 'A' && c2 <= 'Z')
				c2 -= ('A' - 'a');
			if (c1 != c2)
				return false;
		}
		return true;
	}

static int CompareNodeName(std::string_view n1, std::string_view n2)
	{
		for (size_t i = 0; i < std::min(n1.size(), n2.size()); i++)
		{
			char c1 = n1[i];
			char c2 = n2[i];
			if (c1 >= 'A' && c1 <= 'Z')
				c1 -= ('A' - 'a');
			if (c2 >= 'A' && c2 <= 'Z')
				c2 -= ('A' - 'a');
			if (c1 != c2)
				return (int)(uint8_t)c2 - (int)(uint8_t)c1;
		}
		if (n1.size() < n2.size())
			return 1;
		if (n1.size() > n2.size())
			return -1;
		return 0;
	}

};

0707010000000A000081A400000000000000000000000163194526000009E6000000000000000000000000000000000000004500000000ZArchive-0.1.2+git20220908.d2c7177/include/zarchive/zarchivereader.h#pragma once

#include <cstdint>
#include <vector>
#include <string_view>
#include <unordered_map>
#include <mutex>

#include <filesystem>
#include <fstream>

#include "zarchivecommon.h"

using ZArchiveNodeHandle = uint32_t;

static inline ZArchiveNodeHandle ZARCHIVE_INVALID_NODE = 0xFFFFFFFF;

class ZArchiveReader
{
public:
	struct DirEntry
	{
		std::string_view name;
		bool isFile;
		bool isDirectory;
		uint64_t size; // only valid for directories
	};

static ZArchiveReader* OpenFromFile(const std::filesystem::path& path);

~ZArchiveReader();

ZArchiveNodeHandle LookUp(std::string_view path, bool allowFile = true, bool allowDirectory = true);
	bool IsDirectory(ZArchiveNodeHandle nodeHandle) const;
	bool IsFile(ZArchiveNodeHandle nodeHandle) const;

// directory operations
	uint32_t GetDirEntryCount(ZArchiveNodeHandle nodeHandle) const;
	bool GetDirEntry(ZArchiveNodeHandle nodeHandle, uint32_t index, DirEntry& dirEntry) const;

// file operations
	uint64_t GetFileSize(ZArchiveNodeHandle nodeHandle);
	uint64_t ReadFromFile(ZArchiveNodeHandle nodeHandle, uint64_t offset, uint64_t length, void* buffer);

private:
	struct CacheBlock
	{
		uint8_t* data;
		uint64_t blockIndex;
		// linked-list for LRU
		CacheBlock* prev;
		CacheBlock* next;
	};

std::mutex m_accessMutex;

std::vector<uint8_t> m_cacheDataBuffer;
	std::vector<CacheBlock> m_cacheBlocks;
	CacheBlock* m_lruChainFirst;
	CacheBlock* m_lruChainLast;
	std::unordered_map<uint64_t, CacheBlock*> m_blockLookup;

ZArchiveReader(std::ifstream&& file, std::vector<_ZARCHIVE::CompressionOffsetRecord>&& offsetRecords, std::vector<uint8_t>&& nameTable, std::vector<_ZARCHIVE::FileDirectoryEntry>&& fileTree, uint64_t compressedDataOffset, uint64_t compressedDataSize);

CacheBlock* GetCachedBlock(uint64_t blockIndex);
	CacheBlock* RecycleLRUBlock(uint64_t newBlockIndex);
	void MarkBlockAsMRU(CacheBlock* block);

void RegisterBlock(CacheBlock* block, uint64_t blockIndex);
	void UnregisterBlock(CacheBlock* block);
	bool LoadBlock(CacheBlock* block);

static std::string_view GetName(const std::vector<uint8_t>& nameTable, uint32_t nameOffset);

std::ifstream m_file;
	std::vector<_ZARCHIVE::CompressionOffsetRecord> m_offsetRecords;
	std::vector<uint8_t> m_nameTable;
	std::vector<_ZARCHIVE::FileDirectoryEntry> m_fileTree;
	uint64_t m_compressedDataOffset;
	uint64_t m_compressedDataSize;
	uint64_t m_blockCount;

std::vector<uint8_t> m_blockDecompressionBuffer;
};0707010000000B000081A400000000000000000000000163194526000009D1000000000000000000000000000000000000004500000000ZArchive-0.1.2+git20220908.d2c7177/include/zarchive/zarchivewriter.h#pragma once

#include <cstdint>
#include <vector>
#include <string_view>
#include <unordered_map>

#include "zarchivecommon.h"

class ZArchiveWriter
{
	struct PathNode
	{
		PathNode() : isFile(false), nameIndex(0xFFFFFFFF) {};
		PathNode(bool isFile, uint32_t nameIndex) : isFile(isFile), nameIndex(nameIndex) {};

bool isFile;
		uint32_t nameIndex; // index in m_nodeNames

std::vector<PathNode*> subnodes;

// file properties
		uint64_t fileOffset{};
		uint64_t fileSize{};
		// directory properties
		uint32_t nodeStartIndex{};
	};

public:
	typedef void(*CB_NewOutputFile)(const int32_t partIndex, void* ctx);
	typedef void(*CB_WriteOutputData)(const void* data, size_t length, void* ctx);

ZArchiveWriter(CB_NewOutputFile cbNewOutputFile, CB_WriteOutputData cbWriteOutputData, void* ctx);
	~ZArchiveWriter();

bool StartNewFile(const char* path); // creates a new virtual file and makes it active
	void AppendData(const void* data, size_t size); // appends data to currently active file
	bool MakeDir(const char* path, bool recursive = false);
	void Finalize();

private:
	PathNode* GetNodeByPath(PathNode* root, std::string_view path);
	PathNode* FindSubnodeByName(PathNode* parent, std::string_view nodeName);

uint32_t CreateNameEntry(std::string_view name);

void OutputData(const void* data, size_t length);
	uint64_t GetCurrentOutputOffset() const;

void StoreBlock(const uint8_t* uncompressedData);

void WriteOffsetRecords();
	void WriteNameTable();
	void WriteFileTree();
	void WriteMetaData();
	void WriteFooter();

private:
	// callbacks
	CB_NewOutputFile m_cbNewOutputFile;
	CB_WriteOutputData m_cbWriteOutputData;
	void* m_cbCtx;
	// file tree
	PathNode m_rootNode;
	PathNode* m_currentFileNode{ nullptr };
	std::vector<std::string> m_nodeNames;
	std::vector<uint32_t> m_nodeNameOffsets;
	std::unordered_map<std::string, uint32_t> m_nodeNameLookup;
	// footer
	_ZARCHIVE::Footer m_footer;
	// writes and compression
	std::vector<uint8_t> m_currentWriteBuffer;
	std::vector<uint8_t> m_compressionBuffer;
	uint64_t m_currentCompressedWriteIndex{ 0 }; // output file write index
	uint64_t m_currentInputOffset{ 0 }; // current offset within uncompressed file data
	// uncompressed-to-compressed offset records
	uint64_t m_numWrittenOffsetRecords{ 0 };
	std::vector<_ZARCHIVE::CompressionOffsetRecord> m_compressionOffsetRecord;
	// hashing
	struct Sha_256* m_mainShaCtx{};
	uint8_t m_integritySha[32];
};0707010000000C000041ED0000000000000000000000016319452600000000000000000000000000000000000000000000002700000000ZArchive-0.1.2+git20220908.d2c7177/src0707010000000D000081A40000000000000000000000016319452600001CFC000000000000000000000000000000000000003000000000ZArchive-0.1.2+git20220908.d2c7177/src/main.cpp#include "zarchive/zarchivewriter.h"
#include "zarchive/zarchivereader.h"

#include <vector>
#include <fstream>
#include <filesystem>
#include <cassert>
#include <optional>

#include <stdio.h>

namespace fs = std::filesystem;

void PrintHelp()
{
	puts("Usage:\n");
	puts("zarchive.exe input_path [output_path]");
	puts("If input_path is a directory, then output_path will be the ZArchive output file path");
	puts("If input_path is a ZArchive file path, then output_path will be the output directory");
	puts("output_path is optional");
}

bool ExtractFile(ZArchiveReader* reader, std::string_view srcPath, const fs::path& path)
{
	ZArchiveNodeHandle fileHandle = reader->LookUp(srcPath, true, false);
	if (fileHandle == ZARCHIVE_INVALID_NODE)
	{
		puts("Unable to extract file:");
		puts(std::string(srcPath).c_str());
		return false;
	}

std::vector<uint8_t> buffer;
	buffer.resize(64 * 1024);

std::ofstream fileOut(path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
	if (!fileOut.is_open())
	{
		puts("Unable to write file:");
		puts(path.generic_string().c_str());
	}
	uint64_t readOffset = 0;
	while (true)
	{
		uint64_t bytesRead = reader->ReadFromFile(fileHandle, readOffset, buffer.size(), buffer.data());
		if (bytesRead == 0)
			break;
		fileOut.write((const char*)buffer.data(), bytesRead);
		readOffset += bytesRead;
	}
	if (readOffset != reader->GetFileSize(fileHandle))
		return false;

return true;
}

bool ExtractRecursive(ZArchiveReader* reader, std::string srcPath, fs::path outputDirectory)
{
	ZArchiveNodeHandle dirHandle = reader->LookUp(srcPath, false, true);
	if (dirHandle == ZARCHIVE_INVALID_NODE)
		return false;
	std::error_code ec;
	fs::create_directories(outputDirectory);
	uint32_t numEntries = reader->GetDirEntryCount(dirHandle);
	for (uint32_t i = 0; i < numEntries; i++)
	{
		ZArchiveReader::DirEntry dirEntry;
		if (!reader->GetDirEntry(dirHandle, i, dirEntry))
		{
			puts("Directory contains invalid node");
			return false;
		}
		puts(std::string(srcPath).append("/").append(dirEntry.name).c_str());
		if (dirEntry.isDirectory)
		{
			ExtractRecursive(reader, std::string(srcPath).append("/").append(dirEntry.name), outputDirectory / dirEntry.name);
		}
		else
		{
			// extract file
			if (!ExtractFile(reader, std::string(srcPath).append("/").append(dirEntry.name), outputDirectory / dirEntry.name))
				return false;
		}
	}
	return true;
}

int Extract(fs::path inputFile, fs::path outputDirectory)
{
	std::error_code ec;
	if (!fs::exists(inputFile, ec))
	{
		puts("Unable to find archive file");
		return -10;
	}

ZArchiveReader* reader = ZArchiveReader::OpenFromFile(inputFile);
	if (!reader)
	{
		puts("Failed to open ZArchive");
		return -11;
	}
	bool r = ExtractRecursive(reader, "", outputDirectory);
	if (!r)
	{
		puts("Extraction failed");
		delete reader;
		return -12;
	}
	delete reader;
	return 0;
}

struct PackContext
{
	fs::path outputFilePath;
	std::ofstream currentOutputFile;
	bool hasError{false};
};

void _pack_NewOutputFile(const int32_t partIndex, void* ctx)
{
	PackContext* packContext = (PackContext*)ctx;
	packContext->currentOutputFile = std::ofstream(packContext->outputFilePath, std::ios::binary);
	if (!packContext->currentOutputFile.is_open())
	{
		printf("Failed to create output file: %s\n", packContext->outputFilePath.string().c_str());
		packContext->hasError = true;
	}
}

void _pack_WriteOutputData(const void* data, size_t length, void* ctx)
{
	PackContext* packContext = (PackContext*)ctx;
	packContext->currentOutputFile.write((const char*)data, length);
}

int Pack(fs::path inputDirectory, fs::path outputFile)
{
	std::vector<uint8_t> buffer;
	buffer.resize(64 * 1024);

std::error_code ec;
	PackContext packContext;
	packContext.outputFilePath = outputFile;
	ZArchiveWriter zWriter(_pack_NewOutputFile, _pack_WriteOutputData, &packContext);
	if (packContext.hasError)
		return -16;
	for (auto const& dirEntry : fs::recursive_directory_iterator(inputDirectory))
	{
		fs::path pathEntry = fs::relative(dirEntry.path(), inputDirectory, ec);
		if (dirEntry.is_directory())
		{
			if (!zWriter.MakeDir(pathEntry.generic_string().c_str(), false))
			{
				printf("Failed to create directory %s\n", pathEntry.string().c_str());
				return -13;
			}
		}
		else if (dirEntry.is_regular_file())
		{
			printf("Adding %s\n", pathEntry.string().c_str());
			if (!zWriter.StartNewFile(pathEntry.generic_string().c_str()))
			{
				printf("Failed to create archive file %s\n", pathEntry.string().c_str());
				return -14;
			}
			std::ifstream inputFile(inputDirectory / pathEntry, std::ios::binary);
			if (!inputFile.is_open())
			{
				printf("Failed to open input file %s\n", pathEntry.string().c_str());
				return -15;
			}
			while( true )
			{
				inputFile.read((char*)buffer.data(), buffer.size());
				int32_t readBytes = (int32_t)inputFile.gcount();
				if (readBytes <= 0)
					break;
				zWriter.AppendData(buffer.data(), readBytes);
			}
		}
		if (packContext.hasError)
			return -16;
	}
	zWriter.Finalize();
	return 0;
}

int main(int argc, char* argv[])
{
	if (argc <= 1)
	{
		PrintHelp();
		return 0;
	}
	std::optional<std::string> strInput;
	std::optional<std::string> strOutput;
	for (int i = 1; i < argc; i++)
	{
		if (strInput)
		{
			if (strOutput)
			{
				puts("Too many paths specified");
				return -1;
			}
			else
			{
				strOutput = argv[i];
			}
		}
		else
		{
			strInput = argv[i];
		}
	}

if (strInput)
	{
		std::error_code ec;
		fs::path p(*strInput);
		if (fs::is_regular_file(p, ec))
		{
			// extract
			fs::path outputDirectory;
			if (!strOutput)
			{
				fs::path defaultOutputPath = p.parent_path() / (p.stem().filename().string().append("_extracted"));
				outputDirectory = defaultOutputPath;
				printf("Extracting to: %s\n", outputDirectory.generic_string().c_str());
			}
			else
				outputDirectory = *strOutput;
			if (fs::exists(outputDirectory, ec) && !fs::is_directory(outputDirectory, ec))
			{
				puts("The specified output path is not a valid directory");
				return -3;
			}
			fs::create_directories(outputDirectory, ec);
			if (!fs::exists(outputDirectory, ec))
			{
				puts("Failed to create output directory");
				return -4;
			}
			return Extract(p, outputDirectory);
		}
		else if(fs::is_directory(p, ec))
		{
			// pack
			fs::path outputFile;
			if (!strOutput)
			{
				fs::path defaultOutputPath = p.parent_path() / (p.stem().filename().string().append(".zar"));
				outputFile = defaultOutputPath;
				printf("Outputting to: %s\n", outputFile.generic_string().c_str());
			}
			else
				outputFile = *strOutput;
			if ((fs::exists(outputFile, ec) && !fs::is_regular_file(outputFile, ec)))
			{
				puts("The specified output path is not a valid file");
				return -10;
			}
			if ((fs::exists(outputFile, ec) && fs::is_regular_file(outputFile, ec)))
			{
				puts("The output file already exists");
				return -11;
			}
			int r = Pack(p, outputFile);
			if (r != 0)
			{
				// delete incomplete output file
				fs::remove(outputFile, ec);
			}
			return r;
		}
		else
		{
			puts("Input path is not a valid file or directory");
			return -1;
		}
	}
	return 0;
}
0707010000000E000081A40000000000000000000000016319452600001DD4000000000000000000000000000000000000003100000000ZArchive-0.1.2+git20220908.d2c7177/src/sha_256.c#include "sha_256.h"

#define TOTAL_LEN_LEN 8

/* Original code by amosnier. Tweaked for extra performance */
/* https://github.com/amosnier/sha-2 */

/*
 * Comments from pseudo-code at https://en.wikipedia.org/wiki/SHA-2 are reproduced here.
 * When useful for clarification, portions of the pseudo-code are reproduced here too.
 */

/*
 * @brief Rotate a 32-bit value by a number of bits to the right.
 * @param value The value to be rotated.
 * @param count The number of bits to rotate by.
 * @return The rotated value.
 */
static inline uint32_t right_rot(uint32_t value, unsigned int count)
{
	/*
	 * Defined behaviour in standard C for all count where 0 < count < 32, which is what we need here.
	 */
	return value >> count | value << (32 - count);
}

static const uint32_t _sha256_k[] = {
	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
	0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
	0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
	0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
	0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
	0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
	0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
	0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
	0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
	0xc67178f2 };

/*
 * @brief Update a hash value under calculation with a new chunk of data.
 * @param h Pointer to the first hash item, of a total of eight.
 * @param p Pointer to the chunk data, which has a standard length.
 *
 * @note This is the SHA-256 work horse.
 */
static inline void consume_chunk(uint32_t *h, const uint8_t *p)
{
	unsigned i, j;
	uint32_t ah[8];

/* Initialize working variables to current hash value: */
	for (i = 0; i < 8; i++)
		ah[i] = h[i];

/*
	 * The w-array is really w[64], but since we only need 16 of them at a time, we save stack by
	 * calculating 16 at a time.
	 *
	 * This optimization was not there initially and the rest of the comments about w[64] are kept in their
	 * initial state.
	 */

/*
	 * create a 64-entry message schedule array w[0..63] of 32-bit words (The initial values in w[0..63]
	 * don't matter, so many implementations zero them here) copy chunk into first 16 words w[0..15] of the
	 * message schedule array
	 */
	uint32_t w[16];
	const uint32_t* k = _sha256_k;
	/* Unrolled i == 0 */
	for (j = 0; j < 16; j++) {
		w[j] = (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | (uint32_t)p[3];
		p += 4;
		const uint32_t s1 = right_rot(ah[4], 6) ^ right_rot(ah[4], 11) ^ right_rot(ah[4], 25);
		const uint32_t ch = (ah[4] & ah[5]) ^ (~ah[4] & ah[6]);

const uint32_t temp1 = ah[7] + s1 + ch + *k + w[j]; k++;
		const uint32_t s0 = right_rot(ah[0], 2) ^ right_rot(ah[0], 13) ^ right_rot(ah[0], 22);
		const uint32_t maj = (ah[0] & ah[1]) ^ (ah[0] & ah[2]) ^ (ah[1] & ah[2]);
		const uint32_t temp2 = s0 + maj;

ah[7] = ah[6];
		ah[6] = ah[5];
		ah[5] = ah[4];
		ah[4] = ah[3] + temp1;
		ah[3] = ah[2];
		ah[2] = ah[1];
		ah[1] = ah[0];
		ah[0] = temp1 + temp2;
	}
	/* Compression function main loop: */
	for (i = 1; i < 4; i++) {
		for (j = 0; j < 16; j++) {
			uint32_t t = w[(j + 1) & 0xf];
			const uint32_t s0t = right_rot(t, 7) ^ right_rot(t, 18) ^ (t >> 3);
			t = w[(j + 14) & 0xf];
			const uint32_t s1t = right_rot(t, 17) ^ right_rot(t, 19) ^ (t >> 10);
			w[j] = w[j] + s0t + w[(j + 9) & 0xf] + s1t;

const uint32_t s1 = right_rot(ah[4], 6) ^ right_rot(ah[4], 11) ^ right_rot(ah[4], 25);
			const uint32_t ch = (ah[4] & ah[5]) ^ (~ah[4] & ah[6]);

const uint32_t temp1 = ah[7] + s1 + ch + *k + w[j]; k++;
			const uint32_t s0 = right_rot(ah[0], 2) ^ right_rot(ah[0], 13) ^ right_rot(ah[0], 22);
			const uint32_t maj = (ah[0] & ah[1]) ^ (ah[0] & ah[2]) ^ (ah[1] & ah[2]);
			const uint32_t temp2 = s0 + maj;

ah[7] = ah[6];
			ah[6] = ah[5];
			ah[5] = ah[4];
			ah[4] = ah[3] + temp1;
			ah[3] = ah[2];
			ah[2] = ah[1];
			ah[1] = ah[0];
			ah[0] = temp1 + temp2;
		}
	}

/* Add the compressed chunk to the current hash value: */
	for (i = 0; i < 8; i++)
		h[i] += ah[i];
}

/*
 * Public functions. See header file for documentation.
 */

void sha_256_init(struct Sha_256 *sha_256, uint8_t hash[SIZE_OF_SHA_256_HASH])
{
	sha_256->hash = hash;
	sha_256->chunk_pos = sha_256->chunk;
	sha_256->space_left = SIZE_OF_SHA_256_CHUNK;
	sha_256->total_len = 0;
	/*
	 * Initialize hash values (first 32 bits of the fractional parts of the square roots of the first 8 primes
	 * 2..19):
	 */
	sha_256->h[0] = 0x6a09e667;
	sha_256->h[1] = 0xbb67ae85;
	sha_256->h[2] = 0x3c6ef372;
	sha_256->h[3] = 0xa54ff53a;
	sha_256->h[4] = 0x510e527f;
	sha_256->h[5] = 0x9b05688c;
	sha_256->h[6] = 0x1f83d9ab;
	sha_256->h[7] = 0x5be0cd19;
}

void sha_256_write(struct Sha_256 *sha_256, const void *data, size_t len)
{
	sha_256->total_len += len;

const uint8_t *p = data;

while (len > 0) {
		/*
		 * If the input chunks have sizes that are multiples of the calculation chunk size, no copies are
		 * necessary. We operate directly on the input data instead.
		 */
		if (sha_256->space_left == SIZE_OF_SHA_256_CHUNK && len >= SIZE_OF_SHA_256_CHUNK) {
			consume_chunk(sha_256->h, p);
			len -= SIZE_OF_SHA_256_CHUNK;
			p += SIZE_OF_SHA_256_CHUNK;
			continue;
		}
		/* General case, no particular optimization. */
		const size_t consumed_len = len < sha_256->space_left ? len : sha_256->space_left;
		memcpy(sha_256->chunk_pos, p, consumed_len);
		sha_256->space_left -= consumed_len;
		len -= consumed_len;
		p += consumed_len;
		if (sha_256->space_left == 0) {
			consume_chunk(sha_256->h, sha_256->chunk);
			sha_256->chunk_pos = sha_256->chunk;
			sha_256->space_left = SIZE_OF_SHA_256_CHUNK;
		} else {
			sha_256->chunk_pos += consumed_len;
		}
	}
}

uint8_t *sha_256_close(struct Sha_256 *sha_256)
{
	uint8_t *pos = sha_256->chunk_pos;
	size_t space_left = sha_256->space_left;
	uint32_t *const h = sha_256->h;

/*
	 * The current chunk cannot be full. Otherwise, it would already have be consumed. I.e. there is space left for
	 * at least one byte. The next step in the calculation is to add a single one-bit to the data.
	 */
	*pos++ = 0x80;
	--space_left;

/*
	 * Now, the last step is to add the total data length at the end of the last chunk, and zero padding before
	 * that. But we do not necessarily have enough space left. If not, we pad the current chunk with zeroes, and add
	 * an extra chunk at the end.
	 */
	if (space_left < TOTAL_LEN_LEN) {
		memset(pos, 0x00, space_left);
		consume_chunk(h, sha_256->chunk);
		pos = sha_256->chunk;
		space_left = SIZE_OF_SHA_256_CHUNK;
	}
	const size_t left = space_left - TOTAL_LEN_LEN;
	memset(pos, 0x00, left);
	pos += left;
	size_t len = sha_256->total_len;
	pos[7] = (uint8_t)(len << 3);
	len >>= 5;
	int i;
	for (i = 6; i >= 0; --i) {
		pos[i] = (uint8_t)len;
		len >>= 8;
	}
	consume_chunk(h, sha_256->chunk);
	/* Produce the final hash value (big-endian): */
	int j;
	uint8_t *const hash = sha_256->hash;
	for (i = 0, j = 0; i < 8; i++) {
		hash[j++] = (uint8_t)(h[i] >> 24);
		hash[j++] = (uint8_t)(h[i] >> 16);
		hash[j++] = (uint8_t)(h[i] >> 8);
		hash[j++] = (uint8_t)h[i];
	}
	return sha_256->hash;
}

void calc_sha_256(uint8_t hash[SIZE_OF_SHA_256_HASH], const void *input, size_t len)
{
	struct Sha_256 sha_256;
	sha_256_init(&sha_256, hash);
	sha_256_write(&sha_256, input, len);
	(void)sha_256_close(&sha_256);
}
0707010000000F000081A40000000000000000000000016319452600001060000000000000000000000000000000000000003100000000ZArchive-0.1.2+git20220908.d2c7177/src/sha_256.h#ifndef SHA_256_H
#define SHA_256_H

#include <stdint.h>
#include <string.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * @brief Size of the SHA-256 sum. This times eight is 256 bits.
 */
#define SIZE_OF_SHA_256_HASH 32

/*
 * @brief Size of the chunks used for the calculations.
 *
 * @note This should mostly be ignored by the user, although when using the streaming API, it has an impact for
 * performance. Add chunks whose size is a multiple of this, and you will avoid a lot of superfluous copying in RAM!
 */
#define SIZE_OF_SHA_256_CHUNK 64

/*
 * @brief The opaque SHA-256 type, that should be instantiated when using the streaming API.
 *
 * @note Although the details are exposed here, in order to make instantiation easy, you should refrain from directly
 * accessing the fields, as they may change in the future.
 */
struct Sha_256 {
	uint8_t *hash;
	uint8_t chunk[SIZE_OF_SHA_256_CHUNK];
	uint8_t *chunk_pos;
	size_t space_left;
	size_t total_len;
	uint32_t h[8];
};

/*
 * @brief The simple SHA-256 calculation function.
 * @param hash Hash array, where the result is delivered.
 * @param input Pointer to the data the hash shall be calculated on.
 * @param len Length of the input data, in byte.
 *
 * @note If all of the data you are calculating the hash value on is available in a contiguous buffer in memory, this is
 * the function you should use.
 *
 * @note If either of the passed pointers is NULL, the results are unpredictable.
 */
void calc_sha_256(uint8_t hash[SIZE_OF_SHA_256_HASH], const void *input, size_t len);

/*
 * @brief Initialize a SHA-256 streaming calculation.
 * @param sha_256 A pointer to a SHA-256 structure.
 * @param hash Hash array, where the result will be delivered.
 *
 * @note If all of the data you are calculating the hash value on is not available in a contiguous buffer in memory, this is
 * where you should start. Instantiate a SHA-256 structure, for instance by simply declaring it locally, make your hash
 * buffer available, and invoke this function. Once a SHA-256 hash has been calculated (see further below) a SHA-256
 * structure can be initialized again for the next calculation.
 *
 * @note If either of the passed pointers is NULL, the results are unpredictable.
 */
void sha_256_init(struct Sha_256 *sha_256, uint8_t hash[SIZE_OF_SHA_256_HASH]);

/*
 * @brief Stream more input data for an on-going SHA-256 calculation.
 * @param sha_256 A pointer to a previously initialized SHA-256 structure.
 * @param data Pointer to the data to be added to the calculation.
 * @param len Length of the data to add, in byte.
 *
 * @note This function may be invoked an arbitrary number of times between initialization and closing, but the maximum
 * data length is limited by the SHA-256 algorithm: the total number of bits (i.e. the total number of bytes times
 * eight) must be representable by a 64-bit unsigned integer. While that is not a practical limitation, the results are
 * unpredictable if that limit is exceeded.
 *
 * @note This function may be invoked on empty data (zero length), although that obviously will not add any data.
 *
 * @note If either of the passed pointers is NULL, the results are unpredictable.
 */
void sha_256_write(struct Sha_256 *sha_256, const void *data, size_t len);

/*
 * @brief Conclude a SHA-256 streaming calculation, making the hash value available.
 * @param sha_256 A pointer to a previously initialized SHA-256 structure.
 * @return Pointer to the hash array, where the result is delivered.
 *
 * @note After this function has been invoked, the result is available in the hash buffer that initially was provided. A
 * pointer to the hash value is returned for convenience, but you should feel free to ignore it: it is simply a pointer
 * to the first byte of your initially provided hash array.
 *
 * @note If the passed pointer is NULL, the results are unpredictable.
 *
 * @note Invoking this function for a calculation with no data (the writing function has never been invoked, or it only
 * has been invoked with empty data) is legal. It will calculate the SHA-256 value of the empty string.
 */
uint8_t *sha_256_close(struct Sha_256 *sha_256);

#ifdef __cplusplus
}
#endif

#endif
07070100000010000081A40000000000000000000000016319452600003368000000000000000000000000000000000000003A00000000ZArchive-0.1.2+git20220908.d2c7177/src/zarchivereader.cpp#include "zarchive/zarchivereader.h"
#include "zarchive/zarchivecommon.h"

#include <fstream>

#include <zstd.h>
#include <cassert>

static uint64_t _ifstream_getFileSize(std::ifstream& file)
{
	file.seekg(0, std::ios_base::end);
	return (uint64_t)file.tellg();
}

static bool _ifstream_readBytes(std::ifstream& file, uint64_t offset, void* buffer, uint32_t size)
{
	file.seekg(offset, std::ios_base::beg);
	file.read((char*)buffer, size);
	return file.gcount() == size;
}

static uint64_t _getValidElementCount(uint64_t size, uint64_t elementSize)
{
	if ((size % elementSize) != 0)
		return 0;
	return size / elementSize;
}

ZArchiveReader* ZArchiveReader::OpenFromFile(const std::filesystem::path& path)
{
	std::ifstream file;
	file.open(path, std::ios_base::in | std::ios_base::binary);
	if (!file.is_open())
		return nullptr;
	uint64_t fileSize = _ifstream_getFileSize(file);
	if (fileSize <= sizeof(_ZARCHIVE::Footer))
		return nullptr;
	// read footer
	_ZARCHIVE::Footer footer;
	if (!_ifstream_readBytes(file, fileSize - sizeof(_ZARCHIVE::Footer), &footer, sizeof(_ZARCHIVE::Footer)))
		return nullptr;
	_ZARCHIVE::Footer::Deserialize(&footer, &footer);
	// validate footer
	if (footer.magic != _ZARCHIVE::Footer::kMagic)
		return nullptr;
	if (footer.version != _ZARCHIVE::Footer::kVersion1)
		return nullptr;
	if (footer.totalSize != fileSize)
		return nullptr;
	if (!footer.sectionCompressedData.IsWithinValidRange(fileSize) ||
		!footer.sectionOffsetRecords.IsWithinValidRange(fileSize) ||
		!footer.sectionNames.IsWithinValidRange(fileSize) ||
		!footer.sectionFileTree.IsWithinValidRange(fileSize) ||
		!footer.sectionMetaDirectory.IsWithinValidRange(fileSize) ||
		!footer.sectionMetaData.IsWithinValidRange(fileSize))
		return nullptr;
	if (footer.sectionOffsetRecords.size > (uint64_t)0xFFFFFFFF)
		return nullptr;
	if (footer.sectionNames.size > (uint64_t)0x7FFFFFFF)
		return nullptr;
	if (footer.sectionFileTree.size > (uint64_t)0xFFFFFFFF)
		return nullptr;
	// read offset records
	std::vector<_ZARCHIVE::CompressionOffsetRecord> offsetRecords;
	offsetRecords.resize(_getValidElementCount(footer.sectionOffsetRecords.size, sizeof(_ZARCHIVE::CompressionOffsetRecord)));
	if (offsetRecords.empty() || !_ifstream_readBytes(file, footer.sectionOffsetRecords.offset, offsetRecords.data(), (uint32_t)(offsetRecords.size() * sizeof(_ZARCHIVE::CompressionOffsetRecord))))
		return nullptr;
	_ZARCHIVE::CompressionOffsetRecord::Deserialize(offsetRecords.data(), offsetRecords.size(), offsetRecords.data());
	// read name table
	std::vector<uint8_t> nameTable;
	nameTable.resize(footer.sectionNames.size);
	if (!_ifstream_readBytes(file, footer.sectionNames.offset, nameTable.data(), (uint32_t)(nameTable.size() * sizeof(uint8_t))))
		return nullptr;
	// read file tree
	std::vector<_ZARCHIVE::FileDirectoryEntry> fileTree;
	fileTree.resize(_getValidElementCount(footer.sectionFileTree.size, sizeof(_ZARCHIVE::FileDirectoryEntry)));
	if (fileTree.empty() || !_ifstream_readBytes(file, footer.sectionFileTree.offset, fileTree.data(), (uint32_t)(fileTree.size() * sizeof(_ZARCHIVE::FileDirectoryEntry))))
		return nullptr;
	_ZARCHIVE::FileDirectoryEntry::Deserialize(fileTree.data(), fileTree.size(), fileTree.data());
	// verify file tree
	if (fileTree[0].IsFile())
		return nullptr; // first entry must be root directory
	auto rootName = GetName(nameTable, fileTree[0].GetNameOffset());
	if (!rootName.empty())
		return nullptr; // root node must not have a name
	// read meta data
	// todo

ZArchiveReader* cfs = new ZArchiveReader(std::move(file), std::move(offsetRecords), std::move(nameTable), std::move(fileTree), footer.sectionCompressedData.offset, footer.sectionCompressedData.size);
	return cfs;
}

ZArchiveReader::ZArchiveReader(std::ifstream&& file, std::vector<_ZARCHIVE::CompressionOffsetRecord>&& offsetRecords, std::vector<uint8_t>&& nameTable, std::vector<_ZARCHIVE::FileDirectoryEntry>&& fileTree, uint64_t compressedDataOffset, uint64_t compressedDataSize) :
	m_file(std::move(file)), m_offsetRecords(std::move(offsetRecords)), m_nameTable(std::move(nameTable)), m_fileTree(std::move(fileTree)),
	m_compressedDataOffset(compressedDataOffset), m_compressedDataSize(compressedDataSize)
{
	m_blockCount = (uint64_t)m_offsetRecords.size() * _ZARCHIVE::ENTRIES_PER_OFFSETRECORD;
	m_blockDecompressionBuffer.resize(_ZARCHIVE::COMPRESSED_BLOCK_SIZE);
	// init cache
	uint64_t cacheSize = 1024 * 1024 * 4; // 4MiB
	if ((cacheSize % _ZARCHIVE::COMPRESSED_BLOCK_SIZE) != 0)
		cacheSize += (_ZARCHIVE::COMPRESSED_BLOCK_SIZE - (cacheSize % _ZARCHIVE::COMPRESSED_BLOCK_SIZE));
	m_cacheDataBuffer.resize(cacheSize);
	// create cache blocks and init LRU chain
	m_cacheBlocks.resize(cacheSize / _ZARCHIVE::COMPRESSED_BLOCK_SIZE);
	m_lruChainFirst = m_cacheBlocks.data() + 0;
	m_lruChainLast = m_cacheBlocks.data() + m_cacheBlocks.size() - 1;
	CacheBlock* prevBlock = nullptr;
	for (size_t i = 0; i < m_cacheBlocks.size(); i++)
	{
		m_cacheBlocks[i].blockIndex = 0xFFFFFFFFFFFFFFFF;
		m_cacheBlocks[i].data = m_cacheDataBuffer.data() + i * _ZARCHIVE::COMPRESSED_BLOCK_SIZE;
		m_cacheBlocks[i].prev = prevBlock;
		m_cacheBlocks[i].next = m_cacheBlocks.data() + i + 1;
		prevBlock = m_cacheBlocks.data() + i;
	}
	m_cacheBlocks.back().next = nullptr;
}

ZArchiveReader::~ZArchiveReader()
{

}

ZArchiveNodeHandle ZArchiveReader::LookUp(std::string_view path, bool allowFile, bool allowDirectory)
{
	std::string_view pathParser = path;
	uint32_t currentNode = 0;
	while (true)
	{
		std::string_view pathNodeName;
		if (!_ZARCHIVE::GetNextPathNode(pathParser, pathNodeName))
			return (ZArchiveNodeHandle)currentNode; // end of path reached
		_ZARCHIVE::FileDirectoryEntry& entry = m_fileTree.at(currentNode);
		if (entry.IsFile())
			return ZARCHIVE_INVALID_NODE; // trying to iterate a file
		// linear scan
		// todo - we could accelerate this if we use binary search
		uint32_t currentIndex = entry.directoryRecord.nodeStartIndex;
		uint32_t endIndex = entry.directoryRecord.nodeStartIndex + entry.directoryRecord.count;
		_ZARCHIVE::FileDirectoryEntry* match = nullptr;
		while (currentIndex < endIndex)
		{
			_ZARCHIVE::FileDirectoryEntry& it = m_fileTree.at(currentIndex);
			std::string_view itName = GetName(m_nameTable, it.GetNameOffset());
			if (_ZARCHIVE::CompareNodeNameBool(pathNodeName, itName))
			{
				match = &it;
				break;
			}
			currentIndex++;
			continue;
		}
		if (!match)
			return ZARCHIVE_INVALID_NODE; // path not found
		currentNode = (uint32_t)(match - m_fileTree.data());
	}
	return ZARCHIVE_INVALID_NODE;
}

bool ZArchiveReader::IsDirectory(ZArchiveNodeHandle nodeHandle) const
{
	if (nodeHandle >= m_fileTree.size())
		return false;
	return !m_fileTree[nodeHandle].IsFile();
}

bool ZArchiveReader::IsFile(ZArchiveNodeHandle nodeHandle) const
{
	if (nodeHandle >= m_fileTree.size())
		return false;
	return m_fileTree[nodeHandle].IsFile();
}

uint32_t ZArchiveReader::GetDirEntryCount(ZArchiveNodeHandle nodeHandle) const
{
	if (nodeHandle >= m_fileTree.size())
		return 0;
	auto& entry = m_fileTree.at(nodeHandle);
	if (entry.IsFile())
		return 0;
	return entry.directoryRecord.count;
}

bool ZArchiveReader::GetDirEntry(ZArchiveNodeHandle nodeHandle, uint32_t index, DirEntry& dirEntry) const
{
	if (nodeHandle >= m_fileTree.size())
		return false;
	auto& dir = m_fileTree.at(nodeHandle);
	if (dir.IsFile())
		return false;
	if (index >= dir.directoryRecord.count)
		return false;
	auto& it = m_fileTree.at(dir.directoryRecord.nodeStartIndex + index);
	dirEntry.isFile = it.IsFile();
	dirEntry.isDirectory = !dirEntry.isFile;
	if (dirEntry.isFile)
		dirEntry.size = it.GetFileSize();
	else
		dirEntry.size = 0;
	dirEntry.name = GetName(m_nameTable, it.GetNameOffset());
	if (dirEntry.name.empty())
		return false; // bad name
	return true;
}

uint64_t ZArchiveReader::GetFileSize(ZArchiveNodeHandle nodeHandle)
{
	if (nodeHandle >= m_fileTree.size())
		return 0;
	auto& file = m_fileTree.at(nodeHandle);
	if (!file.IsFile())
		return 0;
	return file.GetFileSize();
}

uint64_t ZArchiveReader::ReadFromFile(ZArchiveNodeHandle nodeHandle, uint64_t offset, uint64_t length, void* buffer)
{
	if (nodeHandle >= m_fileTree.size())
		return 0;
	std::unique_lock<std::mutex> _lock(m_accessMutex);
	auto& file = m_fileTree.at(nodeHandle);
	if (!file.IsFile())
		return 0;
	uint64_t fileOffset = file.GetFileOffset();
	uint64_t fileSize = file.GetFileSize();
	if (offset >= fileSize)
		return 0;
	uint64_t bytesToRead = std::min(length, (fileSize - offset));

uint64_t rawReadOffset = fileOffset + offset;
	uint64_t remainingBytes = bytesToRead;
	uint8_t* bufferU8 = (uint8_t*)buffer;
	while (remainingBytes > 0)
	{
		uint64_t blockIdx = rawReadOffset / _ZARCHIVE::COMPRESSED_BLOCK_SIZE;
		uint32_t blockOffset = (uint32_t)(rawReadOffset % _ZARCHIVE::COMPRESSED_BLOCK_SIZE);
		uint32_t stepSize = std::min<uint32_t>(remainingBytes, _ZARCHIVE::COMPRESSED_BLOCK_SIZE - blockOffset);
		CacheBlock* block = GetCachedBlock(blockIdx);
		if (!block)
			return 0;
		std::memcpy(bufferU8, block->data + blockOffset, stepSize);
		rawReadOffset += stepSize;
		remainingBytes -= stepSize;
		bufferU8 += stepSize;
	}
	return bytesToRead;
}

ZArchiveReader::CacheBlock* ZArchiveReader::GetCachedBlock(uint64_t blockIndex)
{
	auto it = m_blockLookup.find(blockIndex);
	if (it != m_blockLookup.end())
	{
		MarkBlockAsMRU(it->second);
		return it->second;
	}
	if (blockIndex >= m_blockCount)
		return nullptr;
	// not in cache
	CacheBlock* newBlock = RecycleLRUBlock(blockIndex);
	if (!LoadBlock(newBlock))
	{
		UnregisterBlock(newBlock);
		return nullptr;
	}
	return newBlock;
}

ZArchiveReader::CacheBlock* ZArchiveReader::RecycleLRUBlock(uint64_t newBlockIndex)
{
	CacheBlock* recycledBlock = m_lruChainFirst;
	UnregisterBlock(recycledBlock);
	RegisterBlock(recycledBlock, newBlockIndex);
	MarkBlockAsMRU(recycledBlock);
	return recycledBlock;
}

void ZArchiveReader::MarkBlockAsMRU(ZArchiveReader::CacheBlock* block)
{
	if (!block->next)
		return; // already at the end of the list (MRU)
	// remove from linked list
	if (!block->prev)
	{
		m_lruChainFirst = block->next;
		block->next->prev = nullptr;
	}
	else if (!block->next)
	{
		m_lruChainLast->next = block;
		m_lruChainLast = block;
	}
	else
	{
		block->prev->next = block->next;
		block->next->prev = block->prev;
	}
	// attach at the end
	block->prev = m_lruChainLast;
	block->next = nullptr;
	m_lruChainLast->next = block;
	m_lruChainLast = block;
}

void ZArchiveReader::RegisterBlock(CacheBlock* block, uint64_t blockIndex)
{
	block->blockIndex = blockIndex;
	m_blockLookup.emplace(blockIndex, block);
}

void ZArchiveReader::UnregisterBlock(CacheBlock* block)
{
	if (block->blockIndex != 0xFFFFFFFFFFFFFFFF)
		m_blockLookup.erase(block->blockIndex);
	block->blockIndex = 0xFFFFFFFFFFFFFFFF;
}

bool ZArchiveReader::LoadBlock(CacheBlock* block)
{
	uint32_t recordIndex = (uint32_t)(block->blockIndex / _ZARCHIVE::ENTRIES_PER_OFFSETRECORD);
	uint32_t recordSubIndex = (uint32_t)(block->blockIndex % _ZARCHIVE::ENTRIES_PER_OFFSETRECORD);
	if (recordIndex >= m_offsetRecords.size())
		return false;
	// determine offset and size of compressed block
	auto& record = m_offsetRecords[recordIndex];
	uint64_t offset = record.baseOffset;
	for (uint32_t i = 0; i < recordSubIndex; i++)
	{
		offset += (uint64_t)record.size[i];
		offset++;
	}
	uint32_t compressedSize = (uint32_t)record.size[recordSubIndex] + 1;
	// load file data
	if ((offset + compressedSize) > m_compressedDataSize)
		return false;
	offset += m_compressedDataOffset;
	if (compressedSize == _ZARCHIVE::COMPRESSED_BLOCK_SIZE)
	{
		// uncompressed block, read directly into cached block
		return _ifstream_readBytes(m_file, offset, block->data, compressedSize);
	}
	if (!_ifstream_readBytes(m_file, offset, m_blockDecompressionBuffer.data(), compressedSize))
		return false;
	// decompress
	size_t outputSize = ZSTD_decompress(block->data, _ZARCHIVE::COMPRESSED_BLOCK_SIZE, m_blockDecompressionBuffer.data(), compressedSize);
	return outputSize == _ZARCHIVE::COMPRESSED_BLOCK_SIZE;
}

// returns empty view on failure
std::string_view ZArchiveReader::GetName(const std::vector<uint8_t>& nameTable, uint32_t nameOffset)
{
	if (nameOffset == 0x7FFFFFFF || nameOffset > nameTable.size())
		return "";
	// parse header
	uint16_t nameLength = nameTable[nameOffset] & 0x7F;
	if (nameTable[nameOffset] & 0x80)
	{
		// extended 2-byte length
		if (nameOffset + 1 >= nameTable.size())
			return "";
		nameLength |= ((uint16_t)nameTable[nameOffset] << 7);
		nameOffset += 2;
	}
	else
		nameOffset++;
	// nameOffset can never exceed 0x7FFFFFFF so we don't have to worry about an overflow here
	if ((nameOffset + (uint32_t)nameLength) > nameTable.size())
		return "";
	return std::basic_string_view<char>((char*)nameTable.data() + nameOffset, nameLength);
}07070100000011000081A40000000000000000000000016319452600002C9D000000000000000000000000000000000000003A00000000ZArchive-0.1.2+git20220908.d2c7177/src/zarchivewriter.cpp#include "zarchive/zarchivewriter.h"
#include "zarchive/zarchivecommon.h"

#include <string>
#include <string_view>
#include <queue>

#include <zstd.h>

#include "sha_256.h"

#include <cassert>
#include <algorithm>

ZArchiveWriter::ZArchiveWriter(CB_NewOutputFile cbNewOutputFile, CB_WriteOutputData cbWriteOutputData, void* ctx) : m_cbCtx(ctx), m_cbNewOutputFile(cbNewOutputFile), m_cbWriteOutputData(cbWriteOutputData)
{
	cbNewOutputFile(-1, ctx);
	m_mainShaCtx = (struct Sha_256*)malloc(sizeof(struct Sha_256));
	sha_256_init(m_mainShaCtx, m_integritySha);
};

ZArchiveWriter::~ZArchiveWriter()
{
	free(m_mainShaCtx);
}

ZArchiveWriter::PathNode* ZArchiveWriter::GetNodeByPath(ZArchiveWriter::PathNode* root, std::string_view path)
{
	PathNode* currentNode = &m_rootNode;

std::string_view pathParser = path;
	while (true)
	{
		std::string_view nodeName;
		if (!_ZARCHIVE::GetNextPathNode(pathParser, nodeName))
			break;
		PathNode* nextSubnode = FindSubnodeByName(currentNode, nodeName);
		if (!nextSubnode || (nextSubnode && nextSubnode->isFile))
			return nullptr;
		currentNode = nextSubnode;
	}
	return currentNode;
}

ZArchiveWriter::PathNode* ZArchiveWriter::FindSubnodeByName(ZArchiveWriter::PathNode* parent, std::string_view nodeName)
{
	for (auto& it : parent->subnodes)
	{
		std::string_view itName = m_nodeNames[it->nameIndex];
		if (_ZARCHIVE::CompareNodeNameBool(itName, nodeName))
			return it;
	}
	return nullptr;
}

bool ZArchiveWriter::StartNewFile(const char* path)
{
	m_currentFileNode = nullptr;
	std::string_view pathParser = path;
	std::string_view filename;
	_ZARCHIVE::SplitFilenameFromPath(pathParser, filename);
	PathNode* dir = GetNodeByPath(&m_rootNode, pathParser);
	if (!dir)
		return false;
	if (FindSubnodeByName(dir, filename))
		return false;
	// add new entry and make it the currently active file for append operations
	PathNode*& r = dir->subnodes.emplace_back(new PathNode(true, CreateNameEntry(filename)));
	m_currentFileNode = r;
	r->fileOffset = m_currentInputOffset;
	return true;
}

bool ZArchiveWriter::MakeDir(const char* path, bool recursive)
{
	std::string_view pathParser = path;
	while (!pathParser.empty() && (pathParser.back() == '/' || pathParser.back() == '\\'))
		pathParser.remove_suffix(1);
	if (!recursive)
	{
		std::string_view dirName;
		_ZARCHIVE::SplitFilenameFromPath(pathParser, dirName);
		PathNode* dir = GetNodeByPath(&m_rootNode, pathParser);
		if (!dir)
			return false;
		if (FindSubnodeByName(dir, dirName))
			return false;
		dir->subnodes.emplace_back(new PathNode(false, CreateNameEntry(dirName)));
	}
	else
	{
		PathNode* currentNode = &m_rootNode;
		while (true)
		{
			std::string_view nodeName;
			if (!_ZARCHIVE::GetNextPathNode(pathParser, nodeName))
				break;
			PathNode* nextSubnode = FindSubnodeByName(currentNode, nodeName);
			if (nextSubnode && nextSubnode->isFile)
				return false;
			if (!nextSubnode)
			{
				PathNode*& r = currentNode->subnodes.emplace_back(new PathNode(false, CreateNameEntry(nodeName)));
				nextSubnode = r;
			}
			currentNode = nextSubnode;
		}
	}
	return true;
}

uint32_t ZArchiveWriter::CreateNameEntry(std::string_view name)
{
	auto it = m_nodeNameLookup.find(std::string(name));
	if (it != m_nodeNameLookup.end())
		return it->second;
	uint32_t nameIndex = (uint32_t)m_nodeNames.size();
	m_nodeNames.emplace_back(name);
	m_nodeNameLookup.emplace(name, nameIndex);
	return nameIndex;
}

void ZArchiveWriter::OutputData(const void* data, size_t length)
{
	m_cbWriteOutputData(data, length, m_cbCtx);
	m_currentCompressedWriteIndex += length;
	// hash the data
	if (m_mainShaCtx)
		sha_256_write(m_mainShaCtx, data, length);
}

uint64_t ZArchiveWriter::GetCurrentOutputOffset() const
{
	return m_currentCompressedWriteIndex;
}

void ZArchiveWriter::StoreBlock(const uint8_t* uncompressedData)
{
	// compress and store
	uint64_t compressedWriteOffset = GetCurrentOutputOffset();
	m_compressionBuffer.resize(ZSTD_compressBound(_ZARCHIVE::COMPRESSED_BLOCK_SIZE));
	size_t outputSize = ZSTD_compress(m_compressionBuffer.data(), m_compressionBuffer.size(), uncompressedData, _ZARCHIVE::COMPRESSED_BLOCK_SIZE, 6);
	assert(outputSize >= 0);
	if (outputSize >= _ZARCHIVE::COMPRESSED_BLOCK_SIZE)
	{
		// store block uncompressed if it is equal or larger than the input after compression
		outputSize = _ZARCHIVE::COMPRESSED_BLOCK_SIZE;
		OutputData(uncompressedData, _ZARCHIVE::COMPRESSED_BLOCK_SIZE);
	}
	else
	{
		OutputData(m_compressionBuffer.data(), outputSize);
	}
	// add offset translation record
	if ((m_numWrittenOffsetRecords % _ZARCHIVE::ENTRIES_PER_OFFSETRECORD) == 0)
		m_compressionOffsetRecord.emplace_back().baseOffset = compressedWriteOffset;
	m_compressionOffsetRecord.back().size[m_numWrittenOffsetRecords % _ZARCHIVE::ENTRIES_PER_OFFSETRECORD] = (uint16_t)outputSize - 1;
	m_numWrittenOffsetRecords++;
}

void ZArchiveWriter::AppendData(const void* data, size_t size)
{
	size_t dataSize = size;
	const uint8_t* input = (const uint8_t*)data;
	while (size > 0)
	{
		size_t bytesToCopy = _ZARCHIVE::COMPRESSED_BLOCK_SIZE - m_currentWriteBuffer.size();
		if (bytesToCopy > size)
			bytesToCopy = size;
		if (bytesToCopy == _ZARCHIVE::COMPRESSED_BLOCK_SIZE)
		{
			// if incoming data is block-aligned we can store it directly without memcpy to temporary buffer
			StoreBlock(input);
			input += bytesToCopy;
			size -= bytesToCopy;
			continue;
		}
		m_currentWriteBuffer.insert(m_currentWriteBuffer.end(), input, input + bytesToCopy);
		input += bytesToCopy;
		size -= bytesToCopy;
		if (m_currentWriteBuffer.size() == _ZARCHIVE::COMPRESSED_BLOCK_SIZE)
		{
			StoreBlock(m_currentWriteBuffer.data());
			m_currentWriteBuffer.clear();
		}
	}
	if (m_currentFileNode)
		m_currentFileNode->fileSize += dataSize;
	m_currentInputOffset += dataSize;
}

void ZArchiveWriter::Finalize()
{
	m_currentFileNode = nullptr; // make sure the padding added below doesn't modify the active file
	// flush write buffer by padding it to the length of a full block
	if (!m_currentWriteBuffer.empty())
	{
		std::vector<uint8_t> padBuffer;
		padBuffer.resize(_ZARCHIVE::COMPRESSED_BLOCK_SIZE - m_currentWriteBuffer.size());
		AppendData(padBuffer.data(), padBuffer.size());
	}
	m_footer.sectionCompressedData.offset = 0;
	m_footer.sectionCompressedData.size = GetCurrentOutputOffset();
	// pad to 8 byte
	while ((GetCurrentOutputOffset() % 8) != 0)
	{
		uint8_t b = 0;
		OutputData(&b, sizeof(uint8_t));
	}
	WriteOffsetRecords();
	WriteNameTable();
	WriteFileTree();
	WriteMetaData();
	WriteFooter();
}

void ZArchiveWriter::WriteOffsetRecords()
{
	m_footer.sectionOffsetRecords.offset = GetCurrentOutputOffset();
	_ZARCHIVE::CompressionOffsetRecord::Serialize(m_compressionOffsetRecord.data(), m_compressionOffsetRecord.size(), m_compressionOffsetRecord.data()); // in-place
	OutputData(m_compressionOffsetRecord.data(), m_compressionOffsetRecord.size() * sizeof(_ZARCHIVE::CompressionOffsetRecord));
	m_footer.sectionOffsetRecords.size = GetCurrentOutputOffset() - m_footer.sectionOffsetRecords.offset;
}

void ZArchiveWriter::WriteNameTable()
{
	m_footer.sectionNames.offset = GetCurrentOutputOffset();
	uint32_t currentNameTableOffset = 0;
	m_nodeNameOffsets.resize(m_nodeNames.size());
	for (size_t i = 0; i < m_nodeNames.size(); i++)
	{
		m_nodeNameOffsets[i] = currentNameTableOffset;
		// Each node name is stored with a length prefix byte. The prefix byte's MSB is used to indicate if an extended 2-byte header is used. The lower 7 bits are used to store the lower bits of the name length
		// If MSB is set, add an extra byte which extends the 7 bit name length field to 15 bit
		std::string_view name = m_nodeNames[i];
		if (name.size() > 0x7FFF)
			name = name.substr(0, 0x7FFF); // cut-off after 2^15-1 characters
		if (name.size() >= 0x80)
		{
			uint8_t header[2];
			header[0] = (uint8_t)(name.size() & 0x7F) | 0x80;
			header[1] = (uint8_t)(name.size() >> 7);
			OutputData(header, 2);
			currentNameTableOffset += 2;
		}
		else
		{
			uint8_t header[1];
			header[0] = (uint8_t)name.size() & 0x7F;
			OutputData(header, 1);
			currentNameTableOffset += 1;
		}
		OutputData(name.data(), name.size());
		currentNameTableOffset += (uint32_t)name.size();
	}
	m_footer.sectionNames.size = GetCurrentOutputOffset() - m_footer.sectionNames.offset;
}

void ZArchiveWriter::WriteFileTree()
{
	std::queue<PathNode*> nodeQueue;
	// first pass - assign a node range to all directories
	nodeQueue.push(&m_rootNode);
	uint32_t currentIndex = 1; // root node is at index 0
	while (!nodeQueue.empty())
	{
		PathNode* node = nodeQueue.front();
		nodeQueue.pop();
		if (node->isFile)
		{
			node->nodeStartIndex = (uint32_t)0xFFFFFFFF;
			continue;
		}
		// order entries lexicographically so we can use binary search in the reader
		std::sort(node->subnodes.begin(), node->subnodes.end(),
			[&](ZArchiveWriter::PathNode*& a, ZArchiveWriter::PathNode*& b) -> int
			{
				return _ZARCHIVE::CompareNodeName(m_nodeNames[a->nameIndex], m_nodeNames[b->nameIndex]) > 0;
			});

node->nodeStartIndex = currentIndex;
		currentIndex += (uint32_t)node->subnodes.size();
		for (auto& it : node->subnodes)
			nodeQueue.push(it);
	}
	// second pass - serialize to file
	m_footer.sectionFileTree.offset = GetCurrentOutputOffset();
	nodeQueue.push(&m_rootNode);
	while (!nodeQueue.empty())
	{
		PathNode* node = nodeQueue.front();
		nodeQueue.pop();

_ZARCHIVE::FileDirectoryEntry tmp;
		if(node == &m_rootNode)
			tmp.SetTypeAndNameOffset(node->isFile, 0x7FFFFFFF);
		else
			tmp.SetTypeAndNameOffset(node->isFile, m_nodeNameOffsets[node->nameIndex]);
		if (node->isFile)
		{
			tmp.SetFileOffset(node->fileOffset);
			tmp.SetFileSize(node->fileSize);
		}
		else
		{
			tmp.directoryRecord.count = (uint32_t)node->subnodes.size();
			tmp.directoryRecord.nodeStartIndex = node->nodeStartIndex;
			tmp.directoryRecord._reserved = 0;
		}
		_ZARCHIVE::FileDirectoryEntry::Serialize(&tmp, 1, &tmp);
		OutputData(&tmp, sizeof(_ZARCHIVE::FileDirectoryEntry));
		for (auto& it : node->subnodes)
			nodeQueue.push(it);
	}
	m_footer.sectionFileTree.size = GetCurrentOutputOffset() - m_footer.sectionFileTree.offset;
}

void ZArchiveWriter::WriteMetaData()
{
	// todo
	m_footer.sectionMetaDirectory.offset = GetCurrentOutputOffset();
	m_footer.sectionMetaDirectory.size = 0;
	m_footer.sectionMetaData.offset = GetCurrentOutputOffset();
	m_footer.sectionMetaData.size = 0;
}

void ZArchiveWriter::WriteFooter()
{
	m_footer.magic = _ZARCHIVE::Footer::kMagic;
	m_footer.version = _ZARCHIVE::Footer::kVersion1;
	m_footer.totalSize = GetCurrentOutputOffset() + sizeof(_ZARCHIVE::Footer);

_ZARCHIVE::Footer tmp;

// serialize and hash the footer with all hash bytes set to zero
	memset(m_footer.integrityHash, 0, 32);
	_ZARCHIVE::Footer::Serialize(&m_footer, &tmp);
	sha_256_write(m_mainShaCtx, &tmp, sizeof(_ZARCHIVE::Footer));
	sha_256_close(m_mainShaCtx);
	free(m_mainShaCtx);
	m_mainShaCtx = nullptr;

// set hash and write footer
	memcpy(m_footer.integrityHash, m_integritySha, 32);
	_ZARCHIVE::Footer::Serialize(&m_footer, &tmp);
	OutputData(&tmp, sizeof(_ZARCHIVE::Footer));
}
07070100000012000081A40000000000000000000000016319452600000119000000000000000000000000000000000000003200000000ZArchive-0.1.2+git20220908.d2c7177/zarchive.pc.inprefix=@CMAKE_INSTALL_PREFIX@
includedir=@PKGCONFIG_INCLUDEDIR@
libdir=@PKGCONFIG_LIBDIR@

Name: @PROJECT_NAME@
Description: @PROJECT_DESCRIPTION@
URL: @PROJECT_HOMEPAGE_URL@
Version: @PROJECT_VERSION@
Requires.private: libzstd
Libs: -L${libdir} -lzarchive
Cflags: -I${includedir}
07070100000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000B00000000TRAILER!!!140 blocks

Places

File ZArchive-0.1.2+git20220908.d2c7177.obscpio of Package ZArchive (Revision 16fffacd1da41e7d0e6f481e9dd3de12)

Places