Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
Please login to access the resource
home:X0F:branches:network
libtorrent-rasterbar
libtorrent-rasterbar_PR7013.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File libtorrent-rasterbar_PR7013.patch of Package libtorrent-rasterbar
From db932d9fa4efaa4e3927015f78494d29ed9d6751 Mon Sep 17 00:00:00 2001 From: arvidn <arvid@libtorrent.org> Date: Sun, 17 Jul 2022 18:15:49 -0700 Subject: [PATCH 1/7] add new multi-threaded disk I/O subsystem using preadv and pwritev --- CMakeLists.txt | 5 + ChangeLog | 1 + Jamfile | 3 + Makefile | 7 + bindings/python/src/session.cpp | 3 + examples/client_test.cpp | 7 +- include/libtorrent/aux_/debug_disk_thread.hpp | 5 + include/libtorrent/aux_/disk_buffer_pool.hpp | 3 + include/libtorrent/aux_/disk_cache.hpp | 380 ++++ .../libtorrent/aux_/disk_completed_queue.hpp | 1 + include/libtorrent/aux_/disk_job.hpp | 8 + include/libtorrent/aux_/disk_job_pool.hpp | 2 + include/libtorrent/aux_/pread_disk_job.hpp | 27 + include/libtorrent/aux_/pread_storage.hpp | 187 ++ include/libtorrent/aux_/store_buffer.hpp | 7 + include/libtorrent/aux_/unique_ptr.hpp | 7 + include/libtorrent/config.hpp | 10 + include/libtorrent/libtorrent.hpp | 1 + include/libtorrent/pread_disk_io.hpp | 28 + src/disk_buffer_pool.cpp | 11 +- src/disk_cache.cpp | 669 +++++++ src/disk_completed_queue.cpp | 27 + src/disk_job.cpp | 2 + src/disk_job_pool.cpp | 2 + src/mmap_disk_io.cpp | 1 + src/pread_disk_io.cpp | 1748 +++++++++++++++++ src/pread_storage.cpp | 797 ++++++++ src/session.cpp | 5 +- src/settings_pack.cpp | 2 +- src/torrent.cpp | 8 +- test/test_add_torrent.cpp | 3 +- test/test_copy_file.cpp | 1 + test/test_file.cpp | 2 + test/test_storage.cpp | 80 +- test/test_torrent_info.cpp | 3 +- test/web_seed_suite.cpp | 6 +- tools/disk_io_stress_test.cpp | 5 +- tools/parse_session_stats.py | 3 +- tools/run_benchmark.py | 9 +- 39 files changed, 4050 insertions(+), 26 deletions(-) create mode 100644 include/libtorrent/aux_/disk_cache.hpp create mode 100644 include/libtorrent/aux_/pread_disk_job.hpp create mode 100644 include/libtorrent/aux_/pread_storage.hpp create mode 100644 include/libtorrent/pread_disk_io.hpp create mode 100644 src/disk_cache.cpp create mode 100644 src/pread_disk_io.cpp create mode 100644 src/pread_storage.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 31768b82e43..5cb31258987 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,6 +166,7 @@ set(libtorrent_aux_include_files disable_warnings_pop.hpp disable_warnings_push.hpp disk_buffer_pool.hpp + disk_cache.hpp disk_completed_queue.hpp mmap_disk_job.hpp disk_job.hpp @@ -226,6 +227,7 @@ set(libtorrent_aux_include_files portmap.hpp posix_part_file.hpp posix_storage.hpp + pread_disk_job.hpp proxy_base.hpp proxy_settings.hpp puff.hpp @@ -327,6 +329,7 @@ set(sources disabled_disk_io.cpp disk_buffer_holder.cpp disk_buffer_pool.cpp + disk_cache.cpp disk_completed_queue.cpp disk_io_thread_pool.cpp disk_job_fence.cpp @@ -384,6 +387,8 @@ set(sources posix_disk_io.cpp posix_part_file.cpp posix_storage.cpp + pread_disk_io.cpp + pread_storage.cpp proxy_base.cpp proxy_settings.cpp puff.cpp diff --git a/ChangeLog b/ChangeLog index 15b0177afaf..e17b67f74e5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ 2.1.0 not released + * add a multi-threaded, pread()-based, disk I/O backend (pread_disk_io) * try harder to bind TCP and UDP sockets to the same port * made disk_interface's status_t type a flags type * optimize resume data format to use less space diff --git a/Jamfile b/Jamfile index f8985d16dad..4a8cc207f33 100644 --- a/Jamfile +++ b/Jamfile @@ -796,6 +796,7 @@ SOURCES = directory disk_buffer_holder disk_buffer_pool + disk_cache disk_completed_queue disk_io_thread_pool disabled_disk_io @@ -900,6 +901,8 @@ SOURCES = mmap mmap_disk_io mmap_storage + pread_disk_io + pread_storage posix_disk_io posix_part_file posix_storage diff --git a/Makefile b/Makefile index 624eafefaa1..9f9d169f54a 100644 --- a/Makefile +++ b/Makefile @@ -321,6 +321,7 @@ SOURCES = \ disabled_disk_io.cpp \ disk_buffer_holder.cpp \ disk_buffer_pool.cpp \ + disk_cache.cpp \ disk_completed_queue.cpp \ disk_io_thread_pool.cpp \ disk_job_fence.cpp \ @@ -379,6 +380,8 @@ SOURCES = \ posix_disk_io.cpp \ posix_part_file.cpp \ posix_storage.cpp \ + pread_disk_io.cpp \ + pread_storage.cpp \ proxy_base.cpp \ proxy_settings.cpp \ puff.cpp \ @@ -495,6 +498,7 @@ HEADERS = \ piece_block.hpp \ portmap.hpp \ posix_disk_io.hpp \ + pread_disk_io.hpp \ read_resume_data.hpp \ session.hpp \ session_handle.hpp \ @@ -560,6 +564,7 @@ HEADERS = \ aux_/disable_warnings_pop.hpp \ aux_/disable_warnings_push.hpp \ aux_/disk_buffer_pool.hpp \ + aux_/disk_cache.hpp \ aux_/disk_completed_queue.hpp \ aux_/disk_io_thread_pool.hpp \ aux_/disk_job_fence.hpp \ @@ -626,6 +631,8 @@ HEADERS = \ aux_/portmap.hpp \ aux_/posix_part_file.hpp \ aux_/posix_storage.hpp \ + aux_/pread_disk_job.hpp \ + aux_/pread_storage.hpp \ aux_/proxy_base.hpp \ aux_/proxy_settings.hpp \ aux_/puff.hpp \ diff --git a/bindings/python/src/session.cpp b/bindings/python/src/session.cpp index 46048393e2f..e979398a58c 100644 --- a/bindings/python/src/session.cpp +++ b/bindings/python/src/session.cpp @@ -30,6 +30,7 @@ #include <libtorrent/mmap_disk_io.hpp> #include <libtorrent/posix_disk_io.hpp> +#include <libtorrent/pread_disk_io.hpp> namespace boost { @@ -882,6 +883,8 @@ namespace #endif if (disk_io == "posix_disk_io_constructor") s.disk_io_constructor = <::posix_disk_io_constructor; + else if (disk_io == "pread_disk_io_constructor") + s.disk_io_constructor = <::pread_disk_io_constructor; else s.disk_io_constructor = <::default_disk_io_constructor; } diff --git a/examples/client_test.cpp b/examples/client_test.cpp index f93d5f07542..ec548cab97a 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -56,6 +56,7 @@ see LICENSE file. #include "libtorrent/mmap_disk_io.hpp" #include "libtorrent/posix_disk_io.hpp" +#include "libtorrent/pread_disk_io.hpp" #include "libtorrent/disabled_disk_io.hpp" #include "torrent_view.hpp" @@ -1347,7 +1348,7 @@ CLIENT OPTIONS -O print session stats counters to the log -1 exit on first torrent completing (useful for benchmarks) -i <disk-io> specify which disk I/O back-end to use. One of: - mmap, posix, disabled + mmap, posix, pread, disabled )" #ifdef TORRENT_UTP_LOG_ENABLE R"( @@ -1561,6 +1562,10 @@ int main(int argc, char* argv[]) #endif if (arg == "posix"_sv) params.disk_io_constructor = lt::posix_disk_io_constructor; +#if TORRENT_HAVE_PREAD || defined TORRENT_WINDOWS + else if (arg == "pread"_sv) + params.disk_io_constructor = lt::pread_disk_io_constructor; +#endif else if (arg == "disabled"_sv) params.disk_io_constructor = lt::disabled_disk_io_constructor; else diff --git a/include/libtorrent/aux_/debug_disk_thread.hpp b/include/libtorrent/aux_/debug_disk_thread.hpp index a1b7a5b3608..f0f9f2865ae 100644 --- a/include/libtorrent/aux_/debug_disk_thread.hpp +++ b/include/libtorrent/aux_/debug_disk_thread.hpp @@ -24,6 +24,7 @@ see LICENSE file. #include <string> #include <sstream> #include <unordered_map> +#include <thread> #include "libtorrent/aux_/disk_job.hpp" #include "libtorrent/disk_interface.hpp" @@ -93,6 +94,10 @@ inline std::string print_job(aux::disk_job const& j) << " buf-offset:" << j.buffer_offset << " size:" << j.buffer_size << " )"; } + void operator()(job::kick_hasher const& j) const { + m_ss << "kick-hasher( piece:" << j.piece << " )"; + } + private: std::stringstream& m_ss; }; diff --git a/include/libtorrent/aux_/disk_buffer_pool.hpp b/include/libtorrent/aux_/disk_buffer_pool.hpp index 99d39c44506..e07eca52a0d 100644 --- a/include/libtorrent/aux_/disk_buffer_pool.hpp +++ b/include/libtorrent/aux_/disk_buffer_pool.hpp @@ -21,6 +21,7 @@ see LICENSE file. #include <mutex> #include <functional> #include <memory> +#include <optional> #include "libtorrent/io_context.hpp" #include "libtorrent/span.hpp" @@ -54,6 +55,8 @@ namespace aux { return m_in_use; } + std::optional<int> flush_request() const; + void set_settings(settings_interface const& sett); private: diff --git a/include/libtorrent/aux_/disk_cache.hpp b/include/libtorrent/aux_/disk_cache.hpp new file mode 100644 index 00000000000..ad5a2b5d095 --- /dev/null +++ b/include/libtorrent/aux_/disk_cache.hpp @@ -0,0 +1,380 @@ +/* + +Copyright (c) 2023, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#ifndef TORRENT_DISK_CACHE +#define TORRENT_DISK_CACHE + +#include <unordered_map> +#include <mutex> + +#include "libtorrent/storage_defs.hpp" +#include "libtorrent/aux_/scope_end.hpp" +#include "libtorrent/aux_/alloca.hpp" +#include "libtorrent/aux_/invariant_check.hpp" +#include "libtorrent/aux_/pread_disk_job.hpp" +#include "libtorrent/aux_/pread_storage.hpp" +#include "libtorrent/aux_/disk_io_thread_pool.hpp" // for jobqueue_t +#include "libtorrent/aux_/unique_ptr.hpp" +#include "libtorrent/disk_buffer_holder.hpp" +#include "libtorrent/hasher.hpp" + +#include "libtorrent/aux_/disable_warnings_push.hpp" +#include <boost/functional/hash.hpp> + +#define BOOST_BIND_NO_PLACEHOLDERS + +#include <boost/multi_index_container.hpp> +#include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/hashed_index.hpp> +#include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index/mem_fun.hpp> +#include <boost/multi_index/member.hpp> + +#include "libtorrent/aux_/disable_warnings_pop.hpp" + + +namespace libtorrent::aux { + +namespace mi = boost::multi_index; + +// uniquely identifies a torrent and piece +struct piece_location +{ + piece_location(storage_index_t const t, piece_index_t const p) + : torrent(t), piece(p) {} + storage_index_t torrent; + piece_index_t piece; + bool operator==(piece_location const& rhs) const + { + return std::tie(torrent, piece) + == std::tie(rhs.torrent, rhs.piece); + } + + bool operator<(piece_location const& rhs) const + { + return std::tie(torrent, piece) + < std::tie(rhs.torrent, rhs.piece); + } +}; + +inline size_t hash_value(piece_location const& l) +{ + std::size_t ret = 0; + boost::hash_combine(ret, std::hash<storage_index_t>{}(l.torrent)); + boost::hash_combine(ret, std::hash<piece_index_t>{}(l.piece)); + return ret; +} + +struct cached_block_entry +{ + span<char const> buf() const; + + // once the write job has been executed, and we've flushed the buffer, we + // move it into buf_holder, to keep the buffer alive until any hash job has + // completed as well. The underlying data can be accessed through buf, but + // the owner moves from the pread_disk_job object to this buf_holder. + // TODO: save space by just storing the buffer pointer here. The + // cached_piece_entry could hold the pointer to the buffer pool to be able + // to free these on destruction + // we would still need to save the *size* of the block, to support the + // shorter last block of a torrent + disk_buffer_holder buf_holder; + pread_disk_job* write_job = nullptr; + + bool flushed_to_disk = false; + + // TODO: only allocate this field for v2 torrents + sha256_hash block_hash; +}; + +struct cached_piece_entry +{ + cached_piece_entry(piece_location const& loc + , int const num_blocks + , int const piece_size_v2); + + span<cached_block_entry> get_blocks() const; + + piece_location piece; + + // this is set to true when the piece has been populated with all blocks + // it will make it prioritized for flushing to disk + // it will be cleared once all blocks have been flushed + bool ready_to_flush = false; + + // when this is true, there is a thread currently hashing blocks and + // updating the hash context in "ph". + bool hashing = false; + + // when a thread is writing this piece to disk, this is true. Only one + // thread at a time should be flushing a piece to disk. + bool flushing = false; + + // this is set to true if the piece hash has been computed and returned + // to the bittorrent engine. + bool piece_hash_returned = false; + + // this indicates that this piece belongs to a v2 torrent, and it has the + // block_hash member of cached_block_entry and we need to compute the block + // hashes as well + bool v1_hashes = false; + bool v2_hashes = false; + + // if this is a v2 torrent, this is the exact size of this piece. The + // end-piece of each file may be truncated for v2 torrents + int piece_size2; + + int blocks_in_piece = 0; + + // the number of blocks that have been hashed so far. Specifically for the + // v1 SHA1 hash of the piece, so all blocks are contiguous starting at block + // 0. + int hasher_cursor = 0; + + // the number of contiguous blocks, starting at 0, that have been flushed to + // disk so far. This is used to determine how many blocks are left to flush + // from this piece without requiring read-back to hash them, by substracting + // flushed_cursor from hasher_cursor. + int flushed_cursor = 0; + + // the number of blocks that have a write job associated with them + int num_jobs = 0; + + // returns the number of blocks in this piece that have been hashed and + // ready to be flushed without requiring reading them back in the future. + int cheap_to_flush() const + { + return int(hasher_cursor) - int(flushed_cursor); + } + + unique_ptr<cached_block_entry[]> blocks; + + hasher ph; + + // if there is a hash_job set on this piece, whenever we complete hashing + // the last block, we should post this + pread_disk_job* hash_job = nullptr; + + // if the piece has been requested to be cleared, but it was locked + // (flushing) at the time. We hang this job here to complete it once the + // thread currently flushing is done with it + pread_disk_job* clear_piece = nullptr; +}; + +struct disk_cache +{ + using piece_container = mi::multi_index_container< + cached_piece_entry, + mi::indexed_by< + // look up ranges of pieces by (torrent, piece-index) + mi::ordered_unique<mi::member<cached_piece_entry, piece_location, &cached_piece_entry::piece>>, + // ordered by the number of contiguous blocks we can flush without + // read-back. large numbers are ordered first + mi::ordered_non_unique<mi::const_mem_fun<cached_piece_entry, int, &cached_piece_entry::cheap_to_flush>, std::greater<void>>, + // ordered by whether the piece is ready to be flushed or not + // true is ordered before false + mi::ordered_non_unique<mi::member<cached_piece_entry, bool, &cached_piece_entry::ready_to_flush>, std::greater<void>>, + // hash-table lookup of individual pieces. faster than index 0 + mi::hashed_unique<mi::member<cached_piece_entry, piece_location, &cached_piece_entry::piece>> + > + >; + + template <typename Fun> + bool get(piece_location const loc, int const block_idx, Fun f) const + { + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto i = view.find(loc); + if (i == view.end()) return false; + + if (i->blocks[block_idx].buf().data()) + { + // TODO: it would be nice if this could be called without holding + // the mutex. It would require being able to lock the piece + f(i->blocks[block_idx].buf()); + return true; + } + return false; + } + + template <typename Fun> + sha256_hash hash2(piece_location const loc, int const block_idx, Fun f) const + { + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto i = view.find(loc); + if (i != view.end()) + { + if (i->hashing) + { + // TODO: it would probably be more efficient to wait here. + // #error we should hang the hash job onto the piece. If there is a + // job already, form a queue + l.unlock(); + return f(); + } + auto const& cbe = i->blocks[block_idx]; + // There's nothing stopping the hash threads from hashing the blocks in + // parallel. This should not depend on the hasher_cursor. That's a v1 + // concept + if (i->hasher_cursor > block_idx) + return cbe.block_hash; + if (cbe.buf().data()) + { + hasher256 h; + h.update(cbe.buf()); + return h.final(); + } + } + l.unlock(); + return f(); + } + + // returns false if the piece is not in the cache + template <typename Fun> + bool hash_piece(piece_location const loc, Fun f) + { + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto piece_iter = view.find(loc); + if (piece_iter == view.end()) return false; + + TORRENT_ALLOCA(blocks, char const*, piece_iter->blocks_in_piece); + TORRENT_ALLOCA(v2_hashes, sha256_hash, piece_iter->blocks_in_piece); + + for (int i = 0; i < piece_iter->blocks_in_piece; ++i) + { + blocks[i] = piece_iter->blocks[i].buf().data(); + v2_hashes[i] = piece_iter->blocks[i].block_hash; + } + + view.modify(piece_iter, [](cached_piece_entry& e) { e.hashing = true; }); + int const hasher_cursor = piece_iter->hasher_cursor; + l.unlock(); + + auto se = scope_end([&] { + l.lock(); + view.modify(piece_iter, [&](cached_piece_entry& e) { + e.hashing = false; + }); + }); + f(const_cast<hasher&>(piece_iter->ph), hasher_cursor, blocks, v2_hashes); + return true; + } + + // If the specified piece exists in the cache, and it's unlocked, clear all + // write jobs (return them in "aborted"). Returns true if the clear_piece + // job should be posted as complete. Returns false if the piece is locked by + // another thread, and the clear_piece job has been queued to be issued once + // the piece is unlocked. + bool try_clear_piece(piece_location const loc, pread_disk_job* j, jobqueue_t& aborted); + + template <typename Fun> + int get2(piece_location const loc, int const block_idx, Fun f) const + { + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto i = view.find(loc); + if (i == view.end()) return 0; + + char const* buf1 = i->blocks[block_idx].buf().data(); + char const* buf2 = i->blocks[block_idx + 1].buf().data(); + + if (buf1 == nullptr && buf2 == nullptr) + return 0; + + return f(buf1, buf2); + } + + // returns true if this piece needs to have its hasher kicked + bool insert(piece_location const loc + , int const block_idx + , pread_disk_job* write_job); + + enum hash_result: std::uint8_t + { + job_completed, + job_queued, + post_job, + }; + + // this call can have 3 outcomes: + // 1. the job is immediately satisfied and should be posted to the + // completion queue + // 2. The piece is in the cache and currently hashing, but it's not done + // yet. We hang the hash job on the piece itself so the hashing thread + // can complete it when hashing finishes + // 3. The piece is not in the cache and should be posted to the disk thread + // to read back the bytes. + hash_result try_hash_piece(piece_location const loc, pread_disk_job* hash_job); + + // this should be called from a hasher thread + void kick_hasher(piece_location const& loc, jobqueue_t& completed_jobs); + + // this should be called by a disk thread + // the callback should return the number of blocks it successfully flushed + // to disk + void flush_to_disk(std::function<int(bitfield&, span<cached_block_entry const>, int)> f + , int const target_blocks + , std::function<void(jobqueue_t, pread_disk_job*)> clear_piece_fun); + + void flush_storage(std::function<int(bitfield&, span<cached_block_entry const>, int)> f + , storage_index_t const storage + , std::function<void(jobqueue_t, pread_disk_job*)> clear_piece_fun); + + std::size_t size() const; + std::size_t num_flushing() const; + +#if TORRENT_USE_INVARIANT_CHECKS + void check_invariant() const; +#endif + +private: + + // this requires the mutex to be locked + void clear_piece_impl(cached_piece_entry& cpe, jobqueue_t& aborted); + + template <typename Iter, typename View> + Iter flush_piece_impl(View& view + , Iter piece_iter + , std::function<int(bitfield&, span<cached_block_entry const>, int)> const& f + , std::unique_lock<std::mutex>& l + , int const num_blocks + , span<cached_block_entry> const blocks + , std::function<void(jobqueue_t, pread_disk_job*)> clear_piece_fun); + + mutable std::mutex m_mutex; + piece_container m_pieces; + + // the number of *dirty* blocks in the cache. i.e. blocks that need to be + // flushed to disk. The cache may (briefly) hold more buffers than this + // while finishing hashing blocks. + int m_blocks = 0; + + // the number of blocks currently being flushed by a disk thread + // we use this to avoid over-shooting flushing blocks + int m_flushing_blocks = 0; +}; + +} + +#endif + diff --git a/include/libtorrent/aux_/disk_completed_queue.hpp b/include/libtorrent/aux_/disk_completed_queue.hpp index 2a307fa6014..cf13c2138f6 100644 --- a/include/libtorrent/aux_/disk_completed_queue.hpp +++ b/include/libtorrent/aux_/disk_completed_queue.hpp @@ -26,6 +26,7 @@ struct disk_completed_queue {} void abort_job(io_context& ioc, aux::disk_job* j); + void abort_jobs(io_context& ioc, jobqueue_t jobs); void append(io_context& ioc, jobqueue_t jobs); private: diff --git a/include/libtorrent/aux_/disk_job.hpp b/include/libtorrent/aux_/disk_job.hpp index 78197185556..dc8d793dd33 100644 --- a/include/libtorrent/aux_/disk_job.hpp +++ b/include/libtorrent/aux_/disk_job.hpp @@ -44,6 +44,7 @@ namespace libtorrent::aux { , file_priority , clear_piece , partial_read + , kick_hasher , num_job_ids }; @@ -234,6 +235,12 @@ namespace job { // the piece to clear piece_index_t piece; }; + + struct kick_hasher + { + // the piece whose hasher to kick + piece_index_t piece; + }; } // disk_job is a generic base class to disk io subsystem-specifit jobs (e.g. @@ -285,6 +292,7 @@ namespace job { , job::file_priority , job::clear_piece , job::partial_read + , job::kick_hasher > action; // the type of job this is diff --git a/include/libtorrent/aux_/disk_job_pool.hpp b/include/libtorrent/aux_/disk_job_pool.hpp index a9d108a60ef..b515b7cadd8 100644 --- a/include/libtorrent/aux_/disk_job_pool.hpp +++ b/include/libtorrent/aux_/disk_job_pool.hpp @@ -79,7 +79,9 @@ namespace aux { }; struct mmap_disk_job; + struct pread_disk_job; extern template struct disk_job_pool<aux::mmap_disk_job>; + extern template struct disk_job_pool<aux::pread_disk_job>; } } diff --git a/include/libtorrent/aux_/pread_disk_job.hpp b/include/libtorrent/aux_/pread_disk_job.hpp new file mode 100644 index 00000000000..fe9896b730b --- /dev/null +++ b/include/libtorrent/aux_/pread_disk_job.hpp @@ -0,0 +1,27 @@ +/* + +Copyright (c) 2022, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#ifndef TORRENT_PREAD_DISK_JOB_HPP +#define TORRENT_PREAD_DISK_JOB_HPP + +#include "libtorrent/aux_/disk_job.hpp" + +namespace libtorrent::aux { + + struct pread_storage; + + struct TORRENT_EXTRA_EXPORT pread_disk_job : disk_job + { + // the disk storage this job applies to (if applicable) + std::shared_ptr<pread_storage> storage; + }; + +} + +#endif // TORRENT_PREAD_DISK_JOB_HPP diff --git a/include/libtorrent/aux_/pread_storage.hpp b/include/libtorrent/aux_/pread_storage.hpp new file mode 100644 index 00000000000..6c8f2b3ef01 --- /dev/null +++ b/include/libtorrent/aux_/pread_storage.hpp @@ -0,0 +1,187 @@ +/* + +Copyright (c) 2022, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#ifndef TORRENT_PREAD_STORAGE_HPP +#define TORRENT_PREAD_STORAGE_HPP + +#include "libtorrent/config.hpp" + +#include <mutex> +#include <memory> + +#include "libtorrent/fwd.hpp" +#include "libtorrent/aux_/disk_job_fence.hpp" +#include "libtorrent/storage_defs.hpp" +#include "libtorrent/aux_/part_file.hpp" +#include "libtorrent/aux_/stat_cache.hpp" +#include "libtorrent/aux_/file_pool.hpp" +#include "libtorrent/bitfield.hpp" +#include "libtorrent/span.hpp" +#include "libtorrent/aux_/vector.hpp" +#include "libtorrent/aux_/open_mode.hpp" // for aux::open_mode_t +#include "libtorrent/disk_interface.hpp" // for disk_job_flags_t + +namespace libtorrent::aux { + + struct session_settings; + struct file_view; + + struct TORRENT_EXTRA_EXPORT pread_storage + : std::enable_shared_from_this<pread_storage> + , aux::disk_job_fence + { + // constructs the pread_storage based on the given storage_params. + // ``file_pool`` is the cache of file handles that the storage will use. + // All files it opens will ask the file_pool to open them. + pread_storage(storage_params const& params, aux::file_pool&); + + // hidden + ~pread_storage(); + pread_storage(pread_storage const&) = delete; + pread_storage& operator=(pread_storage const&) = delete; + + void abort_jobs(); + + bool has_any_file(storage_error&); + void set_file_priority(settings_interface const& + , aux::vector<download_priority_t, file_index_t>& prio + , storage_error&); + void rename_file(file_index_t index, std::string const& new_filename + , storage_error&); + void release_files(storage_error&); + void delete_files(remove_flags_t options, storage_error&); + status_t initialize(settings_interface const&, storage_error&); + std::pair<status_t, std::string> move_storage(std::string save_path + , move_flags_t, storage_error&); + bool verify_resume_data(add_torrent_params const& rd + , aux::vector<std::string, file_index_t> const& links + , storage_error&); + bool tick(); + + int read(settings_interface const&, span<char> buffer + , piece_index_t piece, int offset, aux::open_mode_t mode + , disk_job_flags_t flags + , storage_error&); + int write(settings_interface const&, span<char> buffer + , piece_index_t piece, int offset, aux::open_mode_t mode + , disk_job_flags_t flags + , storage_error&); + int write(settings_interface const& sett + , span<span<char> const> buffers + , piece_index_t const piece, int offset + , open_mode_t const mode + , disk_job_flags_t const flags + , storage_error& error); + int hash(settings_interface const&, hasher& ph, std::ptrdiff_t len + , piece_index_t piece, int offset, aux::open_mode_t mode + , disk_job_flags_t flags, storage_error&); + int hash2(settings_interface const&, hasher256& ph, std::ptrdiff_t len + , piece_index_t piece, int offset, aux::open_mode_t mode + , disk_job_flags_t flags, storage_error&); + + // if the files in this storage are mapped, returns the mapped + // file_storage, otherwise returns the original file_storage object. + file_storage const& files() const { return m_mapped_files ? *m_mapped_files : m_files; } + + bool set_need_tick() + { + bool const prev = m_need_tick; + m_need_tick = true; + return prev; + } + + void do_tick() + { + m_need_tick = false; + tick(); + } + + void set_owner(std::shared_ptr<void> const& tor) { m_torrent = tor; } + + storage_index_t storage_index() const { return m_storage_index; } + void set_storage_index(storage_index_t st) { m_storage_index = st; } + + bool v1() const { return m_v1; } + bool v2() const { return m_v2; } + + private: + + bool m_need_tick = false; + file_storage const& m_files; + + // the reason for this to be a void pointer + // is to avoid creating a dependency on the + // torrent. This shared_ptr is here only + // to keep the torrent object alive until + // the storage destructs. This is because + // the file_storage object is owned by the torrent. + std::shared_ptr<void> m_torrent; + + storage_index_t m_storage_index{0}; + + void need_partfile(); + + std::unique_ptr<file_storage> m_mapped_files; + + // in order to avoid calling stat() on each file multiple times + // during startup, cache the results in here, and clear it all + // out once the torrent starts (to avoid getting stale results) + // each entry represents the size and timestamp of the file + mutable aux::stat_cache m_stat_cache; + + // helper function to open a file in the file pool with the right mode + std::shared_ptr<aux::file_handle> open_file(settings_interface const&, file_index_t + , aux::open_mode_t, storage_error&) const; + std::shared_ptr<aux::file_handle> open_file_impl(settings_interface const& + , file_index_t, aux::open_mode_t, storage_error&) const; + + bool use_partfile(file_index_t index) const; + void use_partfile(file_index_t index, bool b); + + aux::vector<download_priority_t, file_index_t> m_file_priority; + std::string m_save_path; + std::string m_part_file_name; + + // this this is an array indexed by file-index. Each slot represents + // whether this file has the part-file enabled for it. This is used for + // backwards compatibility with pre-partfile versions of libtorrent. If + // this vector is empty, the default is that files *do* use the partfile. + // on startup, any 0-priority file that's found in it's original location + // is expected to be an old-style (pre-partfile) torrent storage, and + // those files have their slot set to false in this vector. + // note that the vector is *sparse*, it's only allocated if a file has its + // entry set to false, and only indices up to that entry. + aux::vector<bool, file_index_t> m_use_partfile; + + // the file pool is a member of the disk_io_thread + // to make all storage instances share the pool + aux::file_pool& m_pool; + + // used for skipped files + std::unique_ptr<part_file> m_part_file; + + // this is a bitfield with one bit per file. A bit being set means + // we've written to that file previously. If we do write to a file + // whose bit is 0, we set the file size, to make the file allocated + // on disk (in full allocation mode) and just sparsely allocated in + // case of sparse allocation mode + mutable std::mutex m_file_created_mutex; + mutable typed_bitfield<file_index_t> m_file_created; + + bool m_allocate_files; + // this is a v1 torrent + bool m_v1; + // this is a v2 torrent. If both v1 and v2 are set, it's a hybrid + // torrent + bool m_v2; + }; + +} + +#endif // TORRENT_PREAD_STORAGE_HPP diff --git a/include/libtorrent/aux_/store_buffer.hpp b/include/libtorrent/aux_/store_buffer.hpp index 48a51448186..3f1cf0aa9dc 100644 --- a/include/libtorrent/aux_/store_buffer.hpp +++ b/include/libtorrent/aux_/store_buffer.hpp @@ -74,6 +74,13 @@ struct store_buffer auto const it = m_store_buffer.find(loc); if (it != m_store_buffer.end()) { + // TODO: it would be nice if this could be called without holding + // the mutex. It would require a reference counter on the store + // buffer entries and that we potentially erases it after this call. + // it would also require the store buffer being able to take over + // ownership of the buffer when the owner erases it. Perhase erase() + // could be made to take a buffer_holder, which is held onto if the + // refcount > 0 f(it->second); return true; } diff --git a/include/libtorrent/aux_/unique_ptr.hpp b/include/libtorrent/aux_/unique_ptr.hpp index cd490554e39..f892043bcba 100644 --- a/include/libtorrent/aux_/unique_ptr.hpp +++ b/include/libtorrent/aux_/unique_ptr.hpp @@ -31,6 +31,8 @@ namespace libtorrent { namespace aux { unique_ptr() = default; explicit unique_ptr(T* arr) : base(arr) {} + unique_ptr(base b): base(std::move(b)) {} + decltype(auto) operator[](IndexType idx) const { TORRENT_ASSERT(idx >= IndexType(0)); @@ -38,6 +40,11 @@ namespace libtorrent { namespace aux { } }; + template <typename T, typename IndexType = std::ptrdiff_t> + unique_ptr<T, IndexType> make_unique(IndexType const num) { + static_assert(std::is_array_v<T>); + return unique_ptr<T, IndexType>(new std::remove_extent_t<T>[std::size_t(num)]); + } }} #endif diff --git a/include/libtorrent/config.hpp b/include/libtorrent/config.hpp index e106e6c3e77..270bfe3abf5 100644 --- a/include/libtorrent/config.hpp +++ b/include/libtorrent/config.hpp @@ -352,6 +352,11 @@ see LICENSE file. #define TORRENT_USE_IFCONF 1 #define TORRENT_USE_GRTTABLE 1 +#ifndef TORRENT_HAVE_PREAD +#define TORRENT_HAVE_PREAD 0 +#endif + + // ==== GNU/Hurd === #elif defined __GNU__ #define TORRENT_HURD @@ -466,6 +471,11 @@ see LICENSE file. #define TORRENT_HAVE_MMAP 0 #endif +#ifndef TORRENT_HAVE_PREAD +#define TORRENT_HAVE_PREAD 1 +#endif + + #ifndef TORRENT_HAVE_MAP_VIEW_OF_FILE #define TORRENT_HAVE_MAP_VIEW_OF_FILE 0 #endif diff --git a/include/libtorrent/libtorrent.hpp b/include/libtorrent/libtorrent.hpp index 1d6a027b67a..f1b4f66dfa3 100644 --- a/include/libtorrent/libtorrent.hpp +++ b/include/libtorrent/libtorrent.hpp @@ -83,6 +83,7 @@ #include "libtorrent/piece_block.hpp" #include "libtorrent/portmap.hpp" #include "libtorrent/posix_disk_io.hpp" +#include "libtorrent/pread_disk_io.hpp" #include "libtorrent/random.hpp" #include "libtorrent/read_resume_data.hpp" #include "libtorrent/session.hpp" diff --git a/include/libtorrent/pread_disk_io.hpp b/include/libtorrent/pread_disk_io.hpp new file mode 100644 index 00000000000..b6ef36772c5 --- /dev/null +++ b/include/libtorrent/pread_disk_io.hpp @@ -0,0 +1,28 @@ +/* + +Copyright (c) 2022, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#ifndef TORRENT_PREAD_DISK_IO_HPP +#define TORRENT_PREAD_DISK_IO_HPP + +#include "libtorrent/config.hpp" +#include "libtorrent/disk_interface.hpp" +#include "libtorrent/io_context.hpp" + +namespace libtorrent { + + struct counters; + struct settings_interface; + + // constructs a multi-threaded file disk I/O using pread()/pwrite() + TORRENT_EXPORT std::unique_ptr<disk_interface> pread_disk_io_constructor( + io_context& ios, settings_interface const&, counters& cnt); + +} + +#endif // TORRENT_PREAD_DISK_IO_HPP diff --git a/src/disk_buffer_pool.cpp b/src/disk_buffer_pool.cpp index 7c7004471ad..ab7205e82a2 100644 --- a/src/disk_buffer_pool.cpp +++ b/src/disk_buffer_pool.cpp @@ -16,6 +16,7 @@ see LICENSE file. #include "libtorrent/io_context.hpp" #include "libtorrent/disk_observer.hpp" #include "libtorrent/disk_interface.hpp" // for default_block_size +#include "libtorrent/aux_/debug_disk_thread.hpp" #include "libtorrent/aux_/disable_warnings_push.hpp" @@ -178,7 +179,7 @@ namespace { int const pool_size = std::max(1, sett.get_int(settings_pack::max_queued_disk_bytes) / default_block_size); m_max_use = pool_size; - m_low_watermark = m_max_use / 2; + m_low_watermark = std::max(2, m_max_use - 32); if (m_in_use >= m_max_use && !m_exceeded_max_size) { m_exceeded_max_size = true; @@ -189,6 +190,14 @@ namespace { #endif } + std::optional<int> disk_buffer_pool::flush_request() const + { + std::unique_lock<std::mutex> l(m_pool_mutex); + if (m_in_use >= m_low_watermark) + return m_in_use - m_low_watermark; + return std::nullopt; + } + void disk_buffer_pool::remove_buffer_in_use(char* buf) { TORRENT_UNUSED(buf); diff --git a/src/disk_cache.cpp b/src/disk_cache.cpp new file mode 100644 index 00000000000..63bd6192084 --- /dev/null +++ b/src/disk_cache.cpp @@ -0,0 +1,669 @@ +/* + +Copyright (c) 2023, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#include "libtorrent/aux_/disk_cache.hpp" + +namespace libtorrent::aux { + +namespace mi = boost::multi_index; + +namespace { +struct compare_storage +{ + bool operator()(piece_location const& lhs, storage_index_t const rhs) const + { + return lhs.torrent < rhs; + } + + bool operator()(storage_index_t const lhs, piece_location const& rhs) const + { + return lhs < rhs.torrent; + } +}; + +bool have_buffers(span<const cached_block_entry> blocks) +{ + for (auto const& b : blocks) + if (b.buf().data() == nullptr) return false; + return true; +} + +bool compute_ready_to_flush(span<const cached_block_entry> blocks) +{ + bool has_job = false; + for (auto const& b : blocks) + { + has_job |= bool(b.write_job); + if (!b.write_job && !b.flushed_to_disk) return false; + } + return has_job; +} + +int compute_flushed_cursor(span<const cached_block_entry> blocks) +{ + int ret = 0; + for (auto const& b : blocks) + { + if (!b.flushed_to_disk) return ret; + ++ret; + } + return ret; +} + +#if TORRENT_USE_ASSERTS +int count_jobs(span<const cached_block_entry> blocks) +{ + return static_cast<int>(std::count_if(blocks.begin(), blocks.end() + , [](cached_block_entry const& b) { return b.write_job; })); +} +#endif + +} + +span<char const> cached_block_entry::buf() const { + if (buf_holder) + return {buf_holder.data(), buf_holder.size()}; + + if (write_job != nullptr) + { + TORRENT_ASSERT(write_job->get_type() == aux::job_action_t::write); + auto const& job = std::get<job::write>(write_job->action); + return {job.buf.data(), job.buffer_size}; + } + return {nullptr, 0}; +} + +cached_piece_entry::cached_piece_entry(piece_location const& loc, int const num_blocks, int const piece_size_v2) + : piece(loc) + , piece_size2(piece_size_v2) + , blocks_in_piece(num_blocks) + , blocks(aux::make_unique<cached_block_entry[], std::ptrdiff_t>(num_blocks)) + , ph(hasher()) +{} + +span<cached_block_entry> cached_piece_entry::get_blocks() const +{ + return {blocks.get(), blocks_in_piece}; +} + +// If the specified piece exists in the cache, and it's unlocked, clear all +// write jobs (return them in "aborted"). Returns true if the clear_piece +// job should be posted as complete. Returns false if the piece is locked by +// another thread, and the clear_piece job has been queued to be issued once +// the piece is unlocked. +bool disk_cache::try_clear_piece(piece_location const loc, pread_disk_job* j, jobqueue_t& aborted) +{ + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto i = view.find(loc); + if (i == view.end()) return true; + if (i->flushing) + { + // postpone the clearing until we're done flushing + view.modify(i, [&](cached_piece_entry& e) { e.clear_piece = j; }); + return false; + } + + // we clear a piece after it fails the hash check. It doesn't make sense + // to be hashing still + TORRENT_ASSERT(!i->hashing); + if (i->hashing) + { + // postpone the clearing until we're done flushing + view.modify(i, [&](cached_piece_entry& e) { e.clear_piece = j; }); + return false; + } + + view.modify(i, [&](cached_piece_entry& e) { + clear_piece_impl(e, aborted); + }); + return true; +} + +// returns true if this piece needs to have its hasher kicked +bool disk_cache::insert(piece_location const loc + , int const block_idx + , pread_disk_job* write_job) +{ + TORRENT_ASSERT(write_job != nullptr); + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto i = view.find(loc); + if (i == view.end()) + { + pread_storage* storage = write_job->storage.get(); + file_storage const& fs = storage->files(); + int const blocks_in_piece = (storage->files().piece_size(loc.piece) + default_block_size - 1) / default_block_size; + int const piece_size2 = fs.piece_size2(loc.piece); + cached_piece_entry pe(loc, blocks_in_piece, piece_size2); + pe.v1_hashes = storage->v1(); + pe.v2_hashes = storage->v2(); + i = m_pieces.insert(std::move(pe)).first; + } + + cached_block_entry& blk = i->blocks[block_idx]; + TORRENT_ASSERT(!blk.buf_holder); + TORRENT_ASSERT(blk.write_job == nullptr); + TORRENT_ASSERT(blk.flushed_to_disk == false); + TORRENT_ASSERT(block_idx >= i->flushed_cursor); + TORRENT_ASSERT(block_idx >= i->hasher_cursor); + + TORRENT_ASSERT(write_job->get_type() == aux::job_action_t::write); + blk.write_job = write_job; + ++m_blocks; + + bool const ready_to_flush = compute_ready_to_flush(i->get_blocks()); + view.modify(i, [&](cached_piece_entry& e) { + e.ready_to_flush = ready_to_flush; + ++e.num_jobs; + }); + + return block_idx == 0 || ready_to_flush; +} + +// this call can have 3 outcomes: +// 1. the job is immediately satisfied and should be posted to the +// completion queue +// 2. The piece is in the cache and currently hashing, but it's not done +// yet. We hang the hash job on the piece itself so the hashing thread +// can complete it when hashing finishes +// 3. The piece is not in the cache and should be posted to the disk thread +// to read back the bytes. +disk_cache::hash_result disk_cache::try_hash_piece(piece_location const loc, pread_disk_job* hash_job) +{ + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto i = view.find(loc); + if (i == view.end()) return hash_result::post_job; + + // we should only ask for the hash once + TORRENT_ASSERT(!i->piece_hash_returned); + + if (!i->hashing && i->hasher_cursor == i->blocks_in_piece) + { + view.modify(i, [&](cached_piece_entry& e) { + e.piece_hash_returned = true; + + job::hash& job = std::get<aux::job::hash>(hash_job->action); + job.piece_hash = e.ph.final(); + if (!job.block_hashes.empty()) + { + TORRENT_ASSERT(i->v2_hashes); + for (int idx = 0; idx < e.blocks_in_piece; ++idx) + job.block_hashes[idx] = e.blocks[idx].block_hash; + } + }); + return hash_result::job_completed; + } + + if (i->hashing + && i->hasher_cursor < i->blocks_in_piece + && have_buffers(i->get_blocks().subspan(i->hasher_cursor)) + ) + { + // We're not done hashing yet, let the hashing thread post the + // completion once it's done + + // We don't expect to ever have simultaneous async_hash() requests + // for the same piece + TORRENT_ASSERT(i->hash_job == nullptr); + view.modify(i, [&](cached_piece_entry& e) { e.hash_job = hash_job; }); + return hash_result::job_queued; + } + + return hash_result::post_job; +} + +// this should be called from a hasher thread +void disk_cache::kick_hasher(piece_location const& loc, jobqueue_t& completed_jobs) +{ + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& view = m_pieces.template get<0>(); + auto piece_iter = view.find(loc); + if (piece_iter == view.end()) + return; + + // some other thread beat us to it + if (piece_iter->hashing) + return; + + TORRENT_ALLOCA(blocks_storage, span<char const>, piece_iter->blocks_in_piece); + int cursor = piece_iter->hasher_cursor; +keep_going: + int block_idx = 0; + int end = cursor; + while (end < piece_iter->blocks_in_piece && piece_iter->blocks[end].buf().data()) + { + blocks_storage[block_idx] = piece_iter->blocks[end].buf(); + ++block_idx; + ++end; + } + auto const blocks = blocks_storage.first(block_idx); + + hasher& ctx = const_cast<hasher&>(piece_iter->ph); + + view.modify(piece_iter, [](cached_piece_entry& e) { e.hashing = true; }); + + bool const need_v1 = piece_iter->v1_hashes; + bool const need_v2 = piece_iter->v2_hashes; + + l.unlock(); + + int bytes_left = piece_iter->piece_size2 - (cursor * default_block_size); + for (auto& buf: blocks) + { + cached_block_entry& cbe = piece_iter->blocks[cursor]; + + if (need_v1) + ctx.update(buf); + + if (need_v2 && bytes_left > 0) + { + int const this_block_size = std::min(bytes_left, default_block_size); + cbe.block_hash = hasher256(buf.first(this_block_size)).final(); + bytes_left -= default_block_size; + } + + ++cursor; + } + + l.lock(); + for (auto& cbe : piece_iter->get_blocks().subspan(piece_iter->hasher_cursor, block_idx)) + { + // TODO: free these in bulk, acquiring the mutex just once + // free them after releasing the mutex, l + if (cbe.buf_holder) + cbe.buf_holder.reset(); + } + + view.modify(piece_iter, [&](cached_piece_entry& e) { + e.hasher_cursor = cursor; + e.hashing = false; + }); + + if (cursor != piece_iter->blocks_in_piece) + { + // if some other thread added the next block, keep going + if (piece_iter->blocks[cursor].buf().data()) + goto keep_going; + } + + if (!piece_iter->hash_job) return; + + // there's a hash job hung on this piece, post it now + pread_disk_job* j = nullptr; + span<cached_block_entry> const cached_blocks = piece_iter->get_blocks(); + view.modify(piece_iter, [&cached_blocks, &j](cached_piece_entry& e) { + j = std::exchange(e.hash_job, nullptr); + e.ready_to_flush = compute_ready_to_flush(cached_blocks); + }); + // we've hashed all blocks, and there's a hash job associated with + // this piece, post it. + sha1_hash const piece_hash = ctx.final(); + + job::hash& job = std::get<job::hash>(j->action); + job.piece_hash = piece_hash; + if (!job.block_hashes.empty()) + { + TORRENT_ASSERT(need_v2); + int const to_copy = std::min( + piece_iter->blocks_in_piece, + int(job.block_hashes.size())); + for (int i = 0; i < to_copy; ++i) + job.block_hashes[i] = piece_iter->blocks[i].block_hash; + } + completed_jobs.push_back(j); +} + +template <typename Iter, typename View> +Iter disk_cache::flush_piece_impl(View& view + , Iter piece_iter + , std::function<int(bitfield&, span<cached_block_entry const>, int)> const& f + , std::unique_lock<std::mutex>& l + , int const num_blocks + , span<cached_block_entry> const blocks + , std::function<void(jobqueue_t, pread_disk_job*)> clear_piece_fun) +{ + view.modify(piece_iter, [](cached_piece_entry& e) { TORRENT_ASSERT(!e.flushing); e.flushing = true; }); + m_flushing_blocks += num_blocks; + TORRENT_ASSERT(num_blocks > 0); + + int const hash_cursor = piece_iter->hasher_cursor; + + // we have to release the lock while flushing, but since we set the + // "flushing" member to true, this piece is pinned to the cache + l.unlock(); + + int count = 0; + bitfield flushed_blocks; + { + auto se = scope_end([&] { + l.lock(); + view.modify(piece_iter, [](cached_piece_entry& e) { + TORRENT_ASSERT(e.flushing); + e.flushing = false; + }); + TORRENT_ASSERT(m_flushing_blocks >= num_blocks); + m_flushing_blocks -= num_blocks; + }); + flushed_blocks.resize(int(blocks.size())); + flushed_blocks.clear_all(); + count = f(flushed_blocks, blocks, hash_cursor); + } + + // now that we hold the mutex again, we can update the entries for + // all the blocks that were flushed + int jobs = 0; + for (int i = 0; i < blocks.size(); ++i) + { + if (!flushed_blocks.get_bit(i)) continue; + cached_block_entry& blk = blocks[i]; + + auto* j = blk.write_job; + TORRENT_ASSERT(j); + TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); + blk.buf_holder = std::move(std::get<aux::job::write>(j->action).buf); + blk.flushed_to_disk = true; + TORRENT_ASSERT(blk.buf_holder); + // TODO: free these in bulk at the end, or something + if (i < hash_cursor) + blk.buf_holder.reset(); + + blk.write_job = nullptr; + ++jobs; + } + auto next_iter = std::next(piece_iter); + view.modify(piece_iter, [&blocks, jobs](cached_piece_entry& e) { + e.flushed_cursor = compute_flushed_cursor(blocks); + e.ready_to_flush = compute_ready_to_flush(blocks); + TORRENT_ASSERT(e.num_jobs >= jobs); + e.num_jobs -= jobs; + }); + TORRENT_ASSERT(count <= blocks.size()); + TORRENT_ASSERT(m_blocks >= count); + m_blocks -= count; + if (piece_iter->clear_piece) + { + jobqueue_t aborted; + pread_disk_job* clear_piece = nullptr; + view.modify(piece_iter, [&](cached_piece_entry& e) { + clear_piece_impl(e, aborted); + clear_piece = std::exchange(e.clear_piece, nullptr); + }); + clear_piece_fun(std::move(aborted), clear_piece); + } + + return next_iter; +} + +// this should be called by a disk thread +// the callback should return the number of blocks it successfully flushed +// to disk +void disk_cache::flush_to_disk( + std::function<int(bitfield&, span<cached_block_entry const>, int)> f + , int const target_blocks + , std::function<void(jobqueue_t, pread_disk_job*)> clear_piece_fun) +{ + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + // first we look for pieces that are ready to be flushed and should be + // updating + auto& view = m_pieces.template get<2>(); + for (auto piece_iter = view.begin(); piece_iter != view.end();) + { + // We want to flush all pieces that are ready to flush regardless of + // the flush target. There's not much value in keeping them in RAM + // when we've completely downloaded the piece and hashed it + // so, we don't check flush target in this loop + + if (piece_iter->flushing) + { + ++piece_iter; + continue; + } + + if (!piece_iter->ready_to_flush) + break; + + int const num_blocks = piece_iter->blocks_in_piece; + TORRENT_ASSERT(num_blocks >= 0); + if (num_blocks == 0) + { + ++piece_iter; + continue; + } + span<cached_block_entry> const blocks = piece_iter->get_blocks(); + + auto const next_iter = flush_piece_impl(view, piece_iter, f, l + , num_blocks, blocks, clear_piece_fun); + + if (piece_iter->piece_hash_returned) + { + TORRENT_ASSERT(!piece_iter->flushing); + TORRENT_ASSERT(!piece_iter->hashing); + view.erase(piece_iter); + } + piece_iter = next_iter; + } + + // if we get here, we have to "force flush" some blocks even though we + // don't have all the blocks yet. Start by flushing pieces that have the + // most contiguous blocks to flush: + auto& view2 = m_pieces.template get<1>(); + for (auto piece_iter = view2.begin(); piece_iter != view2.end();) + { + // We avoid flushing if other threads have already initiated sufficient + // amount of flushing + if (m_blocks - m_flushing_blocks <= target_blocks) + return; + + if (piece_iter->flushing) + { + ++piece_iter; + continue; + } + + int const num_blocks = piece_iter->hasher_cursor - piece_iter->flushed_cursor; + TORRENT_ASSERT(num_blocks >= 0); + + // the pieces are ordered by the number of blocks that are cheap to + // flush (i.e. won't require read-back later) + // if we encounter a 0, all the remaining ones will also be zero + if (num_blocks <= 0) break; + span<cached_block_entry> const blocks = piece_iter->get_blocks().subspan(piece_iter->flushed_cursor); + + piece_iter = flush_piece_impl(view2, piece_iter, f, l + , num_blocks, blocks, clear_piece_fun); + } + + // we may still need to flush blocks at this point, even though we + // would require read-back later to compute the piece hash + auto& view3 = m_pieces.template get<0>(); + for (auto piece_iter = view3.begin(); piece_iter != view3.end();) + { + // We avoid flushing if other threads have already initiated sufficient + // amount of flushing + if (m_blocks - m_flushing_blocks <= target_blocks) + return; + + if (piece_iter->flushing) + { + ++piece_iter; + continue; + } + + int const num_blocks = piece_iter->num_jobs; + TORRENT_ASSERT(count_jobs(piece_iter->get_blocks()) == num_blocks); + if (num_blocks == 0) + { + ++piece_iter; + continue; + } + + span<cached_block_entry> const blocks = piece_iter->get_blocks(); + + piece_iter = flush_piece_impl(view3, piece_iter, f, l + , num_blocks, blocks, clear_piece_fun); + } +} + +void disk_cache::flush_storage(std::function<int(bitfield&, span<cached_block_entry const>, int)> f + , storage_index_t const storage + , std::function<void(jobqueue_t, pread_disk_job*)> clear_piece_fun) +{ + std::unique_lock<std::mutex> l(m_mutex); + + INVARIANT_CHECK; + + auto& range_view = m_pieces.template get<0>(); + auto& view = m_pieces.template get<3>(); + auto const [begin, end] = range_view.equal_range(storage, compare_storage()); + + std::vector<piece_index_t> pieces; + for (auto i = begin; i != end; ++i) + pieces.push_back(i->piece.piece); + + bitfield flushed_blocks; + + for (auto piece : pieces) + { + auto piece_iter = view.find(piece_location{storage, piece}); + if (piece_iter == view.end()) + continue; + + // There's a risk that some other thread is flushing this piece, but + // won't force-flush it completely. In that case parts of the piece + // may not be flushed + // TODO: maybe we should track these pieces and synchronize with + // them later. maybe wait for them to be flushed or hang our job on + // them, but that would really only work if there's only one piece + // left + if (piece_iter->flushing) + continue; + + int const num_blocks = piece_iter->num_jobs; + TORRENT_ASSERT(count_jobs(piece_iter->get_blocks()) == num_blocks); + if (num_blocks == 0) continue; + span<cached_block_entry> const blocks = piece_iter->get_blocks(); + + flush_piece_impl(view, piece_iter, f, l + , num_blocks, blocks, clear_piece_fun); + + TORRENT_ASSERT(!piece_iter->flushing); + TORRENT_ASSERT(!piece_iter->hashing); + piece_iter = view.erase(piece_iter); + } +} + +std::size_t disk_cache::size() const +{ + std::unique_lock<std::mutex> l(m_mutex); + INVARIANT_CHECK; + return static_cast<std::size_t>(m_blocks); +} + +std::size_t disk_cache::num_flushing() const +{ + std::unique_lock<std::mutex> l(m_mutex); + INVARIANT_CHECK; + return static_cast<std::size_t>(m_flushing_blocks); +} + +#if TORRENT_USE_INVARIANT_CHECKS +void disk_cache::check_invariant() const +{ + // mutex must be held by caller + int dirty_blocks = 0; + int flushing_blocks = 0; + + auto& view = m_pieces.template get<2>(); + for (auto const& piece_entry : view) + { + int const num_blocks = piece_entry.blocks_in_piece; + + if (piece_entry.flushing) + flushing_blocks += num_blocks; + + span<cached_block_entry> const blocks = piece_entry.get_blocks(); + + TORRENT_ASSERT(piece_entry.flushed_cursor <= num_blocks); + TORRENT_ASSERT(piece_entry.hasher_cursor <= num_blocks); + + int idx = 0; + for (auto& be : blocks) + { + if (be.write_job) ++dirty_blocks; + // a block holds either a write job or buffer, never both + TORRENT_ASSERT(!(bool(be.write_job) && bool(be.buf_holder))); + if (be.write_job) + TORRENT_ASSERT(be.write_job->get_type() == aux::job_action_t::write); + + if (idx < piece_entry.flushed_cursor) + TORRENT_ASSERT(be.write_job == nullptr); + else if (idx == piece_entry.flushed_cursor) + TORRENT_ASSERT(!be.buf_holder); + +// if (idx < piece_entry.hasher_cursor) +// TORRENT_ASSERT(!be.buf_holder); + + if (piece_entry.ready_to_flush) + TORRENT_ASSERT(be.write_job != nullptr || be.flushed_to_disk); + ++idx; + } + } + // if one or more blocks are being flushed, we cannot know how many blocks + // are in flight. We just know the limit + TORRENT_ASSERT(dirty_blocks == m_blocks); + TORRENT_ASSERT(m_flushing_blocks <= flushing_blocks); +} +#endif + +// this requires the mutex to be locked +void disk_cache::clear_piece_impl(cached_piece_entry& cpe, jobqueue_t& aborted) +{ + TORRENT_ASSERT(!cpe.flushing); + TORRENT_ASSERT(!cpe.hashing); + int jobs = 0; + for (int idx = 0; idx < cpe.blocks_in_piece; ++idx) + { + auto& cbe = cpe.blocks[idx]; + if (cbe.write_job) + { + aborted.push_back(cbe.write_job); + cbe.write_job = nullptr; + cbe.flushed_to_disk = false; + ++jobs; + --m_blocks; + } + cbe.buf_holder.reset(); + } + cpe.ready_to_flush = false; + cpe.piece_hash_returned = false; + cpe.hasher_cursor = 0; + cpe.flushed_cursor = 0; + TORRENT_ASSERT(cpe.num_jobs >= jobs); + cpe.num_jobs -= jobs; + cpe.ph = hasher{}; +} + +} diff --git a/src/disk_completed_queue.cpp b/src/disk_completed_queue.cpp index 18429b1cb16..552a3415f55 100644 --- a/src/disk_completed_queue.cpp +++ b/src/disk_completed_queue.cpp @@ -42,6 +42,33 @@ void disk_completed_queue::abort_job(io_context& ioc, aux::disk_job* j) } } +void disk_completed_queue::abort_jobs(io_context& ioc, jobqueue_t jobs) +{ + if (jobs.empty()) return; + + for (auto i = jobs.iterate(); i.get(); i.next()) + { + auto* j = i.get(); + j->ret = disk_status::fatal_disk_error; + j->error = storage_error(boost::asio::error::operation_aborted); + j->flags |= aux::disk_job::aborted; +#if TORRENT_USE_ASSERTS + TORRENT_ASSERT(j->job_posted == false); + j->job_posted = true; +#endif + } + std::lock_guard<std::mutex> l(m_completed_jobs_mutex); + m_completed_jobs.append(std::move(jobs)); + + if (!m_job_completions_in_flight) + { + DLOG("posting job handlers (%d)\n", m_completed_jobs.size()); + + post(ioc, [this] { this->call_job_handlers(); }); + m_job_completions_in_flight = true; + } +} + void disk_completed_queue::append(io_context& ioc, jobqueue_t jobs) { std::lock_guard<std::mutex> l(m_completed_jobs_mutex); diff --git a/src/disk_job.cpp b/src/disk_job.cpp index 6d2a01d25ac..7cfedf1596f 100644 --- a/src/disk_job.cpp +++ b/src/disk_job.cpp @@ -97,6 +97,8 @@ namespace { j.handler(std::move(j.buf), m_job.error); } + void operator()(job::kick_hasher&) const {} + private: disk_job& m_job; }; diff --git a/src/disk_job_pool.cpp b/src/disk_job_pool.cpp index 11f0571dbae..e08abb0cb73 100644 --- a/src/disk_job_pool.cpp +++ b/src/disk_job_pool.cpp @@ -10,6 +10,7 @@ see LICENSE file. #include "libtorrent/aux_/disk_job_pool.hpp" #include "libtorrent/aux_/mmap_disk_job.hpp" +#include "libtorrent/aux_/pread_disk_job.hpp" namespace libtorrent { namespace aux { @@ -69,5 +70,6 @@ namespace aux { } template struct disk_job_pool<aux::mmap_disk_job>; + template struct disk_job_pool<aux::pread_disk_job>; } } diff --git a/src/mmap_disk_io.cpp b/src/mmap_disk_io.cpp index dea6bf67ae8..ba6cfef722b 100644 --- a/src/mmap_disk_io.cpp +++ b/src/mmap_disk_io.cpp @@ -136,6 +136,7 @@ struct TORRENT_EXTRA_EXPORT mmap_disk_io final // this submits all queued up jobs to the thread void submit_jobs() override; + status_t do_job(aux::job::kick_hasher&, aux::mmap_disk_job*) { return status_t{}; } status_t do_job(aux::job::partial_read& a, aux::mmap_disk_job* j); status_t do_job(aux::job::read& a, aux::mmap_disk_job* j); status_t do_job(aux::job::write& a, aux::mmap_disk_job* j); diff --git a/src/pread_disk_io.cpp b/src/pread_disk_io.cpp new file mode 100644 index 00000000000..63d4c58f3dd --- /dev/null +++ b/src/pread_disk_io.cpp @@ -0,0 +1,1748 @@ +/* + +Copyright (c) 2022, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#include "libtorrent/config.hpp" + +#include "libtorrent/aux_/pread_storage.hpp" +#include "libtorrent/pread_disk_io.hpp" +#include "libtorrent/disk_buffer_holder.hpp" +#include "libtorrent/aux_/throw.hpp" +#include "libtorrent/error_code.hpp" +#include "libtorrent/error.hpp" +#include "libtorrent/aux_/disk_buffer_pool.hpp" +#include "libtorrent/aux_/pread_disk_job.hpp" +#include "libtorrent/performance_counters.hpp" +#include "libtorrent/aux_/debug.hpp" +#include "libtorrent/units.hpp" +#include "libtorrent/hasher.hpp" +#include "libtorrent/aux_/platform_util.hpp" // for set_thread_name +#include "libtorrent/aux_/disk_job_pool.hpp" +#include "libtorrent/aux_/disk_io_thread_pool.hpp" +#include "libtorrent/aux_/disk_cache.hpp" +#include "libtorrent/aux_/time.hpp" +#include "libtorrent/add_torrent_params.hpp" +#include "libtorrent/aux_/numeric_cast.hpp" +#include "libtorrent/settings_pack.hpp" +#include "libtorrent/aux_/storage_array.hpp" +#include "libtorrent/aux_/disk_completed_queue.hpp" +#include "libtorrent/aux_/debug_disk_thread.hpp" + +#include <functional> + +namespace libtorrent { +namespace { + +aux::open_mode_t file_mode_for_job(aux::pread_disk_job* j) +{ + aux::open_mode_t ret = aux::open_mode::read_only; + if (!(j->flags & disk_interface::sequential_access)) ret |= aux::open_mode::random_access; + return ret; +} + +#if TORRENT_USE_ASSERTS +bool valid_flags(disk_job_flags_t const flags) +{ + return (flags & ~(disk_interface::force_copy + | disk_interface::sequential_access + | disk_interface::volatile_read + | disk_interface::v1_hash + | disk_interface::flush_piece)) + == disk_job_flags_t{}; +} +#endif + +template <typename Fun> +void translate_error(aux::disk_job* j, Fun f) +{ + try + { + j->ret = f(); + } + catch (boost::system::system_error const& err) + { + j->ret = disk_status::fatal_disk_error; + j->error.ec = err.code(); + j->error.operation = operation_t::exception; + } + catch (std::bad_alloc const&) + { + j->ret = disk_status::fatal_disk_error; + j->error.ec = errors::no_memory; + j->error.operation = operation_t::exception; + } + catch (std::exception const&) + { + j->ret = disk_status::fatal_disk_error; + j->error.ec = boost::asio::error::fault; + j->error.operation = operation_t::exception; + } +} + +} // anonymous namespace + +// this is a singleton consisting of the thread and a queue +// of disk io jobs +struct TORRENT_EXTRA_EXPORT pread_disk_io final + : disk_interface +{ + pread_disk_io(io_context& ios, settings_interface const&, counters& cnt); +#if TORRENT_USE_ASSERTS + ~pread_disk_io() override; +#endif + + void settings_updated() override; + storage_holder new_torrent(storage_params const& params + , std::shared_ptr<void> const& owner) override; + void remove_torrent(storage_index_t) override; + + void abort(bool wait) override; + + void async_read(storage_index_t storage, peer_request const& r + , std::function<void(disk_buffer_holder, storage_error const&)> handler + , disk_job_flags_t flags = {}) override; + bool async_write(storage_index_t storage, peer_request const& r + , char const* buf, std::shared_ptr<disk_observer> o + , std::function<void(storage_error const&)> handler + , disk_job_flags_t flags = {}) override; + void async_hash(storage_index_t storage, piece_index_t piece, span<sha256_hash> v2 + , disk_job_flags_t flags + , std::function<void(piece_index_t, sha1_hash const&, storage_error const&)> handler) override; + void async_hash2(storage_index_t storage, piece_index_t piece, int offset, disk_job_flags_t flags + , std::function<void(piece_index_t, sha256_hash const&, storage_error const&)> handler) override; + void async_move_storage(storage_index_t storage, std::string p, move_flags_t flags + , std::function<void(status_t, std::string const&, storage_error const&)> handler) override; + void async_release_files(storage_index_t storage + , std::function<void()> handler = std::function<void()>()) override; + void async_delete_files(storage_index_t storage, remove_flags_t options + , std::function<void(storage_error const&)> handler) override; + void async_check_files(storage_index_t storage + , add_torrent_params const* resume_data + , aux::vector<std::string, file_index_t> links + , std::function<void(status_t, storage_error const&)> handler) override; + void async_rename_file(storage_index_t storage, file_index_t index, std::string name + , std::function<void(std::string const&, file_index_t, storage_error const&)> handler) override; + void async_stop_torrent(storage_index_t storage + , std::function<void()> handler) override; + void async_set_file_priority(storage_index_t storage + , aux::vector<download_priority_t, file_index_t> prio + , std::function<void(storage_error const& + , aux::vector<download_priority_t, file_index_t>)> handler) override; + + void async_clear_piece(storage_index_t storage, piece_index_t index + , std::function<void(piece_index_t)> handler) override; + + void update_stats_counters(counters& c) const override; + + std::vector<open_file_state> get_status(storage_index_t) const override; + + // this submits all queued up jobs to the thread + void submit_jobs() override; + + status_t do_job(aux::job::partial_read& a, aux::pread_disk_job* j); + status_t do_job(aux::job::read& a, aux::pread_disk_job* j); + status_t do_job(aux::job::write& a, aux::pread_disk_job* j); + status_t do_job(aux::job::hash& a, aux::pread_disk_job* j); + status_t do_job(aux::job::hash2& a, aux::pread_disk_job* j); + + status_t do_job(aux::job::move_storage& a, aux::pread_disk_job* j); + status_t do_job(aux::job::release_files& a, aux::pread_disk_job* j); + status_t do_job(aux::job::delete_files& a, aux::pread_disk_job* j); + status_t do_job(aux::job::check_fastresume& a, aux::pread_disk_job* j); + status_t do_job(aux::job::rename_file& a, aux::pread_disk_job* j); + status_t do_job(aux::job::stop_torrent& a, aux::pread_disk_job* j); + status_t do_job(aux::job::file_priority& a, aux::pread_disk_job* j); + status_t do_job(aux::job::clear_piece& a, aux::pread_disk_job* j); + status_t do_job(aux::job::kick_hasher& a, aux::pread_disk_job* j); + +private: + + void thread_fun(aux::disk_io_thread_pool& pool + , executor_work_guard<io_context::executor_type> work); + + void add_completed_jobs(jobqueue_t jobs); + void add_completed_jobs_impl(jobqueue_t jobs, jobqueue_t& completed); + + void perform_job(aux::pread_disk_job* j, jobqueue_t& completed_jobs); + + // this queues up another job to be submitted + void add_job(aux::pread_disk_job* j, bool user_add = true); + void add_fence_job(aux::pread_disk_job* j, bool user_add = true); + + void execute_job(aux::pread_disk_job* j); + void immediate_execute(); + void abort_jobs(); + void abort_hash_jobs(storage_index_t storage); + + void try_flush_cache(int target_cache_size + , std::unique_lock<std::mutex>& l); + void flush_storage(std::shared_ptr<aux::pread_storage> const& storage); + + int flush_cache_blocks(bitfield& flushed, span<aux::cached_block_entry const> blocks + , int hash_cursor + , jobqueue_t& completed_jobs); + void clear_piece_jobs(jobqueue_t aborted, aux::pread_disk_job* clear); + + // returns the maximum number of threads + // the actual number of threads may be less + int num_threads() const; + aux::disk_io_thread_pool& pool_for_job(aux::pread_disk_job* j); + + // set to true once we start shutting down + std::atomic<bool> m_abort{false}; + + // this is a counter of how many threads are currently running. + // it's used to identify the last thread still running while + // shutting down. This last thread is responsible for cleanup + // must hold the job mutex to access + int m_num_running_threads = 0; + + aux::disk_job_pool<aux::pread_disk_job> m_job_pool; + + // std::mutex to protect the m_generic_threads and m_hash_threads lists + mutable std::mutex m_job_mutex; + + // when set, it means we're trying to flush the disk cache down to this size + // it's a signal to generic disk threads to start flushing. Once flushing + // starts, m_flush_target is cleared. + std::optional<int> m_flush_target = std::nullopt; + + settings_interface const& m_settings; + + // LRU cache of open files + aux::file_pool m_file_pool; + + // disk cache + aux::disk_buffer_pool m_buffer_pool; + + // total number of blocks in use by both the read + // and the write cache. This is not supposed to + // exceed m_cache_size + + counters& m_stats_counters; + + // this is the main thread io_context. Callbacks are + // posted on this in order to have them execute in + // the main thread. + io_context& m_ios; + + aux::disk_completed_queue m_completed_jobs; + + // storages that have had write activity recently and will get ticked + // soon, for deferred actions (say, flushing partfile metadata) + std::vector<std::pair<time_point, std::weak_ptr<aux::pread_storage>>> m_need_tick; + std::mutex m_need_tick_mutex; + + aux::storage_array<aux::pread_storage> m_torrents; + + std::atomic_flag m_jobs_aborted = ATOMIC_FLAG_INIT; + + // every write job is inserted into this map while it is in the job queue. + // It is removed after the write completes. This will let subsequent reads + // pull the buffers straight out of the queue instead of having to + // synchronize with the writing thread(s) + aux::disk_cache m_cache; + + // most jobs are posted to m_generic_io_jobs + // but hash jobs are posted to m_hash_io_jobs if m_hash_threads + // has a non-zero maximum thread count + aux::disk_io_thread_pool m_generic_threads; + aux::disk_io_thread_pool m_hash_threads; +}; + +TORRENT_EXPORT std::unique_ptr<disk_interface> pread_disk_io_constructor( + io_context& ios, settings_interface const& sett, counters& cnt) +{ + return std::make_unique<pread_disk_io>(ios, sett, cnt); +} + +// ------- pread_disk_io ------ + +// for _1 and _2 +using namespace std::placeholders; + +pread_disk_io::pread_disk_io(io_context& ios, settings_interface const& sett, counters& cnt) + : m_settings(sett) + , m_file_pool(sett.get_int(settings_pack::file_pool_size)) + , m_buffer_pool(ios) + , m_stats_counters(cnt) + , m_ios(ios) + , m_completed_jobs([&](aux::disk_job** j, int const n) { + m_job_pool.free_jobs(reinterpret_cast<aux::pread_disk_job**>(j), n); + }, cnt) + , m_generic_threads(std::bind(&pread_disk_io::thread_fun, this, _1, _2), ios) + , m_hash_threads(std::bind(&pread_disk_io::thread_fun, this, _1, _2), ios) +{ + settings_updated(); +} + +std::vector<open_file_state> pread_disk_io::get_status(storage_index_t const st) const +{ + return m_file_pool.get_status(st); +} + +storage_holder pread_disk_io::new_torrent(storage_params const& params + , std::shared_ptr<void> const& owner) +{ + TORRENT_ASSERT(params.files.is_valid()); + + auto storage = std::make_shared<aux::pread_storage>(params, m_file_pool); + storage->set_owner(owner); + storage_index_t const idx = m_torrents.add(std::move(storage)); + return storage_holder(idx, *this); +} + +void pread_disk_io::remove_torrent(storage_index_t const idx) +{ + m_torrents.remove(idx); +} + +#if TORRENT_USE_ASSERTS +pread_disk_io::~pread_disk_io() +{ + DLOG("destructing pread_disk_io\n"); + + // abort should have been triggered + TORRENT_ASSERT(m_abort); + + // there are not supposed to be any writes in-flight by now + TORRENT_ASSERT(m_cache.size() == 0); + + // all torrents are supposed to have been removed by now + TORRENT_ASSERT(m_torrents.empty()); +} +#endif + +void pread_disk_io::abort(bool const wait) +{ + DLOG("pread_disk_io::abort: (wait: %d)\n", int(wait)); + + // first make sure queued jobs have been submitted + // otherwise the queue may not get processed + submit_jobs(); + + // abuse the job mutex to make setting m_abort and checking the thread count atomic + // see also the comment in thread_fun + std::unique_lock<std::mutex> l(m_job_mutex); + if (m_abort.exchange(true)) return; + bool const no_threads = m_generic_threads.num_threads() == 0 + && m_hash_threads.num_threads() == 0; + // abort outstanding jobs belonging to this torrent + + DLOG("aborting hash jobs\n"); + m_hash_threads.visit_jobs([](aux::disk_job* j) + { + j->flags |= aux::disk_job::aborted; + }); + l.unlock(); + + // if there are no disk threads, we can't wait for the jobs here, because + // we'd stall indefinitely + if (no_threads) + { + abort_jobs(); + } + + DLOG("aborting thread pools\n"); + // even if there are no threads it doesn't hurt to abort the pools + // it prevents threads from being started after an abort which is a good + // defensive programming measure + m_generic_threads.abort(wait); + m_hash_threads.abort(wait); +} + +void pread_disk_io::settings_updated() +{ + m_buffer_pool.set_settings(m_settings); + m_file_pool.resize(m_settings.get_int(settings_pack::file_pool_size)); + + int const num_threads = m_settings.get_int(settings_pack::aio_threads); + int const num_hash_threads = m_settings.get_int(settings_pack::hashing_threads); + DLOG("set max threads(%d, %d)\n", num_threads, num_hash_threads); + + m_generic_threads.set_max_threads(num_threads); + m_hash_threads.set_max_threads(num_hash_threads); +} + +void pread_disk_io::perform_job(aux::pread_disk_job* j, jobqueue_t& completed_jobs) +{ + TORRENT_ASSERT(j->next == nullptr); + +#if DEBUG_DISK_THREAD + { + std::unique_lock<std::mutex> l(m_job_mutex); + + DLOG("perform_job job: %s outstanding: %d\n" + , print_job(*j).c_str() + , j->storage ? j->storage->num_outstanding_jobs() : -1); + } +#endif + + std::shared_ptr<aux::pread_storage> storage = j->storage; + + m_stats_counters.inc_stats_counter(counters::num_running_disk_jobs, 1); + + // call disk function + // TODO: in the future, propagate exceptions back to the handlers + translate_error(j, [&] { + return std::visit([this, j](auto& a) { return this->do_job(a, j); }, j->action); + }); + + // note that -2 errors are OK + TORRENT_ASSERT(j->ret != disk_status::fatal_disk_error + || (j->error.ec && j->error.operation != operation_t::unknown)); + + m_stats_counters.inc_stats_counter(counters::num_running_disk_jobs, -1); + + completed_jobs.push_back(j); +} + +status_t pread_disk_io::do_job(aux::job::partial_read& a, aux::pread_disk_job* j) +{ + TORRENT_ASSERT(a.buf); + time_point const start_time = clock_type::now(); + + span<char> const b = {a.buf.data() + a.buffer_offset, a.buffer_size}; + + int const ret = j->storage->read(m_settings, b + , a.piece, a.offset, file_mode_for_job(j), j->flags, j->error); + + TORRENT_ASSERT(ret >= 0 || j->error.ec); + TORRENT_UNUSED(ret); + + if (!j->error.ec) + { + std::int64_t const read_time = total_microseconds(clock_type::now() - start_time); + + m_stats_counters.inc_stats_counter(counters::num_read_back); + m_stats_counters.inc_stats_counter(counters::num_blocks_read); + m_stats_counters.inc_stats_counter(counters::num_read_ops); + m_stats_counters.inc_stats_counter(counters::disk_read_time, read_time); + m_stats_counters.inc_stats_counter(counters::disk_job_time, read_time); + } + + TORRENT_ASSERT((j->flags & aux::disk_job::in_progress) || !j->storage); + return status_t{}; +} + +status_t pread_disk_io::do_job(aux::job::read& a, aux::pread_disk_job* j) +{ + a.buf = disk_buffer_holder(m_buffer_pool, m_buffer_pool.allocate_buffer("send buffer"), default_block_size); + if (!a.buf) + { + j->error.ec = error::no_memory; + j->error.operation = operation_t::alloc_cache_piece; + return disk_status::fatal_disk_error; + } + + time_point const start_time = clock_type::now(); + + aux::open_mode_t const file_mode = file_mode_for_job(j); + span<char> const b = {a.buf.data(), a.buffer_size}; + + int const ret = j->storage->read(m_settings, b + , a.piece, a.offset, file_mode, j->flags, j->error); + + TORRENT_ASSERT(ret >= 0 || j->error.ec); + TORRENT_UNUSED(ret); + + if (!j->error.ec) + { + std::int64_t const read_time = total_microseconds(clock_type::now() - start_time); + + m_stats_counters.inc_stats_counter(counters::num_read_back); + m_stats_counters.inc_stats_counter(counters::num_blocks_read); + m_stats_counters.inc_stats_counter(counters::num_read_ops); + m_stats_counters.inc_stats_counter(counters::disk_read_time, read_time); + m_stats_counters.inc_stats_counter(counters::disk_job_time, read_time); + } + TORRENT_ASSERT((j->flags & aux::disk_job::in_progress) || !j->storage); + return status_t{}; +} + +status_t pread_disk_io::do_job(aux::job::write&, aux::pread_disk_job*) +{ + TORRENT_ASSERT_FAIL(); + return status_t{}; +} + +void pread_disk_io::async_read(storage_index_t storage, peer_request const& r + , std::function<void(disk_buffer_holder, storage_error const&)> handler + , disk_job_flags_t const flags) +{ + TORRENT_ASSERT(valid_flags(flags)); + TORRENT_ASSERT(r.length <= default_block_size); + TORRENT_ASSERT(r.length > 0); + TORRENT_ASSERT(r.start >= 0); + + storage_error ec; + if (r.length <= 0 || r.start < 0) + { + // this is an invalid read request. + ec.ec = errors::invalid_request; + ec.operation = operation_t::file_read; + handler(disk_buffer_holder{}, ec); + return; + } + + // in case r.start is not aligned to a block, calculate that offset, + // since that's how the disk_cache is indexed. block_offset is the + // aligned offset to the first block this read touches. In the case the + // request is aligned, it's the same as r.start + int const block_offset = r.start - (r.start % default_block_size); + int const block_idx = r.start / default_block_size; + // this is the offset into the block that we're reading from + int const read_offset = r.start - block_offset; + + DLOG("async_read piece: %d block: %d (read-offset: %d)\n", static_cast<int>(r.piece) + , block_offset / default_block_size, read_offset); + + disk_buffer_holder buffer; + + if (read_offset + r.length > default_block_size) + { + // This is an unaligned request spanning two blocks. One of the two + // blocks may be in the cache, or neither. + // If neither is in the cache, we can just issue a normal + // read job for the unaligned request. + + aux::piece_location const loc{storage, r.piece}; + std::ptrdiff_t const len1 = default_block_size - read_offset; + + TORRENT_ASSERT(r.length > len1); + + int const ret = m_cache.get2(loc, block_idx, [&](char const* buf1, char const* buf2) + { + buffer = disk_buffer_holder(m_buffer_pool + , m_buffer_pool.allocate_buffer("send buffer") + , r.length); + if (!buffer) + { + ec.ec = error::no_memory; + ec.operation = operation_t::alloc_cache_piece; + return 3; + } + + if (buf1) + std::memcpy(buffer.data(), buf1 + read_offset, std::size_t(len1)); + if (buf2) + std::memcpy(buffer.data() + len1, buf2, std::size_t(r.length - len1)); + return (buf1 ? 2 : 0) | (buf2 ? 1 : 0); + }); + + if (ret == 3) + { + // both sides were found in the store buffer and the read request + // was satisfied immediately + handler(std::move(buffer), ec); + return; + } + + if (ret != 0) + { + TORRENT_ASSERT(ret == 1 || ret == 2); + // only one side of the read request was found in the store + // buffer, and we need to issue a partial read for the remaining + // bytes + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::partial_read>( + flags, + m_torrents[storage]->shared_from_this(), + std::move(handler), + std::move(buffer), + std::uint16_t((ret == 1) ? 0 : len1), // buffer_offset + std::uint16_t((ret == 1) ? len1 : r.length - len1), // buffer_size + r.piece, + (ret == 1) ? r.start : block_offset + default_block_size // offset + ); + + add_job(j); + return; + } + + // if we couldn't find any block in the cache, fall through and post it + // as a normal read job + } + else + { + // this is an aligned read request for one block + if (m_cache.get({ storage, r.piece }, block_idx, [&](span<char const> buf) + { + TORRENT_ASSERT(buf.size() <= read_offset + r.length); + buffer = disk_buffer_holder(m_buffer_pool, m_buffer_pool.allocate_buffer("send buffer"), r.length); + if (!buffer) + { + ec.ec = error::no_memory; + ec.operation = operation_t::alloc_cache_piece; + return; + } + + std::memcpy(buffer.data(), buf.data() + read_offset, std::size_t(r.length)); + })) + { + handler(std::move(buffer), ec); + return; + } + } + + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::read>( + flags, + m_torrents[storage]->shared_from_this(), + std::move(handler), + disk_buffer_holder{}, + std::uint16_t(r.length), // buffer_size + r.piece, + r.start // offset + ); + + add_job(j); +} + +bool pread_disk_io::async_write(storage_index_t const storage, peer_request const& r + , char const* buf, std::shared_ptr<disk_observer> o + , std::function<void(storage_error const&)> handler + , disk_job_flags_t const flags) +{ + TORRENT_ASSERT(valid_flags(flags)); + bool exceeded = false; + disk_buffer_holder buffer(m_buffer_pool, m_buffer_pool.allocate_buffer( + exceeded, o, "receive buffer"), r.length); + if (!buffer) aux::throw_ex<std::bad_alloc>(); + std::memcpy(buffer.data(), buf, aux::numeric_cast<std::size_t>(r.length)); + + TORRENT_ASSERT(r.start % default_block_size == 0); + TORRENT_ASSERT(r.length <= default_block_size); + + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::write>( + flags, + m_torrents[storage]->shared_from_this(), + std::move(handler), + std::move(buffer), + r.piece, + r.start, + std::uint16_t(r.length) + ); + + DLOG("async_write: piece: %d offset: %d\n", int(r.piece), int(r.start)); + bool const need_kick = m_cache.insert({j->storage->storage_index(), r.piece}, r.start / default_block_size, j); + + if (need_kick) + { + // TODO: if the most recently added job to the hash thread pool is a + // kick-hasher job for the same piece, skip this + aux::pread_disk_job* khj = m_job_pool.allocate_job<aux::job::kick_hasher>( + flags, + m_torrents[storage]->shared_from_this(), + r.piece + ); + add_job(khj); + } + + std::unique_lock<std::mutex> l(m_job_mutex); + if (!m_flush_target) + { + // if the disk buffer wants to free up blocks, notify the thread + // pool that we may need to flush blocks + auto req = m_buffer_pool.flush_request(); + if (req) + { + m_flush_target = std::max(0, int(m_cache.size()) - *req); + DLOG("async_write: set flush_target: %d\n", *m_flush_target); + // wake up a thread + m_generic_threads.interrupt(); + } + } + + return exceeded; +} + +void pread_disk_io::async_hash(storage_index_t const storage + , piece_index_t const piece, span<sha256_hash> const v2, disk_job_flags_t const flags + , std::function<void(piece_index_t, sha1_hash const&, storage_error const&)> handler) +{ + TORRENT_ASSERT(valid_flags(flags)); + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::hash>( + flags, + m_torrents[storage]->shared_from_this(), + std::move(handler), + piece, + v2, + sha1_hash{} + ); + + aux::disk_cache::hash_result const ret = m_cache.try_hash_piece({j->storage->storage_index(), piece}, j); + + // if we have already computed the piece hash, just post the completion + // immediately + if (ret == aux::disk_cache::job_completed) + { + jobqueue_t jobs; + jobs.push_back(j); + add_completed_jobs(std::move(jobs)); + return; + } + + // In this case the job has been queued on the piece, and will be posted + // once the hashing completes + if (ret == aux::disk_cache::job_queued) + return; + + add_job(j); +} + +void pread_disk_io::async_hash2(storage_index_t const storage + , piece_index_t const piece, int const offset, disk_job_flags_t const flags + , std::function<void(piece_index_t, sha256_hash const&, storage_error const&)> handler) +{ + TORRENT_ASSERT(valid_flags(flags)); + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::hash2>( + flags, + m_torrents[storage]->shared_from_this(), + std::move(handler), + piece, + offset, + sha256_hash{} + ); + + // TODO: check the disk cache here + + add_job(j); +} + +void pread_disk_io::async_move_storage(storage_index_t const storage + , std::string p, move_flags_t const flags + , std::function<void(status_t, std::string const&, storage_error const&)> handler) +{ + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::move_storage>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler), + std::move(p), // path + flags + ); + + add_fence_job(j); +} + +void pread_disk_io::async_release_files(storage_index_t const storage + , std::function<void()> handler) +{ + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::release_files>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler) + ); + + add_fence_job(j); +} + +void pread_disk_io::abort_hash_jobs(storage_index_t const storage) +{ + // abort outstanding hash jobs belonging to this torrent + std::unique_lock<std::mutex> l(m_job_mutex); + + auto st = m_torrents[storage]->shared_from_this(); + // hash jobs + m_hash_threads.visit_jobs([&](aux::disk_job* gj) + { + auto* j = static_cast<aux::pread_disk_job*>(gj); + if (j->storage != st) return; + // only cancel volatile-read jobs. This means only full checking + // jobs. These jobs are likely to have a pretty deep queue and + // really gain from being cancelled. They can also be restarted + // easily. + if (j->flags & disk_interface::volatile_read) + j->flags |= aux::disk_job::aborted; + }); +} + +void pread_disk_io::async_delete_files(storage_index_t const storage + , remove_flags_t const options + , std::function<void(storage_error const&)> handler) +{ + abort_hash_jobs(storage); + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::delete_files>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler), + options + ); + add_fence_job(j); +} + +void pread_disk_io::async_check_files(storage_index_t const storage + , add_torrent_params const* resume_data + , aux::vector<std::string, file_index_t> links + , std::function<void(status_t, storage_error const&)> handler) +{ + aux::vector<std::string, file_index_t>* links_vector = nullptr; + if (!links.empty()) links_vector = new aux::vector<std::string, file_index_t>(std::move(links)); + + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::check_fastresume>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler), + links_vector, + resume_data + ); + + add_fence_job(j); +} + +void pread_disk_io::async_rename_file(storage_index_t const storage + , file_index_t const index, std::string name + , std::function<void(std::string const&, file_index_t, storage_error const&)> handler) +{ + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::rename_file>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler), + index, + std::move(name) + ); + add_fence_job(j); +} + +void pread_disk_io::async_stop_torrent(storage_index_t const storage + , std::function<void()> handler) +{ + auto st = m_torrents[storage]->shared_from_this(); + abort_hash_jobs(storage); + + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::stop_torrent>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler) + ); + add_fence_job(j); +} + +void pread_disk_io::async_set_file_priority(storage_index_t const storage + , aux::vector<download_priority_t, file_index_t> prios + , std::function<void(storage_error const& + , aux::vector<download_priority_t, file_index_t>)> handler) +{ + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::file_priority>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler), + std::move(prios) + ); + + add_fence_job(j); +} + +void pread_disk_io::async_clear_piece(storage_index_t const storage + , piece_index_t const index, std::function<void(piece_index_t)> handler) +{ + aux::pread_disk_job* j = m_job_pool.allocate_job<aux::job::clear_piece>( + {}, + m_torrents[storage]->shared_from_this(), + std::move(handler), + index + ); + + DLOG("async_clear_piece: piece: %d\n", int(index)); + // regular jobs are not executed in-order. + // clear piece must wait for all write jobs issued to the piece finish + // before it completes. + jobqueue_t aborted_jobs; + bool const immediate_completion = m_cache.try_clear_piece( + {j->storage->storage_index(), index}, j, aborted_jobs); + + m_completed_jobs.abort_jobs(m_ios, std::move(aborted_jobs)); + if (immediate_completion) + { + DLOG("immediate clear\n"); + jobqueue_t jobs; + jobs.push_back(j); + add_completed_jobs(std::move(jobs)); + } + else + { + DLOG("deferred clear\n"); + } +} + +status_t pread_disk_io::do_job(aux::job::hash& a, aux::pread_disk_job* j) +{ + // we're not using a cache. This is the simple path + // just read straight from the file + bool const v1 = bool(j->flags & disk_interface::v1_hash); + bool const v2 = !a.block_hashes.empty(); + + int const piece_size = v1 ? j->storage->files().piece_size(a.piece) : 0; + int const piece_size2 = v2 ? j->storage->files().piece_size2(a.piece) : 0; + int const blocks_in_piece = v1 ? (piece_size + default_block_size - 1) / default_block_size : 0; + int const blocks_in_piece2 = v2 ? j->storage->files().blocks_in_piece2(a.piece) : 0; + aux::open_mode_t const file_mode = file_mode_for_job(j); + + TORRENT_ASSERT(!v2 || int(a.block_hashes.size()) >= blocks_in_piece2); + TORRENT_ASSERT(v1 || v2); + + int const blocks_to_read = std::max(blocks_in_piece, blocks_in_piece2); + + // this creates a function object, ready to be passed to + // m_cache.hash_piece() + auto hash_partial_piece = [&] (lt::hasher& ph + , int const hasher_cursor + , span<char const*> const blocks + , span<sha256_hash> const v2_hashes) + { + time_point const start_time = clock_type::now(); + + if (v2 && hasher_cursor > 0) + { + for (int i = 0; i < hasher_cursor; ++i) + { + TORRENT_ASSERT(!v2_hashes[i].is_all_zeros()); + a.block_hashes[i] = v2_hashes[i]; + } + } + + int offset = hasher_cursor * default_block_size; + int blocks_read_from_disk = 0; + for (int i = hasher_cursor; i < blocks_to_read; ++i) + { + bool const v2_block = i < blocks_in_piece2; + + std::ptrdiff_t const len = v1 ? std::min(default_block_size, piece_size - offset) : 0; + std::ptrdiff_t const len2 = v2_block ? std::min(default_block_size, piece_size2 - offset) : 0; + + hasher256 ph2; + char const* buf = blocks[i]; + if (buf == nullptr) + { + DLOG("do_hash: reading (piece: %d block: %d)\n", int(a.piece), i); + + j->error.ec.clear(); + + if (v1) + { + auto const flags = v2_block + ? (j->flags & ~disk_interface::flush_piece) + : j->flags; + + j->storage->hash(m_settings, ph, len, a.piece + , offset, file_mode, flags, j->error); + } + if (v2_block) + { + j->storage->hash2(m_settings, ph2, len2, a.piece, offset + , file_mode, j->flags, j->error); + } + if (j->error) break; + ++blocks_read_from_disk; + } + else + { + if (v1) + ph.update({ buf, len }); + if (v2_block) + ph2.update({buf, len2}); + } + offset += default_block_size; + + if (v2_block) + a.block_hashes[i] = ph2.final(); + } + + if (v1) + a.piece_hash = ph.final(); + + if (!j->error.ec) + { + std::int64_t const read_time = total_microseconds(clock_type::now() - start_time); + + m_stats_counters.inc_stats_counter(counters::num_blocks_read, blocks_read_from_disk); + m_stats_counters.inc_stats_counter(counters::num_read_ops, blocks_read_from_disk); + m_stats_counters.inc_stats_counter(counters::disk_hash_time, read_time); + m_stats_counters.inc_stats_counter(counters::disk_job_time, read_time); + } + }; + + if (!m_cache.hash_piece({ j->storage->storage_index(), a.piece} + , hash_partial_piece)) + { + // fall back to reading everything from disk + + TORRENT_ALLOCA(blocks, char const*, blocks_to_read); + TORRENT_ALLOCA(v2_hashes, sha256_hash, blocks_in_piece2); + for (char const*& b : blocks) b = nullptr; + hasher ph; + hash_partial_piece(ph, 0, blocks, v2_hashes); + } + return j->error ? disk_status::fatal_disk_error : status_t{}; +} + +status_t pread_disk_io::do_job(aux::job::hash2& a, aux::pread_disk_job* j) +{ + int const piece_size = j->storage->files().piece_size2(a.piece); + aux::open_mode_t const file_mode = file_mode_for_job(j); + + DLOG("do_hash2: reading (piece: %d offset: %d)\n", int(a.piece), int(a.offset)); + + time_point const start_time = clock_type::now(); + + TORRENT_ASSERT(piece_size > a.offset); + std::ptrdiff_t const len = std::min(default_block_size, piece_size - a.offset); + + int ret = 0; + a.piece_hash2 = m_cache.hash2({ j->storage->storage_index(), a.piece } + , a.offset / default_block_size + , [&] { + hasher256 h; + ret = j->storage->hash2(m_settings, h, len, a.piece, a.offset + , file_mode, j->flags, j->error); + return h.final(); + }); + + if (!j->error.ec) + { + std::int64_t const read_time = total_microseconds(clock_type::now() - start_time); + + m_stats_counters.inc_stats_counter(counters::num_blocks_read); + m_stats_counters.inc_stats_counter(counters::num_read_ops); + m_stats_counters.inc_stats_counter(counters::disk_hash_time, read_time); + m_stats_counters.inc_stats_counter(counters::disk_job_time, read_time); + } + + return ret >= 0 ? status_t{} : disk_status::fatal_disk_error; +} + +status_t pread_disk_io::do_job(aux::job::move_storage& a, aux::pread_disk_job* j) +{ + // if this assert fails, something's wrong with the fence logic + TORRENT_ASSERT(j->storage->num_outstanding_jobs() == 1); + flush_storage(j->storage); + + // if files have to be closed, that's the storage's responsibility + auto const [ret, p] = j->storage->move_storage(std::move(a.path), a.move_flags, j->error); + + a.path = std::move(p); + return ret; +} + +status_t pread_disk_io::do_job(aux::job::release_files&, aux::pread_disk_job* j) +{ + // if this assert fails, something's wrong with the fence logic + TORRENT_ASSERT(j->storage->num_outstanding_jobs() == 1); + flush_storage(j->storage); + j->storage->release_files(j->error); + return j->error ? disk_status::fatal_disk_error : status_t{}; +} + +status_t pread_disk_io::do_job(aux::job::delete_files& a, aux::pread_disk_job* j) +{ + TORRENT_ASSERT(a.flags); + + // if this assert fails, something's wrong with the fence logic + TORRENT_ASSERT(j->storage->num_outstanding_jobs() == 1); + + // TODO: maybe we don't need to write to files we're about to delete + flush_storage(j->storage); + + j->storage->delete_files(a.flags, j->error); + return j->error ? disk_status::fatal_disk_error : status_t{}; +} + +status_t pread_disk_io::do_job(aux::job::check_fastresume& a, aux::pread_disk_job* j) +{ + // if this assert fails, something's wrong with the fence logic + TORRENT_ASSERT(j->storage->num_outstanding_jobs() == 1); + flush_storage(j->storage); + add_torrent_params const* rd = a.resume_data; + add_torrent_params tmp; + if (rd == nullptr) rd = &tmp; + + std::unique_ptr<aux::vector<std::string, file_index_t>> links(a.links); + // check if the fastresume data is up to date + // if it is, use it and return true. If it + // isn't return false and the full check + // will be run. If the links pointer is non-empty, it has the same number + // of elements as there are files. Each element is either empty or contains + // the absolute path to a file identical to the corresponding file in this + // torrent. The storage must create hard links (or copy) those files. If + // any file does not exist or is inaccessible, the disk job must fail. + + TORRENT_ASSERT(j->storage->files().piece_length() > 0); + + // always initialize the storage + auto const ret_flag = j->storage->initialize(m_settings, j->error); + if (j->error) return disk_status::fatal_disk_error | ret_flag; + + // we must call verify_resume() unconditionally of the setting below, in + // order to set up the links (if present) + bool const verify_success = j->storage->verify_resume_data(*rd + , links ? *links : aux::vector<std::string, file_index_t>(), j->error); + + // j->error may have been set at this point, by verify_resume_data() + // it's important to not have it cleared out subsequent calls, as long + // as they succeed. + + if (m_settings.get_bool(settings_pack::no_recheck_incomplete_resume)) + return status_t{} | ret_flag; + + if (!aux::contains_resume_data(*rd)) + { + // if we don't have any resume data, we still may need to trigger a + // full re-check, if there are *any* files. + storage_error ignore; + return ((j->storage->has_any_file(ignore)) + ? disk_status::need_full_check + : status_t{}) + | ret_flag; + } + + return (verify_success + ? status_t{} + : disk_status::need_full_check) + | ret_flag; +} + +status_t pread_disk_io::do_job(aux::job::rename_file& a, aux::pread_disk_job* j) +{ + // if this assert fails, something's wrong with the fence logic + TORRENT_ASSERT(j->storage->num_outstanding_jobs() == 1); + + // if files need to be closed, that's the storage's responsibility + j->storage->rename_file(a.file_index, a.name, j->error); + return j->error ? disk_status::fatal_disk_error : status_t{}; +} + +status_t pread_disk_io::do_job(aux::job::stop_torrent&, aux::pread_disk_job* j) +{ + // if this assert fails, something's wrong with the fence logic + TORRENT_ASSERT(j->storage->num_outstanding_jobs() == 1); + flush_storage(j->storage); + j->storage->release_files(j->error); + return j->error ? disk_status::fatal_disk_error : status_t{}; +} + +void pread_disk_io::update_stats_counters(counters& c) const +{ + // These are atomic_counts, so it's safe to access them from + // a different thread + std::unique_lock<std::mutex> jl(m_job_mutex); + + c.set_value(counters::num_read_jobs, m_job_pool.read_jobs_in_use()); + c.set_value(counters::num_write_jobs, m_job_pool.write_jobs_in_use()); + c.set_value(counters::num_jobs, m_job_pool.jobs_in_use()); + c.set_value(counters::queued_disk_jobs, m_generic_threads.queue_size() + + m_hash_threads.queue_size()); + + jl.unlock(); + + // gauges + c.set_value(counters::disk_blocks_in_use, m_buffer_pool.in_use()); +} + +status_t pread_disk_io::do_job(aux::job::file_priority& a, aux::pread_disk_job* j) +{ + j->storage->set_file_priority(m_settings + , a.prio + , j->error); + return status_t{}; +} + +status_t pread_disk_io::do_job(aux::job::clear_piece&, aux::pread_disk_job*) +{ + TORRENT_ASSERT_FAIL(); + return {}; +} + +status_t pread_disk_io::do_job(aux::job::kick_hasher& a, aux::pread_disk_job* j) +{ + jobqueue_t jobs; + m_cache.kick_hasher({j->storage->storage_index(), a.piece}, jobs); + add_completed_jobs(std::move(jobs)); + return {}; +} + +void pread_disk_io::add_fence_job(aux::pread_disk_job* j, bool const user_add) +{ + // if this happens, it means we started to shut down + // the disk threads too early. We have to post all jobs + // before the disk threads are shut down + if (m_abort) + { + m_completed_jobs.abort_job(m_ios, j); + return; + } + + DLOG("add_fence:job: %s (outstanding: %d)\n" + , print_job(*j).c_str() + , j->storage->num_outstanding_jobs()); + + TORRENT_ASSERT(j->storage); + m_stats_counters.inc_stats_counter(counters::num_fenced_read + static_cast<int>(j->get_type())); + + int const ret = j->storage->raise_fence(j, m_stats_counters); + if (ret == aux::disk_job_fence::fence_post_fence) + { + std::unique_lock<std::mutex> l(m_job_mutex); + TORRENT_ASSERT((j->flags & aux::disk_job::in_progress) || !j->storage); + m_generic_threads.push_back(j); + l.unlock(); + } + + if (num_threads() == 0 && user_add) + immediate_execute(); +} + +void pread_disk_io::add_job(aux::pread_disk_job* j, bool const user_add) +{ + TORRENT_ASSERT(!j->storage || j->storage->files().is_valid()); + TORRENT_ASSERT(j->next == nullptr); + // if this happens, it means we started to shut down + // the disk threads too early. We have to post all jobs + // before the disk threads are shut down + if (m_abort) + { + m_completed_jobs.abort_job(m_ios, j); + return; + } + + TORRENT_ASSERT(!(j->flags & aux::disk_job::in_progress)); + + DLOG("add_job: %s (outstanding: %d)\n" + , print_job(*j).c_str() + , j->storage ? j->storage->num_outstanding_jobs() : 0); + + // is the fence up for this storage? + // jobs that are instantaneous are not affected by the fence, is_blocked() + // will take ownership of the job and queue it up, in case the fence is up + // if the fence flag is set, this job just raised the fence on the storage + // and should be scheduled + if (j->storage && j->storage->is_blocked(j)) + { + m_stats_counters.inc_stats_counter(counters::blocked_disk_jobs); + DLOG("blocked job: %s (torrent: %d total: %d)\n" + , print_job(*j).c_str(), j->storage ? j->storage->num_blocked() : 0 + , int(m_stats_counters[counters::blocked_disk_jobs])); + return; + } + + std::unique_lock<std::mutex> l(m_job_mutex); + + TORRENT_ASSERT((j->flags & aux::disk_job::in_progress) || !j->storage); + + auto& q = pool_for_job(j); + q.push_back(j); + l.unlock(); + // if we literally have 0 disk threads, we have to execute the jobs + // immediately. If add job is called internally by the pread_disk_io, + // we need to defer executing it. We only want the top level to loop + // over the job queue (as is done below) + if (pool_for_job(j).max_threads() == 0 && user_add) + immediate_execute(); +} + +void pread_disk_io::immediate_execute() +{ + while (!m_generic_threads.empty()) + { + auto* j = static_cast<aux::pread_disk_job*>(m_generic_threads.pop_front()); + execute_job(j); + } +} + +void pread_disk_io::submit_jobs() +{ + std::unique_lock<std::mutex> l(m_job_mutex); + m_generic_threads.submit_jobs(); + m_hash_threads.submit_jobs(); +} + +void pread_disk_io::execute_job(aux::pread_disk_job* j) +{ + jobqueue_t completed_jobs; + if (j->flags & aux::disk_job::aborted) + { + j->ret = disk_status::fatal_disk_error; + j->error = storage_error(boost::asio::error::operation_aborted); + completed_jobs.push_back(j); + add_completed_jobs(std::move(completed_jobs)); + return; + } + + perform_job(j, completed_jobs); + if (!completed_jobs.empty()) + add_completed_jobs(std::move(completed_jobs)); +} + +int pread_disk_io::flush_cache_blocks(bitfield& flushed + , span<aux::cached_block_entry const> blocks + , int const hash_cursor, jobqueue_t& completed_jobs) +{ + if (blocks.empty()) return 0; + +#if DEBUG_DISK_THREAD + { + auto piece = piece_index_t(-1); + std::string blocks_str; + blocks_str.reserve(blocks.size()); + for (auto const& blk : blocks) + { + blocks_str += blk.write_job ? '*' : ' '; + if (blk.write_job) + piece = std::get<aux::job::write>(blk.write_job->action).piece; + } + // If this assert fires, it means we were asked to flush a piece + // that doesn't have any jobs to flush + TORRENT_ASSERT(piece != piece_index_t(-1)); + DLOG("flush_cache_blocks: piece: %d hash_cursor: %d blocks: [%s]\n", int(piece), hash_cursor, blocks_str.c_str()); + } +#else + TORRENT_UNUSED(hash_cursor); +#endif + + // blocks may be sparse. We need to skip any block entry where write_job is null + m_stats_counters.inc_stats_counter(counters::num_running_disk_jobs, 1); + m_stats_counters.inc_stats_counter(counters::num_writing_threads, 1); + time_point const start_time = clock_type::now(); + + TORRENT_ALLOCA(iovec, span<char>, blocks.size()); + bool failed = false; + int count = 0; + int start_idx = 0; + int idx = 0; + + // the total number of blocks we ended up flushing to disk + int ret = 0; + + // the piece offset of the start of the range of contiguous blocks we're + // currently assembling into iovec + int start_offset = 0; + + // the offset of the end of the range of contiguous blocks we're currently + // assembing + int end_offset = 0; + + aux::open_mode_t file_mode; + auto piece = piece_index_t(-1); + disk_job_flags_t flags; + + std::shared_ptr<aux::pread_storage> storage; + + storage_error error; + // TODO: refactor this loop into an iterator adapter that returns + // contiguous ranges of blocks. Then de-duplicate the write-to-disk logic + // into the loop + TORRENT_ASSERT(blocks.size() > 0); + for (auto& be : blocks) + { + auto* j = be.write_job; + + auto const job_offset = [&] { + if (j != nullptr) + return std::get<aux::job::write>(j->action).offset; + else + return 0; + }(); + + if (!storage && j) storage = j->storage; + if (count > 0 && (j == nullptr || job_offset > end_offset)) + { + TORRENT_ASSERT(piece != piece_index_t(-1)); + DLOG("write: blocks: %d (piece: %d)\n", count, int(piece)); + storage->write(m_settings, iovec.first(count) + , piece, start_offset, file_mode, flags, error); + + int i = start_idx; + for (aux::cached_block_entry const& blk : blocks.subspan(start_idx, count)) + { + auto* j2 = blk.write_job; + TORRENT_ASSERT(j2); + TORRENT_ASSERT(j2->get_type() == aux::job_action_t::write); + j2->error = error; + flushed.set_bit(i); + completed_jobs.push_back(j2); + ++i; + } + + if (error) { + // if there was a failure, fail the remaining jobs as well + for (int k = start_idx + count; k < blocks.size(); ++k) + { + auto* j2 = be.write_job; + if (j2 == nullptr) continue; + j2->error = error; + // TODO: should we free the job's buffer here? + completed_jobs.push_back(j2); + } + failed = true; + break; + } + + ret += count; + + start_offset = job_offset; + start_idx = idx; + count = 0; + } + + if (j == nullptr) + { + ++idx; + start_idx = idx; + continue; + } + + TORRENT_ASSERT(j->storage == storage); + TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); + auto& a = std::get<aux::job::write>(j->action); + + if (count == 0) start_offset = job_offset; + iovec[count] = span<char>{ a.buf.data(), a.buffer_size}; + ++count; + flags = j->flags; + piece = a.piece; + file_mode = file_mode_for_job(j); + end_offset = job_offset + a.buffer_size; + ++idx; + } + + if (count > 0) + { + DLOG("write: blocks: %d (piece: %d)\n", count, int(piece)); + storage->write(m_settings, iovec.first(count) + , piece, start_offset, file_mode, flags, error); + + int i = start_idx; + for (aux::cached_block_entry const& blk : blocks.subspan(start_idx, count)) + { + auto* j = blk.write_job; + TORRENT_ASSERT(j); + TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); + j->error = error; + flushed.set_bit(i); + completed_jobs.push_back(j); + ++i; + } + // TODO: if we failed, post the remaining block's jobs as failures too + if (error) failed = true; + else ret += count; + } + + if (!failed) + { + std::int64_t const write_time = total_microseconds(clock_type::now() - start_time); + + m_stats_counters.inc_stats_counter(counters::num_blocks_written, blocks.size()); + m_stats_counters.inc_stats_counter(counters::num_write_ops); + m_stats_counters.inc_stats_counter(counters::disk_write_time, write_time); + m_stats_counters.inc_stats_counter(counters::disk_job_time, write_time); + } + + // TODO: put this in an RAII object + m_stats_counters.inc_stats_counter(counters::num_writing_threads, -1); + m_stats_counters.inc_stats_counter(counters::num_running_disk_jobs, -1); + + return ret; +} + +void pread_disk_io::clear_piece_jobs(jobqueue_t aborted, aux::pread_disk_job* clear) +{ + m_completed_jobs.abort_jobs(m_ios, std::move(aborted)); + jobqueue_t jobs; + jobs.push_back(clear); + add_completed_jobs(std::move(jobs)); +} + +void pread_disk_io::try_flush_cache(int const target_cache_size + , std::unique_lock<std::mutex>& l) +{ + DLOG("flushing, cache target: %d (current size: %d currently flushing: %d)\n" + , target_cache_size, m_cache.size(), m_cache.num_flushing()); + l.unlock(); + jobqueue_t completed_jobs; + m_cache.flush_to_disk( + [&](bitfield& flushed, span<aux::cached_block_entry const> blocks, int const hash_cursor) { + return flush_cache_blocks(flushed, blocks, hash_cursor, completed_jobs); + } + , target_cache_size + , [&](jobqueue_t aborted, aux::pread_disk_job* clear) { + clear_piece_jobs(std::move(aborted), clear); + }); + l.lock(); + DLOG("flushed blocks (%d blocks left), return to disk loop\n", m_cache.size()); + if (!completed_jobs.empty()) + add_completed_jobs(std::move(completed_jobs)); +} + +void pread_disk_io::flush_storage(std::shared_ptr<aux::pread_storage> const& storage) +{ + storage_index_t const torrent = storage->storage_index(); + DLOG("flush_storage (%d)\n", torrent); + jobqueue_t completed_jobs; + m_cache.flush_storage( + [&](bitfield& flushed, span<aux::cached_block_entry const> blocks, int const hash_cursor) { + return flush_cache_blocks(flushed, blocks, hash_cursor, completed_jobs); + } + , torrent + , [&](jobqueue_t aborted, aux::pread_disk_job* clear) { + clear_piece_jobs(std::move(aborted), clear); + }); + DLOG("flush_storage - done (%d left)\n", m_cache.size()); + if (!completed_jobs.empty()) + add_completed_jobs(std::move(completed_jobs)); +} + +void pread_disk_io::thread_fun(aux::disk_io_thread_pool& pool + , executor_work_guard<io_context::executor_type> work) +{ + // work is used to keep the io_context alive + TORRENT_UNUSED(work); + + ADD_OUTSTANDING_ASYNC("pread_disk_io::work"); + std::thread::id const thread_id = std::this_thread::get_id(); + + aux::set_thread_name("Disk"); + + DLOG("started disk thread\n"); + + std::unique_lock<std::mutex> l(m_job_mutex); + + ++m_num_running_threads; + m_stats_counters.inc_stats_counter(counters::num_running_threads, 1); + + // we call close_oldest_file on the file_pool regularly. This is the next + // time we should call it + time_point next_close_oldest_file = min_time(); + + for (;;) + { + auto const res = pool.wait_for_job(l); + + // if we need to flush the cache, let one of the generic threads do + // that + if (m_flush_target && &pool == &m_generic_threads) + { + int const target_cache_size = *std::exchange(m_flush_target, std::nullopt); + DLOG("try_flush_cache(%d)\n", target_cache_size); + try_flush_cache(target_cache_size, l); + continue; + } + + if (res == aux::wait_result::exit_thread) + { + DLOG("exit disk loop\n"); + break; + } + + if (res != aux::wait_result::new_job) + { + DLOG("continue disk loop\n"); + continue; + } + + auto* j = static_cast<aux::pread_disk_job*>(pool.pop_front()); + l.unlock(); + + TORRENT_ASSERT((j->flags & aux::disk_job::in_progress) || !j->storage); + + if (&pool == &m_generic_threads && thread_id == pool.first_thread_id()) + { + time_point const now = aux::time_now(); + { + std::unique_lock<std::mutex> l2(m_need_tick_mutex); + while (!m_need_tick.empty() && m_need_tick.front().first < now) + { + std::shared_ptr<aux::pread_storage> st = m_need_tick.front().second.lock(); + m_need_tick.erase(m_need_tick.begin()); + if (st) + { + l2.unlock(); + st->tick(); + l2.lock(); + } + } + } + + if (now > next_close_oldest_file) + { + seconds const interval(m_settings.get_int(settings_pack::close_file_interval)); + if (interval <= seconds(0)) + { + // check again in one minute, in case the setting changed + next_close_oldest_file = now + minutes(1); + } + else + { + next_close_oldest_file = now + interval; + m_file_pool.close_oldest(); + } + } + } + + execute_job(j); + + l.lock(); + } + + // do cleanup in the last running thread + // if we're not aborting, that means we just configured the thread pool to + // not have any threads (i.e. perform all disk operations in the network + // thread). In this case, the cleanup will happen in abort(). + + int const threads_left = --m_num_running_threads; + if (threads_left > 0 || !m_abort) + { + DLOG("exiting disk thread. num_threads: %d aborting: %d\n" + , threads_left, int(m_abort)); + m_stats_counters.inc_stats_counter(counters::num_running_threads, -1); + COMPLETE_ASYNC("pread_disk_io::work"); + return; + } + + DLOG("last thread alive. (left: %d) cleaning up. (generic-jobs: %d hash-jobs: %d)\n" + , threads_left + , m_generic_threads.queue_size() + , m_hash_threads.queue_size()); + + // flush everything before exiting this thread + try_flush_cache(0, l); + + // it is important to hold the job mutex while calling try_thread_exit() + // and continue to hold it until checking m_abort above so that abort() + // doesn't inadvertently trigger the code below when it thinks there are no + // more disk I/O threads running + l.unlock(); + + // at this point, there are no queued jobs left. However, main + // thread is still running and may still have peer_connections + // that haven't fully destructed yet, reclaiming their references + // to read blocks in the disk cache. We need to wait until all + // references are removed from other threads before we can go + // ahead with the cleanup. + // This is not supposed to happen because the disk thread is now scheduled + // for shut down after all peers have shut down (see + // session_impl::abort_stage2()). + + DLOG("the last disk thread alive. cleaning up\n"); + + abort_jobs(); + + m_stats_counters.inc_stats_counter(counters::num_running_threads, -1); + COMPLETE_ASYNC("pread_disk_io::work"); +} + +void pread_disk_io::abort_jobs() +{ + DLOG("pread_disk_io::abort_jobs\n"); + + if (m_jobs_aborted.test_and_set()) return; + + // close all files. This may take a long + // time on certain OSes (i.e. Mac OS) + // that's why it's important to do this in + // the disk thread in parallel with stopping + // trackers. + m_file_pool.release(); +} + +int pread_disk_io::num_threads() const +{ + return m_generic_threads.max_threads() + m_hash_threads.max_threads(); +} + +aux::disk_io_thread_pool& pread_disk_io::pool_for_job(aux::pread_disk_job* j) +{ + if (m_hash_threads.max_threads() > 0 + && (j->get_type() == aux::job_action_t::hash + || j->get_type() == aux::job_action_t::hash2 + || j->get_type() == aux::job_action_t::kick_hasher)) + return m_hash_threads; + else + return m_generic_threads; +} + +void pread_disk_io::add_completed_jobs(jobqueue_t jobs) +{ + jobqueue_t completed = std::move(jobs); + do + { + // when a job completes, it's possible for it to cause + // a fence to be lowered, issuing the jobs queued up + // behind the fence + jobqueue_t new_jobs; + add_completed_jobs_impl(std::move(completed), new_jobs); + TORRENT_ASSERT(completed.empty()); + completed = std::move(new_jobs); + } while (!completed.empty()); +} + +void pread_disk_io::add_completed_jobs_impl(jobqueue_t jobs, jobqueue_t& completed) +{ + jobqueue_t new_jobs; + int ret = 0; + for (auto i = jobs.iterate(); i.get(); i.next()) + { + auto* j = static_cast<aux::pread_disk_job*>(i.get()); + + if (j->flags & aux::disk_job::fence) + { + m_stats_counters.inc_stats_counter( + counters::num_fenced_read + static_cast<int>(j->get_type()), -1); + } + + if (j->flags & aux::disk_job::in_progress) + { + TORRENT_ASSERT(j->storage); + if (j->storage) + ret += j->storage->job_complete(j, new_jobs); + } + + TORRENT_ASSERT(ret == new_jobs.size()); + TORRENT_ASSERT(!(j->flags & aux::disk_job::in_progress)); +#if TORRENT_USE_ASSERTS + TORRENT_ASSERT(j->job_posted == false); + j->job_posted = true; +#endif + } + + if (ret) + { + DLOG("unblocked %d jobs (%d left)\n", ret + , int(m_stats_counters[counters::blocked_disk_jobs]) - ret); + } + + m_stats_counters.inc_stats_counter(counters::blocked_disk_jobs, -ret); + TORRENT_ASSERT(int(m_stats_counters[counters::blocked_disk_jobs]) >= 0); + + if (m_abort.load()) + { + while (!new_jobs.empty()) + { + auto* j = static_cast<aux::pread_disk_job*>(new_jobs.pop_front()); + TORRENT_ASSERT((j->flags & aux::disk_job::in_progress) || !j->storage); + j->ret = disk_status::fatal_disk_error; + j->error = storage_error(boost::asio::error::operation_aborted); + completed.push_back(j); + } + } + else + { + if (!new_jobs.empty()) + { + { + std::lock_guard<std::mutex> l(m_job_mutex); + m_generic_threads.append(std::move(new_jobs)); + } + + { + std::lock_guard<std::mutex> l(m_job_mutex); + m_generic_threads.submit_jobs(); + } + } + } + + m_completed_jobs.append(m_ios, std::move(jobs)); +} + +} diff --git a/src/pread_storage.cpp b/src/pread_storage.cpp new file mode 100644 index 00000000000..a5e11b99817 --- /dev/null +++ b/src/pread_storage.cpp @@ -0,0 +1,797 @@ +/* + +Copyright (c) 2022, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#include "libtorrent/config.hpp" +#include "libtorrent/error_code.hpp" +#include "libtorrent/aux_/storage_utils.hpp" +#include "libtorrent/hasher.hpp" + +#include <ctime> +#include <algorithm> +#include <numeric> +#include <set> +#include <functional> +#include <cstdio> + +#include "libtorrent/aux_/pread_storage.hpp" +#include "libtorrent/aux_/torrent.hpp" +#include "libtorrent/aux_/path.hpp" +#include "libtorrent/aux_/invariant_check.hpp" +#include "libtorrent/aux_/session_impl.hpp" +#include "libtorrent/aux_/file_pool.hpp" +#include "libtorrent/aux_/file.hpp" // for file_handle, pread_all, pwrite_all +#include "libtorrent/disk_buffer_holder.hpp" +#include "libtorrent/aux_/stat_cache.hpp" +#include "libtorrent/hex.hpp" // to_hex + +#include <sys/types.h> + +#if (TORRENT_HAS_FADVISE && defined POSIX_FADV_DONTNEED) +#include <fcntl.h> +#endif + +#if defined TORRENT_LINUX && defined SYNC_FILE_RANGE_WRITE +#include <fcntl.h> // for sync_file_range +#elif defined TORRENT_WINDOWS +#include "libtorrent/aux_/windows.hpp" // for FlushFileBuffers +#elif defined TORRENT_BSD && ! defined __APPLE__ +#include <unistd.h> // for fsync_range +#else +#include <unistd.h> // for fsync +#endif + +namespace libtorrent::aux { + +namespace { + + // TODO: move this o aux_/file.hpp + void advise_dont_need(handle_type handle, std::int64_t offset, std::int64_t len) + { +#if (TORRENT_HAS_FADVISE && defined POSIX_FADV_DONTNEED) + ::posix_fadvise(handle, offset, len, POSIX_FADV_DONTNEED); +#else + TORRENT_UNUSED(handle); + TORRENT_UNUSED(offset); + TORRENT_UNUSED(len); +#endif + } + + // TODO: move this o aux_/file.hpp + void sync_file(handle_type handle, std::int64_t offset, std::int64_t len) + { +#if defined TORRENT_LINUX && defined SYNC_FILE_RANGE_WRITE + ::sync_file_range(handle, offset, len, SYNC_FILE_RANGE_WRITE); +#elif defined TORRENT_WINDOWS + ::FlushFileBuffers(handle); + TORRENT_UNUSED(offset); + TORRENT_UNUSED(len); +#elif defined TORRENT_BSD && ! defined __APPLE__ + ::fsync_range(handle, FFILESYNC, offset, len); +#else + ::fsync(handle); + TORRENT_UNUSED(offset); + TORRENT_UNUSED(len); +#endif + } +} + + pread_storage::pread_storage(storage_params const& params + , file_pool& pool) + : m_files(params.files) + , m_file_priority(params.priorities) + , m_save_path(complete(params.path)) + , m_part_file_name("." + to_hex(params.info_hash) + ".parts") + , m_pool(pool) + , m_allocate_files(params.mode == storage_mode_allocate) + , m_v1(params.v1) + , m_v2(params.v2) + { + // a torrent must be either v1 or v2 (or both) + TORRENT_ASSERT(m_v1 || m_v2); + if (params.mapped_files) m_mapped_files = std::make_unique<file_storage>(*params.mapped_files); + + TORRENT_ASSERT(files().num_files() > 0); + } + + pread_storage::~pread_storage() + { + error_code ec; + if (m_part_file) m_part_file->flush_metadata(ec); + + // this may be called from a different + // thread than the disk thread + m_pool.release(storage_index()); + } + + void pread_storage::need_partfile() + { + if (m_part_file) return; + + m_part_file = std::make_unique<part_file>( + m_save_path, m_part_file_name + , files().num_pieces(), files().piece_length()); + } + + void pread_storage::set_file_priority(settings_interface const& sett + , vector<download_priority_t, file_index_t>& prio + , storage_error& ec) + { + // extend our file priorities in case it's truncated + // the default assumed priority is 4 (the default) + if (prio.size() > m_file_priority.size()) + m_file_priority.resize(prio.size(), default_priority); + + file_storage const& fs = files(); + for (file_index_t i(0); i < prio.end_index(); ++i) + { + // pad files always have priority 0. + if (fs.pad_file_at(i)) continue; + + download_priority_t const old_prio = m_file_priority[i]; + download_priority_t new_prio = prio[i]; + if (old_prio == dont_download && new_prio != dont_download) + { + // move stuff out of the part file + auto f = open_file(sett, i, open_mode::write, ec); + if (ec) + { + prio = m_file_priority; + return; + } + TORRENT_ASSERT(f); + + if (m_part_file && use_partfile(i)) + { + try + { + m_part_file->export_file([&f](std::int64_t file_offset, span<char> buf) + { + do { + error_code err; + int const r = pwrite_all(f->fd(), buf, file_offset, err); + if (err) + throw_ex<lt::system_error>(err); + buf = buf.subspan(r); + file_offset += r; + } while (buf.size() > 0); + }, fs.file_offset(i), fs.file_size(i), ec.ec); + if (ec) + { + ec.file(i); + ec.operation = operation_t::partfile_write; + prio = m_file_priority; + return; + } + } + catch (lt::system_error const& err) + { + ec.file(i); + ec.operation = operation_t::partfile_write; + ec.ec = err.code(); + return; + } + } + } + else if (old_prio != dont_download && new_prio == dont_download) + { + // move stuff into the part file + // this is not implemented yet. + // so we just don't use a partfile for this file + + std::string const fp = fs.file_path(i, m_save_path); + bool const file_exists = exists(fp, ec.ec); + if (ec.ec) + { + ec.file(i); + ec.operation = operation_t::file_stat; + prio = m_file_priority; + return; + } + use_partfile(i, !file_exists); + } + ec.ec.clear(); + m_file_priority[i] = new_prio; + + if (m_file_priority[i] == dont_download && use_partfile(i)) + { + need_partfile(); + } + } + if (m_part_file) m_part_file->flush_metadata(ec.ec); + if (ec) + { + ec.file(torrent_status::error_file_partfile); + ec.operation = operation_t::partfile_write; + } + } + + bool pread_storage::use_partfile(file_index_t const index) const + { + TORRENT_ASSERT_VAL(index >= file_index_t{}, index); + if (index >= m_use_partfile.end_index()) return true; + return m_use_partfile[index]; + } + + void pread_storage::use_partfile(file_index_t const index, bool const b) + { + if (index >= m_use_partfile.end_index()) + { + // no need to extend this array if we're just setting it to "true", + // that's default already + if (b) return; + m_use_partfile.resize(static_cast<int>(index) + 1, true); + } + m_use_partfile[index] = b; + } + + status_t pread_storage::initialize(settings_interface const& sett, storage_error& ec) + { + m_stat_cache.reserve(files().num_files()); + +#ifdef TORRENT_WINDOWS + // don't do full file allocations on network drives + auto const file_name = convert_to_native_path_string(m_save_path); + int const drive_type = GetDriveTypeW(file_name.c_str()); + + if (drive_type == DRIVE_REMOTE) + m_allocate_files = false; +#endif + { + std::unique_lock<std::mutex> l(m_file_created_mutex); + m_file_created.resize(files().num_files(), false); + } + + file_storage const& fs = files(); + status_t ret{}; + // if some files have priority 0, we need to check if they exist on the + // filesystem, in which case we won't use a partfile for them. + // this is to be backwards compatible with previous versions of + // libtorrent, when part files were not supported. + for (file_index_t i(0); i < m_file_priority.end_index(); ++i) + { + if (m_file_priority[i] != dont_download || fs.pad_file_at(i)) + continue; + + error_code err; + auto const size = m_stat_cache.get_filesize(i, fs, m_save_path, err); + if (!err && size > 0) + { + use_partfile(i, false); + if (size > fs.file_size(i)) + ret = ret | disk_status::oversized_file; + } + else + { + // we may have earlier determined we *can't* use a partfile for + // this file, we need to be able to change our mind in case the + // file disappeared + use_partfile(i, true); + need_partfile(); + } + } + + initialize_storage(fs, m_save_path, m_stat_cache, m_file_priority + , [&sett, this](file_index_t const file_index, storage_error& e) + { open_file(sett, file_index, open_mode::write, e); } + , create_symlink + , [&ret](file_index_t, std::int64_t) { ret = ret | disk_status::oversized_file; } + , ec); + + // close files that were opened in write mode + m_pool.release(storage_index()); + return ret; + } + + bool pread_storage::has_any_file(storage_error& ec) + { + m_stat_cache.reserve(files().num_files()); + + if (aux::has_any_file(files(), m_save_path, m_stat_cache, ec)) + return true; + + if (ec) return false; + + file_status s; + stat_file(combine_path(m_save_path, m_part_file_name), &s, ec.ec); + if (!ec) return true; + + // the part file not existing is expected + if (ec.ec == boost::system::errc::no_such_file_or_directory) + ec.ec.clear(); + + if (ec) + { + ec.file(torrent_status::error_file_partfile); + ec.operation = operation_t::file_stat; + } + return false; + } + + void pread_storage::rename_file(file_index_t const index, std::string const& new_filename + , storage_error& ec) + { + if (index < file_index_t(0) || index >= files().end_file()) return; + std::string const old_name = files().file_path(index, m_save_path); + m_pool.release(storage_index(), index); + + // if the old file doesn't exist, just succeed and change the filename + // that will be created. This shortcut is important because the + // destination directory may not exist yet, which would cause a failure + // even though we're not moving a file (yet). It's better for it to + // fail later when we try to write to the file the first time, because + // the user then will have had a chance to make the destination directory + // valid. + if (exists(old_name, ec.ec)) + { + std::string new_path; + if (is_complete(new_filename)) new_path = new_filename; + else new_path = combine_path(m_save_path, new_filename); + std::string new_dir = parent_path(new_path); + + // create any missing directories that the new filename + // lands in + create_directories(new_dir, ec.ec); + if (ec.ec) + { + ec.file(index); + ec.operation = operation_t::file_rename; + return; + } + + rename(old_name, new_path, ec.ec); + + // if old_name doesn't exist, that's not an error + // here. Once we start writing to the file, it will + // be written to the new filename + if (ec.ec == boost::system::errc::no_such_file_or_directory) + ec.ec.clear(); + + if (ec) + { + ec.ec.clear(); + copy_file(old_name, new_path, ec); + + if (ec) + { + ec.file(index); + return; + } + + error_code ignore; + remove(old_name, ignore); + } + } + else if (ec.ec) + { + // if exists fails, report that error + ec.file(index); + ec.operation = operation_t::file_rename; + return; + } + + // if old path doesn't exist, just rename the file + // in our file_storage, so that when it is created + // it will get the new name + if (!m_mapped_files) + { m_mapped_files = std::make_unique<file_storage>(files()); } + m_mapped_files->rename_file(index, new_filename); + } + + void pread_storage::release_files(storage_error&) + { + if (m_part_file) + { + error_code ignore; + m_part_file->flush_metadata(ignore); + } + + // make sure we don't have the files open + m_pool.release(storage_index()); + + // make sure we can pick up new files added to the download directory when + // we start the torrent again + m_stat_cache.clear(); + } + + void pread_storage::delete_files(remove_flags_t const options, storage_error& ec) + { + // make sure we don't have the files open + m_pool.release(storage_index()); + + // if there's a part file open, make sure to destruct it to have it + // release the underlying part file. Otherwise we may not be able to + // delete it + if (m_part_file) m_part_file.reset(); + + aux::delete_files(files(), m_save_path, m_part_file_name, options, ec); + } + + bool pread_storage::verify_resume_data(add_torrent_params const& rd + , aux::vector<std::string, file_index_t> const& links + , storage_error& ec) + { + return aux::verify_resume_data(rd, links, files() + , m_file_priority, m_stat_cache, m_save_path, ec); + } + + std::pair<status_t, std::string> pread_storage::move_storage(std::string save_path + , move_flags_t const flags, storage_error& ec) + { + m_pool.release(storage_index()); + + status_t ret; + auto move_partfile = [&](std::string const& new_save_path, error_code& e) + { + if (!m_part_file) return; + m_part_file->move_partfile(new_save_path, e); + }; + std::tie(ret, m_save_path) = aux::move_storage(files(), m_save_path, std::move(save_path) + , std::move(move_partfile), flags, ec); + + // clear the stat cache in case the new location has new files + m_stat_cache.clear(); + + return { ret, m_save_path }; + } + + int pread_storage::read(settings_interface const& sett + , span<char> buffer + , piece_index_t const piece, int const offset + , open_mode_t const mode + , disk_job_flags_t const flags + , storage_error& error) + { +#ifdef TORRENT_SIMULATE_SLOW_READ + std::this_thread::sleep_for(seconds(1)); +#endif + return readwrite(files(), buffer, piece, offset, error + , [this, mode, flags, &sett](file_index_t const file_index + , std::int64_t const file_offset + , span<char> buf, storage_error& ec) + { + // reading from a pad file yields zeroes + if (files().pad_file_at(file_index)) return read_zeroes(buf); + + if (file_index < m_file_priority.end_index() + && m_file_priority[file_index] == dont_download + && use_partfile(file_index)) + { + TORRENT_ASSERT(m_part_file); + + error_code e; + peer_request map = files().map_file(file_index, file_offset, 0); + int const ret = m_part_file->read(buf, map.piece, map.start, e); + + if (e) + { + ec.ec = e; + ec.file(file_index); + ec.operation = operation_t::partfile_read; + return -1; + } + return ret; + } + + auto handle = open_file(sett, file_index, mode, ec); + if (ec) return -1; + + // set this unconditionally in case the upper layer would like to treat + // short reads as errors + ec.operation = operation_t::file_read; + + int const ret = pread_all(handle->fd(), buf, file_offset, ec.ec); + if (ec.ec) { + ec.file(file_index); + return ret; + } + if (flags & disk_interface::volatile_read) + advise_dont_need(handle->fd(), file_offset, buf.size()); + + return ret; + }); + } + + int pread_storage::write(settings_interface const& sett + , span<span<char> const> buffers + , piece_index_t const piece, int offset + , open_mode_t const mode + , disk_job_flags_t const flags + , storage_error& error) + { + for (auto const& buf : buffers) + { + write(sett, buf, piece, offset, mode, flags, error); + offset += int(buf.size()); + if (error) return offset; + } + return offset; + } + + int pread_storage::write(settings_interface const& sett + , span<char> buffer + , piece_index_t const piece, int const offset + , open_mode_t const mode + , disk_job_flags_t + , storage_error& error) + { + auto const write_mode = sett.get_int(settings_pack::disk_io_write_mode); + return readwrite(files(), buffer, piece, offset, error + , [this, mode, &sett, write_mode](file_index_t const file_index + , std::int64_t const file_offset + , span<char> buf, storage_error& ec) + { + // writing to a pad-file is a no-op + if (files().pad_file_at(file_index)) + return int(buf.size()); + + if (file_index < m_file_priority.end_index() + && m_file_priority[file_index] == dont_download + && use_partfile(file_index)) + { + TORRENT_ASSERT(m_part_file); + + error_code e; + peer_request map = files().map_file(file_index + , file_offset, 0); + int const ret = m_part_file->write(buf, map.piece, map.start, e); + + if (e) + { + ec.ec = e; + ec.file(file_index); + ec.operation = operation_t::partfile_write; + return -1; + } + return ret; + } + + // invalidate our stat cache for this file, since + // we're writing to it + m_stat_cache.set_dirty(file_index); + + auto handle = open_file(sett, file_index, open_mode::write | mode, ec); + if (ec) return -1; + TORRENT_ASSERT(handle); + + // set this unconditionally in case the upper layer would like to treat + // short reads as errors + ec.operation = operation_t::file_write; + + int const ret = pwrite_all(handle->fd(), buf, file_offset, ec.ec); + if (ec.ec) + { + ec.file(file_index); + ec.operation = operation_t::file_write; + return ret; + } + if (write_mode == settings_pack::write_through) + sync_file(handle->fd(), file_offset, buf.size()); + return ret; + }); + } + + int pread_storage::hash(settings_interface const& sett + , hasher& ph, std::ptrdiff_t const len + , piece_index_t const piece, int const offset + , open_mode_t const mode + , disk_job_flags_t const flags + , storage_error& error) + { +#ifdef TORRENT_SIMULATE_SLOW_READ + std::this_thread::sleep_for(seconds(1)); +#endif + char dummy = 0; + + std::vector<char> scratch_buffer; + + return readwrite(files(), {&dummy, len}, piece, offset, error + , [this, mode, flags, &ph, &sett, &scratch_buffer]( + file_index_t const file_index + , std::int64_t const file_offset + , span<char> buf, storage_error& ec) + { + if (files().pad_file_at(file_index)) + return hash_zeroes(ph, buf.size()); + + if (file_index < m_file_priority.end_index() + && m_file_priority[file_index] == dont_download + && use_partfile(file_index)) + { + error_code e; + peer_request map = files().map_file(file_index, file_offset, 0); + int const ret = m_part_file->hash(ph, buf.size() + , map.piece, map.start, e); + + if (e) + { + ec.ec = e; + ec.file(file_index); + ec.operation = operation_t::partfile_read; + } + return ret; + } + + auto handle = open_file(sett, file_index, mode, ec); + if (ec) return -1; + + scratch_buffer.resize(std::size_t(buf.size())); + int ret = pread_all(handle->fd(), scratch_buffer, file_offset, ec.ec); + if (ec.ec) + { + ec.file(file_index); + ec.operation = operation_t::file_read; + return ret; + } + if (ret >= 0) + { + ph.update(scratch_buffer); + if (flags & disk_interface::volatile_read) + advise_dont_need(handle->fd(), file_offset, buf.size()); + if (flags & disk_interface::flush_piece) + sync_file(handle->fd(), file_offset, buf.size()); + } + + return ret; + }); + } + + int pread_storage::hash2(settings_interface const& sett + , hasher256& ph, std::ptrdiff_t const len + , piece_index_t const piece, int const offset + , open_mode_t const mode + , disk_job_flags_t const flags + , storage_error& error) + { + std::int64_t const start_offset = static_cast<int>(piece) * std::int64_t(files().piece_length()) + offset; + file_index_t const file_index = files().file_index_at_offset(start_offset); + std::int64_t const file_offset = start_offset - files().file_offset(file_index); + TORRENT_ASSERT(file_offset >= 0); + TORRENT_ASSERT(!files().pad_file_at(file_index)); + + if (file_index < m_file_priority.end_index() + && m_file_priority[file_index] == dont_download + && use_partfile(file_index)) + { + error_code e; + peer_request map = files().map_file(file_index, file_offset, 0); + int const ret = m_part_file->hash2(ph, len, map.piece, map.start, e); + + if (e) + { + error.ec = e; + error.file(file_index); + error.operation = operation_t::partfile_read; + return -1; + } + return ret; + } + + auto handle = open_file(sett, file_index, mode, error); + if (error) return -1; + + std::unique_ptr<char[]> scratch_buffer(new char[std::size_t(len)]); + span<char> b = {scratch_buffer.get(), len}; + int const ret = pread_all(handle->fd(), b, file_offset, error.ec); + if (error.ec) + { + error.operation = operation_t::file_read; + error.file(file_index); + return ret; + } + ph.update(b); + if (flags & disk_interface::volatile_read) + advise_dont_need(handle->fd(), file_offset, len); + if (flags & disk_interface::flush_piece) + sync_file(handle->fd(), file_offset, len); + + return static_cast<int>(len); + } + + // a wrapper around open_file_impl that, if it fails, makes sure the + // directories have been created and retries + std::shared_ptr<file_handle> pread_storage::open_file(settings_interface const& sett + , file_index_t const file + , open_mode_t mode, storage_error& ec) const + { + if (mode & open_mode::write + && !(mode & open_mode::truncate)) + { + std::unique_lock<std::mutex> l(m_file_created_mutex); + if (m_file_created.size() != files().num_files()) + m_file_created.resize(files().num_files(), false); + + // if we haven't created this file already, make sure to truncate it to + // its final size + mode |= (m_file_created[file] == false) ? open_mode::truncate : open_mode::read_only; + } + + if (files().file_flags(file) & file_storage::flag_executable) + mode |= open_mode::executable; + + if (files().file_flags(file) & file_storage::flag_hidden) + mode |= open_mode::hidden; + +#ifdef _WIN32 + if (sett.get_bool(settings_pack::enable_set_file_valid_data)) + { + mode |= open_mode::allow_set_file_valid_data; + } +#endif + + auto h = open_file_impl(sett, file, mode, ec); + if (ec.ec) + { + ec.file(file); + return {}; + } + TORRENT_ASSERT(h); + + if (mode & open_mode::truncate) + { + // remember that we've truncated this file, so we don't have to do it + // again + std::unique_lock<std::mutex> l(m_file_created_mutex); + m_file_created.set_bit(file); + } + + // the optional should be set here + TORRENT_ASSERT(static_cast<bool>(h)); + return h; + } + + std::shared_ptr<file_handle> pread_storage::open_file_impl(settings_interface const& sett + , file_index_t file + , open_mode_t mode + , storage_error& ec) const + { + TORRENT_ASSERT(!files().pad_file_at(file)); + if (!m_allocate_files) mode |= open_mode::sparse; + + // files with priority 0 should always be sparse + if (m_file_priority.end_index() > file && m_file_priority[file] == dont_download) + mode |= open_mode::sparse; + + if (sett.get_bool(settings_pack::no_atime_storage)) + { + mode |= open_mode::no_atime; + } + + // if we have a cache already, don't store the data twice by leaving it in the OS cache as well + auto const write_mode = sett.get_int(settings_pack::disk_io_write_mode); + if (write_mode == settings_pack::disable_os_cache + || write_mode == settings_pack::write_through) + { + mode |= open_mode::no_cache; + } + + try { + return m_pool.open_file(storage_index(), m_save_path, file + , files(), mode +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + , nullptr +#endif + ); + } + catch (storage_error const& se) + { + ec = se; + ec.file(file); + TORRENT_ASSERT(ec); + return {}; + } + } + + bool pread_storage::tick() + { + error_code ec; + if (m_part_file) m_part_file->flush_metadata(ec); + + return false; + } +} // namespace libtorrent::aux diff --git a/src/session.cpp b/src/session.cpp index 9714d039ea6..ed44ba97b2c 100644 --- a/src/session.cpp +++ b/src/session.cpp @@ -19,6 +19,7 @@ see LICENSE file. #include "libtorrent/disk_interface.hpp" #include "libtorrent/mmap_disk_io.hpp" #include "libtorrent/posix_disk_io.hpp" +#include "libtorrent/pread_disk_io.hpp" #include "libtorrent/aux_/platform_util.hpp" namespace libtorrent { @@ -491,7 +492,9 @@ namespace { TORRENT_EXPORT std::unique_ptr<disk_interface> default_disk_io_constructor( io_context& ios, settings_interface const& sett, counters& cnt) { -#if TORRENT_HAVE_MMAP || TORRENT_HAVE_MAP_VIEW_OF_FILE +#if TORRENT_HAVE_PREAD || defined TORRENT_WINDOWS + return pread_disk_io_constructor(ios, sett, cnt); +#elif TORRENT_HAVE_MMAP || TORRENT_HAVE_MAP_VIEW_OF_FILE // TODO: In C++17. use if constexpr instead #include "libtorrent/aux_/disable_deprecation_warnings_push.hpp" if (sizeof(void*) == 8) diff --git a/src/settings_pack.cpp b/src/settings_pack.cpp index dc622e5a5ca..459d8b4c843 100644 --- a/src/settings_pack.cpp +++ b/src/settings_pack.cpp @@ -244,7 +244,7 @@ namespace { SET(initial_picker_threshold, 4, nullptr), SET(allowed_fast_set_size, 5, nullptr), SET(suggest_mode, settings_pack::no_piece_suggestions, nullptr), - SET(max_queued_disk_bytes, 1024 * 1024, nullptr), + SET(max_queued_disk_bytes, 50 * 1024 * 1024, nullptr), SET(handshake_timeout, 10, nullptr), SET(send_buffer_low_watermark, 10 * 1024, nullptr), SET(send_buffer_watermark, 500 * 1024, nullptr), diff --git a/src/torrent.cpp b/src/torrent.cpp index f0422a7a0d3..70462988748 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -2545,9 +2545,7 @@ bool is_downloading_state(int const st) // if the v1 hash failed the check, don't add the v2 hashes to the // merkle tree. They are most likely invalid. if (torrent_file().info_hashes().has_v2() && !bool(hash_passed[0] == false)) - { - hash_passed[1] = on_blocks_hashed(piece, block_hashes); - } + { hash_passed[1] = on_blocks_hashed(piece, block_hashes); } } else { @@ -6949,7 +6947,9 @@ namespace { return result.valid; } - if (m_picker && m_picker->is_downloading(p) && m_picker->is_piece_finished(p) + if (m_picker + && m_picker->is_downloading(p) + && !m_picker->has_piece_passed(p) && !m_picker->is_hashing(p)) { piece_passed(p); diff --git a/test/test_add_torrent.cpp b/test/test_add_torrent.cpp index cdf64bd5c41..5b19994ae0a 100644 --- a/test/test_add_torrent.cpp +++ b/test/test_add_torrent.cpp @@ -9,6 +9,7 @@ see LICENSE file. #include "test.hpp" #include "setup_transfer.hpp" // for load_file +#include "settings.hpp" // for settings() #include "libtorrent/flags.hpp" #include "libtorrent/alert_types.hpp" @@ -77,7 +78,7 @@ lt::error_code test_add_torrent(std::string file, add_torrent_test_flag_t const atp.ti.reset(); } - lt::session_params p; + lt::session_params p = settings(); p.settings.set_int(lt::settings_pack::alert_mask, lt::alert_category::error | lt::alert_category::status); p.settings.set_str(lt::settings_pack::listen_interfaces, "127.0.0.1:6881"); lt::session ses(p); diff --git a/test/test_copy_file.cpp b/test/test_copy_file.cpp index 789fce4eb16..fe6815dba67 100644 --- a/test/test_copy_file.cpp +++ b/test/test_copy_file.cpp @@ -11,6 +11,7 @@ see LICENSE file. #include "libtorrent/error_code.hpp" #include "libtorrent/aux_/mmap.hpp" #include "libtorrent/aux_/open_mode.hpp" +#include "libtorrent/aux_/storage_utils.hpp" #include "test.hpp" #include <fstream> diff --git a/test/test_file.cpp b/test/test_file.cpp index ce6c32b451a..bd06b52b011 100644 --- a/test/test_file.cpp +++ b/test/test_file.cpp @@ -18,6 +18,8 @@ see LICENSE file. #include "libtorrent/string_view.hpp" #include "libtorrent/aux_/file_view_pool.hpp" #include "libtorrent/aux_/numeric_cast.hpp" +#include "libtorrent/aux_/storage_utils.hpp" +#include "libtorrent/aux_/file_pool_impl.hpp" #include "test.hpp" #include "test_utils.hpp" #include <vector> diff --git a/test/test_storage.cpp b/test/test_storage.cpp index c51a81476b7..1594b098167 100644 --- a/test/test_storage.cpp +++ b/test/test_storage.cpp @@ -19,7 +19,9 @@ see LICENSE file. #include "libtorrent/aux_/mmap_storage.hpp" #include "libtorrent/aux_/posix_storage.hpp" +#include "libtorrent/aux_/pread_storage.hpp" #include "libtorrent/aux_/file_view_pool.hpp" +#include "libtorrent/aux_/file_pool.hpp" #include "libtorrent/hasher.hpp" #include "libtorrent/session.hpp" #include "libtorrent/session_params.hpp" @@ -45,20 +47,13 @@ see LICENSE file. using namespace std::placeholders; using namespace lt; -#if ! TORRENT_HAVE_MMAP && ! TORRENT_HAVE_MAP_VIEW_OF_FILE -namespace libtorrent { -namespace aux { - struct file_view_pool {}; -} -} -#endif - namespace { #if TORRENT_HAVE_MMAP || TORRENT_HAVE_MAP_VIEW_OF_FILE using lt::aux::mmap_storage; #endif using lt::aux::posix_storage; +using lt::aux::pread_storage; constexpr int piece_size = 16 * 1024 * 16; constexpr int half = piece_size / 2; @@ -186,6 +181,12 @@ struct file_pool_type<posix_storage> using type = int; }; +template <> +struct file_pool_type<pread_storage> +{ + using type = aux::file_pool; +}; + template <typename StorageType> std::shared_ptr<StorageType> make_storage(storage_params const& p , typename file_pool_type<StorageType>::type& fp); @@ -206,6 +207,13 @@ std::shared_ptr<posix_storage> make_storage(storage_params const& p return std::make_shared<posix_storage>(p); } +template <> +std::shared_ptr<pread_storage> make_storage(storage_params const& p + , aux::file_pool& fp) +{ + return std::make_shared<pread_storage>(p, fp); +} + template <typename StorageType, typename FilePool> std::pair<std::shared_ptr<StorageType>, std::shared_ptr<torrent_info>> setup_torrent( @@ -295,6 +303,33 @@ int read(std::shared_ptr<posix_storage> s void release_files(std::shared_ptr<posix_storage>, storage_error&) {} +int write(std::shared_ptr<pread_storage> s + , aux::session_settings const& sett + , span<char> buf + , piece_index_t const piece + , int const offset + , aux::open_mode_t mode + , storage_error& error) +{ + return s->write(sett, buf, piece, offset, mode, disk_job_flags_t{}, error); +} + +int read(std::shared_ptr<pread_storage> s + , aux::session_settings const& sett + , span<char> buf + , piece_index_t piece + , int offset + , aux::open_mode_t mode + , storage_error& ec) +{ + return s->read(sett, buf, piece, offset, mode, disk_job_flags_t{}, ec); +} + +void release_files(std::shared_ptr<pread_storage> s, storage_error& ec) +{ + s->release_files(ec); +} + std::vector<char> new_piece(std::size_t const size) { std::vector<char> ret(size); @@ -732,6 +767,17 @@ TORRENT_TEST(remove_posix_disk_io) test_remove<posix_storage>(current_working_directory()); } +TORRENT_TEST(rename_pread_disk_io) +{ + test_rename<pread_storage>(current_working_directory()); +} + +TORRENT_TEST(remove_pread_disk_io) +{ + test_remove<pread_storage>(current_working_directory()); +} + + void test_fastresume(bool const test_deprecated) { std::string test_path = current_working_directory(); @@ -769,6 +815,7 @@ void test_fastresume(bool const test_deprecated) { print_alerts(ses, "ses"); s = h.status(); + std::cout << "progress: " << s.progress << std::endl; if (s.progress == 1.0f) { std::cout << "progress: 1.0f" << std::endl; @@ -1079,6 +1126,7 @@ bool check_pattern(std::vector<char> const& buf, int counter) TORRENT_TEST(mmap_disk_io) { run_test<mmap_storage>(); } #endif TORRENT_TEST(posix_disk_io) { run_test<posix_storage>(); } +TORRENT_TEST(pread_disk_io) { run_test<pread_storage>(); } namespace { @@ -1433,6 +1481,22 @@ TORRENT_TEST(move_posix_storage_reset) test_move_storage_reset<posix_storage>(move_flags_t::reset_save_path_unchecked); } +TORRENT_TEST(move_pread_storage_to_self) +{ + test_move_storage_to_self<pread_storage>(); +} + +TORRENT_TEST(move_pread_storage_into_self) +{ + test_move_storage_into_self<pread_storage>(); +} + +TORRENT_TEST(move_pread_storage_reset) +{ + test_move_storage_reset<pread_storage>(move_flags_t::reset_save_path); + test_move_storage_reset<pread_storage>(move_flags_t::reset_save_path_unchecked); +} + TORRENT_TEST(storage_paths_string_pooling) { file_storage file_storage; diff --git a/test/test_torrent_info.cpp b/test/test_torrent_info.cpp index 94496bcc36c..3ec0545a66d 100644 --- a/test/test_torrent_info.cpp +++ b/test/test_torrent_info.cpp @@ -14,7 +14,8 @@ see LICENSE file. #include "test.hpp" #include "setup_transfer.hpp" // for load_file #include "test_utils.hpp" -#include "settings.hpp" +#include "settings.hpp" // for settings() + #include "libtorrent/file_storage.hpp" #include "libtorrent/load_torrent.hpp" #include "libtorrent/aux_/path.hpp" diff --git a/test/web_seed_suite.cpp b/test/web_seed_suite.cpp index de5ac01c9dc..e7a28325534 100644 --- a/test/web_seed_suite.cpp +++ b/test/web_seed_suite.cpp @@ -220,7 +220,10 @@ void test_transfer(lt::session& ses, lt::add_torrent_params p { bool const expect = !fs.pad_file_at(i); std::string file_path = combine_path(save_path, fs.file_path(i)); - std::printf("checking file: %s\n", file_path.c_str()); + std::printf("checking file: %s (pad-file: %d size: %" PRId64 ")\n" + , file_path.c_str() + , !expect + , fs.file_size(i)); TEST_EQUAL(exists(file_path), expect); } } @@ -338,7 +341,6 @@ int EXPORT run_http_suite(int proxy, char const* protocol { settings_pack pack = settings(); - pack.set_int(settings_pack::max_queued_disk_bytes, 256 * 1024); pack.set_str(settings_pack::listen_interfaces, test_listen_interface()); pack.set_int(settings_pack::max_retry_port_bind, 1000); pack.set_bool(settings_pack::enable_lsd, false); diff --git a/tools/disk_io_stress_test.cpp b/tools/disk_io_stress_test.cpp index 14ef3bab141..dba004db19f 100644 --- a/tools/disk_io_stress_test.cpp +++ b/tools/disk_io_stress_test.cpp @@ -10,6 +10,7 @@ see LICENSE file. #include "libtorrent/session.hpp" // for default_disk_io_constructor #include "libtorrent/disabled_disk_io.hpp" #include "libtorrent/mmap_disk_io.hpp" +#include "libtorrent/pread_disk_io.hpp" #include "libtorrent/posix_disk_io.hpp" #include "libtorrent/disk_interface.hpp" @@ -167,6 +168,8 @@ int run_test(test_case const& t) { if (t.disk_backend == "posix"_sv) disk_io = lt::posix_disk_io_constructor(ioc, pack, cnt); + else if (t.disk_backend == "pread"_sv) + disk_io = lt::pread_disk_io_constructor(ioc, pack, cnt); else if (t.disk_backend == "disabled"_sv) disk_io = lt::disabled_disk_io_constructor(ioc, pack, cnt); else @@ -253,7 +256,7 @@ int run_test(test_case const& t) { if ((job_counter & 0x1fff) == 0) { - printf("o: %d w: %d r: %d\r" + printf("o: %d w: %d r: %d \r" , outstanding , int(blocks_to_write.size()) , int(blocks_to_read.size())); diff --git a/tools/parse_session_stats.py b/tools/parse_session_stats.py index 4ef185348f2..bf27d4133c3 100755 --- a/tools/parse_session_stats.py +++ b/tools/parse_session_stats.py @@ -130,7 +130,8 @@ def process_color(c, op): def plot_fun(script): try: - ret = os.system('gnuplot "%s" 2>/dev/null' % script) + print('gnuplot "%s"' % script) + ret = os.system('gnuplot "%s"' % script) except Exception as e: print('please install gnuplot: sudo apt install gnuplot') raise e diff --git a/tools/run_benchmark.py b/tools/run_benchmark.py index 0e2dcb42048..a05e50dfc1b 100755 --- a/tools/run_benchmark.py +++ b/tools/run_benchmark.py @@ -49,10 +49,11 @@ def main(): rm_file_or_dir('t') - run_test('download-write-through', 'upload', ['-1', '--disk_io_write_mode=write_through', '-s', args.save_path], args.download_peers) - reset_download(args.save_path) - run_test('download-full-cache', 'upload', ['-1', '--disk_io_write_mode=enable_os_cache', '-s', args.save_path], args.download_peers) - run_test('upload', 'download', ['-G', '-e', '240', '-s', args.save_path], args.upload_peers) + for io_backend in ["mmap", "pread", "posix"]: + run_test(f'download-write-through-{io_backend}', 'upload', ['-i', io_backend, '-1', '--disk_io_write_mode=write_through', '-s', args.save_path], args.download_peers) + reset_download(args.save_path) + run_test(f'download-full-cache-{io_backend}', 'upload', ['-i', io_backend, '-1', '--disk_io_write_mode=enable_os_cache', '-s', args.save_path], args.download_peers) + run_test(f'upload-{io_backend}', 'download', ['-i', io_backend, '-G', '-e', '240', '-s', args.save_path], args.upload_peers) def run_test(name, test_cmd, client_arg, num_peers): From ba4567a3ff25b99e96419e115911d98d0ac53d53 Mon Sep 17 00:00:00 2001 From: arvidn <arvid@libtorrent.org> Date: Sat, 24 Feb 2024 19:33:43 +0100 Subject: [PATCH 2/7] fix clang-tidy warnings --- src/disk_cache.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/disk_cache.cpp b/src/disk_cache.cpp index 63bd6192084..3bdde5b26f3 100644 --- a/src/disk_cache.cpp +++ b/src/disk_cache.cpp @@ -84,7 +84,6 @@ cached_piece_entry::cached_piece_entry(piece_location const& loc, int const num_ , piece_size2(piece_size_v2) , blocks_in_piece(num_blocks) , blocks(aux::make_unique<cached_block_entry[], std::ptrdiff_t>(num_blocks)) - , ph(hasher()) {} span<cached_block_entry> cached_piece_entry::get_blocks() const @@ -199,7 +198,7 @@ disk_cache::hash_result disk_cache::try_hash_piece(piece_location const loc, pre view.modify(i, [&](cached_piece_entry& e) { e.piece_hash_returned = true; - job::hash& job = std::get<aux::job::hash>(hash_job->action); + auto& job = std::get<aux::job::hash>(hash_job->action); job.piece_hash = e.ph.final(); if (!job.block_hashes.empty()) { @@ -258,7 +257,7 @@ void disk_cache::kick_hasher(piece_location const& loc, jobqueue_t& completed_jo } auto const blocks = blocks_storage.first(block_idx); - hasher& ctx = const_cast<hasher&>(piece_iter->ph); + auto& ctx = const_cast<hasher&>(piece_iter->ph); view.modify(piece_iter, [](cached_piece_entry& e) { e.hashing = true; }); @@ -319,7 +318,7 @@ void disk_cache::kick_hasher(piece_location const& loc, jobqueue_t& completed_jo // this piece, post it. sha1_hash const piece_hash = ctx.final(); - job::hash& job = std::get<job::hash>(j->action); + auto& job = std::get<job::hash>(j->action); job.piece_hash = piece_hash; if (!job.block_hashes.empty()) { From 9ad0cfd263d9a969cf26561f54b4a8aa6dd4e269 Mon Sep 17 00:00:00 2001 From: arvidn <arvid@libtorrent.org> Date: Sun, 25 Feb 2024 01:53:32 +0100 Subject: [PATCH 3/7] fixup comments --- include/libtorrent/aux_/disk_cache.hpp | 8 -------- src/pread_disk_io.cpp | 5 +++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/include/libtorrent/aux_/disk_cache.hpp b/include/libtorrent/aux_/disk_cache.hpp index ad5a2b5d095..1255e506b78 100644 --- a/include/libtorrent/aux_/disk_cache.hpp +++ b/include/libtorrent/aux_/disk_cache.hpp @@ -316,14 +316,6 @@ struct disk_cache post_job, }; - // this call can have 3 outcomes: - // 1. the job is immediately satisfied and should be posted to the - // completion queue - // 2. The piece is in the cache and currently hashing, but it's not done - // yet. We hang the hash job on the piece itself so the hashing thread - // can complete it when hashing finishes - // 3. The piece is not in the cache and should be posted to the disk thread - // to read back the bytes. hash_result try_hash_piece(piece_location const loc, pread_disk_job* hash_job); // this should be called from a hasher thread diff --git a/src/pread_disk_io.cpp b/src/pread_disk_io.cpp index 63d4c58f3dd..621c1e60e6f 100644 --- a/src/pread_disk_io.cpp +++ b/src/pread_disk_io.cpp @@ -708,8 +708,9 @@ void pread_disk_io::async_hash2(storage_index_t const storage sha256_hash{} ); - // TODO: check the disk cache here - + // In theory, we could check the cache for this block hash, but we + // only retain cached_piece_entries until the main piece hash has been + // returned, asking for individual blocks may not be available add_job(j); } From a9149de627d34d53da982d29f7648390b963f90e Mon Sep 17 00:00:00 2001 From: arvidn <arvid@libtorrent.org> Date: Sun, 25 Feb 2024 13:35:36 +0100 Subject: [PATCH 4/7] factor out common code from flush_cache_blocks --- include/libtorrent/aux_/disk_cache.hpp | 46 +++++++++ src/pread_disk_io.cpp | 138 ++++++------------------- 2 files changed, 78 insertions(+), 106 deletions(-) diff --git a/include/libtorrent/aux_/disk_cache.hpp b/include/libtorrent/aux_/disk_cache.hpp index 1255e506b78..e241f2a7aa1 100644 --- a/include/libtorrent/aux_/disk_cache.hpp +++ b/include/libtorrent/aux_/disk_cache.hpp @@ -167,6 +167,52 @@ struct cached_piece_entry pread_disk_job* clear_piece = nullptr; }; +// TODO: add unit test for this function +template <typename Fun> +void visit_block_iovecs(span<aux::cached_block_entry const> blocks + , Fun const& f) +{ + TORRENT_ASSERT(blocks.size() > 0); + TORRENT_ALLOCA(iovec, span<char>, blocks.size()); + + int count = 0; + + int start_idx = 0; + int idx = 0; + + for (auto& be : blocks) + { + auto* j = be.write_job; + if (count > 0 && j == nullptr) + { + bool const interrupt = f(iovec.first(count), start_idx); + if (interrupt) return; + + start_idx = idx; + count = 0; + } + + if (j == nullptr) + { + ++idx; + start_idx = idx; + continue; + } + + TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); + auto& a = std::get<aux::job::write>(j->action); + + iovec[count] = span<char>{ a.buf.data(), a.buffer_size}; + ++count; + ++idx; + } + + if (count > 0) + { + f(iovec.first(count), start_idx); + } +} + struct disk_cache { using piece_container = mi::multi_index_container< diff --git a/src/pread_disk_io.cpp b/src/pread_disk_io.cpp index 621c1e60e6f..436b4626be7 100644 --- a/src/pread_disk_io.cpp +++ b/src/pread_disk_io.cpp @@ -1307,128 +1307,54 @@ int pread_disk_io::flush_cache_blocks(bitfield& flushed m_stats_counters.inc_stats_counter(counters::num_writing_threads, 1); time_point const start_time = clock_type::now(); - TORRENT_ALLOCA(iovec, span<char>, blocks.size()); bool failed = false; - int count = 0; - int start_idx = 0; - int idx = 0; // the total number of blocks we ended up flushing to disk int ret = 0; - // the piece offset of the start of the range of contiguous blocks we're - // currently assembling into iovec - int start_offset = 0; - - // the offset of the end of the range of contiguous blocks we're currently - // assembing - int end_offset = 0; - - aux::open_mode_t file_mode; - auto piece = piece_index_t(-1); - disk_job_flags_t flags; - - std::shared_ptr<aux::pread_storage> storage; - - storage_error error; - // TODO: refactor this loop into an iterator adapter that returns - // contiguous ranges of blocks. Then de-duplicate the write-to-disk logic - // into the loop - TORRENT_ASSERT(blocks.size() > 0); - for (auto& be : blocks) - { - auto* j = be.write_job; - - auto const job_offset = [&] { - if (j != nullptr) - return std::get<aux::job::write>(j->action).offset; - else - return 0; - }(); - - if (!storage && j) storage = j->storage; - if (count > 0 && (j == nullptr || job_offset > end_offset)) - { - TORRENT_ASSERT(piece != piece_index_t(-1)); - DLOG("write: blocks: %d (piece: %d)\n", count, int(piece)); - storage->write(m_settings, iovec.first(count) - , piece, start_offset, file_mode, flags, error); - - int i = start_idx; - for (aux::cached_block_entry const& blk : blocks.subspan(start_idx, count)) - { - auto* j2 = blk.write_job; - TORRENT_ASSERT(j2); - TORRENT_ASSERT(j2->get_type() == aux::job_action_t::write); - j2->error = error; - flushed.set_bit(i); - completed_jobs.push_back(j2); - ++i; - } - - if (error) { - // if there was a failure, fail the remaining jobs as well - for (int k = start_idx + count; k < blocks.size(); ++k) - { - auto* j2 = be.write_job; - if (j2 == nullptr) continue; - j2->error = error; - // TODO: should we free the job's buffer here? - completed_jobs.push_back(j2); - } - failed = true; - break; - } - - ret += count; - - start_offset = job_offset; - start_idx = idx; - count = 0; - } - - if (j == nullptr) - { - ++idx; - start_idx = idx; - continue; - } - - TORRENT_ASSERT(j->storage == storage); + visit_block_iovecs(blocks, [&] (span<span<char>> iovec, int const start_idx) { + auto* j = blocks[start_idx].write_job; TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); auto& a = std::get<aux::job::write>(j->action); + aux::open_mode_t const file_mode = file_mode_for_job(j); + aux::pread_storage* storage = j->storage.get(); - if (count == 0) start_offset = job_offset; - iovec[count] = span<char>{ a.buf.data(), a.buffer_size}; - ++count; - flags = j->flags; - piece = a.piece; - file_mode = file_mode_for_job(j); - end_offset = job_offset + a.buffer_size; - ++idx; - } + TORRENT_ASSERT(a.piece != piece_index_t(-1)); + int const count = static_cast<int>(iovec.size()); + DLOG("write: blocks: %d (piece: %d)\n", count, int(a.piece)); - if (count > 0) - { - DLOG("write: blocks: %d (piece: %d)\n", count, int(piece)); - storage->write(m_settings, iovec.first(count) - , piece, start_offset, file_mode, flags, error); + storage_error error; + storage->write(m_settings, iovec + , a.piece, a.offset, file_mode, j->flags, error); int i = start_idx; for (aux::cached_block_entry const& blk : blocks.subspan(start_idx, count)) { - auto* j = blk.write_job; - TORRENT_ASSERT(j); - TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); - j->error = error; + auto* j2 = blk.write_job; + TORRENT_ASSERT(j2); + TORRENT_ASSERT(j2->get_type() == aux::job_action_t::write); + j2->error = error; flushed.set_bit(i); - completed_jobs.push_back(j); + completed_jobs.push_back(j2); ++i; } - // TODO: if we failed, post the remaining block's jobs as failures too - if (error) failed = true; - else ret += count; - } + + ret += count; + + if (error) { + // if there was a failure, fail the remaining jobs as well + for (aux::cached_block_entry const& blk : blocks.subspan(start_idx + count)) + { + auto* j2 = blk.write_job; + if (j2 == nullptr) continue; + j2->error = error; + // TODO: should we free the job's buffer here? + completed_jobs.push_back(j2); + } + failed = true; + } + return failed; + }); if (!failed) { From b12ea03ca475d99201aeb58c1e1dad5e7dc66957 Mon Sep 17 00:00:00 2001 From: arvidn <arvid@libtorrent.org> Date: Sun, 10 Mar 2024 14:21:58 +0100 Subject: [PATCH 5/7] factor out visit_block_iovecs into its own header file --- CMakeLists.txt | 1 + Makefile | 1 + include/libtorrent/aux_/disk_cache.hpp | 54 +++------------- include/libtorrent/aux_/pread_storage.hpp | 4 +- .../libtorrent/aux_/visit_block_iovecs.hpp | 62 +++++++++++++++++++ src/disk_cache.cpp | 14 ++++- src/pread_disk_io.cpp | 3 +- src/pread_storage.cpp | 8 +-- test/Jamfile | 1 + 9 files changed, 94 insertions(+), 54 deletions(-) create mode 100644 include/libtorrent/aux_/visit_block_iovecs.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5cb31258987..1f7cc231868 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,7 @@ set(libtorrent_aux_include_files disable_warnings_push.hpp disk_buffer_pool.hpp disk_cache.hpp + visit_block_iovecs.hpp disk_completed_queue.hpp mmap_disk_job.hpp disk_job.hpp diff --git a/Makefile b/Makefile index 9f9d169f54a..8ed34d4c47d 100644 --- a/Makefile +++ b/Makefile @@ -565,6 +565,7 @@ HEADERS = \ aux_/disable_warnings_push.hpp \ aux_/disk_buffer_pool.hpp \ aux_/disk_cache.hpp \ + aux_/visit_block_iovecs.hpp \ aux_/disk_completed_queue.hpp \ aux_/disk_io_thread_pool.hpp \ aux_/disk_job_fence.hpp \ diff --git a/include/libtorrent/aux_/disk_cache.hpp b/include/libtorrent/aux_/disk_cache.hpp index e241f2a7aa1..c23a57a7008 100644 --- a/include/libtorrent/aux_/disk_cache.hpp +++ b/include/libtorrent/aux_/disk_cache.hpp @@ -73,8 +73,16 @@ inline size_t hash_value(piece_location const& l) struct cached_block_entry { + // returns the buffer associated with this block. It either picks it from + // the write job that's hung on this block, or from the buffer in the block + // object, if it has been flushed to disk already. + // If there is no buffer, it returns an empty span. span<char const> buf() const; + // returns the buffer associated with the write job hanging on this block. + // If there is no write job, it returns an empty span. + span<char const> write_buf() const; + // once the write job has been executed, and we've flushed the buffer, we // move it into buf_holder, to keep the buffer alive until any hash job has // completed as well. The underlying data can be accessed through buf, but @@ -167,52 +175,6 @@ struct cached_piece_entry pread_disk_job* clear_piece = nullptr; }; -// TODO: add unit test for this function -template <typename Fun> -void visit_block_iovecs(span<aux::cached_block_entry const> blocks - , Fun const& f) -{ - TORRENT_ASSERT(blocks.size() > 0); - TORRENT_ALLOCA(iovec, span<char>, blocks.size()); - - int count = 0; - - int start_idx = 0; - int idx = 0; - - for (auto& be : blocks) - { - auto* j = be.write_job; - if (count > 0 && j == nullptr) - { - bool const interrupt = f(iovec.first(count), start_idx); - if (interrupt) return; - - start_idx = idx; - count = 0; - } - - if (j == nullptr) - { - ++idx; - start_idx = idx; - continue; - } - - TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); - auto& a = std::get<aux::job::write>(j->action); - - iovec[count] = span<char>{ a.buf.data(), a.buffer_size}; - ++count; - ++idx; - } - - if (count > 0) - { - f(iovec.first(count), start_idx); - } -} - struct disk_cache { using piece_container = mi::multi_index_container< diff --git a/include/libtorrent/aux_/pread_storage.hpp b/include/libtorrent/aux_/pread_storage.hpp index 6c8f2b3ef01..18c187d5cd2 100644 --- a/include/libtorrent/aux_/pread_storage.hpp +++ b/include/libtorrent/aux_/pread_storage.hpp @@ -68,12 +68,12 @@ namespace libtorrent::aux { , piece_index_t piece, int offset, aux::open_mode_t mode , disk_job_flags_t flags , storage_error&); - int write(settings_interface const&, span<char> buffer + int write(settings_interface const&, span<char const> buffer , piece_index_t piece, int offset, aux::open_mode_t mode , disk_job_flags_t flags , storage_error&); int write(settings_interface const& sett - , span<span<char> const> buffers + , span<span<char const> const> buffers , piece_index_t const piece, int offset , open_mode_t const mode , disk_job_flags_t const flags diff --git a/include/libtorrent/aux_/visit_block_iovecs.hpp b/include/libtorrent/aux_/visit_block_iovecs.hpp new file mode 100644 index 00000000000..84f738d8843 --- /dev/null +++ b/include/libtorrent/aux_/visit_block_iovecs.hpp @@ -0,0 +1,62 @@ +/* + +Copyright (c) 2023, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#ifndef TORRENT_VISIT_BLOCK_IOVECS +#define TORRENT_VISIT_BLOCK_IOVECS + +#include "libtorrent/span.hpp" + +namespace libtorrent::aux { + +// Fun is a function object that's called with f(span<span<char const>>, int) +template <typename Fun, typename BlockEntry> +void visit_block_iovecs(span<BlockEntry const> blocks + , Fun const& f) +{ + TORRENT_ASSERT(blocks.size() > 0); + TORRENT_ALLOCA(iovec, span<char const>, blocks.size()); + + int count = 0; + + int start_idx = 0; + int idx = 0; + + for (auto& be : blocks) + { + auto const buf = be.write_buf(); + if (count > 0 && buf.empty()) + { + bool const interrupt = f(iovec.first(count), start_idx); + if (interrupt) return; + + start_idx = idx; + count = 0; + } + + if (buf.empty()) + { + ++idx; + start_idx = idx; + continue; + } + + iovec[count] = buf; + ++count; + ++idx; + } + + if (count > 0) + { + f(iovec.first(count), start_idx); + } +} + +} + +#endif diff --git a/src/disk_cache.cpp b/src/disk_cache.cpp index 3bdde5b26f3..1342c126ba2 100644 --- a/src/disk_cache.cpp +++ b/src/disk_cache.cpp @@ -66,7 +66,8 @@ int count_jobs(span<const cached_block_entry> blocks) } -span<char const> cached_block_entry::buf() const { +span<char const> cached_block_entry::buf() const +{ if (buf_holder) return {buf_holder.data(), buf_holder.size()}; @@ -79,6 +80,17 @@ span<char const> cached_block_entry::buf() const { return {nullptr, 0}; } +span<char const> cached_block_entry::write_buf() const +{ + if (write_job != nullptr) + { + TORRENT_ASSERT(write_job->get_type() == aux::job_action_t::write); + auto const& job = std::get<job::write>(write_job->action); + return {job.buf.data(), job.buffer_size}; + } + return {nullptr, 0}; +} + cached_piece_entry::cached_piece_entry(piece_location const& loc, int const num_blocks, int const piece_size_v2) : piece(loc) , piece_size2(piece_size_v2) diff --git a/src/pread_disk_io.cpp b/src/pread_disk_io.cpp index 436b4626be7..286334bf1c2 100644 --- a/src/pread_disk_io.cpp +++ b/src/pread_disk_io.cpp @@ -25,6 +25,7 @@ see LICENSE file. #include "libtorrent/aux_/disk_job_pool.hpp" #include "libtorrent/aux_/disk_io_thread_pool.hpp" #include "libtorrent/aux_/disk_cache.hpp" +#include "libtorrent/aux_/visit_block_iovecs.hpp" #include "libtorrent/aux_/time.hpp" #include "libtorrent/add_torrent_params.hpp" #include "libtorrent/aux_/numeric_cast.hpp" @@ -1312,7 +1313,7 @@ int pread_disk_io::flush_cache_blocks(bitfield& flushed // the total number of blocks we ended up flushing to disk int ret = 0; - visit_block_iovecs(blocks, [&] (span<span<char>> iovec, int const start_idx) { + visit_block_iovecs(blocks, [&] (span<span<char const>> iovec, int const start_idx) { auto* j = blocks[start_idx].write_job; TORRENT_ASSERT(j->get_type() == aux::job_action_t::write); auto& a = std::get<aux::job::write>(j->action); diff --git a/src/pread_storage.cpp b/src/pread_storage.cpp index a5e11b99817..1427a5005bc 100644 --- a/src/pread_storage.cpp +++ b/src/pread_storage.cpp @@ -498,7 +498,7 @@ namespace { } int pread_storage::write(settings_interface const& sett - , span<span<char> const> buffers + , span<span<char const> const> buffers , piece_index_t const piece, int offset , open_mode_t const mode , disk_job_flags_t const flags @@ -514,17 +514,17 @@ namespace { } int pread_storage::write(settings_interface const& sett - , span<char> buffer + , span<char const> buffer , piece_index_t const piece, int const offset , open_mode_t const mode , disk_job_flags_t , storage_error& error) { auto const write_mode = sett.get_int(settings_pack::disk_io_write_mode); - return readwrite(files(), buffer, piece, offset, error + return readwrite(files(), reinterpret_cast<span<char>>(buffer), piece, offset, error , [this, mode, &sett, write_mode](file_index_t const file_index , std::int64_t const file_offset - , span<char> buf, storage_error& ec) + , span<char const> buf, storage_error& ec) { // writing to a pad-file is a no-op if (files().pad_file_at(file_index)) diff --git a/test/Jamfile b/test/Jamfile index 4f3b892dd76..c8a5a4b83c6 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -200,6 +200,7 @@ run test_remap_files.cpp ; run test_similar_torrent.cpp ; run test_truncate.cpp ; run test_copy_file.cpp ; +run test_disk_cache.cpp ; # turn these tests into simulations run test_resume.cpp ; From afe049ea33d5889b93335c7c57a325ae9529c5da Mon Sep 17 00:00:00 2001 From: arvidn <arvid@libtorrent.org> Date: Sun, 10 Mar 2024 16:03:19 +0100 Subject: [PATCH 6/7] add test for visit_block_iovecs --- .../libtorrent/aux_/visit_block_iovecs.hpp | 2 + src/pread_storage.cpp | 7 +- test/test_disk_cache.cpp | 171 ++++++++++++++++++ 3 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 test/test_disk_cache.cpp diff --git a/include/libtorrent/aux_/visit_block_iovecs.hpp b/include/libtorrent/aux_/visit_block_iovecs.hpp index 84f738d8843..fa6da043ead 100644 --- a/include/libtorrent/aux_/visit_block_iovecs.hpp +++ b/include/libtorrent/aux_/visit_block_iovecs.hpp @@ -11,10 +11,12 @@ see LICENSE file. #define TORRENT_VISIT_BLOCK_IOVECS #include "libtorrent/span.hpp" +#include "libtorrent/aux_/alloca.hpp" namespace libtorrent::aux { // Fun is a function object that's called with f(span<span<char const>>, int) +// and is expected to return a bool. true=interrupt, false=continue template <typename Fun, typename BlockEntry> void visit_block_iovecs(span<BlockEntry const> blocks , Fun const& f) diff --git a/src/pread_storage.cpp b/src/pread_storage.cpp index 1427a5005bc..563c83b026e 100644 --- a/src/pread_storage.cpp +++ b/src/pread_storage.cpp @@ -28,6 +28,7 @@ see LICENSE file. #include "libtorrent/aux_/file.hpp" // for file_handle, pread_all, pwrite_all #include "libtorrent/disk_buffer_holder.hpp" #include "libtorrent/aux_/stat_cache.hpp" +#include "libtorrent/aux_/readwrite.hpp" #include "libtorrent/hex.hpp" // to_hex #include <sys/types.h> @@ -521,7 +522,7 @@ namespace { , storage_error& error) { auto const write_mode = sett.get_int(settings_pack::disk_io_write_mode); - return readwrite(files(), reinterpret_cast<span<char>>(buffer), piece, offset, error + return readwrite(files(), buffer, piece, offset, error , [this, mode, &sett, write_mode](file_index_t const file_index , std::int64_t const file_offset , span<char const> buf, storage_error& ec) @@ -590,11 +591,11 @@ namespace { std::vector<char> scratch_buffer; - return readwrite(files(), {&dummy, len}, piece, offset, error + return readwrite(files(), span<char const>{&dummy, len}, piece, offset, error , [this, mode, flags, &ph, &sett, &scratch_buffer]( file_index_t const file_index , std::int64_t const file_offset - , span<char> buf, storage_error& ec) + , span<char const> buf, storage_error& ec) { if (files().pad_file_at(file_index)) return hash_zeroes(ph, buf.size()); diff --git a/test/test_disk_cache.cpp b/test/test_disk_cache.cpp new file mode 100644 index 00000000000..3a1d22c52e8 --- /dev/null +++ b/test/test_disk_cache.cpp @@ -0,0 +1,171 @@ +/* + +Copyright (c) 2024, Arvid Norberg +All rights reserved. + +You may use, distribute and modify this code under the terms of the BSD license, +see LICENSE file. +*/ + +#include "libtorrent/aux_/visit_block_iovecs.hpp" +#include <array> +#include "test.hpp" + +using lt::span; + +namespace { + +struct tbe +{ + span<char const> write_buf() const + { + return _buf; + } + span<char const> _buf; +}; + +template <size_t N> +tbe b(char const (&literal)[N]) +{ + auto buf = span<char const>{&literal[0], N - 1}; + return tbe{buf}; +} + +std::string join(span<span<char const>> iovec) +{ + std::string ret; + for (span<char const> const& b : iovec) + { + ret.append(b.begin(), b.end()); + } + return ret; +} + +} + +TORRENT_TEST(visit_block_iovecs_full) +{ + std::array<tbe, 5> const blocks{b("a"), b("b"), b("c"), b("d"), b("e")}; + + int cnt = 0; + lt::aux::visit_block_iovecs(span<tbe const>(blocks) + , [&cnt] (span<span<char const>> iovec, int start_idx) { + TEST_EQUAL(cnt, 0); + TEST_EQUAL(start_idx, 0); + TEST_EQUAL(iovec.size(), 5); + TEST_EQUAL(join(iovec), "abcde"); + ++cnt; + return false; + }); +} + +TORRENT_TEST(visit_block_iovecs_one_hole) +{ + std::array<tbe, 5> const blocks{b("a"), b("b"), b(""), b("d"), b("e")}; + + int cnt = 0; + lt::aux::visit_block_iovecs(span<tbe const>(blocks) + , [&cnt] (span<span<char const>> iovec, int start_idx) { + switch (cnt) { + case 0: + TEST_EQUAL(start_idx, 0); + TEST_EQUAL(iovec.size(), 2); + TEST_EQUAL(join(iovec), "ab"); + break; + case 1: + TEST_EQUAL(start_idx, 3); + TEST_EQUAL(iovec.size(), 2); + TEST_EQUAL(join(iovec), "de"); + break; + default: + TORRENT_ASSERT_FAIL(); + } + ++cnt; + return false; + }); +} + +TORRENT_TEST(visit_block_iovecs_two_holes) +{ + std::array<tbe, 5> const blocks{b("a"), b(""), b("c"), b(""), b("e")}; + + int cnt = 0; + lt::aux::visit_block_iovecs(span<tbe const>(blocks) + , [&cnt] (span<span<char const>> iovec, int start_idx) { + switch (cnt) { + case 0: + TEST_EQUAL(start_idx, 0); + TEST_EQUAL(iovec.size(), 1); + TEST_EQUAL(join(iovec), "a"); + break; + case 1: + TEST_EQUAL(start_idx, 2); + TEST_EQUAL(iovec.size(), 1); + TEST_EQUAL(join(iovec), "c"); + break; + case 2: + TEST_EQUAL(start_idx, 4); + TEST_EQUAL(iovec.size(), 1); + TEST_EQUAL(join(iovec), "e"); + break; + default: + TORRENT_ASSERT_FAIL(); + } + ++cnt; + return false; + }); +} + + +TORRENT_TEST(visit_block_iovecs_interrupt) +{ + std::array<tbe, 3> const blocks{b("a"), b(""), b("c")}; + + int cnt = 0; + lt::aux::visit_block_iovecs(span<tbe const>(blocks) + , [&cnt] (span<span<char const>> iovec, int start_idx) { + switch (cnt) { + case 0: + TEST_EQUAL(start_idx, 0); + TEST_EQUAL(iovec.size(), 1); + TEST_EQUAL(join(iovec), "a"); + break; + default: + TORRENT_ASSERT_FAIL(); + } + ++cnt; + return true; + }); +} + +TORRENT_TEST(visit_block_iovecs_leading_hole) +{ + std::array<tbe, 5> const blocks{b(""), b("a"), b("b"), b("c"), b("d")}; + + int cnt = 0; + lt::aux::visit_block_iovecs(span<tbe const>(blocks) + , [&cnt] (span<span<char const>> iovec, int start_idx) { + TEST_EQUAL(cnt, 0); + TEST_EQUAL(start_idx, 1); + TEST_EQUAL(iovec.size(), 4); + TEST_EQUAL(join(iovec), "abcd"); + ++cnt; + return false; + }); +} + +TORRENT_TEST(visit_block_iovecs_trailing_hole) +{ + std::array<tbe, 5> const blocks{b("a"), b("b"), b("c"), b("d"), b("")}; + + int cnt = 0; + lt::aux::visit_block_iovecs(span<tbe const>(blocks) + , [&cnt] (span<span<char const>> iovec, int start_idx) { + TEST_EQUAL(cnt, 0); + TEST_EQUAL(start_idx, 0); + TEST_EQUAL(iovec.size(), 4); + TEST_EQUAL(join(iovec), "abcd"); + ++cnt; + return false; + }); +} From 0d41aed6cc3d8a0ed622e78f80e8a1222ead8142 Mon Sep 17 00:00:00 2001 From: arvidn <arvid@libtorrent.org> Date: Tue, 12 Mar 2024 01:04:56 +0100 Subject: [PATCH 7/7] extend the disk_io_stress_test to assert job callbacks --- tools/disk_io_stress_test.cpp | 50 +++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/tools/disk_io_stress_test.cpp b/tools/disk_io_stress_test.cpp index dba004db19f..2b8027f5d5e 100644 --- a/tools/disk_io_stress_test.cpp +++ b/tools/disk_io_stress_test.cpp @@ -234,12 +234,21 @@ int run_test(test_case const& t) std::vector<char> write_buffer(lt::default_block_size); int outstanding = 0; + std::set<int> in_flight; lt::add_torrent_params atp; - disk_io->async_check_files(tor, &atp, lt::aux::vector<std::string, lt::file_index_t>{} - , [&](lt::status_t, lt::storage_error const&) { --outstanding; }); + int job_idx = 0; + in_flight.insert(job_idx); ++outstanding; + disk_io->async_check_files(tor, &atp, lt::aux::vector<std::string, lt::file_index_t>{} + , [&, job_idx](lt::status_t, lt::storage_error const&) { + TORRENT_ASSERT(in_flight.count(job_idx)); + in_flight.erase(job_idx); + TORRENT_ASSERT(outstanding > 0); + --outstanding; + }); + ++job_idx; disk_io->submit_jobs(); while (outstanding > 0) @@ -269,8 +278,13 @@ int run_test(test_case const& t) auto const req = blocks_to_read.back(); blocks_to_read.erase(blocks_to_read.end() - 1); - disk_io->async_read(tor, req, [&, req](lt::disk_buffer_holder h, lt::storage_error const& ec) + in_flight.insert(job_idx); + ++outstanding; + disk_io->async_read(tor, req, [&, req, job_idx](lt::disk_buffer_holder h, lt::storage_error const& ec) { + TORRENT_ASSERT(in_flight.count(job_idx)); + in_flight.erase(job_idx); + TORRENT_ASSERT(outstanding > 0); --outstanding; ++job_counter; if (ec) @@ -288,8 +302,7 @@ int run_test(test_case const& t) throw std::runtime_error("read buffer mismatch!"); } }); - - ++outstanding; + ++job_idx; } } @@ -300,9 +313,14 @@ int run_test(test_case const& t) generate_block_fill(req, {write_buffer.data(), lt::default_block_size}); + in_flight.insert(job_idx); + ++outstanding; disk_io->async_write(tor, req, write_buffer.data() - , {}, [&](lt::storage_error const& ec) + , {}, [&, job_idx](lt::storage_error const& ec) { + TORRENT_ASSERT(in_flight.count(job_idx)); + in_flight.erase(job_idx); + TORRENT_ASSERT(outstanding > 0); --outstanding; ++job_counter; if (ec) @@ -313,6 +331,7 @@ int run_test(test_case const& t) throw std::runtime_error("async_write failed"); } }); + ++job_idx; if (t.flags & test_mode::read_random_order) { std::uniform_int_distribution<> d(0, blocks_to_read.end_index()); @@ -329,28 +348,37 @@ int run_test(test_case const& t) std::uniform_int_distribution<> d(0, blocks_to_read.end_index()); blocks_to_read.insert(blocks_to_read.begin() + d(random_engine), req); } - - ++outstanding; } if ((t.flags & test_mode::flush_files) && (job_counter % 500) == 499) { - disk_io->async_release_files(tor, [&]() + in_flight.insert(job_idx); + ++outstanding; + disk_io->async_release_files(tor, [&, job_idx]() { + TORRENT_ASSERT(in_flight.count(job_idx)); + in_flight.erase(job_idx); + TORRENT_ASSERT(outstanding > 0); --outstanding; ++job_counter; }); + ++job_idx; } if ((t.flags & test_mode::clear_pieces) && (job_counter % 300) == 299) { lt::piece_index_t const p = blocks_to_write.front().piece; - disk_io->async_clear_piece(tor, p, [&](lt::piece_index_t) + in_flight.insert(job_idx); + ++outstanding; + disk_io->async_clear_piece(tor, p, [&, job_idx](lt::piece_index_t) { + TORRENT_ASSERT(in_flight.count(job_idx)); + in_flight.erase(job_idx); + TORRENT_ASSERT(outstanding > 0); --outstanding; ++job_counter; }); - ++outstanding; + ++job_idx; // TODO: technically all blocks for this piece should be added // to blocks_to_write again here }
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
