merge RC_2_0 into master

.github/workflows/macos.yml

@@ -44,7 +44,7 @@ jobs:
 
       - name: install boost
         run: |
-          brew install boost-build boost openssl@1.1
+          brew install boost-build boost openssl@3
           pip3 install websockets --break-system-packages
           echo "using darwin ;" >>~/user-config.jam
 
@@ -80,7 +80,7 @@ jobs:
 
       - name: install boost
         run: |
-          brew install boost-build boost openssl@1.1
+          brew install boost-build boost openssl@3
           echo "using darwin ;" >>~/user-config.jam
 
       - name: build and run simulations
@@ -114,7 +114,7 @@ jobs:
 
       - name: install boost
         run: |
-          brew install boost-build boost openssl@1.1
+          brew install boost-build boost openssl@3
           echo "using darwin ;" >>~/user-config.jam
 
       - name: build library

.github/workflows/python.yml

@@ -80,7 +80,7 @@ jobs:
     - name: dependencies (MacOS)
       if: runner.os == 'macOS'
       run: |
-        brew install boost-build boost boost-python3 [email protected] openssl@1.1 python-setuptools
+        brew install boost-build boost boost-python3 [email protected] openssl@3 python-setuptools
         export PATH=$(brew --prefix)/opt/[email protected]/bin:$PATH
 
     - name: update package lists (linux)

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ default_language_version:
     python: python3
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
     -   id: trailing-whitespace
     #-   id: end-of-file-fixer
@@ -62,12 +62,12 @@ repos:
             tools/update_copyright.py
           )$
 - repo: https://github.com/PyCQA/autoflake
-  rev: v2.2.1
+  rev: v2.3.1
   hooks:
     - id: autoflake
       args: [--in-place, --remove-unused-variables, --remove-all-unused-imports, --remove-duplicate-keys]
 - repo: https://github.com/psf/black
-  rev: 24.2.0
+  rev: 24.8.0
   hooks:
     - id: black
       # Avoiding PR bloat
@@ -107,7 +107,7 @@ repos:
       name: black (pyi)
       types: [pyi]
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.8.0
+  rev: v1.11.2
   hooks:
     - id: mypy
       # Avoiding PR bloat
@@ -140,7 +140,7 @@ repos:
               tools/update_copyright.py
           )$
 - repo: https://github.com/PyCQA/flake8
-  rev: 7.0.0
+  rev: 7.1.1
   hooks:
   - id: flake8
     exclude: |

ChangeLog

@@ -15,6 +15,9 @@
 
 2.0.11 not released
 
+	* don't hint FADV_RANDOM on posix systems. May improve seeding performance
+	* allow boost connect while checking resume data if no_verify_files flag is set
+	* fix BEP-40 peer priority for IPv6
 	* limit piece size in torrent creator
 	* fix file pre-allocation when changing file priority (HanabishiRecca)
 	* fix uTP issue where closing the connection could corrupt the payload

docs/building.rst

@@ -76,7 +76,7 @@ Linux::
 
 Mac OS::
 
-	brew install boost-build boost openssl@1.1
+	brew install boost-build boost openssl@3
 	echo "using darwin ;" >>~/user-config.jam
 	b2 crypto=openssl release
 

include/libtorrent/alert_types.hpp

@@ -1338,7 +1338,7 @@ TORRENT_VERSION_NAMESPACE_3
 	// handle, like this:
 	// .. code:: c++
 	//
-	//	torrent_handle h = alert->handle();
+	//	torrent_handle h = alert->handle;
 	//	h.save_resume_data(torrent_handle::save_info_dict);
 	//
 	// In the handler for the save_resume_alert, you can save the resume data

include/libtorrent/aux_/open_mode.hpp

@@ -25,7 +25,7 @@ namespace aux {
 		constexpr open_mode_t no_cache = 1_bit;
 		constexpr open_mode_t truncate = 2_bit;
 		constexpr open_mode_t no_atime = 3_bit;
-		constexpr open_mode_t random_access = 4_bit;
+		constexpr open_mode_t sequential_access = 4_bit;
 		constexpr open_mode_t hidden = 5_bit;
 		constexpr open_mode_t sparse = 6_bit;
 		constexpr open_mode_t executable = 7_bit;

include/libtorrent/disk_interface.hpp

@@ -94,8 +94,8 @@ namespace file_open_mode {
 	// this generally improves disk performance.
 	constexpr file_open_mode_t no_atime = 3_bit;
 
-	// open the file for random access. This disables read-ahead
-	// logic
+	// When this is not set, the kernel is hinted that access to this file will
+	// be made sequentially.
 	constexpr file_open_mode_t random_access = 5_bit;
 
 #if TORRENT_ABI_VERSION == 1

src/file.cpp

@@ -290,7 +290,7 @@ typedef struct _FILE_ALLOCATED_RANGE_BUFFER {
 	DWORD file_flags(open_mode_t const mode)
 	{
 		return ((mode & open_mode::no_cache) ? FILE_FLAG_WRITE_THROUGH : 0)
-			| ((mode & open_mode::random_access) ? 0 : FILE_FLAG_SEQUENTIAL_SCAN)
+			| ((mode & open_mode::sequential_access) ? FILE_FLAG_SEQUENTIAL_SCAN : 0)
 			;
 	}
 
@@ -324,7 +324,8 @@ typedef struct _FILE_ALLOCATED_RANGE_BUFFER {
 		// http://support.microsoft.com/kb/2549369
 		return ((mode & open_mode::hidden) ? FILE_ATTRIBUTE_HIDDEN : FILE_ATTRIBUTE_NORMAL)
 			| ((mode & open_mode::no_cache) ? FILE_FLAG_WRITE_THROUGH : 0)
-			| ((mode & open_mode::random_access) ? 0 : FILE_FLAG_SEQUENTIAL_SCAN);
+			| ((mode & open_mode::sequential_access) ? FILE_FLAG_SEQUENTIAL_SCAN : 0)
+			;
 	}
 
 	auto create_file(native_path_string const& name, open_mode_t const mode)
@@ -537,11 +538,11 @@ file_handle::file_handle(string_view name, std::int64_t const size
 	}
 #endif
 
-#if (TORRENT_HAS_FADVISE && defined POSIX_FADV_RANDOM)
-	if (mode & aux::open_mode::random_access)
+#if (TORRENT_HAS_FADVISE && defined POSIX_FADV_SEQUENTIAL)
+	if (mode & aux::open_mode::sequential_access)
 	{
 		// disable read-ahead
-		::posix_fadvise(m_fd, 0, 0, POSIX_FADV_RANDOM);
+		::posix_fadvise(m_fd, 0, 0, POSIX_FADV_SEQUENTIAL);
 	}
 #endif
 }

src/mmap.cpp

@@ -164,7 +164,7 @@ file_mapping::file_mapping(file_handle file, open_mode_t const mode, std::int64_
 #if TORRENT_USE_MADVISE
 	if (file_size > 0)
 	{
-		int const advise = ((mode & open_mode::random_access) ? 0 : MADV_SEQUENTIAL)
+		int const advise = ((mode & open_mode::sequential_access) ? MADV_SEQUENTIAL : 0)
 #ifdef MADV_DONTDUMP
 		// on versions of linux that support it, ask for this region to not be
 		// included in coredumps (mostly to make the coredumps more manageable

src/mmap_disk_io.cpp

@@ -61,7 +61,7 @@ namespace {
 	aux::open_mode_t file_mode_for_job(aux::mmap_disk_job* j)
 	{
 		aux::open_mode_t ret = aux::open_mode::read_only;
-		if (!(j->flags & disk_interface::sequential_access)) ret |= aux::open_mode::random_access;
+		if (j->flags & disk_interface::sequential_access) ret |= aux::open_mode::sequential_access;
 		return ret;
 	}
 

src/torrent.cpp

@@ -25,6 +25,7 @@ Copyright (c) 2021, AdvenT
 Copyright (c) 2021, Joris CARRIER
 Copyright (c) 2021, Mark Scott
 Copyright (c) 2021, thrnz
+Copyright (c) 2024, Elyas EL IDRISSI
 All rights reserved.
 
 You may use, distribute and modify this code under the terms of the BSD license,
@@ -8242,8 +8243,11 @@ namespace {
 		// if we're paused, obviously we're not connecting to peers
 		if (is_paused() || m_abort || m_graceful_pause_mode) return false;
 
+		// if metadata are valid and we are either checking files or checking resume data without no_verify_files flag,
+		// we don't want peers
 		if ((m_state == torrent_status::checking_files
-			|| m_state == torrent_status::checking_resume_data)
+			|| (m_state == torrent_status::checking_resume_data
+				&& !(m_add_torrent_params && m_add_torrent_params->flags & torrent_flags::no_verify_files)))
 			&& valid_metadata())
 			return false;
 

src/torrent_peer.cpp

@@ -42,16 +42,16 @@ namespace libtorrent::aux {
 	// 4. if IPs are not in the same /16, mask the IPs by 0xffff5555, hash them
 	//    ordered, lowest first.
 	//
-	// * for IPv6 peers, just use the first 64 bits and widen the masks.
-	//   like this: 0xffff5555 -> 0xffffffff55555555
-	//   the lower 64 bits are always unmasked
+	// * for IPv6 addresses, the lower 48 bits are always unmasked
 	//
-	// * for IPv6 addresses, compare /32 and /48 instead of /16 and /24
+	// * for IPv6 addresses, compare /48, /56, /64, /72 and so on.
 	//
 	// * the two IP addresses that are used to calculate the rank must
 	//   always be of the same address family
 	//
 	// * all IP addresses are in network byte order when hashed
+	// The full specification is here:
+	// https://www.bittorrent.org/beps/bep_0040.html
 	std::uint32_t peer_priority(tcp::endpoint e1, tcp::endpoint e2)
 	{
 		TORRENT_ASSERT(aux::is_v4(e1) == aux::is_v4(e2));
@@ -73,19 +73,30 @@ namespace libtorrent::aux {
 		}
 		else if (aux::is_v6(e1))
 		{
-			static const std::uint8_t v6mask[][8] = {
-				{ 0xff, 0xff, 0xff, 0xff, 0x55, 0x55, 0x55, 0x55 },
-				{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x55, 0x55 },
-				{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }
-			};
-
 			if (e1 > e2) swap(e1, e2);
 			address_v6::bytes_type b1 = e1.address().to_v6().to_bytes();
 			address_v6::bytes_type b2 = e2.address().to_v6().to_bytes();
-			int const mask = std::memcmp(b1.data(), b2.data(), 4) ? 0
-				: std::memcmp(b1.data(), b2.data(), 6) ? 1 : 2;
-			apply_mask(b1.data(), v6mask[mask], 8);
-			apply_mask(b2.data(), v6mask[mask], 8);
+			size_t offset = 0xff;
+			for (size_t i = 0; i < b1.size(); ++i)
+			{
+				// we never mask the first 6 bytes, index 6 (the 7th byte)
+				// is the earliest we start masking at. But if the prefix is
+				// identical, we keep pushing out where we start masking
+				if (offset == 0xff && b1[i] != b2[i])
+					offset = std::max(i + 1, size_t(5));
+				else if (i > offset)
+				{
+#if defined __GNUC__ && __GNUC__ >= 7
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstringop-overflow"
+#endif
+					b1[i] &= 0x55;
+					b2[i] &= 0x55;
+#if defined __GNUC__ && __GNUC__ >= 7
+#pragma GCC diagnostic pop
+#endif
+				}
+			}
 			std::uint64_t addrbuf[4];
 			std::memcpy(&addrbuf[0], b1.data(), 16);
 			std::memcpy(&addrbuf[2], b2.data(), 16);

test/test_crc32.cpp

@@ -14,11 +14,25 @@ see LICENSE file.
 #include "libtorrent/aux_/byteswap.hpp"
 #include "test.hpp"
 
+#include "libtorrent/aux_/disable_warnings_push.hpp"
+#include <boost/crc.hpp>
+#include "libtorrent/aux_/disable_warnings_pop.hpp"
+
+namespace {
+std::uint32_t crc32c_buffer(char const* buf, int const len)
+{
+	boost::crc_optimal<32, 0x1EDC6F41, 0xFFFFFFFF, 0xFFFFFFFF, true, true> crc;
+	crc.process_block(buf, buf + len);
+	return crc.checksum();
+}
+}
+
 TORRENT_TEST(crc32)
 {
 	using namespace lt;
 
 	std::uint32_t out;
+	std::uint32_t out2;
 
 	std::uint32_t in1 = aux::host_to_network(0xeffea55a);
 	out = aux::crc32c_32(in1);
@@ -31,16 +45,22 @@ TORRENT_TEST(crc32)
 	// https://tools.ietf.org/html/rfc3720#appendix-B.4
 	out = aux::crc32c(buf, 4);
 	TEST_EQUAL(out, 0x8a9136aaU);
+	out2 = crc32c_buffer(reinterpret_cast<char const*>(buf), 32);
+	TEST_EQUAL(out, out2);
 
 	memcpy(buf, "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
 		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 32);
 	out = aux::crc32c(buf, 4);
 	TEST_EQUAL(out, 0x62a8ab43U);
+	out2 = crc32c_buffer(reinterpret_cast<char const*>(buf), 32);
+	TEST_EQUAL(out, out2);
 
 	memcpy(buf, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
 		"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", 32);
 	out = aux::crc32c(buf, 4);
 	TEST_EQUAL(out, 0x46dd794eU);
+	out2 = crc32c_buffer(reinterpret_cast<char const*>(buf), 32);
+	TEST_EQUAL(out, out2);
 
 #if !TORRENT_HAS_ARM
 	TORRENT_ASSERT(!aux::arm_crc32c_support);