diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 550c7b8c92..ff9206fb6f 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2022 The Bitcoin Core developers +# Copyright (c) 2018-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -export PATH=$PWD/ci/retry:$PATH +export CI_RETRY_EXE="/ci_retry --" + +pushd "/" ${CI_RETRY_EXE} apt-get update # Lint dependencies: @@ -19,7 +21,7 @@ ${CI_RETRY_EXE} apt-get install -y automake pkg-config libtool curl xz-utils git PYTHON_PATH="/python_build" if [ ! -d "${PYTHON_PATH}/bin" ]; then ( - ${CI_RETRY_EXE} git clone https://github.com/pyenv/pyenv.git + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/pyenv/pyenv.git cd pyenv/plugins/python-build || exit 1 ./install.sh ) @@ -28,7 +30,7 @@ if [ ! -d "${PYTHON_PATH}/bin" ]; then libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ clang - env CC=clang python-build "$(cat "./.python-version")" "${PYTHON_PATH}" + env CC=clang python-build "$(cat "/.python-version")" "${PYTHON_PATH}" fi export PATH="${PYTHON_PATH}/bin:${PATH}" command -v python3 @@ -38,7 +40,7 @@ export LINT_RUNNER_PATH="/lint_test_runner" if [ ! -d "${LINT_RUNNER_PATH}" ]; then ${CI_RETRY_EXE} apt-get install -y cargo ( - cd ./test/lint/test_runner || exit 1 + cd "/test/lint/test_runner" || exit 1 cargo build mkdir -p "${LINT_RUNNER_PATH}" mv target/debug/test_runner "${LINT_RUNNER_PATH}" @@ -62,3 +64,5 @@ MLC_VERSION=v0.18.0 MLC_BIN=mlc-x86_64-linux curl -sL "https://github.com/becheran/mlc/releases/download/${MLC_VERSION}/${MLC_BIN}" -o "/usr/bin/mlc" chmod +x /usr/bin/mlc + +popd || exit diff --git a/ci/lint_imagefile b/ci/lint_imagefile index d32b35b19d..0e7ede5204 100644 --- a/ci/lint_imagefile +++ b/ci/lint_imagefile @@ -4,11 +4,12 @@ # See test/lint/README.md for usage. -FROM debian:bookworm +FROM docker.io/debian:bookworm ENV DEBIAN_FRONTEND=noninteractive ENV LC_ALL=C.UTF-8 +COPY ./ci/retry/retry /ci_retry COPY ./.python-version /.python-version COPY ./ci/lint/container-entrypoint.sh /entrypoint.sh COPY ./ci/lint/04_install.sh /install.sh diff --git a/ci/lint_run_all.sh b/ci/lint_run_all.sh index b56ee0d303..c57261d21a 100755 --- a/ci/lint_run_all.sh +++ b/ci/lint_run_all.sh @@ -1,12 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -set -o errexit; source ./ci/test/00_setup_env.sh +# Only used in .cirrus.yml. Refer to test/lint/README.md on how to run locally. + +cp "./ci/retry/retry" "/ci_retry" +cp "./.python-version" "/.python-version" +mkdir --parents "/test/lint" +cp --recursive "./test/lint/test_runner" "/test/lint/" set -o errexit; source ./ci/lint/04_install.sh set -o errexit ./ci/lint/06_script.sh diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 23d9180f96..9ab08fb4a3 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -19,7 +19,7 @@ else fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 3fcaa8c6c6..5bfbb4cbf1 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -11,4 +11,4 @@ export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' --with-sanitizers=thread" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES' --with-sanitizers=thread" diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index f57e9abfec..46f9ee915f 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -38,13 +38,13 @@ def check_ELF_RELRO(binary) -> bool: return have_gnu_relro and have_bindnow -def check_ELF_Canary(binary) -> bool: +def check_ELF_CANARY(binary) -> bool: ''' Check for use of stack canary ''' return binary.has_symbol('__stack_chk_fail') -def check_ELF_separate_code(binary): +def check_ELF_SEPARATE_CODE(binary): ''' Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code @@ -105,7 +105,7 @@ def check_ELF_separate_code(binary): return False return True -def check_ELF_control_flow(binary) -> bool: +def check_ELF_CONTROL_FLOW(binary) -> bool: ''' Check for control flow instrumentation ''' @@ -130,7 +130,7 @@ def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' return binary.has_relocations -def check_PE_control_flow(binary) -> bool: +def check_PE_CONTROL_FLOW(binary) -> bool: ''' Check for control flow instrumentation ''' @@ -145,7 +145,7 @@ def check_PE_control_flow(binary) -> bool: return True return False -def check_PE_Canary(binary) -> bool: +def check_PE_CANARY(binary) -> bool: ''' Check for use of stack canary ''' @@ -163,7 +163,7 @@ def check_MACHO_FIXUP_CHAINS(binary) -> bool: ''' return binary.has_dyld_chained_fixups -def check_MACHO_Canary(binary) -> bool: +def check_MACHO_CANARY(binary) -> bool: ''' Check for use of stack canary ''' @@ -182,7 +182,7 @@ def check_NX(binary) -> bool: ''' return binary.has_nx -def check_MACHO_control_flow(binary) -> bool: +def check_MACHO_CONTROL_FLOW(binary) -> bool: ''' Check for control flow instrumentation ''' @@ -192,7 +192,7 @@ def check_MACHO_control_flow(binary) -> bool: return True return False -def check_MACHO_branch_protection(binary) -> bool: +def check_MACHO_BRANCH_PROTECTION(binary) -> bool: ''' Check for branch protection instrumentation ''' @@ -206,8 +206,8 @@ def check_MACHO_branch_protection(binary) -> bool: ('PIE', check_PIE), ('NX', check_NX), ('RELRO', check_ELF_RELRO), - ('Canary', check_ELF_Canary), - ('separate_code', check_ELF_separate_code), + ('CANARY', check_ELF_CANARY), + ('SEPARATE_CODE', check_ELF_SEPARATE_CODE), ] BASE_PE = [ @@ -216,19 +216,19 @@ def check_MACHO_branch_protection(binary) -> bool: ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_NX), ('RELOC_SECTION', check_PE_RELOC_SECTION), - ('CONTROL_FLOW', check_PE_control_flow), - ('Canary', check_PE_Canary), + ('CONTROL_FLOW', check_PE_CONTROL_FLOW), + ('CANARY', check_PE_CANARY), ] BASE_MACHO = [ ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('Canary', check_MACHO_Canary), + ('CANARY', check_MACHO_CANARY), ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), ] CHECKS = { lief.EXE_FORMATS.ELF: { - lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW)], lief.ARCHITECTURES.ARM: BASE_ELF, lief.ARCHITECTURES.ARM64: BASE_ELF, lief.ARCHITECTURES.PPC: BASE_ELF, @@ -240,39 +240,24 @@ def check_MACHO_branch_protection(binary) -> bool: lief.EXE_FORMATS.MACHO: { lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), ('NX', check_NX), - ('CONTROL_FLOW', check_MACHO_control_flow)], - lief.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_branch_protection)], + ('CONTROL_FLOW', check_MACHO_CONTROL_FLOW)], + lief.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)], } } if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: - try: - binary = lief.parse(filename) - etype = binary.format - arch = binary.abstract.header.architecture - binary.concrete - - if etype == lief.EXE_FORMATS.UNKNOWN: - print(f'{filename}: unknown executable format') - retval = 1 - continue - - if arch == lief.ARCHITECTURES.NONE: - print(f'{filename}: unknown architecture') - retval = 1 - continue - - failed: list[str] = [] - for (name, func) in CHECKS[etype][arch]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - except IOError: - print(f'{filename}: cannot open') + binary = lief.parse(filename) + etype = binary.format + arch = binary.abstract.header.architecture + binary.concrete + + failed: list[str] = [] + for (name, func) in CHECKS[etype][arch]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') retval = 1 sys.exit(retval) - diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index c4e6bc81e1..cff5a9b480 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -299,22 +299,14 @@ def check_ELF_ABI(binary) -> bool: if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: - try: - binary = lief.parse(filename) - etype = binary.format - if etype == lief.EXE_FORMATS.UNKNOWN: - print(f'{filename}: unknown executable format') - retval = 1 - continue - - failed: list[str] = [] - for (name, func) in CHECKS[etype]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - except IOError: - print(f'{filename}: cannot open') + binary = lief.parse(filename) + etype = binary.format + + failed: list[str] = [] + for (name, func) in CHECKS[etype]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') retval = 1 sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index de372cbd39..4bec6bfe7c 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -59,33 +59,20 @@ def test_ELF(self): arch = get_arch(cxx, source, executable) if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-zexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), - (0, '')) + pass_flags = ['-Wl,-znoexecstack', '-Wl,-zrelro', '-Wl,-z,now', '-pie', '-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-zexecstack']), (1, executable + ': failed NX')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-no-pie','-fno-PIE']), (1, executable + ': failed PIE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-znorelro']), (1, executable + ': failed RELRO')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-z,noseparate-code']), (1, executable + ': failed SEPARATE_CODE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fcf-protection=none']), (1, executable + ': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) else: - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-zexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (0, '')) + pass_flags = ['-Wl,-znoexecstack', '-Wl,-zrelro', '-Wl,-z,now', '-pie', '-fPIE', '-Wl,-z,separate-code'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-zexecstack']), (1, executable + ': failed NX')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-no-pie','-fno-PIE']), (1, executable + ': failed PIE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-znorelro']), (1, executable + ': failed RELRO')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-z,noseparate-code']), (1, executable + ': failed SEPARATE_CODE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) clean_files(source, executable) @@ -95,20 +82,16 @@ def test_PE(self): cxx = determine_wellknown_cmd('CXX', 'x86_64-w64-mingw32-g++') write_testcode(source) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fno-stack-protector']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW Canary')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full','-fstack-protector-all', '-lssp']), - (0, '')) + pass_flags = ['-Wl,--nxcompat', '-Wl,--enable-reloc-section', '-Wl,--dynamicbase', '-Wl,--high-entropy-va', '-pie', '-fPIE', '-fcf-protection=full', '-fstack-protector-all', '-lssp'] + + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fno-stack-protector']), (1, executable + ': failed CANARY')) + # https://github.com/lief-project/LIEF/issues/1076 - in future, we could test this individually. + # self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-reloc-section']), (1, executable + ': failed RELOC_SECTION')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-nxcompat']), (1, executable + ': failed NX')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-dynamicbase']), (1, executable + ': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing without --dynamicbase + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-high-entropy-va']), (1, executable + ': failed HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fcf-protection=none']), (1, executable + ': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) clean_files(source, executable) @@ -120,27 +103,21 @@ def test_MACHO(self): arch = get_arch(cxx, source, executable) if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']), - (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains']), - (1, executable+': failed NOUNDEFS Canary CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed NOUNDEFS CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cxx, source, executable, ['-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), - (0, '')) + pass_flags = ['-Wl,-pie', '-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-no_pie', '-Wl,-no_fixup_chains']), (1, executable+': failed FIXUP_CHAINS PIE')) # -fixup_chains is incompatible with -no_pie + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-no_fixup_chains']), (1, executable + ': failed FIXUP_CHAINS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fno-stack-protector']), (1, executable + ': failed CANARY')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-flat_namespace']), (1, executable + ': failed NOUNDEFS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fcf-protection=none']), (1, executable + ': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) else: - # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']), - (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS BRANCH_PROTECTION')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains', '-mbranch-protection=bti']), - (1, executable+': failed NOUNDEFS Canary')) - self.assertEqual(call_security_check(cxx, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']), - (1, executable+': failed NOUNDEFS')) - self.assertEqual(call_security_check(cxx, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']), - (0, '')) - + # arm64 darwin doesn't support non-PIE binaries or executable stacks + pass_flags = ['-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-mbranch-protection=none']), (1, executable + ': failed BRANCH_PROTECTION')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-no_fixup_chains']), (1, executable + ': failed FIXUP_CHAINS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fno-stack-protector']), (1, executable + ': failed CANARY')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-flat_namespace']), (1, executable + ': failed NOUNDEFS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) clean_files(source, executable) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 44fbfa1c0b..732f65da30 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -25,6 +25,7 @@ (guix build-system gnu) (guix build-system python) (guix build-system trivial) + (guix download) (guix gexp) (guix git-download) ((guix licenses) #:prefix license:) @@ -91,7 +92,18 @@ chain for " target " development.")) (home-page (package-home-page xgcc)) (license (package-license xgcc))))) -(define base-gcc gcc-12) +(define base-gcc + (package + (inherit gcc-12) ;; 12.3.0 + (version "12.4.0") + (source (origin + (method url-fetch) + (uri (string-append "mirror://gnu/gcc/gcc-" + version "/gcc-" version ".tar.xz")) + (sha256 + (base32 + "0xcida8l2wykvvzvpcrcn649gj0ijn64gwxbplacpg6c0hk6akvh")))))) + (define base-linux-kernel-headers linux-libre-headers-6.1) (define* (make-bitcoin-cross-toolchain target @@ -119,7 +131,10 @@ desirable for building Bitcoin Core release binaries." (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) - (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (machine (substring target 0 (string-index target #\-))) + (pthreads-xlibc (make-mingw-w64 machine + #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) + #:with-winpthreads? #t)) (pthreads-xgcc (cross-gcc target #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) #:xbinutils xbinutils @@ -500,6 +515,7 @@ inspecting signatures in Mach-O binaries.") gzip xz ;; Build tools + gcc-toolchain-12 cmake-minimal gnu-make libtool @@ -515,22 +531,16 @@ inspecting signatures in Mach-O binaries.") python-lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) - (list ;; Native GCC 12 toolchain - gcc-toolchain-12 - zip + (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64 nss-certs osslsigncode)) ((string-contains target "-linux-") - (list ;; Native GCC 12 toolchain - gcc-toolchain-12 - (list gcc-toolchain-12 "static") + (list (list gcc-toolchain-12 "static") (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list ;; Native GCC 11 toolchain - gcc-toolchain-11 - clang-toolchain-18 + (list clang-toolchain-18 lld-18 (make-lld-wrapper lld-18 #:lld-as-ld? #t) python-signapple diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 2ec660109c..fb7d509938 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -3,19 +3,25 @@ $(package)_version=2.4.8 $(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/ $(package)_file_name=$(package)-$($(package)_version).tar.xz $(package)_sha256_hash=f79b8f904b749e3e0d20afeadecf8249c55b2e32d4ebb089ae378df479dcaf25 +$(package)_build_subdir=build +$(package)_patches += cmake_minimum.patch # -D_DEFAULT_SOURCE defines __USE_MISC, which exposes additional # definitions in endian.h, which are required for a working # endianness check in configure when building with -flto. define $(package)_set_vars - $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples - $(package)_config_opts += --disable-dependency-tracking --enable-option-checking - $(package)_config_opts += --without-xmlwf + $(package)_config_opts := -DCMAKE_BUILD_TYPE=None -DEXPAT_BUILD_TOOLS=OFF + $(package)_config_opts += -DEXPAT_BUILD_EXAMPLES=OFF -DEXPAT_BUILD_TESTS=OFF + $(package)_config_opts += -DBUILD_SHARED_LIBS=OFF $(package)_cppflags += -D_DEFAULT_SOURCE endef +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/cmake_minimum.patch +endef + define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_cmake) -S .. -B . endef define $(package)_build_cmds @@ -27,5 +33,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf share lib/cmake lib/*.la + rm -rf share lib/cmake endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index c942cbb936..fef0beaa7b 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -24,6 +24,3 @@ define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef -define $(package)_postprocess_cmds - rm -rf share/man -endef diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 60e1ee469e..24e940eaa0 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -4,6 +4,7 @@ $(package)_download_path=https://github.com/libevent/libevent/releases/download/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb $(package)_patches=cmake_fixups.patch +$(package)_patches+=fix_mingw_link.patch $(package)_build_subdir=build # When building for Windows, we set _WIN32_WINNT to target the same Windows @@ -21,7 +22,8 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch + patch -p1 < $($(package)_patch_dir)/cmake_fixups.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_mingw_link.patch endef define $(package)_config_cmds @@ -37,7 +39,7 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm bin/event_rpcgen.py && \ + rm -rf bin && \ rm include/ev*.h && \ rm include/event2/*_compat.h endef diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 341031b5f8..91d71a71b6 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -29,3 +29,8 @@ endef define $(package)_stage_cmds cmake --install . --prefix $($(package)_staging_prefix_dir) endef + +define $(package)_postprocess_cmds + rm -rf bin && \ + rm -rf share +endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index ea08eefea6..2e30be434c 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=8b8a4766ce0a1892b9e8a5eb73dc39821005e520 +$(package)_version=6aca5f389bacf2942394b8738bbe15d6c9edfb9b $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=475c0dc2357a2ff30e9a164e4c16dc8a6597a57c9193d646fa9cbf0a55c45d78 +$(package)_sha256_hash=2efeed53542bc1d8af3291f2b6f0e5d430d86a5e04e415ce33c136f2c226a51d $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 4216646063..e3f614091d 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -28,3 +28,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm -rf share +endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index bfa5e97c60..295c9fdec0 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -3,34 +3,44 @@ $(package)_version=4.3.5 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=6653ef5910f17954861fe72332e68b03ca6e4d9c7160eb3a8de5a5a913bfab43 -$(package)_patches=remove_libstd_link.patch +$(package)_build_subdir=build +$(package)_patches = remove_libstd_link.patch +$(package)_patches += macos_mktemp_check.patch +$(package)_patches += builtin_sha1.patch +$(package)_patches += fix_have_windows.patch +$(package)_patches += cmake_minimum.patch +$(package)_patches += no_librt.patch define $(package)_set_vars - $(package)_config_opts = --without-docs --disable-shared --disable-valgrind - $(package)_config_opts += --disable-perf --disable-curve-keygen --disable-curve --disable-libbsd - $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci - $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking - $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking + $(package)_config_opts := -DCMAKE_BUILD_TYPE=None -DWITH_DOCS=OFF -DWITH_LIBSODIUM=OFF + $(package)_config_opts += -DWITH_LIBBSD=OFF -DENABLE_CURVE=OFF -DENABLE_CPACK=OFF + $(package)_config_opts += -DBUILD_SHARED=OFF -DBUILD_TESTS=OFF -DZMQ_BUILD_TESTS=OFF + $(package)_config_opts += -DENABLE_DRAFTS=OFF -DZMQ_BUILD_TESTS=OFF + $(package)_cxxflags += -ffile-prefix-map=$($(package)_extract_dir)=/usr + $(package)_config_opts_mingw32 += -DZMQ_WIN32_WINNT=0x0601 -DZMQ_HAVE_IPC=OFF endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch + patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + patch -p1 < $($(package)_patch_dir)/macos_mktemp_check.patch && \ + patch -p1 < $($(package)_patch_dir)/builtin_sha1.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_have_windows.patch && \ + patch -p1 < $($(package)_patch_dir)/cmake_minimum.patch && \ + patch -p1 < $($(package)_patch_dir)/no_librt.patch endef define $(package)_config_cmds - ./autogen.sh && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config && \ - $($(package)_autoconf) + $($(package)_cmake) -S .. -B . endef define $(package)_build_cmds - $(MAKE) src/libzmq.la + $(MAKE) endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install-libLTLIBRARIES install-includeHEADERS install-pkgconfigDATA + $(MAKE) DESTDIR=$($(package)_staging_dir) install endef define $(package)_postprocess_cmds - rm -rf bin share lib/*.la + rm -rf share endef diff --git a/depends/patches/expat/cmake_minimum.patch b/depends/patches/expat/cmake_minimum.patch new file mode 100644 index 0000000000..a849a82a33 --- /dev/null +++ b/depends/patches/expat/cmake_minimum.patch @@ -0,0 +1,13 @@ +build: set minimum required CMake to 3.16 + +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -33,7 +33,7 @@ + # Unlike most of Expat, + # this file is copyrighted under the BSD-license for buildsystem files of KDE. + +-cmake_minimum_required(VERSION 3.1.3) ++cmake_minimum_required(VERSION 3.16) + + # This allows controlling documented build time switches + # when Expat is pulled in using the add_subdirectory function, e.g. diff --git a/depends/patches/libevent/fix_mingw_link.patch b/depends/patches/libevent/fix_mingw_link.patch new file mode 100644 index 0000000000..41cbd463c9 --- /dev/null +++ b/depends/patches/libevent/fix_mingw_link.patch @@ -0,0 +1,25 @@ +commit d108099913c5fdbe518f3f4d711f248f8522bd10 +Author: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> +Date: Mon Apr 22 06:39:35 2024 +0100 + + build: Add `Iphlpapi` to `Libs.private` in `*.pc` files on Windows + + It has been required since https://github.com/libevent/libevent/pull/923 + at least for the `if_nametoindex` call. + + See https://github.com/libevent/libevent/pull/1622. + + +diff --git a/configure.ac b/configure.ac +index d00e063a..cd1fce37 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -906,7 +906,7 @@ if(WIN32) + list(APPEND HDR_PRIVATE WIN32-Code/getopt.h) + + set(EVENT__DNS_USE_FTIME_FOR_ID 1) +- set(LIB_PLATFORM ws2_32 shell32 advapi32) ++ set(LIB_PLATFORM ws2_32 shell32 advapi32 iphlpapi) + add_definitions( + -D_CRT_SECURE_NO_WARNINGS + -D_CRT_NONSTDC_NO_DEPRECATE) diff --git a/depends/patches/zeromq/builtin_sha1.patch b/depends/patches/zeromq/builtin_sha1.patch new file mode 100644 index 0000000000..5481c9dbdd --- /dev/null +++ b/depends/patches/zeromq/builtin_sha1.patch @@ -0,0 +1,17 @@ +Don't use builtin sha1 if not using ws + +The builtin SHA1 (ZMQ_USE_BUILTIN_SHA1) is only used in the websocket +engine (ws_engine.cpp). +Upstreamed in https://github.com/zeromq/libzmq/pull/4670. + +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -234,7 +234,7 @@ if(NOT ZMQ_USE_GNUTLS) + endif() + endif() + endif() +- if(NOT ZMQ_USE_NSS) ++ if(ENABLE_WS AND NOT ZMQ_USE_NSS) + list(APPEND sources ${CMAKE_CURRENT_SOURCE_DIR}/external/sha1/sha1.c + ${CMAKE_CURRENT_SOURCE_DIR}/external/sha1/sha1.h) + message(STATUS "Using builtin sha1") diff --git a/depends/patches/zeromq/cmake_minimum.patch b/depends/patches/zeromq/cmake_minimum.patch new file mode 100644 index 0000000000..d6b6b5fae7 --- /dev/null +++ b/depends/patches/zeromq/cmake_minimum.patch @@ -0,0 +1,18 @@ +Set a more sane cmake_minimum_required. + +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1,12 +1,7 @@ + # CMake build script for ZeroMQ ++cmake_minimum_required(VERSION 3.16) + project(ZeroMQ) + +-if(${CMAKE_SYSTEM_NAME} STREQUAL Darwin) +- cmake_minimum_required(VERSION 3.0.2) +-else() +- cmake_minimum_required(VERSION 2.8.12) +-endif() +- + include(CheckIncludeFiles) + include(CheckCCompilerFlag) + include(CheckCXXCompilerFlag) diff --git a/depends/patches/zeromq/fix_have_windows.patch b/depends/patches/zeromq/fix_have_windows.patch new file mode 100644 index 0000000000..e77ef31adf --- /dev/null +++ b/depends/patches/zeromq/fix_have_windows.patch @@ -0,0 +1,54 @@ +This fixes several instances where _MSC_VER was +used to determine whether to use afunix.h or not. + +See https://github.com/zeromq/libzmq/pull/4678. +--- a/src/ipc_address.hpp ++++ b/src/ipc_address.hpp +@@ -7,7 +7,7 @@ + + #include + +-#if defined _MSC_VER ++#if defined ZMQ_HAVE_WINDOWS + #include + #else + #include +diff --git a/src/ipc_connecter.cpp b/src/ipc_connecter.cpp +index 3f988745..ed2a0645 100644 +--- a/src/ipc_connecter.cpp ++++ b/src/ipc_connecter.cpp +@@ -16,7 +16,7 @@ + #include "ipc_address.hpp" + #include "session_base.hpp" + +-#ifdef _MSC_VER ++#if defined ZMQ_HAVE_WINDOWS + #include + #else + #include +diff --git a/src/ipc_listener.cpp b/src/ipc_listener.cpp +index 50126040..5428579b 100644 +--- a/src/ipc_listener.cpp ++++ b/src/ipc_listener.cpp +@@ -17,7 +17,7 @@ + #include "socket_base.hpp" + #include "address.hpp" + +-#ifdef _MSC_VER ++#ifdef ZMQ_HAVE_WINDOWS + #ifdef ZMQ_IOTHREAD_POLLER_USE_SELECT + #error On Windows, IPC does not work with POLLER=select, use POLLER=epoll instead, or disable IPC transport + #endif +diff --git a/tests/testutil.cpp b/tests/testutil.cpp +index bdc80283..6f21e8f6 100644 +--- a/tests/testutil.cpp ++++ b/tests/testutil.cpp +@@ -7,7 +7,7 @@ + + #if defined _WIN32 + #include "../src/windows.hpp" +-#if defined _MSC_VER ++#if defined ZMQ_HAVE_WINDOWS + #if defined ZMQ_HAVE_IPC + #include + #include diff --git a/depends/patches/zeromq/macos_mktemp_check.patch b/depends/patches/zeromq/macos_mktemp_check.patch new file mode 100644 index 0000000000..c703abcd71 --- /dev/null +++ b/depends/patches/zeromq/macos_mktemp_check.patch @@ -0,0 +1,16 @@ +build: fix mkdtemp check on macOS + +On macOS, mkdtemp is in unistd.h. Fix the CMake check so that is works. +Upstreamed in https://github.com/zeromq/libzmq/pull/4668. + +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -599,7 +599,7 @@ if(NOT MSVC) + + check_cxx_symbol_exists(fork unistd.h HAVE_FORK) + check_cxx_symbol_exists(gethrtime sys/time.h HAVE_GETHRTIME) +- check_cxx_symbol_exists(mkdtemp stdlib.h HAVE_MKDTEMP) ++ check_cxx_symbol_exists(mkdtemp "stdlib.h;unistd.h" HAVE_MKDTEMP) + check_cxx_symbol_exists(accept4 sys/socket.h HAVE_ACCEPT4) + check_cxx_symbol_exists(strnlen string.h HAVE_STRNLEN) + else() diff --git a/depends/patches/zeromq/no_librt.patch b/depends/patches/zeromq/no_librt.patch new file mode 100644 index 0000000000..b63854c95b --- /dev/null +++ b/depends/patches/zeromq/no_librt.patch @@ -0,0 +1,54 @@ +We don't use librt, so don't try and link against it. + +Related to: https://github.com/zeromq/libzmq/pull/4702. + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 03462271..87ceab3c 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -564,13 +564,6 @@ else() + check_cxx_symbol_exists(SO_BUSY_POLL sys/socket.h ZMQ_HAVE_BUSY_POLL) + endif() + +-if(NOT MINGW) +- find_library(RT_LIBRARY rt) +- if(RT_LIBRARY) +- set(pkg_config_libs_private "${pkg_config_libs_private} -lrt") +- endif() +-endif() +- + find_package(Threads) + + if(WIN32 AND NOT CYGWIN) +@@ -588,9 +581,7 @@ if(WIN32 AND NOT CYGWIN) + endif() + + if(NOT MSVC) +- set(CMAKE_REQUIRED_LIBRARIES rt) + check_cxx_symbol_exists(clock_gettime time.h HAVE_CLOCK_GETTIME) +- set(CMAKE_REQUIRED_LIBRARIES) + + check_cxx_symbol_exists(fork unistd.h HAVE_FORK) + check_cxx_symbol_exists(gethrtime sys/time.h HAVE_GETHRTIME) +@@ -1503,10 +1494,6 @@ if(BUILD_SHARED) + target_link_libraries(libzmq iphlpapi) + endif() + +- if(RT_LIBRARY) +- target_link_libraries(libzmq -lrt) +- endif() +- + if(norm_FOUND) + target_link_libraries(libzmq norm::norm) + endif() +@@ -1553,10 +1540,6 @@ if(BUILD_STATIC) + target_link_libraries(libzmq-static iphlpapi) + endif() + +- if(RT_LIBRARY) +- target_link_libraries(libzmq-static -lrt) +- endif() +- + if(CMAKE_SYSTEM_NAME MATCHES "QNX") + add_definitions(-DUNITY_EXCLUDE_MATH_H) + endif() diff --git a/doc/design/libraries.md b/doc/design/libraries.md index aa8034ab37..58a0469eb7 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -97,7 +97,7 @@ class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold - *libbitcoin_consensus* should only depend on *libbitcoin_crypto*, and all other libraries besides *libbitcoin_crypto* should be allowed to depend on it. -- *libbitcoin_util* should be a standalone dependency that any library can depend on, and it should not depend on other libraries except *libbitcoin_crypto*. It provides basic utilities that fill in gaps in the C++ standard library and provide lightweight abstractions over platform-specific features. Since the util library is distributed with the kernel and is usable by kernel applications, it shouldn't contain functions that external code shouldn't call, like higher level code targetted at the node or wallet. (*libbitcoin_common* is a better place for higher level code, or code that is meant to be used by internal applications only.) +- *libbitcoin_util* should be a standalone dependency that any library can depend on, and it should not depend on other libraries except *libbitcoin_crypto*. It provides basic utilities that fill in gaps in the C++ standard library and provide lightweight abstractions over platform-specific features. Since the util library is distributed with the kernel and is usable by kernel applications, it shouldn't contain functions that external code shouldn't call, like higher level code targeted at the node or wallet. (*libbitcoin_common* is a better place for higher level code, or code that is meant to be used by internal applications only.) - *libbitcoin_common* is a home for miscellaneous shared code used by different Bitcoin Core applications. It should not depend on anything other than *libbitcoin_util*, *libbitcoin_consensus*, and *libbitcoin_crypto*. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index d9d5b392c5..0445921026 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -492,7 +492,7 @@ make cov # unit and functional tests. ``` -Additional LCOV options can be specified using `LCOV_OPTS`, but may be dependant +Additional LCOV options can be specified using `LCOV_OPTS`, but may be dependent on the version of LCOV. For example, when using LCOV `2.x`, branch coverage can be enabled by setting `LCOV_OPTS="--rc branch_coverage=1"`, when configuring. diff --git a/doc/release-notes-30482.md b/doc/release-notes-30482.md new file mode 100644 index 0000000000..fb625c2fa9 --- /dev/null +++ b/doc/release-notes-30482.md @@ -0,0 +1,6 @@ +Updated REST APIs +----------------- +- Parameter validation for `/rest/getutxos` has been improved by rejecting + truncated or overly large txids and malformed outpoint indices by raising an + HTTP_BAD_REQUEST "Parse error". Previously, these malformed requests would be + silently handled. (#30482, #30444) diff --git a/src/Makefile.am b/src/Makefile.am index 017084145f..c2fbc1a470 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -133,6 +133,7 @@ BITCOIN_CORE_H = \ chainparamsseeds.h \ checkqueue.h \ clientversion.h \ + cluster_linearize.h \ coins.h \ common/args.h \ common/bloom.h \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 7e3aa369c7..fe6333d8c0 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -25,6 +25,7 @@ bench_bench_bitcoin_SOURCES = \ bench/checkblock.cpp \ bench/checkblockindex.cpp \ bench/checkqueue.cpp \ + bench/cluster_linearize.cpp \ bench/crypto_hash.cpp \ bench/data.cpp \ bench/data.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 8f35db6a1a..e53a729ed0 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -84,6 +84,7 @@ BITCOIN_TESTS =\ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ + test/cluster_linearize_tests.cpp \ test/coins_tests.cpp \ test/coinstatsindex_tests.cpp \ test/common_url_tests.cpp \ @@ -304,6 +305,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/buffered_file.cpp \ test/fuzz/chain.cpp \ test/fuzz/checkqueue.cpp \ + test/fuzz/cluster_linearize.cpp \ test/fuzz/coins_view.cpp \ test/fuzz/coinscache_sim.cpp \ test/fuzz/connman.cpp \ diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 960eb078c8..0c0e849fba 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -10,6 +10,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/chainstate.h \ + test/util/cluster_linearize.h \ test/util/coins.h \ test/util/index.h \ test/util/json.h \ diff --git a/src/bench/cluster_linearize.cpp b/src/bench/cluster_linearize.cpp new file mode 100644 index 0000000000..9987d376a5 --- /dev/null +++ b/src/bench/cluster_linearize.cpp @@ -0,0 +1,214 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include + +using namespace cluster_linearize; + +namespace { + +/** Construct a linear graph. These are pessimal for AncestorCandidateFinder, as they maximize + * the number of ancestor set feerate updates. The best ancestor set is always the topmost + * remaining transaction, whose removal requires updating all remaining transactions' ancestor + * set feerates. */ +template +DepGraph MakeLinearGraph(ClusterIndex ntx) +{ + DepGraph depgraph; + for (ClusterIndex i = 0; i < ntx; ++i) { + depgraph.AddTransaction({-int32_t(i), 1}); + if (i > 0) depgraph.AddDependency(i - 1, i); + } + return depgraph; +} + +/** Construct a wide graph (one root, with N-1 children that are otherwise unrelated, with + * increasing feerates). These graphs are pessimal for the LIMO step in Linearize, because + * rechunking is needed after every candidate (the last transaction gets picked every time). + */ +template +DepGraph MakeWideGraph(ClusterIndex ntx) +{ + DepGraph depgraph; + for (ClusterIndex i = 0; i < ntx; ++i) { + depgraph.AddTransaction({int32_t(i) + 1, 1}); + if (i > 0) depgraph.AddDependency(0, i); + } + return depgraph; +} + +// Construct a difficult graph. These need at least sqrt(2^(n-1)) iterations in the best +// known algorithms (purely empirically determined). +template +DepGraph MakeHardGraph(ClusterIndex ntx) +{ + DepGraph depgraph; + for (ClusterIndex i = 0; i < ntx; ++i) { + if (ntx & 1) { + // Odd cluster size. + // + // Mermaid diagram code for the resulting cluster for 11 transactions: + // ```mermaid + // graph BT + // T0["T0: 1/2"];T1["T1: 14/2"];T2["T2: 6/1"];T3["T3: 5/1"];T4["T4: 7/1"]; + // T5["T5: 5/1"];T6["T6: 7/1"];T7["T7: 5/1"];T8["T8: 7/1"];T9["T9: 5/1"]; + // T10["T10: 7/1"]; + // T1-->T0;T1-->T2;T3-->T2;T4-->T3;T4-->T5;T6-->T5;T4-->T7;T8-->T7;T4-->T9;T10-->T9; + // ``` + if (i == 0) { + depgraph.AddTransaction({1, 2}); + } else if (i == 1) { + depgraph.AddTransaction({14, 2}); + depgraph.AddDependency(0, 1); + } else if (i == 2) { + depgraph.AddTransaction({6, 1}); + depgraph.AddDependency(2, 1); + } else if (i == 3) { + depgraph.AddTransaction({5, 1}); + depgraph.AddDependency(2, 3); + } else if ((i & 1) == 0) { + depgraph.AddTransaction({7, 1}); + depgraph.AddDependency(i - 1, i); + } else { + depgraph.AddTransaction({5, 1}); + depgraph.AddDependency(i, 4); + } + } else { + // Even cluster size. + // + // Mermaid diagram code for the resulting cluster for 10 transactions: + // ```mermaid + // graph BT + // T0["T0: 1"];T1["T1: 3"];T2["T2: 1"];T3["T3: 4"];T4["T4: 0"];T5["T5: 4"];T6["T6: 0"]; + // T7["T7: 4"];T8["T8: 0"];T9["T9: 4"]; + // T1-->T0;T2-->T0;T3-->T2;T3-->T4;T5-->T4;T3-->T6;T7-->T6;T3-->T8;T9-->T8; + // ``` + if (i == 0) { + depgraph.AddTransaction({1, 1}); + } else if (i == 1) { + depgraph.AddTransaction({3, 1}); + depgraph.AddDependency(0, 1); + } else if (i == 2) { + depgraph.AddTransaction({1, 1}); + depgraph.AddDependency(0, 2); + } else if (i & 1) { + depgraph.AddTransaction({4, 1}); + depgraph.AddDependency(i - 1, i); + } else { + depgraph.AddTransaction({0, 1}); + depgraph.AddDependency(i, 3); + } + } + } + return depgraph; +} + +/** Benchmark that does search-based candidate finding with 10000 iterations. + * + * Its goal is measuring how much time every additional search iteration in linearization costs. + */ +template +void BenchLinearizePerIterWorstCase(ClusterIndex ntx, benchmark::Bench& bench) +{ + const auto depgraph = MakeHardGraph(ntx); + const auto iter_limit = std::min(10000, uint64_t{1} << (ntx / 2 - 1)); + uint64_t rng_seed = 0; + bench.batch(iter_limit).unit("iters").run([&] { + SearchCandidateFinder finder(depgraph, rng_seed++); + auto [candidate, iters_performed] = finder.FindCandidateSet(iter_limit, {}); + assert(iters_performed == iter_limit); + }); +} + +/** Benchmark for linearization improvement of a trivial linear graph using just ancestor sort. + * + * Its goal is measuring how much time linearization may take without any search iterations. + * + * If P is the resulting time of BenchLinearizePerIterWorstCase, and N is the resulting time of + * BenchLinearizeNoItersWorstCase*, then an invocation of Linearize with max_iterations=m should + * take no more than roughly N+m*P time. This may however be an overestimate, as the worst cases + * do not coincide (the ones that are worst for linearization without any search happen to be ones + * that do not need many search iterations). + * + * This benchmark exercises a worst case for AncestorCandidateFinder, but for which improvement is + * cheap. + */ +template +void BenchLinearizeNoItersWorstCaseAnc(ClusterIndex ntx, benchmark::Bench& bench) +{ + const auto depgraph = MakeLinearGraph(ntx); + uint64_t rng_seed = 0; + std::vector old_lin(ntx); + for (ClusterIndex i = 0; i < ntx; ++i) old_lin[i] = i; + bench.run([&] { + Linearize(depgraph, /*max_iterations=*/0, rng_seed++, old_lin); + }); +} + +/** Benchmark for linearization improvement of a trivial wide graph using just ancestor sort. + * + * Its goal is measuring how much time improving a linearization may take without any search + * iterations, similar to the previous function. + * + * This benchmark exercises a worst case for improving an existing linearization, but for which + * AncestorCandidateFinder is cheap. + */ +template +void BenchLinearizeNoItersWorstCaseLIMO(ClusterIndex ntx, benchmark::Bench& bench) +{ + const auto depgraph = MakeWideGraph(ntx); + uint64_t rng_seed = 0; + std::vector old_lin(ntx); + for (ClusterIndex i = 0; i < ntx; ++i) old_lin[i] = i; + bench.run([&] { + Linearize(depgraph, /*max_iterations=*/0, rng_seed++, old_lin); + }); +} + +} // namespace + +static void LinearizePerIter16TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(16, bench); } +static void LinearizePerIter32TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(32, bench); } +static void LinearizePerIter48TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(48, bench); } +static void LinearizePerIter64TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(64, bench); } +static void LinearizePerIter75TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(75, bench); } +static void LinearizePerIter99TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(99, bench); } + +static void LinearizeNoIters16TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(16, bench); } +static void LinearizeNoIters32TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(32, bench); } +static void LinearizeNoIters48TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(48, bench); } +static void LinearizeNoIters64TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(64, bench); } +static void LinearizeNoIters75TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(75, bench); } +static void LinearizeNoIters99TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(99, bench); } + +static void LinearizeNoIters16TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(16, bench); } +static void LinearizeNoIters32TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(32, bench); } +static void LinearizeNoIters48TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(48, bench); } +static void LinearizeNoIters64TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(64, bench); } +static void LinearizeNoIters75TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(75, bench); } +static void LinearizeNoIters99TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(99, bench); } + +BENCHMARK(LinearizePerIter16TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter32TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter48TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter64TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter75TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter99TxWorstCase, benchmark::PriorityLevel::HIGH); + +BENCHMARK(LinearizeNoIters16TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters32TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters48TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters64TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters75TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters99TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); + +BENCHMARK(LinearizeNoIters16TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters32TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters48TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters64TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters75TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters99TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 98af162b4d..ebe013b638 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,12 @@ int main(int argc, char* argv[]) { + // We do not enable logging for this app, so explicitly disable it. + // To enable logging instead, replace with: + // LogInstance().m_print_to_console = true; + // LogInstance().StartLogging(); + LogInstance().DisableLogging(); + // SETUP: Argument parsing and handling if (argc != 2) { std::cerr diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 89faa0123a..89c03c1647 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -264,8 +264,8 @@ static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInpu throw std::runtime_error("TX input missing separator"); // extract and validate TXID - uint256 txid; - if (!ParseHashStr(vStrInputParts[0], txid)) { + auto txid{Txid::FromHex(vStrInputParts[0])}; + if (!txid) { throw std::runtime_error("invalid TX input txid"); } @@ -285,7 +285,7 @@ static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInpu } // append to transaction input list - CTxIn txin(Txid::FromUint256(txid), vout, CScript(), nSequenceIn); + CTxIn txin(*txid, vout, CScript(), nSequenceIn); tx.vin.push_back(txin); } @@ -625,8 +625,8 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (!prevOut.checkObject(types)) throw std::runtime_error("prevtxs internal object typecheck fail"); - uint256 txid; - if (!ParseHashStr(prevOut["txid"].get_str(), txid)) { + auto txid{Txid::FromHex(prevOut["txid"].get_str())}; + if (!txid) { throw std::runtime_error("txid must be hexadecimal string (not '" + prevOut["txid"].get_str() + "')"); } @@ -634,7 +634,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout cannot be negative"); - COutPoint out(Txid::FromUint256(txid), nOut); + COutPoint out(*txid, nOut); std::vector pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); diff --git a/src/cluster_linearize.h b/src/cluster_linearize.h new file mode 100644 index 0000000000..07d28a9aa5 --- /dev/null +++ b/src/cluster_linearize.h @@ -0,0 +1,743 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CLUSTER_LINEARIZE_H +#define BITCOIN_CLUSTER_LINEARIZE_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace cluster_linearize { + +/** Data type to represent cluster input. + * + * cluster[i].first is tx_i's fee and size. + * cluster[i].second[j] is true iff tx_i spends one or more of tx_j's outputs. + */ +template +using Cluster = std::vector>; + +/** Data type to represent transaction indices in clusters. */ +using ClusterIndex = uint32_t; + +/** Data structure that holds a transaction graph's preprocessed data (fee, size, ancestors, + * descendants). */ +template +class DepGraph +{ + /** Information about a single transaction. */ + struct Entry + { + /** Fee and size of transaction itself. */ + FeeFrac feerate; + /** All ancestors of the transaction (including itself). */ + SetType ancestors; + /** All descendants of the transaction (including itself). */ + SetType descendants; + + /** Equality operator (primarily for for testing purposes). */ + friend bool operator==(const Entry&, const Entry&) noexcept = default; + + /** Construct an empty entry. */ + Entry() noexcept = default; + /** Construct an entry with a given feerate, ancestor set, descendant set. */ + Entry(const FeeFrac& f, const SetType& a, const SetType& d) noexcept : feerate(f), ancestors(a), descendants(d) {} + }; + + /** Data for each transaction, in the same order as the Cluster it was constructed from. */ + std::vector entries; + +public: + /** Equality operator (primarily for testing purposes). */ + friend bool operator==(const DepGraph&, const DepGraph&) noexcept = default; + + // Default constructors. + DepGraph() noexcept = default; + DepGraph(const DepGraph&) noexcept = default; + DepGraph(DepGraph&&) noexcept = default; + DepGraph& operator=(const DepGraph&) noexcept = default; + DepGraph& operator=(DepGraph&&) noexcept = default; + + /** Construct a DepGraph object for ntx transactions, with no dependencies. + * + * Complexity: O(N) where N=ntx. + **/ + explicit DepGraph(ClusterIndex ntx) noexcept + { + Assume(ntx <= SetType::Size()); + entries.resize(ntx); + for (ClusterIndex i = 0; i < ntx; ++i) { + entries[i].ancestors = SetType::Singleton(i); + entries[i].descendants = SetType::Singleton(i); + } + } + + /** Construct a DepGraph object given a cluster. + * + * Complexity: O(N^2) where N=cluster.size(). + */ + explicit DepGraph(const Cluster& cluster) noexcept : entries(cluster.size()) + { + for (ClusterIndex i = 0; i < cluster.size(); ++i) { + // Fill in fee and size. + entries[i].feerate = cluster[i].first; + // Fill in direct parents as ancestors. + entries[i].ancestors = cluster[i].second; + // Make sure transactions are ancestors of themselves. + entries[i].ancestors.Set(i); + } + + // Propagate ancestor information. + for (ClusterIndex i = 0; i < entries.size(); ++i) { + // At this point, entries[a].ancestors[b] is true iff b is an ancestor of a and there + // is a path from a to b through the subgraph consisting of {a, b} union + // {0, 1, ..., (i-1)}. + SetType to_merge = entries[i].ancestors; + for (ClusterIndex j = 0; j < entries.size(); ++j) { + if (entries[j].ancestors[i]) { + entries[j].ancestors |= to_merge; + } + } + } + + // Fill in descendant information by transposing the ancestor information. + for (ClusterIndex i = 0; i < entries.size(); ++i) { + for (auto j : entries[i].ancestors) { + entries[j].descendants.Set(i); + } + } + } + + /** Get the number of transactions in the graph. Complexity: O(1). */ + auto TxCount() const noexcept { return entries.size(); } + /** Get the feerate of a given transaction i. Complexity: O(1). */ + const FeeFrac& FeeRate(ClusterIndex i) const noexcept { return entries[i].feerate; } + /** Get the ancestors of a given transaction i. Complexity: O(1). */ + const SetType& Ancestors(ClusterIndex i) const noexcept { return entries[i].ancestors; } + /** Get the descendants of a given transaction i. Complexity: O(1). */ + const SetType& Descendants(ClusterIndex i) const noexcept { return entries[i].descendants; } + + /** Add a new unconnected transaction to this transaction graph (at the end), and return its + * ClusterIndex. + * + * Complexity: O(1) (amortized, due to resizing of backing vector). + */ + ClusterIndex AddTransaction(const FeeFrac& feefrac) noexcept + { + Assume(TxCount() < SetType::Size()); + ClusterIndex new_idx = TxCount(); + entries.emplace_back(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + return new_idx; + } + + /** Modify this transaction graph, adding a dependency between a specified parent and child. + * + * Complexity: O(N) where N=TxCount(). + **/ + void AddDependency(ClusterIndex parent, ClusterIndex child) noexcept + { + // Bail out if dependency is already implied. + if (entries[child].ancestors[parent]) return; + // To each ancestor of the parent, add as descendants the descendants of the child. + const auto& chl_des = entries[child].descendants; + for (auto anc_of_par : Ancestors(parent)) { + entries[anc_of_par].descendants |= chl_des; + } + // To each descendant of the child, add as ancestors the ancestors of the parent. + const auto& par_anc = entries[parent].ancestors; + for (auto dec_of_chl : Descendants(child)) { + entries[dec_of_chl].ancestors |= par_anc; + } + } + + /** Compute the aggregate feerate of a set of nodes in this graph. + * + * Complexity: O(N) where N=elems.Count(). + **/ + FeeFrac FeeRate(const SetType& elems) const noexcept + { + FeeFrac ret; + for (auto pos : elems) ret += entries[pos].feerate; + return ret; + } + + /** Append the entries of select to list in a topologically valid order. + * + * Complexity: O(select.Count() * log(select.Count())). + */ + void AppendTopo(std::vector& list, const SetType& select) const noexcept + { + ClusterIndex old_len = list.size(); + for (auto i : select) list.push_back(i); + std::sort(list.begin() + old_len, list.end(), [&](ClusterIndex a, ClusterIndex b) noexcept { + const auto a_anc_count = entries[a].ancestors.Count(); + const auto b_anc_count = entries[b].ancestors.Count(); + if (a_anc_count != b_anc_count) return a_anc_count < b_anc_count; + return a < b; + }); + } +}; + +/** A set of transactions together with their aggregate feerate. */ +template +struct SetInfo +{ + /** The transactions in the set. */ + SetType transactions; + /** Their combined fee and size. */ + FeeFrac feerate; + + /** Construct a SetInfo for the empty set. */ + SetInfo() noexcept = default; + + /** Construct a SetInfo for a specified set and feerate. */ + SetInfo(const SetType& txn, const FeeFrac& fr) noexcept : transactions(txn), feerate(fr) {} + + /** Construct a SetInfo for a given transaction in a depgraph. */ + explicit SetInfo(const DepGraph& depgraph, ClusterIndex pos) noexcept : + transactions(SetType::Singleton(pos)), feerate(depgraph.FeeRate(pos)) {} + + /** Construct a SetInfo for a set of transactions in a depgraph. */ + explicit SetInfo(const DepGraph& depgraph, const SetType& txn) noexcept : + transactions(txn), feerate(depgraph.FeeRate(txn)) {} + + /** Add the transactions of other to this SetInfo (no overlap allowed). */ + SetInfo& operator|=(const SetInfo& other) noexcept + { + Assume(!transactions.Overlaps(other.transactions)); + transactions |= other.transactions; + feerate += other.feerate; + return *this; + } + + /** Construct a new SetInfo equal to this, with more transactions added (which may overlap + * with the existing transactions in the SetInfo). */ + [[nodiscard]] SetInfo Add(const DepGraph& depgraph, const SetType& txn) const noexcept + { + return {transactions | txn, feerate + depgraph.FeeRate(txn - transactions)}; + } + + /** Swap two SetInfo objects. */ + friend void swap(SetInfo& a, SetInfo& b) noexcept + { + swap(a.transactions, b.transactions); + swap(a.feerate, b.feerate); + } + + /** Permit equality testing. */ + friend bool operator==(const SetInfo&, const SetInfo&) noexcept = default; +}; + +/** Compute the feerates of the chunks of linearization. */ +template +std::vector ChunkLinearization(const DepGraph& depgraph, Span linearization) noexcept +{ + std::vector ret; + for (ClusterIndex i : linearization) { + /** The new chunk to be added, initially a singleton. */ + auto new_chunk = depgraph.FeeRate(i); + // As long as the new chunk has a higher feerate than the last chunk so far, absorb it. + while (!ret.empty() && new_chunk >> ret.back()) { + new_chunk += ret.back(); + ret.pop_back(); + } + // Actually move that new chunk into the chunking. + ret.push_back(std::move(new_chunk)); + } + return ret; +} + +/** Data structure encapsulating the chunking of a linearization, permitting removal of subsets. */ +template +class LinearizationChunking +{ + /** The depgraph this linearization is for. */ + const DepGraph& m_depgraph; + + /** The linearization we started from. */ + Span m_linearization; + + /** Chunk sets and their feerates, of what remains of the linearization. */ + std::vector> m_chunks; + + /** Which transactions remain in the linearization. */ + SetType m_todo; + + /** Fill the m_chunks variable. */ + void BuildChunks() noexcept + { + // Caller must clear m_chunks. + Assume(m_chunks.empty()); + + // Iterate over the entries in m_linearization. This is effectively the same + // algorithm as ChunkLinearization, but supports skipping parts of the linearization and + // keeps track of the sets themselves instead of just their feerates. + for (auto idx : m_linearization) { + if (!m_todo[idx]) continue; + // Start with an initial chunk containing just element idx. + SetInfo add(m_depgraph, idx); + // Absorb existing final chunks into add while they have lower feerate. + while (!m_chunks.empty() && add.feerate >> m_chunks.back().feerate) { + add |= m_chunks.back(); + m_chunks.pop_back(); + } + // Remember new chunk. + m_chunks.push_back(std::move(add)); + } + } + +public: + /** Initialize a LinearizationSubset object for a given length of linearization. */ + explicit LinearizationChunking(const DepGraph& depgraph LIFETIMEBOUND, Span lin LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), m_linearization(lin) + { + // Mark everything in lin as todo still. + for (auto i : m_linearization) m_todo.Set(i); + // Compute the initial chunking. + m_chunks.reserve(depgraph.TxCount()); + BuildChunks(); + } + + /** Determine how many chunks remain in the linearization. */ + ClusterIndex NumChunksLeft() const noexcept { return m_chunks.size(); } + + /** Access a chunk. Chunk 0 is the highest-feerate prefix of what remains. */ + const SetInfo& GetChunk(ClusterIndex n) const noexcept + { + Assume(n < m_chunks.size()); + return m_chunks[n]; + } + + /** Remove some subset of transactions from the linearization. */ + void MarkDone(SetType subset) noexcept + { + Assume(subset.Any()); + Assume(subset.IsSubsetOf(m_todo)); + m_todo -= subset; + // Rechunk what remains of m_linearization. + m_chunks.clear(); + BuildChunks(); + } + + /** Find the shortest intersection between subset and the prefixes of remaining chunks + * of the linearization that has a feerate not below subset's. + * + * This is a crucial operation in guaranteeing improvements to linearizations. If subset has + * a feerate not below GetChunk(0)'s, then moving Intersect(subset) to the front of (what + * remains of) the linearization is guaranteed not to make it worse at any point. + * + * See https://delvingbitcoin.org/t/introduction-to-cluster-linearization/1032 for background. + */ + SetInfo Intersect(const SetInfo& subset) const noexcept + { + Assume(subset.transactions.IsSubsetOf(m_todo)); + SetInfo accumulator; + // Iterate over all chunks of the remaining linearization. + for (ClusterIndex i = 0; i < NumChunksLeft(); ++i) { + // Find what (if any) intersection the chunk has with subset. + const SetType to_add = GetChunk(i).transactions & subset.transactions; + if (to_add.Any()) { + // If adding that to accumulator makes us hit all of subset, we are done as no + // shorter intersection with higher/equal feerate exists. + accumulator.transactions |= to_add; + if (accumulator.transactions == subset.transactions) break; + // Otherwise update the accumulator feerate. + accumulator.feerate += m_depgraph.FeeRate(to_add); + // If that does result in something better, or something with the same feerate but + // smaller, return that. Even if a longer, higher-feerate intersection exists, it + // does not hurt to return the shorter one (the remainder of the longer intersection + // will generally be found in the next call to Intersect, but even if not, it is not + // required for the improvement guarantee this function makes). + if (!(accumulator.feerate << subset.feerate)) return accumulator; + } + } + return subset; + } +}; + +/** Class encapsulating the state needed to find the best remaining ancestor set. + * + * It is initialized for an entire DepGraph, and parts of the graph can be dropped by calling + * MarkDone. + * + * As long as any part of the graph remains, FindCandidateSet() can be called which will return a + * SetInfo with the highest-feerate ancestor set that remains (an ancestor set is a single + * transaction together with all its remaining ancestors). + */ +template +class AncestorCandidateFinder +{ + /** Internal dependency graph. */ + const DepGraph& m_depgraph; + /** Which transaction are left to include. */ + SetType m_todo; + /** Precomputed ancestor-set feerates (only kept up-to-date for indices in m_todo). */ + std::vector m_ancestor_set_feerates; + +public: + /** Construct an AncestorCandidateFinder for a given cluster. + * + * Complexity: O(N^2) where N=depgraph.TxCount(). + */ + AncestorCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), + m_todo{SetType::Fill(depgraph.TxCount())}, + m_ancestor_set_feerates(depgraph.TxCount()) + { + // Precompute ancestor-set feerates. + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + /** The remaining ancestors for transaction i. */ + SetType anc_to_add = m_depgraph.Ancestors(i); + FeeFrac anc_feerate; + // Reuse accumulated feerate from first ancestor, if usable. + Assume(anc_to_add.Any()); + ClusterIndex first = anc_to_add.First(); + if (first < i) { + anc_feerate = m_ancestor_set_feerates[first]; + Assume(!anc_feerate.IsEmpty()); + anc_to_add -= m_depgraph.Ancestors(first); + } + // Add in other ancestors (which necessarily include i itself). + Assume(anc_to_add[i]); + anc_feerate += m_depgraph.FeeRate(anc_to_add); + // Store the result. + m_ancestor_set_feerates[i] = anc_feerate; + } + } + + /** Remove a set of transactions from the set of to-be-linearized ones. + * + * The same transaction may not be MarkDone()'d twice. + * + * Complexity: O(N*M) where N=depgraph.TxCount(), M=select.Count(). + */ + void MarkDone(SetType select) noexcept + { + Assume(select.Any()); + Assume(select.IsSubsetOf(m_todo)); + m_todo -= select; + for (auto i : select) { + auto feerate = m_depgraph.FeeRate(i); + for (auto j : m_depgraph.Descendants(i) & m_todo) { + m_ancestor_set_feerates[j] -= feerate; + } + } + } + + /** Check whether any unlinearized transactions remain. */ + bool AllDone() const noexcept + { + return m_todo.None(); + } + + /** Find the best (highest-feerate, smallest among those in case of a tie) ancestor set + * among the remaining transactions. Requires !AllDone(). + * + * Complexity: O(N) where N=depgraph.TxCount(); + */ + SetInfo FindCandidateSet() const noexcept + { + Assume(!AllDone()); + std::optional best; + for (auto i : m_todo) { + if (best.has_value()) { + Assume(!m_ancestor_set_feerates[i].IsEmpty()); + if (!(m_ancestor_set_feerates[i] > m_ancestor_set_feerates[*best])) continue; + } + best = i; + } + Assume(best.has_value()); + return {m_depgraph.Ancestors(*best) & m_todo, m_ancestor_set_feerates[*best]}; + } +}; + +/** Class encapsulating the state needed to perform search for good candidate sets. + * + * It is initialized for an entire DepGraph, and parts of the graph can be dropped by calling + * MarkDone(). + * + * As long as any part of the graph remains, FindCandidateSet() can be called to perform a search + * over the set of topologically-valid subsets of that remainder, with a limit on how many + * combinations are tried. + */ +template +class SearchCandidateFinder +{ + /** Internal RNG. */ + InsecureRandomContext m_rng; + /** Internal dependency graph for the cluster. */ + const DepGraph& m_depgraph; + /** Which transactions are left to do (sorted indices). */ + SetType m_todo; + +public: + /** Construct a candidate finder for a graph. + * + * @param[in] depgraph Dependency graph for the to-be-linearized cluster. + * @param[in] rng_seed A random seed to control the search order. + * + * Complexity: O(1). + */ + SearchCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND, uint64_t rng_seed) noexcept : + m_rng(rng_seed), + m_depgraph(depgraph), + m_todo(SetType::Fill(depgraph.TxCount())) {} + + /** Check whether any unlinearized transactions remain. */ + bool AllDone() const noexcept + { + return m_todo.None(); + } + + /** Find a high-feerate topologically-valid subset of what remains of the cluster. + * Requires !AllDone(). + * + * @param[in] max_iterations The maximum number of optimization steps that will be performed. + * @param[in] best A set/feerate pair with an already-known good candidate. This may + * be empty. + * @return A pair of: + * - The best (highest feerate, smallest size as tiebreaker) + * topologically valid subset (and its feerate) that was + * encountered during search. It will be at least as good as the + * best passed in (if not empty). + * - The number of optimization steps that were performed. This will + * be <= max_iterations. If strictly < max_iterations, the + * returned subset is optimal. + * + * Complexity: O(N * min(max_iterations, 2^N)) where N=depgraph.TxCount(). + */ + std::pair, uint64_t> FindCandidateSet(uint64_t max_iterations, SetInfo best) noexcept + { + Assume(!AllDone()); + + /** Type for work queue items. */ + struct WorkItem + { + /** Set of transactions definitely included (and its feerate). This must be a subset + * of m_todo, and be topologically valid (includes all in-m_todo ancestors of + * itself). */ + SetInfo inc; + /** Set of undecided transactions. This must be a subset of m_todo, and have no overlap + * with inc. The set (inc | und) must be topologically valid. */ + SetType und; + + /** Construct a new work item. */ + WorkItem(SetInfo&& i, SetType&& u) noexcept : + inc(std::move(i)), und(std::move(u)) {} + + /** Swap two WorkItems. */ + void Swap(WorkItem& other) noexcept + { + swap(inc, other.inc); + swap(und, other.und); + } + }; + + /** The queue of work items. */ + VecDeque queue; + queue.reserve(std::max(256, 2 * m_todo.Count())); + + // Create an initial entry with m_todo as undecided. Also use it as best if not provided, + // so that during the work processing loop below, and during the add_fn/split_fn calls, we + // do not need to deal with the best=empty case. + if (best.feerate.IsEmpty()) best = SetInfo(m_depgraph, m_todo); + queue.emplace_back(SetInfo{}, SetType{m_todo}); + + /** Local copy of the iteration limit. */ + uint64_t iterations_left = max_iterations; + + /** Internal function to add an item to the queue of elements to explore if there are any + * transactions left to split on, and to update best. + * + * - inc: the "inc" value for the new work item (must be topological). + * - und: the "und" value for the new work item ((inc | und) must be topological). + */ + auto add_fn = [&](SetInfo inc, SetType und) noexcept { + if (!inc.feerate.IsEmpty()) { + // If inc's feerate is better than best's, remember it as our new best. + if (inc.feerate > best.feerate) { + best = inc; + } + } else { + Assume(inc.transactions.None()); + } + + // Make sure there are undecided transactions left to split on. + if (und.None()) return; + + // Actually construct a new work item on the queue. Due to the switch to DFS when queue + // space runs out (see below), we know that no reallocation of the queue should ever + // occur. + Assume(queue.size() < queue.capacity()); + queue.emplace_back(std::move(inc), std::move(und)); + }; + + /** Internal process function. It takes an existing work item, and splits it in two: one + * with a particular transaction (and its ancestors) included, and one with that + * transaction (and its descendants) excluded. */ + auto split_fn = [&](WorkItem&& elem) noexcept { + // Any queue element must have undecided transactions left, otherwise there is nothing + // to explore anymore. + Assume(elem.und.Any()); + // The included and undecided set are all subsets of m_todo. + Assume(elem.inc.transactions.IsSubsetOf(m_todo) && elem.und.IsSubsetOf(m_todo)); + // Included transactions cannot be undecided. + Assume(!elem.inc.transactions.Overlaps(elem.und)); + + // Pick the first undecided transaction as the one to split on. + const ClusterIndex split = elem.und.First(); + + // Add a work item corresponding to exclusion of the split transaction. + const auto& desc = m_depgraph.Descendants(split); + add_fn(/*inc=*/elem.inc, + /*und=*/elem.und - desc); + + // Add a work item corresponding to inclusion of the split transaction. + const auto anc = m_depgraph.Ancestors(split) & m_todo; + add_fn(/*inc=*/elem.inc.Add(m_depgraph, anc), + /*und=*/elem.und - anc); + + // Account for the performed split. + --iterations_left; + }; + + // Work processing loop. + // + // New work items are always added at the back of the queue, but items to process use a + // hybrid approach where they can be taken from the front or the back. + // + // Depth-first search (DFS) corresponds to always taking from the back of the queue. This + // is very memory-efficient (linear in the number of transactions). Breadth-first search + // (BFS) corresponds to always taking from the front, which potentially uses more memory + // (up to exponential in the transaction count), but seems to work better in practice. + // + // The approach here combines the two: use BFS (plus random swapping) until the queue grows + // too large, at which point we temporarily switch to DFS until the size shrinks again. + while (!queue.empty()) { + // Randomly swap the first two items to randomize the search order. + if (queue.size() > 1 && m_rng.randbool()) { + queue[0].Swap(queue[1]); + } + + // Processing the first queue item, and then using DFS for everything it gives rise to, + // may increase the queue size by the number of undecided elements in there, minus 1 + // for the first queue item being removed. Thus, only when that pushes the queue over + // its capacity can we not process from the front (BFS), and should we use DFS. + while (queue.size() - 1 + queue.front().und.Count() > queue.capacity()) { + if (!iterations_left) break; + auto elem = queue.back(); + queue.pop_back(); + split_fn(std::move(elem)); + } + + // Process one entry from the front of the queue (BFS exploration) + if (!iterations_left) break; + auto elem = queue.front(); + queue.pop_front(); + split_fn(std::move(elem)); + } + + // Return the found best set and the number of iterations performed. + return {std::move(best), max_iterations - iterations_left}; + } + + /** Remove a subset of transactions from the cluster being linearized. + * + * Complexity: O(N) where N=done.Count(). + */ + void MarkDone(const SetType& done) noexcept + { + Assume(done.Any()); + Assume(done.IsSubsetOf(m_todo)); + m_todo -= done; + } +}; + +/** Find or improve a linearization for a cluster. + * + * @param[in] depgraph Dependency graph of the cluster to be linearized. + * @param[in] max_iterations Upper bound on the number of optimization steps that will be done. + * @param[in] rng_seed A random number seed to control search order. This prevents peers + * from predicting exactly which clusters would be hard for us to + * linearize. + * @param[in] old_linearization An existing linearization for the cluster (which must be + * topologically valid), or empty. + * @return A pair of: + * - The resulting linearization. It is guaranteed to be at least as + * good (in the feerate diagram sense) as old_linearization. + * - A boolean indicating whether the result is guaranteed to be + * optimal. + * + * Complexity: O(N * min(max_iterations + N, 2^N)) where N=depgraph.TxCount(). + */ +template +std::pair, bool> Linearize(const DepGraph& depgraph, uint64_t max_iterations, uint64_t rng_seed, Span old_linearization = {}) noexcept +{ + Assume(old_linearization.empty() || old_linearization.size() == depgraph.TxCount()); + if (depgraph.TxCount() == 0) return {{}, true}; + + uint64_t iterations_left = max_iterations; + std::vector linearization; + + AncestorCandidateFinder anc_finder(depgraph); + SearchCandidateFinder src_finder(depgraph, rng_seed); + linearization.reserve(depgraph.TxCount()); + bool optimal = true; + + /** Chunking of what remains of the old linearization. */ + LinearizationChunking old_chunking(depgraph, old_linearization); + + while (true) { + // Find the highest-feerate prefix of the remainder of old_linearization. + SetInfo best_prefix; + if (old_chunking.NumChunksLeft()) best_prefix = old_chunking.GetChunk(0); + + // Then initialize best to be either the best remaining ancestor set, or the first chunk. + auto best = anc_finder.FindCandidateSet(); + if (!best_prefix.feerate.IsEmpty() && best_prefix.feerate >= best.feerate) best = best_prefix; + + // Invoke bounded search to update best, with up to half of our remaining iterations as + // limit. + uint64_t max_iterations_now = (iterations_left + 1) / 2; + uint64_t iterations_done_now = 0; + std::tie(best, iterations_done_now) = src_finder.FindCandidateSet(max_iterations_now, best); + iterations_left -= iterations_done_now; + + if (iterations_done_now == max_iterations_now) { + optimal = false; + // If the search result is not (guaranteed to be) optimal, run intersections to make + // sure we don't pick something that makes us unable to reach further diagram points + // of the old linearization. + if (old_chunking.NumChunksLeft() > 0) { + best = old_chunking.Intersect(best); + } + } + + // Add to output in topological order. + depgraph.AppendTopo(linearization, best.transactions); + + // Update state to reflect best is no longer to be linearized. + anc_finder.MarkDone(best.transactions); + if (anc_finder.AllDone()) break; + src_finder.MarkDone(best.transactions); + if (old_chunking.NumChunksLeft() > 0) { + old_chunking.MarkDone(best.transactions); + } + } + + return {std::move(linearization), optimal}; +} + +} // namespace cluster_linearize + +#endif // BITCOIN_CLUSTER_LINEARIZE_H diff --git a/src/core_io.h b/src/core_io.h index 4405f5c8f8..9305bb7239 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -37,15 +37,6 @@ std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDeco [[nodiscard]] bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header); -/** - * Parse a hex string into 256 bits - * @param[in] strHex a hex-formatted, 64-character string - * @param[out] result the result of the parsing - * @returns true if successful, false if not - * - * @see ParseHashV for an RPC-oriented version of this - */ -bool ParseHashStr(const std::string& strHex, uint256& result); [[nodiscard]] util::Result SighashFromStr(const std::string& sighash); // core_write.cpp diff --git a/src/core_read.cpp b/src/core_read.cpp index 0ba271a8d2..23f341c230 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -234,15 +234,6 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return true; } -bool ParseHashStr(const std::string& strHex, uint256& result) -{ - if ((strHex.size() != 64) || !IsHex(strHex)) - return false; - - result.SetHex(strHex); - return true; -} - util::Result SighashFromStr(const std::string& sighash) { static const std::map map_sighash_values = { diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 89d7204808..deedc0a6d1 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -7,8 +7,9 @@ #include #include -#include -#include +#include +#include +#include #if !defined(DISABLE_OPTIMIZED_SHA256) #include diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 2bff663d8b..6aba0c371d 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -30,7 +30,7 @@ fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const return m_dir / fs::u8path(strprintf("%s%05u.dat", m_prefix, pos.nFile)); } -FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) +FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) const { if (pos.IsNull()) { return nullptr; @@ -52,7 +52,7 @@ FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) return file; } -size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) +size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) const { out_of_space = false; @@ -78,7 +78,7 @@ size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_ return 0; } -bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) +bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) const { FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos if (!file) { diff --git a/src/flatfile.h b/src/flatfile.h index a9d7edd306..3edb0b85da 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -63,7 +63,7 @@ class FlatFileSeq fs::path FileName(const FlatFilePos& pos) const; /** Open a handle to the file at the given position. */ - FILE* Open(const FlatFilePos& pos, bool read_only = false); + FILE* Open(const FlatFilePos& pos, bool read_only = false) const; /** * Allocate additional space in a file after the given starting position. The amount allocated @@ -74,7 +74,7 @@ class FlatFileSeq * @param[out] out_of_space Whether the allocation failed due to insufficient disk space. * @return The number of bytes successfully allocated. */ - size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space); + size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) const; /** * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final. @@ -83,7 +83,7 @@ class FlatFileSeq * @param[in] finalize True if no more data will be written to this file. * @return true on success, false on failure. */ - bool Flush(const FlatFilePos& pos, bool finalize = false); + bool Flush(const FlatFilePos& pos, bool finalize = false) const; }; #endif // BITCOIN_FLATFILE_H diff --git a/src/init.h b/src/init.h index ead5f5e0d2..40a5da3c0b 100644 --- a/src/init.h +++ b/src/init.h @@ -6,9 +6,7 @@ #ifndef BITCOIN_INIT_H #define BITCOIN_INIT_H -#include -#include -#include +#include //! Default value for -daemon option static constexpr bool DEFAULT_DAEMON = false; diff --git a/src/logging.cpp b/src/logging.cpp index a9fea433be..9c87cfd2b7 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include #include @@ -14,8 +15,7 @@ #include using util::Join; -using util::RemovePrefix; -using util::ToString; +using util::RemovePrefixView; const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; constexpr auto MAX_USER_SETABLE_SEVERITY_LEVEL{BCLog::Level::Info}; @@ -43,7 +43,7 @@ BCLog::Logger& LogInstance() bool fLogIPs = DEFAULT_LOGIPS; -static int FileWriteStr(const std::string &str, FILE *fp) +static int FileWriteStr(std::string_view str, FILE *fp) { return fwrite(str.data(), 1, str.size(), fp); } @@ -71,17 +71,22 @@ bool BCLog::Logger::StartLogging() // dump buffered messages from before we opened the log m_buffering = false; + if (m_buffer_lines_discarded > 0) { + LogPrintStr_(strprintf("Early logging buffer overflowed, %d log lines discarded.\n", m_buffer_lines_discarded), __func__, __FILE__, __LINE__, BCLog::ALL, Level::Info); + } while (!m_msgs_before_open.empty()) { - const std::string& s = m_msgs_before_open.front(); + const auto& buflog = m_msgs_before_open.front(); + std::string s{buflog.str}; + FormatLogStrInPlace(s, buflog.category, buflog.level, buflog.source_file, buflog.source_line, buflog.logging_function, buflog.threadname, buflog.now, buflog.mocktime); + m_msgs_before_open.pop_front(); if (m_print_to_file) FileWriteStr(s, m_fileout); if (m_print_to_console) fwrite(s.data(), 1, s.size(), stdout); for (const auto& cb : m_print_callbacks) { cb(s); } - - m_msgs_before_open.pop_front(); } + m_cur_buffer_memusage = 0; if (m_print_to_console) fflush(stdout); return true; @@ -94,6 +99,23 @@ void BCLog::Logger::DisconnectTestLogger() if (m_fileout != nullptr) fclose(m_fileout); m_fileout = nullptr; m_print_callbacks.clear(); + m_max_buffer_memusage = DEFAULT_MAX_LOG_BUFFER; + m_cur_buffer_memusage = 0; + m_buffer_lines_discarded = 0; + m_msgs_before_open.clear(); + +} + +void BCLog::Logger::DisableLogging() +{ + { + StdLockGuard scoped_lock(m_cs); + assert(m_buffering); + assert(m_print_callbacks.empty()); + } + m_print_to_file = false; + m_print_to_console = false; + StartLogging(); } void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) @@ -101,7 +123,7 @@ void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) m_categories |= flag; } -bool BCLog::Logger::EnableCategory(const std::string& str) +bool BCLog::Logger::EnableCategory(std::string_view str) { BCLog::LogFlags flag; if (!GetLogCategory(flag, str)) return false; @@ -114,7 +136,7 @@ void BCLog::Logger::DisableCategory(BCLog::LogFlags flag) m_categories &= ~flag; } -bool BCLog::Logger::DisableCategory(const std::string& str) +bool BCLog::Logger::DisableCategory(std::string_view str) { BCLog::LogFlags flag; if (!GetLogCategory(flag, str)) return false; @@ -145,7 +167,7 @@ bool BCLog::Logger::DefaultShrinkDebugFile() const return m_categories == BCLog::NONE; } -static const std::map LOG_CATEGORIES_BY_STR{ +static const std::map> LOG_CATEGORIES_BY_STR{ {"0", BCLog::NONE}, {"", BCLog::NONE}, {"net", BCLog::NET}, @@ -185,7 +207,7 @@ static const std::map LOG_CATEGORIES_BY_STR{ static const std::unordered_map LOG_CATEGORIES_BY_FLAG{ // Swap keys and values from LOG_CATEGORIES_BY_STR. - [](const std::map& in) { + [](const auto& in) { std::unordered_map out; for (const auto& [k, v] : in) { switch (v) { @@ -198,7 +220,7 @@ static const std::unordered_map LOG_CATEGORIES_BY_ }(LOG_CATEGORIES_BY_STR) }; -bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) +bool GetLogCategory(BCLog::LogFlags& flag, std::string_view str) { if (str.empty()) { flag = BCLog::ALL; @@ -236,7 +258,7 @@ std::string LogCategoryToStr(BCLog::LogFlags category) return it->second; } -static std::optional GetLogLevel(const std::string& level_str) +static std::optional GetLogLevel(std::string_view level_str) { if (level_str == "trace") { return BCLog::Level::Trace; @@ -276,28 +298,23 @@ std::string BCLog::Logger::LogLevelsString() const return Join(std::vector{levels.begin(), levels.end()}, ", ", [](BCLog::Level level) { return LogLevelToStr(level); }); } -std::string BCLog::Logger::LogTimestampStr(const std::string& str) +std::string BCLog::Logger::LogTimestampStr(SystemClock::time_point now, std::chrono::seconds mocktime) const { std::string strStamped; if (!m_log_timestamps) - return str; - - if (m_started_new_line) { - const auto now{SystemClock::now()}; - const auto now_seconds{std::chrono::time_point_cast(now)}; - strStamped = FormatISO8601DateTime(TicksSinceEpoch(now_seconds)); - if (m_log_time_micros && !strStamped.empty()) { - strStamped.pop_back(); - strStamped += strprintf(".%06dZ", Ticks(now - now_seconds)); - } - std::chrono::seconds mocktime = GetMockTime(); - if (mocktime > 0s) { - strStamped += " (mocktime: " + FormatISO8601DateTime(count_seconds(mocktime)) + ")"; - } - strStamped += ' ' + str; - } else - strStamped = str; + return strStamped; + + const auto now_seconds{std::chrono::time_point_cast(now)}; + strStamped = FormatISO8601DateTime(TicksSinceEpoch(now_seconds)); + if (m_log_time_micros && !strStamped.empty()) { + strStamped.pop_back(); + strStamped += strprintf(".%06dZ", Ticks(now - now_seconds)); + } + if (mocktime > 0s) { + strStamped += " (mocktime: " + FormatISO8601DateTime(count_seconds(mocktime)) + ")"; + } + strStamped += ' '; return strStamped; } @@ -310,7 +327,7 @@ namespace BCLog { * It escapes instead of removes them to still allow for troubleshooting * issues where they accidentally end up in strings. */ - std::string LogEscapeMessage(const std::string& str) { + std::string LogEscapeMessage(std::string_view str) { std::string ret; for (char ch_in : str) { uint8_t ch = (uint8_t)ch_in; @@ -350,34 +367,84 @@ std::string BCLog::Logger::GetLogPrefix(BCLog::LogFlags category, BCLog::Level l return s; } -void BCLog::Logger::LogPrintStr(const std::string& str, const std::string& logging_function, const std::string& source_file, int source_line, BCLog::LogFlags category, BCLog::Level level) +static size_t MemUsage(const BCLog::Logger::BufferedLog& buflog) { - StdLockGuard scoped_lock(m_cs); - std::string str_prefixed = LogEscapeMessage(str); + return buflog.str.size() + buflog.logging_function.size() + buflog.source_file.size() + buflog.threadname.size() + memusage::MallocUsage(sizeof(memusage::list_node)); +} - if (m_started_new_line) { - str_prefixed.insert(0, GetLogPrefix(category, level)); - } +void BCLog::Logger::FormatLogStrInPlace(std::string& str, BCLog::LogFlags category, BCLog::Level level, std::string_view source_file, int source_line, std::string_view logging_function, std::string_view threadname, SystemClock::time_point now, std::chrono::seconds mocktime) const +{ + str.insert(0, GetLogPrefix(category, level)); - if (m_log_sourcelocations && m_started_new_line) { - str_prefixed.insert(0, "[" + RemovePrefix(source_file, "./") + ":" + ToString(source_line) + "] [" + logging_function + "] "); + if (m_log_sourcelocations) { + str.insert(0, strprintf("[%s:%d] [%s] ", RemovePrefixView(source_file, "./"), source_line, logging_function)); } - if (m_log_threadnames && m_started_new_line) { - const auto& threadname = util::ThreadGetInternalName(); - str_prefixed.insert(0, "[" + (threadname.empty() ? "unknown" : threadname) + "] "); + if (m_log_threadnames) { + str.insert(0, strprintf("[%s] ", (threadname.empty() ? "unknown" : threadname))); } - str_prefixed = LogTimestampStr(str_prefixed); + str.insert(0, LogTimestampStr(now, mocktime)); +} + +void BCLog::Logger::LogPrintStr(std::string_view str, std::string_view logging_function, std::string_view source_file, int source_line, BCLog::LogFlags category, BCLog::Level level) +{ + StdLockGuard scoped_lock(m_cs); + return LogPrintStr_(str, logging_function, source_file, source_line, category, level); +} +void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_function, std::string_view source_file, int source_line, BCLog::LogFlags category, BCLog::Level level) +{ + std::string str_prefixed = LogEscapeMessage(str); + + const bool starts_new_line = m_started_new_line; m_started_new_line = !str.empty() && str[str.size()-1] == '\n'; if (m_buffering) { - // buffer if we haven't started logging yet - m_msgs_before_open.push_back(str_prefixed); + if (!starts_new_line) { + if (!m_msgs_before_open.empty()) { + m_msgs_before_open.back().str += str_prefixed; + m_cur_buffer_memusage += str_prefixed.size(); + return; + } else { + // unlikely edge case; add a marker that something was trimmed + str_prefixed.insert(0, "[...] "); + } + } + + { + BufferedLog buf{ + .now=SystemClock::now(), + .mocktime=GetMockTime(), + .str=str_prefixed, + .logging_function=std::string(logging_function), + .source_file=std::string(source_file), + .threadname=util::ThreadGetInternalName(), + .source_line=source_line, + .category=category, + .level=level, + }; + m_cur_buffer_memusage += MemUsage(buf); + m_msgs_before_open.push_back(std::move(buf)); + } + + while (m_cur_buffer_memusage > m_max_buffer_memusage) { + if (m_msgs_before_open.empty()) { + m_cur_buffer_memusage = 0; + break; + } + m_cur_buffer_memusage -= MemUsage(m_msgs_before_open.front()); + m_msgs_before_open.pop_front(); + ++m_buffer_lines_discarded; + } + return; } + if (starts_new_line) { + FormatLogStrInPlace(str_prefixed, category, level, source_file, source_line, logging_function, util::ThreadGetInternalName(), SystemClock::now(), GetMockTime()); + } + if (m_print_to_console) { // print to console fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); @@ -444,7 +511,7 @@ void BCLog::Logger::ShrinkDebugFile() fclose(file); } -bool BCLog::Logger::SetLogLevel(const std::string& level_str) +bool BCLog::Logger::SetLogLevel(std::string_view level_str) { const auto level = GetLogLevel(level_str); if (!level.has_value() || level.value() > MAX_USER_SETABLE_SEVERITY_LEVEL) return false; @@ -452,7 +519,7 @@ bool BCLog::Logger::SetLogLevel(const std::string& level_str) return true; } -bool BCLog::Logger::SetCategoryLogLevel(const std::string& category_str, const std::string& level_str) +bool BCLog::Logger::SetCategoryLogLevel(std::string_view category_str, std::string_view level_str) { BCLog::LogFlags flag; if (!GetLogCategory(flag, category_str)) return false; diff --git a/src/logging.h b/src/logging.h index fe6b7051ba..91f919e822 100644 --- a/src/logging.h +++ b/src/logging.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -79,15 +80,29 @@ namespace BCLog { Error, }; constexpr auto DEFAULT_LOG_LEVEL{Level::Debug}; + constexpr size_t DEFAULT_MAX_LOG_BUFFER{1'000'000}; // buffer up to 1MB of log data prior to StartLogging class Logger { + public: + struct BufferedLog { + SystemClock::time_point now; + std::chrono::seconds mocktime; + std::string str, logging_function, source_file, threadname; + int source_line; + LogFlags category; + Level level; + }; + private: mutable StdMutex m_cs; // Can not use Mutex from sync.h because in debug mode it would cause a deadlock when a potential deadlock was detected FILE* m_fileout GUARDED_BY(m_cs) = nullptr; - std::list m_msgs_before_open GUARDED_BY(m_cs); + std::list m_msgs_before_open GUARDED_BY(m_cs); bool m_buffering GUARDED_BY(m_cs) = true; //!< Buffer messages before logging can be started. + size_t m_max_buffer_memusage GUARDED_BY(m_cs){DEFAULT_MAX_LOG_BUFFER}; + size_t m_cur_buffer_memusage GUARDED_BY(m_cs){0}; + size_t m_buffer_lines_discarded GUARDED_BY(m_cs){0}; /** * m_started_new_line is a state variable that will suppress printing of @@ -106,11 +121,17 @@ namespace BCLog { /** Log categories bitfield. */ std::atomic m_categories{0}; - std::string LogTimestampStr(const std::string& str); + void FormatLogStrInPlace(std::string& str, LogFlags category, Level level, std::string_view source_file, int source_line, std::string_view logging_function, std::string_view threadname, SystemClock::time_point now, std::chrono::seconds mocktime) const; + + std::string LogTimestampStr(SystemClock::time_point now, std::chrono::seconds mocktime) const; /** Slots that connect to the print signal */ std::list> m_print_callbacks GUARDED_BY(m_cs) {}; + /** Send a string to the log output (internal) */ + void LogPrintStr_(std::string_view str, std::string_view logging_function, std::string_view source_file, int source_line, BCLog::LogFlags category, BCLog::Level level) + EXCLUSIVE_LOCKS_REQUIRED(m_cs); + public: bool m_print_to_console = false; bool m_print_to_file = false; @@ -127,17 +148,18 @@ namespace BCLog { std::string GetLogPrefix(LogFlags category, Level level) const; /** Send a string to the log output */ - void LogPrintStr(const std::string& str, const std::string& logging_function, const std::string& source_file, int source_line, BCLog::LogFlags category, BCLog::Level level); + void LogPrintStr(std::string_view str, std::string_view logging_function, std::string_view source_file, int source_line, BCLog::LogFlags category, BCLog::Level level) + EXCLUSIVE_LOCKS_REQUIRED(!m_cs); /** Returns whether logs will be written to any output */ - bool Enabled() const + bool Enabled() const EXCLUSIVE_LOCKS_REQUIRED(!m_cs) { StdLockGuard scoped_lock(m_cs); return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); } /** Connect a slot to the print signal and return the connection */ - std::list>::iterator PushBackCallback(std::function fun) + std::list>::iterator PushBackCallback(std::function fun) EXCLUSIVE_LOCKS_REQUIRED(!m_cs) { StdLockGuard scoped_lock(m_cs); m_print_callbacks.push_back(std::move(fun)); @@ -145,44 +167,52 @@ namespace BCLog { } /** Delete a connection */ - void DeleteCallback(std::list>::iterator it) + void DeleteCallback(std::list>::iterator it) EXCLUSIVE_LOCKS_REQUIRED(!m_cs) { StdLockGuard scoped_lock(m_cs); m_print_callbacks.erase(it); } /** Start logging (and flush all buffered messages) */ - bool StartLogging(); + bool StartLogging() EXCLUSIVE_LOCKS_REQUIRED(!m_cs); /** Only for testing */ - void DisconnectTestLogger(); + void DisconnectTestLogger() EXCLUSIVE_LOCKS_REQUIRED(!m_cs); + + /** Disable logging + * This offers a slight speedup and slightly smaller memory usage + * compared to leaving the logging system in its default state. + * Mostly intended for libbitcoin-kernel apps that don't want any logging. + * Should be used instead of StartLogging(). + */ + void DisableLogging() EXCLUSIVE_LOCKS_REQUIRED(!m_cs); void ShrinkDebugFile(); - std::unordered_map CategoryLevels() const + std::unordered_map CategoryLevels() const EXCLUSIVE_LOCKS_REQUIRED(!m_cs) { StdLockGuard scoped_lock(m_cs); return m_category_log_levels; } - void SetCategoryLogLevel(const std::unordered_map& levels) + void SetCategoryLogLevel(const std::unordered_map& levels) EXCLUSIVE_LOCKS_REQUIRED(!m_cs) { StdLockGuard scoped_lock(m_cs); m_category_log_levels = levels; } - bool SetCategoryLogLevel(const std::string& category_str, const std::string& level_str); + bool SetCategoryLogLevel(std::string_view category_str, std::string_view level_str) EXCLUSIVE_LOCKS_REQUIRED(!m_cs); Level LogLevel() const { return m_log_level.load(); } void SetLogLevel(Level level) { m_log_level = level; } - bool SetLogLevel(const std::string& level); + bool SetLogLevel(std::string_view level); uint32_t GetCategoryMask() const { return m_categories.load(); } void EnableCategory(LogFlags flag); - bool EnableCategory(const std::string& str); + bool EnableCategory(std::string_view str); void DisableCategory(LogFlags flag); - bool DisableCategory(const std::string& str); + bool DisableCategory(std::string_view str); bool WillLogCategory(LogFlags category) const; - bool WillLogCategoryLevel(LogFlags category, Level level) const; + bool WillLogCategoryLevel(LogFlags category, Level level) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs); /** Returns a vector of the log categories in alphabetical order. */ std::vector LogCategoriesList() const; @@ -212,14 +242,14 @@ static inline bool LogAcceptCategory(BCLog::LogFlags category, BCLog::Level leve } /** Return true if str parses as a log category and set the flag */ -bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str); +bool GetLogCategory(BCLog::LogFlags& flag, std::string_view str); // Be conservative when using functions that // unconditionally log to debug.log! It should not be the case that an inbound // peer can fill up a user's disk with debug.log entries. template -static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const BCLog::LogFlags flag, const BCLog::Level level, const char* fmt, const Args&... args) +static inline void LogPrintf_(std::string_view logging_function, std::string_view source_file, const int source_line, const BCLog::LogFlags flag, const BCLog::Level level, const char* fmt, const Args&... args) { if (LogInstance().Enabled()) { std::string log_msg; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 374497bfd1..70ef00abc3 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -505,10 +505,12 @@ class PeerManagerImpl final : public PeerManager CTxMemPool& pool, node::Warnings& warnings, Options opts); /** Overridden from CValidationInterface. */ + void ActiveTipChange(const CBlockIndex& new_tip, bool) override + EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); void BlockConnected(ChainstateRole role, const std::shared_ptr& pblock, const CBlockIndex* pindexConnected) override - EXCLUSIVE_LOCKS_REQUIRED(!m_recent_confirmed_transactions_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); void BlockDisconnected(const std::shared_ptr &block, const CBlockIndex* pindex) override - EXCLUSIVE_LOCKS_REQUIRED(!m_recent_confirmed_transactions_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void BlockChecked(const CBlock& block, const BlockValidationState& state) override @@ -517,13 +519,13 @@ class PeerManagerImpl final : public PeerManager EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex); /** Implement NetEventsInterface */ - void InitializeNode(const CNode& node, ServiceFlags our_services) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); - void FinalizeNode(const CNode& node) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_headers_presync_mutex); + void InitializeNode(const CNode& node, ServiceFlags our_services) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_tx_download_mutex); + void FinalizeNode(const CNode& node) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_headers_presync_mutex, !m_tx_download_mutex); bool HasAllDesirableServiceFlags(ServiceFlags services) const override; bool ProcessMessages(CNode* pfrom, std::atomic& interrupt) override - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_recent_confirmed_transactions_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex, !m_tx_download_mutex); bool SendMessages(CNode* pto) override - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_recent_confirmed_transactions_mutex, !m_most_recent_block_mutex, g_msgproc_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_most_recent_block_mutex, g_msgproc_mutex, !m_tx_download_mutex); /** Implement PeerManager */ void StartScheduledTasks(CScheduler& scheduler) override; @@ -542,7 +544,7 @@ class PeerManagerImpl final : public PeerManager void UnitTestMisbehaving(NodeId peer_id) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex) { Misbehaving(*Assert(GetPeerRef(peer_id)), ""); }; void ProcessMessage(CNode& pfrom, const std::string& msg_type, DataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic& interruptMsgProc) override - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_recent_confirmed_transactions_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex, !m_tx_download_mutex); void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) override; ServiceFlags GetDesirableServiceFlags(ServiceFlags services) const override; @@ -601,12 +603,12 @@ class PeerManagerImpl final : public PeerManager * Updates m_txrequest, m_recent_rejects, m_recent_rejects_reconsiderable, m_orphanage, and vExtraTxnForCompact. */ void ProcessInvalidTx(NodeId nodeid, const CTransactionRef& tx, const TxValidationState& result, bool maybe_add_extra_compact_tx) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); /** Handle a transaction whose result was MempoolAcceptResult::ResultType::VALID. * Updates m_txrequest, m_orphanage, and vExtraTxnForCompact. Also queues the tx for relay. */ void ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, const std::list& replaced_transactions) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); struct PackageToValidate { const Package m_txns; @@ -636,13 +638,13 @@ class PeerManagerImpl final : public PeerManager * individual transactions, and caches rejection for the package as a group. */ void ProcessPackageResult(const PackageToValidate& package_to_validate, const PackageMempoolAcceptResult& package_result) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); /** Look for a child of this transaction in the orphanage to form a 1-parent-1-child package, * skipping any combinations that have already been tried. Return the resulting package along with * the senders of its respective transactions, or std::nullopt if no package is found. */ std::optional Find1P1CPackage(const CTransactionRef& ptx, NodeId nodeid) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, m_tx_download_mutex); /** * Reconsider orphan transactions after a parent has been accepted to the mempool. @@ -656,7 +658,7 @@ class PeerManagerImpl final : public PeerManager * will be empty. */ bool ProcessOrphanTx(Peer& peer) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, !m_tx_download_mutex); /** Process a single headers message from a peer. * @@ -738,7 +740,7 @@ class PeerManagerImpl final : public PeerManager * peer. The announcement parameters are decided in PeerManager and then * passed to TxRequestTracker. */ void AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_tx_download_mutex); /** Send a message to a peer */ void PushMessage(CNode& node, CSerializedNetMsg&& msg) const { m_connman.PushMessage(&node, std::move(msg)); } @@ -786,7 +788,17 @@ class PeerManagerImpl final : public PeerManager BanMan* const m_banman; ChainstateManager& m_chainman; CTxMemPool& m_mempool; - TxRequestTracker m_txrequest GUARDED_BY(::cs_main); + + /** Synchronizes tx download including TxRequestTracker, rejection filters, and TxOrphanage. + * Lock invariants: + * - A txhash (txid or wtxid) in m_txrequest is not also in m_orphanage. + * - A txhash (txid or wtxid) in m_txrequest is not also in m_recent_rejects. + * - A txhash (txid or wtxid) in m_txrequest is not also in m_recent_rejects_reconsiderable. + * - A txhash (txid or wtxid) in m_txrequest is not also in m_recent_confirmed_transactions. + * - Each data structure's limits hold (m_orphanage max size, m_txrequest per-peer limits, etc). + */ + Mutex m_tx_download_mutex ACQUIRED_BEFORE(m_mempool.cs); + TxRequestTracker m_txrequest GUARDED_BY(m_tx_download_mutex); std::unique_ptr m_txreconciliation; /** The height of the best chain */ @@ -863,11 +875,9 @@ class PeerManagerImpl final : public PeerManager * - m_recent_rejects * - m_recent_rejects_reconsiderable (if include_reconsiderable = true) * - m_recent_confirmed_transactions - * Also responsible for resetting m_recent_rejects and m_recent_rejects_reconsiderable if the - * chain tip has changed. * */ bool AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable) - EXCLUSIVE_LOCKS_REQUIRED(cs_main, !m_recent_confirmed_transactions_mutex); + EXCLUSIVE_LOCKS_REQUIRED(m_tx_download_mutex); /** * Filter for transactions that were recently rejected by the mempool. @@ -903,10 +913,7 @@ class PeerManagerImpl final : public PeerManager * * Memory used: 1.3 MB */ - CRollingBloomFilter m_recent_rejects GUARDED_BY(::cs_main){120'000, 0.000'001}; - /** Block hash of chain tip the last time we reset m_recent_rejects and - * m_recent_rejects_reconsiderable. */ - uint256 hashRecentRejectsChainTip GUARDED_BY(cs_main); + CRollingBloomFilter m_recent_rejects GUARDED_BY(m_tx_download_mutex){120'000, 0.000'001}; /** * Filter for: @@ -928,7 +935,7 @@ class PeerManagerImpl final : public PeerManager * * Parameters are picked to be the same as m_recent_rejects, with the same rationale. */ - CRollingBloomFilter m_recent_rejects_reconsiderable GUARDED_BY(::cs_main){120'000, 0.000'001}; + CRollingBloomFilter m_recent_rejects_reconsiderable GUARDED_BY(m_tx_download_mutex){120'000, 0.000'001}; /* * Filter for transactions that have been recently confirmed. @@ -945,8 +952,7 @@ class PeerManagerImpl final : public PeerManager * transaction per day that would be inadvertently ignored (which is the * same probability that we have in the reject filter). */ - Mutex m_recent_confirmed_transactions_mutex; - CRollingBloomFilter m_recent_confirmed_transactions GUARDED_BY(m_recent_confirmed_transactions_mutex){48'000, 0.000'001}; + CRollingBloomFilter m_recent_confirmed_transactions GUARDED_BY(m_tx_download_mutex){48'000, 0.000'001}; /** * For sending `inv`s to inbound peers, we use a single (exponentially @@ -1083,7 +1089,7 @@ class PeerManagerImpl final : public PeerManager int m_peers_downloading_from GUARDED_BY(cs_main) = 0; /** Storage for orphan information */ - TxOrphanage m_orphanage; + TxOrphanage m_orphanage GUARDED_BY(m_tx_download_mutex); void AddToCompactExtraTransactions(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex); @@ -1646,7 +1652,8 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) void PeerManagerImpl::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) { - AssertLockHeld(::cs_main); // For m_txrequest + AssertLockHeld(::cs_main); // for State + AssertLockHeld(m_tx_download_mutex); // For m_txrequest NodeId nodeid = node.GetId(); if (!node.HasPermission(NetPermissionFlags::Relay) && m_txrequest.Count(nodeid) >= MAX_PEER_TX_ANNOUNCEMENTS) { // Too many queued announcements from this peer @@ -1682,8 +1689,11 @@ void PeerManagerImpl::InitializeNode(const CNode& node, ServiceFlags our_service { NodeId nodeid = node.GetId(); { - LOCK(cs_main); + LOCK(cs_main); // For m_node_states m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(node.IsInboundConn())); + } + { + LOCK(m_tx_download_mutex); assert(m_txrequest.Count(nodeid) == 0); } @@ -1751,8 +1761,11 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) } } } - m_orphanage.EraseForPeer(nodeid); - m_txrequest.DisconnectedPeer(nodeid); + { + LOCK(m_tx_download_mutex); + m_orphanage.EraseForPeer(nodeid); + m_txrequest.DisconnectedPeer(nodeid); + } if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; m_peers_downloading_from -= (!state->vBlocksInFlight.empty()); @@ -1769,6 +1782,7 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) assert(m_peers_downloading_from == 0); assert(m_outbound_peers_with_protect_from_disconnect == 0); assert(m_wtxid_relay_peers == 0); + LOCK(m_tx_download_mutex); assert(m_txrequest.Size() == 0); assert(m_orphanage.Size() == 0); } @@ -2070,6 +2084,23 @@ void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } +void PeerManagerImpl::ActiveTipChange(const CBlockIndex& new_tip, bool is_ibd) +{ + // Ensure mempool mutex was released, otherwise deadlock may occur if another thread holding + // m_tx_download_mutex waits on the mempool mutex. + AssertLockNotHeld(m_mempool.cs); + AssertLockNotHeld(m_tx_download_mutex); + + if (!is_ibd) { + LOCK(m_tx_download_mutex); + // If the chain tip has changed, previously rejected transactions might now be valid, e.g. due + // to a timelock. Reset the rejection filters to give those transactions another chance if we + // see them again. + m_recent_rejects.reset(); + m_recent_rejects_reconsiderable.reset(); + } +} + /** * Evict orphan txn pool entries based on a newly connected * block, remember the recently confirmed transactions, and delete tracked @@ -2100,23 +2131,16 @@ void PeerManagerImpl::BlockConnected( if (role == ChainstateRole::BACKGROUND) { return; } + LOCK(m_tx_download_mutex); m_orphanage.EraseForBlock(*pblock); - { - LOCK(m_recent_confirmed_transactions_mutex); - for (const auto& ptx : pblock->vtx) { - m_recent_confirmed_transactions.insert(ptx->GetHash().ToUint256()); - if (ptx->HasWitness()) { - m_recent_confirmed_transactions.insert(ptx->GetWitnessHash().ToUint256()); - } - } - } - { - LOCK(cs_main); - for (const auto& ptx : pblock->vtx) { - m_txrequest.ForgetTxHash(ptx->GetHash()); - m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); + for (const auto& ptx : pblock->vtx) { + m_recent_confirmed_transactions.insert(ptx->GetHash().ToUint256()); + if (ptx->HasWitness()) { + m_recent_confirmed_transactions.insert(ptx->GetWitnessHash().ToUint256()); } + m_txrequest.ForgetTxHash(ptx->GetHash()); + m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); } } @@ -2130,7 +2154,7 @@ void PeerManagerImpl::BlockDisconnected(const std::shared_ptr &blo // block's worth of transactions in it, but that should be fine, since // presumably the most common case of relaying a confirmed transaction // should be just after a new block containing it is found. - LOCK(m_recent_confirmed_transactions_mutex); + LOCK(m_tx_download_mutex); m_recent_confirmed_transactions.reset(); } @@ -2270,15 +2294,7 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable) { - if (m_chainman.ActiveChain().Tip()->GetBlockHash() != hashRecentRejectsChainTip) { - // If the chain tip has changed previously rejected transactions - // might be now valid, e.g. due to a nLockTime'd tx becoming valid, - // or a double-spend. Reset the rejects filter and give those - // txs a second chance. - hashRecentRejectsChainTip = m_chainman.ActiveChain().Tip()->GetBlockHash(); - m_recent_rejects.reset(); - m_recent_rejects_reconsiderable.reset(); - } + AssertLockHeld(m_tx_download_mutex); const uint256& hash = gtxid.GetHash(); @@ -2302,10 +2318,7 @@ bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid, bool include_reconside if (include_reconsiderable && m_recent_rejects_reconsiderable.contains(hash)) return true; - { - LOCK(m_recent_confirmed_transactions_mutex); - if (m_recent_confirmed_transactions.contains(hash)) return true; - } + if (m_recent_confirmed_transactions.contains(hash)) return true; return m_recent_rejects.contains(hash) || m_mempool.exists(gtxid); } @@ -3204,7 +3217,7 @@ void PeerManagerImpl::ProcessInvalidTx(NodeId nodeid, const CTransactionRef& ptx { AssertLockNotHeld(m_peer_mutex); AssertLockHeld(g_msgproc_mutex); - AssertLockHeld(cs_main); + AssertLockHeld(m_tx_download_mutex); LogDebug(BCLog::MEMPOOLREJ, "%s (wtxid=%s) from peer=%d was not accepted: %s\n", ptx->GetHash().ToString(), @@ -3269,7 +3282,7 @@ void PeerManagerImpl::ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, c { AssertLockNotHeld(m_peer_mutex); AssertLockHeld(g_msgproc_mutex); - AssertLockHeld(cs_main); + AssertLockHeld(m_tx_download_mutex); // As this version of the transaction was acceptable, we can forget about any requests for it. // No-op if the tx is not in txrequest. @@ -3297,7 +3310,7 @@ void PeerManagerImpl::ProcessPackageResult(const PackageToValidate& package_to_v { AssertLockNotHeld(m_peer_mutex); AssertLockHeld(g_msgproc_mutex); - AssertLockHeld(cs_main); + AssertLockHeld(m_tx_download_mutex); const auto& package = package_to_validate.m_txns; const auto& senders = package_to_validate.m_senders; @@ -3353,7 +3366,7 @@ std::optional PeerManagerImpl::Find1P1CPacka { AssertLockNotHeld(m_peer_mutex); AssertLockHeld(g_msgproc_mutex); - AssertLockHeld(cs_main); + AssertLockHeld(m_tx_download_mutex); const auto& parent_wtxid{ptx->GetWitnessHash()}; @@ -3406,7 +3419,7 @@ std::optional PeerManagerImpl::Find1P1CPacka bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) { AssertLockHeld(g_msgproc_mutex); - LOCK(cs_main); + LOCK2(::cs_main, m_tx_download_mutex); CTransactionRef porphanTx = nullptr; @@ -4223,7 +4236,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const bool reject_tx_invs{RejectIncomingTxs(pfrom)}; - LOCK(cs_main); + LOCK2(cs_main, m_tx_download_mutex); const auto current_time{GetTime()}; uint256* best_block{nullptr}; @@ -4587,7 +4600,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const uint256& hash = peer->m_wtxid_relay ? wtxid : txid; AddKnownTx(*peer, hash); - LOCK(cs_main); + LOCK2(cs_main, m_tx_download_mutex); m_txrequest.ReceivedResponse(pfrom.GetId(), txid); if (tx.HasWitness()) m_txrequest.ReceivedResponse(pfrom.GetId(), wtxid); @@ -5344,7 +5357,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::vector vInv; vRecv >> vInv; if (vInv.size() <= MAX_PEER_TX_ANNOUNCEMENTS + MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - LOCK(::cs_main); + LOCK(m_tx_download_mutex); for (CInv &inv : vInv) { if (inv.IsGenTxMsg()) { // If we receive a NOTFOUND message for a tx we requested, mark the announcement for it as @@ -5402,6 +5415,7 @@ bool PeerManagerImpl::MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer) bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic& interruptMsgProc) { + AssertLockNotHeld(m_tx_download_mutex); AssertLockHeld(g_msgproc_mutex); PeerRef peer = GetPeerRef(pfrom->GetId()); @@ -5469,6 +5483,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic& interrupt // by another peer that was already processed; in that case, // the extra work may not be noticed, possibly resulting in an // unnecessary 100ms delay) + LOCK(m_tx_download_mutex); if (m_orphanage.HaveTxToReconsider(peer->m_id)) fMoreWork = true; } catch (const std::exception& e) { LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg.m_type), msg.m_message_size, e.what(), typeid(e).name()); @@ -5892,6 +5907,7 @@ bool PeerManagerImpl::SetupAddressRelay(const CNode& node, Peer& peer) bool PeerManagerImpl::SendMessages(CNode* pto) { + AssertLockNotHeld(m_tx_download_mutex); AssertLockHeld(g_msgproc_mutex); PeerRef peer = GetPeerRef(pto->GetId()); @@ -6362,31 +6378,33 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // // Message: getdata (transactions) // - std::vector> expired; - auto requestable = m_txrequest.GetRequestable(pto->GetId(), current_time, &expired); - for (const auto& entry : expired) { - LogPrint(BCLog::NET, "timeout of inflight %s %s from peer=%d\n", entry.second.IsWtxid() ? "wtx" : "tx", - entry.second.GetHash().ToString(), entry.first); - } - for (const GenTxid& gtxid : requestable) { - // Exclude m_recent_rejects_reconsiderable: we may be requesting a missing parent - // that was previously rejected for being too low feerate. - if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) { - LogPrint(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", - gtxid.GetHash().ToString(), pto->GetId()); - vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); - if (vGetData.size() >= MAX_GETDATA_SZ) { - MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); - vGetData.clear(); + { + LOCK(m_tx_download_mutex); + std::vector> expired; + auto requestable = m_txrequest.GetRequestable(pto->GetId(), current_time, &expired); + for (const auto& entry : expired) { + LogPrint(BCLog::NET, "timeout of inflight %s %s from peer=%d\n", entry.second.IsWtxid() ? "wtx" : "tx", + entry.second.GetHash().ToString(), entry.first); + } + for (const GenTxid& gtxid : requestable) { + // Exclude m_recent_rejects_reconsiderable: we may be requesting a missing parent + // that was previously rejected for being too low feerate. + if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) { + LogPrint(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", + gtxid.GetHash().ToString(), pto->GetId()); + vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); + if (vGetData.size() >= MAX_GETDATA_SZ) { + MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); + vGetData.clear(); + } + m_txrequest.RequestedTx(pto->GetId(), gtxid.GetHash(), current_time + GETDATA_TX_INTERVAL); + } else { + // We have already seen this transaction, no need to download. This is just a belt-and-suspenders, as + // this should already be called whenever a transaction becomes AlreadyHaveTx(). + m_txrequest.ForgetTxHash(gtxid.GetHash()); } - m_txrequest.RequestedTx(pto->GetId(), gtxid.GetHash(), current_time + GETDATA_TX_INTERVAL); - } else { - // We have already seen this transaction, no need to download. This is just a belt-and-suspenders, as - // this should already be called whenever a transaction becomes AlreadyHaveTx(). - m_txrequest.ForgetTxHash(gtxid.GetHash()); } - } - + } // release m_tx_download_mutex if (!vGetData.empty()) MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 80360f1dae..caf2b5f7aa 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -734,7 +734,7 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in bool BlockManager::FlushUndoFile(int block_file, bool finalize) { FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize); - if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { + if (!m_undo_file_seq.Flush(undo_pos_old, finalize)) { m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error.")); return false; } @@ -756,7 +756,7 @@ bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finali assert(static_cast(m_blockfile_info.size()) > blockfile_num); FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize); - if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { + if (!m_block_file_seq.Flush(block_pos_old, fFinalize)) { m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error.")); success = false; } @@ -808,38 +808,28 @@ void BlockManager::UnlinkPrunedFiles(const std::set& setFilesToPrune) const std::error_code ec; for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { FlatFilePos pos(*it, 0); - const bool removed_blockfile{fs::remove(BlockFileSeq().FileName(pos), ec)}; - const bool removed_undofile{fs::remove(UndoFileSeq().FileName(pos), ec)}; + const bool removed_blockfile{fs::remove(m_block_file_seq.FileName(pos), ec)}; + const bool removed_undofile{fs::remove(m_undo_file_seq.FileName(pos), ec)}; if (removed_blockfile || removed_undofile) { LogPrint(BCLog::BLOCKSTORAGE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } } -FlatFileSeq BlockManager::BlockFileSeq() const -{ - return FlatFileSeq(m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); -} - -FlatFileSeq BlockManager::UndoFileSeq() const -{ - return FlatFileSeq(m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE); -} - AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { - return AutoFile{BlockFileSeq().Open(pos, fReadOnly)}; + return AutoFile{m_block_file_seq.Open(pos, fReadOnly)}; } /** Open an undo file (rev?????.dat) */ AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { - return AutoFile{UndoFileSeq().Open(pos, fReadOnly)}; + return AutoFile{m_undo_file_seq.Open(pos, fReadOnly)}; } fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const { - return BlockFileSeq().FileName(pos); + return m_block_file_seq.FileName(pos); } FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime) @@ -919,7 +909,7 @@ FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int n m_blockfile_info[nFile].nSize += nAddSize; bool out_of_space; - size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + size_t bytes_allocated = m_block_file_seq.Allocate(pos, nAddSize, out_of_space); if (out_of_space) { m_opts.notifications.fatalError(_("Disk space is too low!")); return {}; @@ -965,7 +955,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP m_dirty_fileinfo.insert(nFile); bool out_of_space; - size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + size_t bytes_allocated = m_undo_file_seq.Allocate(pos, nAddSize, out_of_space); if (out_of_space) { return FatalError(m_opts.notifications, state, _("Disk space is too low!")); } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 32211c80fe..a6fa6453f5 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -170,9 +170,6 @@ class BlockManager [[nodiscard]] bool FlushChainstateBlockFile(int tip_height); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); - FlatFileSeq BlockFileSeq() const; - FlatFileSeq UndoFileSeq() const; - AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; /** @@ -265,12 +262,17 @@ class BlockManager const kernel::BlockManagerOpts m_opts; + const FlatFileSeq m_block_file_seq; + const FlatFileSeq m_undo_file_seq; + public: using Options = kernel::BlockManagerOpts; explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts) : m_prune_mode{opts.prune_target > 0}, m_opts{std::move(opts)}, + m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}}, + m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}}, m_interrupt{interrupt} {} const util::SignalInterrupt& m_interrupt; diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 949b1b7775..935d9d2b59 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -442,7 +442,7 @@ void OptionsDialog::updateProxyValidationState() QValidatedLineEdit *otherProxyWidget = (pUiProxyIp == ui->proxyIpTor) ? ui->proxyIp : ui->proxyIpTor; if (pUiProxyIp->isValid() && (!ui->proxyPort->isEnabled() || ui->proxyPort->text().toInt() > 0) && (!ui->proxyPortTor->isEnabled() || ui->proxyPortTor->text().toInt() > 0)) { - setOkButtonState(otherProxyWidget->isValid()); //only enable ok button if both proxys are valid + setOkButtonState(otherProxyWidget->isValid()); //only enable ok button if both proxies are valid clearStatusLabel(); } else diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index d4267fcf61..9214e7723d 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -277,7 +277,7 @@ void TransactionTableModel::updateAmountColumnTitle() void TransactionTableModel::updateTransaction(const QString &hash, int status, bool showTransaction) { uint256 updated; - updated.SetHex(hash.toStdString()); + updated.SetHexDeprecated(hash.toStdString()); priv->updateWallet(walletModel->wallet(), updated, status, showTransaction); } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 7e24dbd3ec..2aaa65c6f7 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -396,7 +396,7 @@ void TransactionView::contextualMenu(const QPoint &point) // check if transaction can be abandoned, disable context menu action in case it doesn't uint256 hash; - hash.SetHex(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); + hash.SetHexDeprecated(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); abandonAction->setEnabled(model->wallet().transactionCanBeAbandoned(hash)); bumpFeeAction->setEnabled(model->wallet().transactionCanBeBumped(hash)); copyAddressAction->setEnabled(GUIUtil::hasEntryData(transactionView, 0, TransactionTableModel::AddressRole)); @@ -416,7 +416,7 @@ void TransactionView::abandonTx() // get the hash from the TxHashRole (QVariant / QString) uint256 hash; QString hashQStr = selection.at(0).data(TransactionTableModel::TxHashRole).toString(); - hash.SetHex(hashQStr.toStdString()); + hash.SetHexDeprecated(hashQStr.toStdString()); // Abandon the wallet transaction over the walletModel model->wallet().abandonTransaction(hash); @@ -431,7 +431,7 @@ void TransactionView::bumpFee([[maybe_unused]] bool checked) // get the hash from the TxHashRole (QVariant / QString) uint256 hash; QString hashQStr = selection.at(0).data(TransactionTableModel::TxHashRole).toString(); - hash.SetHex(hashQStr.toStdString()); + hash.SetHexDeprecated(hashQStr.toStdString()); // Bump tx fee over the walletModel uint256 newHash; diff --git a/src/rest.cpp b/src/rest.cpp index 3ef888a180..034c0ff0cb 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -217,9 +217,10 @@ static bool rest_headers(const std::any& context, return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, raw_count)); } - uint256 hash; - if (!ParseHashStr(hashStr, hash)) + auto hash{uint256::FromHex(hashStr)}; + if (!hash) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + } const CBlockIndex* tip = nullptr; std::vector headers; @@ -233,7 +234,7 @@ static bool rest_headers(const std::any& context, LOCK(cs_main); CChain& active_chain = chainman.ActiveChain(); tip = active_chain.Tip(); - const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + const CBlockIndex* pindex{chainman.m_blockman.LookupBlockIndex(*hash)}; while (pindex != nullptr && active_chain.Contains(pindex)) { headers.push_back(pindex); if (headers.size() == *parsed_count) { @@ -292,9 +293,10 @@ static bool rest_block(const std::any& context, std::string hashStr; const RESTResponseFormat rf = ParseDataFormat(hashStr, strURIPart); - uint256 hash; - if (!ParseHashStr(hashStr, hash)) + auto hash{uint256::FromHex(hashStr)}; + if (!hash) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + } FlatFilePos pos{}; const CBlockIndex* pblockindex = nullptr; @@ -305,7 +307,7 @@ static bool rest_block(const std::any& context, { LOCK(cs_main); tip = chainman.ActiveChain().Tip(); - pblockindex = chainman.m_blockman.LookupBlockIndex(hash); + pblockindex = chainman.m_blockman.LookupBlockIndex(*hash); if (!pblockindex) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -392,8 +394,8 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count is invalid or out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, raw_count)); } - uint256 block_hash; - if (!ParseHashStr(raw_blockhash, block_hash)) { + auto block_hash{uint256::FromHex(raw_blockhash)}; + if (!block_hash) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + raw_blockhash); } @@ -415,7 +417,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const ChainstateManager& chainman = *maybe_chainman; LOCK(cs_main); CChain& active_chain = chainman.ActiveChain(); - const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block_hash); + const CBlockIndex* pindex{chainman.m_blockman.LookupBlockIndex(*block_hash)}; while (pindex != nullptr && active_chain.Contains(pindex)) { headers.push_back(pindex); if (headers.size() == *parsed_count) @@ -496,8 +498,8 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilter//"); } - uint256 block_hash; - if (!ParseHashStr(uri_parts[1], block_hash)) { + auto block_hash{uint256::FromHex(uri_parts[1])}; + if (!block_hash) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[1]); } @@ -518,7 +520,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s if (!maybe_chainman) return false; ChainstateManager& chainman = *maybe_chainman; LOCK(cs_main); - block_index = chainman.m_blockman.LookupBlockIndex(block_hash); + block_index = chainman.m_blockman.LookupBlockIndex(*block_hash); if (!block_index) { return RESTERR(req, HTTP_NOT_FOUND, uri_parts[1] + " not found"); } @@ -618,14 +620,14 @@ static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const jsonRequest.params = UniValue(UniValue::VARR); if (!hash_str.empty()) { - uint256 hash; - if (!ParseHashStr(hash_str, hash)) { + auto hash{uint256::FromHex(hash_str)}; + if (!hash) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hash_str); } const ChainstateManager* chainman = GetChainman(context, req); if (!chainman) return false; - if (!WITH_LOCK(::cs_main, return chainman->m_blockman.LookupBlockIndex(ParseHashV(hash_str, "blockhash")))) { + if (!WITH_LOCK(::cs_main, return chainman->m_blockman.LookupBlockIndex(*hash))) { return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); } @@ -706,9 +708,10 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string std::string hashStr; const RESTResponseFormat rf = ParseDataFormat(hashStr, strURIPart); - uint256 hash; - if (!ParseHashStr(hashStr, hash)) + auto hash{uint256::FromHex(hashStr)}; + if (!hash) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + } if (g_txindex) { g_txindex->BlockUntilSyncedToCurrentChain(); @@ -717,7 +720,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string const NodeContext* const node = GetNodeContext(context, req); if (!node) return false; uint256 hashBlock = uint256(); - const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, hashBlock, node->chainman->m_blockman); + const CTransactionRef tx{GetTransaction(/*block_index=*/nullptr, node->mempool.get(), *hash, hashBlock, node->chainman->m_blockman)}; if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -794,13 +797,14 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: if (txid_out.size() != 2) { return RESTERR(req, HTTP_BAD_REQUEST, "Parse error"); } + auto txid{Txid::FromHex(txid_out.at(0))}; auto output{ToIntegral(txid_out.at(1))}; - if (!output || !IsHex(txid_out.at(0))) { + if (!txid || !output) { return RESTERR(req, HTTP_BAD_REQUEST, "Parse error"); } - vOutPoints.emplace_back(TxidFromString(txid_out.at(0)), *output); + vOutPoints.emplace_back(*txid, *output); } if (vOutPoints.size() > 0) diff --git a/src/rpc/auxpow_miner.cpp b/src/rpc/auxpow_miner.cpp index 1b5eaadbb9..1c56764a94 100644 --- a/src/rpc/auxpow_miner.cpp +++ b/src/rpc/auxpow_miner.cpp @@ -124,10 +124,11 @@ AuxpowMiner::lookupSavedBlock (const std::string& hashHex) const { AssertLockHeld (cs); - uint256 hash; - hash.SetHex (hashHex); + const auto hash = uint256::FromHex (hashHex); + if (!hash) + throw JSONRPCError (RPC_INVALID_PARAMETER, "invalid block hash hex"); - const auto iter = blocks.find (hash); + const auto iter = blocks.find (*hash); if (iter == blocks.end ()) throw JSONRPCError (RPC_INVALID_PARAMETER, "block hash unknown"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4b30e362f7..92db241f37 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -732,9 +732,9 @@ const RPCResult getblock_vin{ {RPCResult::Type::STR_AMOUNT, "value", "The value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, }}, @@ -1165,9 +1165,9 @@ static RPCHelpMan gettxout() {RPCResult::Type::NUM, "confirmations", "The number of confirmations"}, {RPCResult::Type::STR_AMOUNT, "value", "The transaction value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg pubkeyhash"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, @@ -2277,7 +2277,7 @@ static RPCHelpMan scantxoutset() RPCResult{"when action=='start'; only returns after scan completes", RPCResult::Type::OBJ, "", "", { {RPCResult::Type::BOOL, "success", "Whether the scan was completed"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs scanned"}, - {RPCResult::Type::NUM, "height", "The current block height (index)"}, + {RPCResult::Type::NUM, "height", "The block height at which the scan was done"}, {RPCResult::Type::STR_HEX, "bestblock", "The hash of the block at the tip of the chain"}, {RPCResult::Type::ARR, "unspents", "", { @@ -2290,6 +2290,8 @@ static RPCHelpMan scantxoutset() {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the unspent output"}, {RPCResult::Type::BOOL, "coinbase", "Whether this is a coinbase output"}, {RPCResult::Type::NUM, "height", "Height of the unspent transaction output"}, + {RPCResult::Type::STR_HEX, "blockhash", "Blockhash of the unspent transaction output"}, + {RPCResult::Type::NUM, "confirmations", "Number of confirmations of the unspent transaction output when the scan was done"}, }}, }}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of all found unspent outputs in " + CURRENCY_UNIT}, @@ -2379,17 +2381,20 @@ static RPCHelpMan scantxoutset() const COutPoint& outpoint = it.first; const Coin& coin = it.second; const CTxOut& txo = coin.out; + const CBlockIndex& coinb_block{*CHECK_NONFATAL(tip->GetAncestor(coin.nHeight))}; input_txos.push_back(txo); total_in += txo.nValue; UniValue unspent(UniValue::VOBJ); unspent.pushKV("txid", outpoint.hash.GetHex()); - unspent.pushKV("vout", (int32_t)outpoint.n); + unspent.pushKV("vout", outpoint.n); unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); unspent.pushKV("coinbase", coin.IsCoinBase()); - unspent.pushKV("height", (int32_t)coin.nHeight); + unspent.pushKV("height", coin.nHeight); + unspent.pushKV("blockhash", coinb_block.GetBlockHash().GetHex()); + unspent.pushKV("confirmations", tip->nHeight - coin.nHeight + 1); unspents.push_back(std::move(unspent)); } diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp index aefe78162b..662d24ef81 100644 --- a/src/rpc/fees.cpp +++ b/src/rpc/fees.cpp @@ -36,7 +36,7 @@ static RPCHelpMan estimatesmartfee() "in BIP 141 (witness data is discounted).\n", { {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"conservative"}, "The fee estimate mode.\n" + {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"economical"}, "The fee estimate mode.\n" "Whether to return a more conservative estimate which also satisfies\n" "a longer history. A conservative estimate potentially returns a\n" "higher feerate and is more likely to be sufficient for the desired\n" @@ -71,13 +71,13 @@ static RPCHelpMan estimatesmartfee() CHECK_NONFATAL(mempool.m_opts.signals)->SyncWithValidationInterfaceQueue(); unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); - bool conservative = true; + bool conservative = false; if (!request.params[1].isNull()) { FeeEstimateMode fee_mode; if (!FeeModeFromString(request.params[1].get_str(), fee_mode)) { throw JSONRPCError(RPC_INVALID_PARAMETER, InvalidEstimateModeErrorMessage()); } - if (fee_mode == FeeEstimateMode::ECONOMICAL) conservative = false; + if (fee_mode == FeeEstimateMode::CONSERVATIVE) conservative = true; } UniValue result(UniValue::VOBJ); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 3aa671b148..bde9968fc6 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -348,13 +348,11 @@ static RPCHelpMan generateblock() std::vector txs; const auto raw_txs_or_txids = request.params[1].get_array(); for (size_t i = 0; i < raw_txs_or_txids.size(); i++) { - const auto str(raw_txs_or_txids[i].get_str()); + const auto& str{raw_txs_or_txids[i].get_str()}; - uint256 hash; CMutableTransaction mtx; - if (ParseHashStr(str, hash)) { - - const auto tx = mempool.get(hash); + if (auto hash{uint256::FromHex(str)}) { + const auto tx{mempool.get(*hash)}; if (!tx) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Transaction %s not in mempool.", str)); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 390fa2c42d..e834286093 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -84,9 +84,9 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, static std::vector ScriptPubKeyDoc() { return { - {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, }; @@ -505,18 +505,18 @@ static RPCHelpMan decodescript() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "asm", "Script public key"}, + {RPCResult::Type::STR, "asm", "Disassembly of the script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the script"}, {RPCResult::Type::STR, "type", "The output type (e.g. " + GetAllOutputTypes() + ")"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "p2sh", /*optional=*/true, "address of P2SH script wrapping this redeem script (not returned for types that should not be wrapped)"}, {RPCResult::Type::OBJ, "segwit", /*optional=*/true, - "Result of a witness script public key wrapping this redeem script (not returned for types that should not be wrapped)", + "Result of a witness output script wrapping this redeem script (not returned for types that should not be wrapped)", { - {RPCResult::Type::STR, "asm", "String representation of the script public key"}, - {RPCResult::Type::STR_HEX, "hex", "Hex string of the script public key"}, - {RPCResult::Type::STR, "type", "The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)"}, + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, + {RPCResult::Type::STR, "type", "The type of the output script (e.g. witness_v0_keyhash or witness_v0_scripthash)"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the script"}, {RPCResult::Type::STR, "p2sh-segwit", "address of the P2SH script wrapping this witness redeem script"}, @@ -832,9 +832,9 @@ const RPCResult decodepsbt_inputs{ {RPCResult::Type::NUM, "amount", "The value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, diff --git a/src/support/cleanse.cpp b/src/support/cleanse.cpp index a8ddcd793f..4e370f1516 100644 --- a/src/support/cleanse.cpp +++ b/src/support/cleanse.cpp @@ -7,14 +7,14 @@ #include -#if defined(_MSC_VER) -#include // For SecureZeroMemory. +#if defined(WIN32) +#include #endif void memory_cleanse(void *ptr, size_t len) { -#if defined(_MSC_VER) - /* SecureZeroMemory is guaranteed not to be optimized out by MSVC. */ +#if defined(WIN32) + /* SecureZeroMemory is guaranteed not to be optimized out. */ SecureZeroMemory(ptr, len); #else std::memset(ptr, 0, len); diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index b372f25ea9..470fdde30a 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -149,8 +149,7 @@ BOOST_AUTO_TEST_CASE(blockfilters_json_test) unsigned int pos = 0; /*int block_height =*/ test[pos++].getInt(); - uint256 block_hash; - BOOST_CHECK(ParseHashStr(test[pos++].get_str(), block_hash)); + BOOST_CHECK(uint256::FromHex(test[pos++].get_str())); CBlock block; BOOST_REQUIRE(DecodeHexBlk(block, test[pos++].get_str())); @@ -165,11 +164,9 @@ BOOST_AUTO_TEST_CASE(blockfilters_json_test) tx_undo.vprevout.emplace_back(txout, 0, false); } - uint256 prev_filter_header_basic; - BOOST_CHECK(ParseHashStr(test[pos++].get_str(), prev_filter_header_basic)); + uint256 prev_filter_header_basic{*Assert(uint256::FromHex(test[pos++].get_str()))}; std::vector filter_basic = ParseHex(test[pos++].get_str()); - uint256 filter_header_basic; - BOOST_CHECK(ParseHashStr(test[pos++].get_str(), filter_header_basic)); + uint256 filter_header_basic{*Assert(uint256::FromHex(test[pos++].get_str()))}; BlockFilter computed_filter_basic(BlockFilterType::BASIC, block, block_undo); BOOST_CHECK(computed_filter_basic.GetFilter().GetEncoded() == filter_basic); diff --git a/src/test/cluster_linearize_tests.cpp b/src/test/cluster_linearize_tests.cpp new file mode 100644 index 0000000000..d15e783ea1 --- /dev/null +++ b/src/test/cluster_linearize_tests.cpp @@ -0,0 +1,138 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(cluster_linearize_tests, BasicTestingSetup) + +using namespace cluster_linearize; + +namespace { + +template +void TestDepGraphSerialization(const Cluster& cluster, const std::string& hexenc) +{ + DepGraph depgraph(cluster); + + // Run normal sanity and correspondence checks, which includes a round-trip test. + VerifyDepGraphFromCluster(cluster, depgraph); + + // There may be multiple serializations of the same graph, but DepGraphFormatter's serializer + // only produces one of those. Verify that hexenc matches that canonical serialization. + std::vector encoding; + VectorWriter writer(encoding, 0); + writer << Using(depgraph); + BOOST_CHECK_EQUAL(HexStr(encoding), hexenc); + + // Test that deserializing that encoding yields depgraph. This is effectively already implied + // by the round-trip test above (if depgraph is acyclic), but verify it explicitly again here. + SpanReader reader(encoding); + DepGraph depgraph_read; + reader >> Using(depgraph_read); + BOOST_CHECK(depgraph == depgraph_read); +} + +} // namespace + +BOOST_AUTO_TEST_CASE(depgraph_ser_tests) +{ + // Empty cluster. + TestDepGraphSerialization( + {}, + "00" /* end of graph */); + + // Transactions: A(fee=0,size=1). + TestDepGraphSerialization( + {{{0, 1}, {}}}, + "01" /* A size */ + "00" /* A fee */ + "00" /* A insertion position (no skips): A */ + "00" /* end of graph */); + + // Transactions: A(fee=42,size=11), B(fee=-13,size=7), B depends on A. + TestDepGraphSerialization( + {{{42, 11}, {}}, {{-13, 7}, {0}}}, + "0b" /* A size */ + "54" /* A fee */ + "00" /* A insertion position (no skips): A */ + "07" /* B size */ + "19" /* B fee */ + "00" /* B->A dependency (no skips) */ + "00" /* B insertion position (no skips): A,B */ + "00" /* end of graph */); + + // Transactions: A(64,128), B(128,256), C(1,1), C depends on A and B. + TestDepGraphSerialization( + {{{64, 128}, {}}, {{128, 256}, {}}, {{1, 1}, {0, 1}}}, + "8000" /* A size */ + "8000" /* A fee */ + "00" /* A insertion position (no skips): A */ + "8100" /* B size */ + "8100" /* B fee */ + "01" /* B insertion position (skip B->A dependency): A,B */ + "01" /* C size */ + "02" /* C fee */ + "00" /* C->B dependency (no skips) */ + "00" /* C->A dependency (no skips) */ + "00" /* C insertion position (no skips): A,B,C */ + "00" /* end of graph */); + + // Transactions: A(-57,113), B(57,114), C(-58,115), D(58,116). Deps: B->A, C->A, D->C, in order + // [B,A,C,D]. This exercises non-topological ordering (internally serialized as A,B,C,D). + TestDepGraphSerialization( + {{{57, 114}, {1}}, {{-57, 113}, {}}, {{-58, 115}, {1}}, {{58, 116}, {2}}}, + "71" /* A size */ + "71" /* A fee */ + "00" /* A insertion position (no skips): A */ + "72" /* B size */ + "72" /* B fee */ + "00" /* B->A dependency (no skips) */ + "01" /* B insertion position (skip A): B,A */ + "73" /* C size */ + "73" /* C fee */ + "01" /* C->A dependency (skip C->B dependency) */ + "00" /* C insertion position (no skips): B,A,C */ + "74" /* D size */ + "74" /* D fee */ + "00" /* D->C dependency (no skips) */ + "01" /* D insertion position (skip D->B dependency, D->A is implied): B,A,C,D */ + "00" /* end of graph */); + + // Transactions: A(1,2), B(3,1), C(2,1), D(1,3), E(1,1). Deps: C->A, D->A, D->B, E->D. + // In order: [D,A,B,E,C]. Internally serialized in order A,B,C,D,E. + TestDepGraphSerialization( + {{{1, 3}, {1, 2}}, {{1, 2}, {}}, {{3, 1}, {}}, {{1, 1}, {0}}, {{2, 1}, {1}}}, + "02" /* A size */ + "02" /* A fee */ + "00" /* A insertion position (no skips): A */ + "01" /* B size */ + "06" /* B fee */ + "01" /* B insertion position (skip B->A dependency): A,B */ + "01" /* C size */ + "04" /* C fee */ + "01" /* C->A dependency (skip C->B dependency) */ + "00" /* C insertion position (no skips): A,B,C */ + "03" /* D size */ + "02" /* D fee */ + "01" /* D->B dependency (skip D->C dependency) */ + "00" /* D->A dependency (no skips) */ + "03" /* D insertion position (skip C,B,A): D,A,B,C */ + "01" /* E size */ + "02" /* E fee */ + "00" /* E->D dependency (no skips) */ + "02" /* E insertion position (skip E->C dependency, E->B and E->A are implied, + skip insertion C): D,A,B,E,C */ + "00" /* end of graph */ + ); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/cluster_linearize.cpp b/src/test/fuzz/cluster_linearize.cpp new file mode 100644 index 0000000000..031cb04559 --- /dev/null +++ b/src/test/fuzz/cluster_linearize.cpp @@ -0,0 +1,689 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace cluster_linearize; + +namespace { + +/** A simple finder class for candidate sets. + * + * This class matches SearchCandidateFinder in interface and behavior, though with fewer + * optimizations. + */ +template +class SimpleCandidateFinder +{ + /** Internal dependency graph. */ + const DepGraph& m_depgraph; + /** Which transaction are left to include. */ + SetType m_todo; + +public: + /** Construct an SimpleCandidateFinder for a given graph. */ + SimpleCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + + /** Remove a set of transactions from the set of to-be-linearized ones. */ + void MarkDone(SetType select) noexcept { m_todo -= select; } + + /** Determine whether unlinearized transactions remain. */ + bool AllDone() const noexcept { return m_todo.None(); } + + /** Find a candidate set using at most max_iterations iterations, and the number of iterations + * actually performed. If that number is less than max_iterations, then the result is optimal. + * + * Complexity: O(N * M), where M is the number of connected topological subsets of the cluster. + * That number is bounded by M <= 2^(N-1). + */ + std::pair, uint64_t> FindCandidateSet(uint64_t max_iterations) const noexcept + { + uint64_t iterations_left = max_iterations; + // Queue of work units. Each consists of: + // - inc: set of transactions definitely included + // - und: set of transactions that can be added to inc still + std::vector> queue; + // Initially we have just one queue element, with the entire graph in und. + queue.emplace_back(SetType{}, m_todo); + // Best solution so far. + SetInfo best(m_depgraph, m_todo); + // Process the queue. + while (!queue.empty() && iterations_left) { + --iterations_left; + // Pop top element of the queue. + auto [inc, und] = queue.back(); + queue.pop_back(); + // Look for a transaction to consider adding/removing. + bool inc_none = inc.None(); + for (auto split : und) { + // If inc is empty, consider any split transaction. Otherwise only consider + // transactions that share ancestry with inc so far (which means only connected + // sets will be considered). + if (inc_none || inc.Overlaps(m_depgraph.Ancestors(split))) { + // Add a queue entry with split included. + SetInfo new_inc(m_depgraph, inc | (m_todo & m_depgraph.Ancestors(split))); + queue.emplace_back(new_inc.transactions, und - new_inc.transactions); + // Add a queue entry with split excluded. + queue.emplace_back(inc, und - m_depgraph.Descendants(split)); + // Update statistics to account for the candidate new_inc. + if (new_inc.feerate > best.feerate) best = new_inc; + break; + } + } + } + return {std::move(best), max_iterations - iterations_left}; + } +}; + +/** A very simple finder class for optimal candidate sets, which tries every subset. + * + * It is even simpler than SimpleCandidateFinder, and is primarily included here to test the + * correctness of SimpleCandidateFinder, which is then used to test the correctness of + * SearchCandidateFinder. + */ +template +class ExhaustiveCandidateFinder +{ + /** Internal dependency graph. */ + const DepGraph& m_depgraph; + /** Which transaction are left to include. */ + SetType m_todo; + +public: + /** Construct an ExhaustiveCandidateFinder for a given graph. */ + ExhaustiveCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + + /** Remove a set of transactions from the set of to-be-linearized ones. */ + void MarkDone(SetType select) noexcept { m_todo -= select; } + + /** Determine whether unlinearized transactions remain. */ + bool AllDone() const noexcept { return m_todo.None(); } + + /** Find the optimal remaining candidate set. + * + * Complexity: O(N * 2^N). + */ + SetInfo FindCandidateSet() const noexcept + { + // Best solution so far. + SetInfo best{m_todo, m_depgraph.FeeRate(m_todo)}; + // The number of combinations to try. + uint64_t limit = (uint64_t{1} << m_todo.Count()) - 1; + // Try the transitive closure of every non-empty subset of m_todo. + for (uint64_t x = 1; x < limit; ++x) { + // If bit number b is set in x, then the remaining ancestors of the b'th remaining + // transaction in m_todo are included. + SetType txn; + auto x_shifted{x}; + for (auto i : m_todo) { + if (x_shifted & 1) txn |= m_depgraph.Ancestors(i); + x_shifted >>= 1; + } + SetInfo cur(m_depgraph, txn & m_todo); + if (cur.feerate > best.feerate) best = cur; + } + return best; + } +}; + +/** A simple linearization algorithm. + * + * This matches Linearize() in interface and behavior, though with fewer optimizations, lacking + * the ability to pass in an existing linearization, and using just SimpleCandidateFinder rather + * than AncestorCandidateFinder and SearchCandidateFinder. + */ +template +std::pair, bool> SimpleLinearize(const DepGraph& depgraph, uint64_t max_iterations) +{ + std::vector linearization; + SimpleCandidateFinder finder(depgraph); + SetType todo = SetType::Fill(depgraph.TxCount()); + bool optimal = true; + while (todo.Any()) { + auto [candidate, iterations_done] = finder.FindCandidateSet(max_iterations); + if (iterations_done == max_iterations) optimal = false; + depgraph.AppendTopo(linearization, candidate.transactions); + todo -= candidate.transactions; + finder.MarkDone(candidate.transactions); + max_iterations -= iterations_done; + } + return {std::move(linearization), optimal}; +} + +/** Given a dependency graph, and a todo set, read a topological subset of todo from reader. */ +template +SetType ReadTopologicalSet(const DepGraph& depgraph, const SetType& todo, SpanReader& reader) +{ + uint64_t mask{0}; + try { + reader >> VARINT(mask); + } catch(const std::ios_base::failure&) {} + SetType ret; + for (auto i : todo) { + if (!ret[i]) { + if (mask & 1) ret |= depgraph.Ancestors(i); + mask >>= 1; + } + } + return ret & todo; +} + +/** Given a dependency graph, construct any valid linearization for it, reading from a SpanReader. */ +template +std::vector ReadLinearization(const DepGraph& depgraph, SpanReader& reader) +{ + std::vector linearization; + TestBitSet todo = TestBitSet::Fill(depgraph.TxCount()); + // In every iteration one topologically-valid transaction is appended to linearization. + while (todo.Any()) { + // Compute the set of transactions with no not-yet-included ancestors. + TestBitSet potential_next; + for (auto j : todo) { + if ((depgraph.Ancestors(j) & todo) == TestBitSet::Singleton(j)) { + potential_next.Set(j); + } + } + // There must always be one (otherwise there is a cycle in the graph). + assert(potential_next.Any()); + // Read a number from reader, and interpret it as index into potential_next. + uint64_t idx{0}; + try { + reader >> VARINT(idx); + } catch (const std::ios_base::failure&) {} + idx %= potential_next.Count(); + // Find out which transaction that corresponds to. + for (auto j : potential_next) { + if (idx == 0) { + // When found, add it to linearization and remove it from todo. + linearization.push_back(j); + assert(todo[j]); + todo.Reset(j); + break; + } + --idx; + } + } + return linearization; +} + +} // namespace + +FUZZ_TARGET(clusterlin_add_dependency) +{ + // Verify that computing a DepGraph from a cluster, or building it step by step using AddDependency + // have the same effect. + + // Construct a cluster of a certain length, with no dependencies. + FuzzedDataProvider provider(buffer.data(), buffer.size()); + auto num_tx = provider.ConsumeIntegralInRange(2, 32); + Cluster cluster(num_tx, std::pair{FeeFrac{0, 1}, TestBitSet{}}); + // Construct the corresponding DepGraph object (also no dependencies). + DepGraph depgraph(cluster); + SanityCheck(depgraph); + // Read (parent, child) pairs, and add them to the cluster and depgraph. + LIMITED_WHILE(provider.remaining_bytes() > 0, TestBitSet::Size() * TestBitSet::Size()) { + auto parent = provider.ConsumeIntegralInRange(0, num_tx - 1); + auto child = provider.ConsumeIntegralInRange(0, num_tx - 2); + child += (child >= parent); + cluster[child].second.Set(parent); + depgraph.AddDependency(parent, child); + assert(depgraph.Ancestors(child)[parent]); + assert(depgraph.Descendants(parent)[child]); + } + // Sanity check the result. + SanityCheck(depgraph); + // Verify that the resulting DepGraph matches one recomputed from the cluster. + assert(DepGraph(cluster) == depgraph); +} + +FUZZ_TARGET(clusterlin_cluster_serialization) +{ + // Verify that any graph of transactions has its ancestry correctly computed by DepGraph, and + // if it is a DAG, that it can be serialized as a DepGraph in a way that roundtrips. This + // guarantees that any acyclic cluster has a corresponding DepGraph serialization. + + FuzzedDataProvider provider(buffer.data(), buffer.size()); + + // Construct a cluster in a naive way (using a FuzzedDataProvider-based serialization). + Cluster cluster; + auto num_tx = provider.ConsumeIntegralInRange(1, 32); + cluster.resize(num_tx); + for (ClusterIndex i = 0; i < num_tx; ++i) { + cluster[i].first.size = provider.ConsumeIntegralInRange(1, 0x3fffff); + cluster[i].first.fee = provider.ConsumeIntegralInRange(-0x8000000000000, 0x7ffffffffffff); + for (ClusterIndex j = 0; j < num_tx; ++j) { + if (i == j) continue; + if (provider.ConsumeBool()) cluster[i].second.Set(j); + } + } + + // Construct dependency graph, and verify it matches the cluster (which includes a round-trip + // check for the serialization). + DepGraph depgraph(cluster); + VerifyDepGraphFromCluster(cluster, depgraph); +} + +FUZZ_TARGET(clusterlin_depgraph_serialization) +{ + // Verify that any deserialized depgraph is acyclic and roundtrips to an identical depgraph. + + // Construct a graph by deserializing. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + SanityCheck(depgraph); + + // Verify the graph is a DAG. + assert(IsAcyclic(depgraph)); +} + +FUZZ_TARGET(clusterlin_chunking) +{ + // Verify the correctness of the ChunkLinearization function. + + // Construct a graph by deserializing. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + + // Read a valid linearization for depgraph. + auto linearization = ReadLinearization(depgraph, reader); + + // Invoke the chunking function. + auto chunking = ChunkLinearization(depgraph, linearization); + + // Verify that chunk feerates are monotonically non-increasing. + for (size_t i = 1; i < chunking.size(); ++i) { + assert(!(chunking[i] >> chunking[i - 1])); + } + + // Naively recompute the chunks (each is the highest-feerate prefix of what remains). + auto todo = TestBitSet::Fill(depgraph.TxCount()); + for (const auto& chunk_feerate : chunking) { + assert(todo.Any()); + SetInfo accumulator, best; + for (ClusterIndex idx : linearization) { + if (todo[idx]) { + accumulator |= SetInfo(depgraph, idx); + if (best.feerate.IsEmpty() || accumulator.feerate >> best.feerate) { + best = accumulator; + } + } + } + assert(chunk_feerate == best.feerate); + assert(best.transactions.IsSubsetOf(todo)); + todo -= best.transactions; + } + assert(todo.None()); +} + +FUZZ_TARGET(clusterlin_ancestor_finder) +{ + // Verify that AncestorCandidateFinder works as expected. + + // Retrieve a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + + AncestorCandidateFinder anc_finder(depgraph); + auto todo = TestBitSet::Fill(depgraph.TxCount()); + while (todo.Any()) { + // Call the ancestor finder's FindCandidateSet for what remains of the graph. + assert(!anc_finder.AllDone()); + auto best_anc = anc_finder.FindCandidateSet(); + // Sanity check the result. + assert(best_anc.transactions.Any()); + assert(best_anc.transactions.IsSubsetOf(todo)); + assert(depgraph.FeeRate(best_anc.transactions) == best_anc.feerate); + // Check that it is topologically valid. + for (auto i : best_anc.transactions) { + assert((depgraph.Ancestors(i) & todo).IsSubsetOf(best_anc.transactions)); + } + + // Compute all remaining ancestor sets. + std::optional> real_best_anc; + for (auto i : todo) { + SetInfo info(depgraph, todo & depgraph.Ancestors(i)); + if (!real_best_anc.has_value() || info.feerate > real_best_anc->feerate) { + real_best_anc = info; + } + } + // The set returned by anc_finder must equal the real best ancestor sets. + assert(real_best_anc.has_value()); + assert(*real_best_anc == best_anc); + + // Find a topologically valid subset of transactions to remove from the graph. + auto del_set = ReadTopologicalSet(depgraph, todo, reader); + // If we did not find anything, use best_anc itself, because we should remove something. + if (del_set.None()) del_set = best_anc.transactions; + todo -= del_set; + anc_finder.MarkDone(del_set); + } + assert(anc_finder.AllDone()); +} + +static constexpr auto MAX_SIMPLE_ITERATIONS = 300000; + +FUZZ_TARGET(clusterlin_search_finder) +{ + // Verify that SearchCandidateFinder works as expected by sanity checking the results + // and comparing with the results from SimpleCandidateFinder, ExhaustiveCandidateFinder, and + // AncestorCandidateFinder. + + // Retrieve an RNG seed and a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + uint64_t rng_seed{0}; + try { + reader >> Using(depgraph) >> rng_seed; + } catch (const std::ios_base::failure&) {} + + // Instantiate ALL the candidate finders. + SearchCandidateFinder src_finder(depgraph, rng_seed); + SimpleCandidateFinder smp_finder(depgraph); + ExhaustiveCandidateFinder exh_finder(depgraph); + AncestorCandidateFinder anc_finder(depgraph); + + auto todo = TestBitSet::Fill(depgraph.TxCount()); + while (todo.Any()) { + assert(!src_finder.AllDone()); + assert(!smp_finder.AllDone()); + assert(!exh_finder.AllDone()); + assert(!anc_finder.AllDone()); + + // For each iteration, read an iteration count limit from the fuzz input. + uint64_t max_iterations = 1; + try { + reader >> VARINT(max_iterations); + } catch (const std::ios_base::failure&) {} + max_iterations &= 0xfffff; + + // Read an initial subset from the fuzz input. + SetInfo init_best(depgraph, ReadTopologicalSet(depgraph, todo, reader)); + + // Call the search finder's FindCandidateSet for what remains of the graph. + auto [found, iterations_done] = src_finder.FindCandidateSet(max_iterations, init_best); + + // Sanity check the result. + assert(iterations_done <= max_iterations); + assert(found.transactions.Any()); + assert(found.transactions.IsSubsetOf(todo)); + assert(depgraph.FeeRate(found.transactions) == found.feerate); + if (!init_best.feerate.IsEmpty()) assert(found.feerate >= init_best.feerate); + // Check that it is topologically valid. + for (auto i : found.transactions) { + assert(found.transactions.IsSupersetOf(depgraph.Ancestors(i) & todo)); + } + + // At most 2^N-1 iterations can be required: the number of non-empty subsets a graph with N + // transactions has. + assert(iterations_done <= ((uint64_t{1} << todo.Count()) - 1)); + + // Perform quality checks only if SearchCandidateFinder claims an optimal result. + if (iterations_done < max_iterations) { + // Compare with SimpleCandidateFinder. + auto [simple, simple_iters] = smp_finder.FindCandidateSet(MAX_SIMPLE_ITERATIONS); + assert(found.feerate >= simple.feerate); + if (simple_iters < MAX_SIMPLE_ITERATIONS) { + assert(found.feerate == simple.feerate); + } + + // Compare with AncestorCandidateFinder; + auto anc = anc_finder.FindCandidateSet(); + assert(found.feerate >= anc.feerate); + + // Compare with ExhaustiveCandidateFinder. This quickly gets computationally expensive + // for large clusters (O(2^n)), so only do it for sufficiently small ones. + if (todo.Count() <= 12) { + auto exhaustive = exh_finder.FindCandidateSet(); + assert(exhaustive.feerate == found.feerate); + // Also compare ExhaustiveCandidateFinder with SimpleCandidateFinder (this is + // primarily a test for SimpleCandidateFinder's correctness). + assert(exhaustive.feerate >= simple.feerate); + if (simple_iters < MAX_SIMPLE_ITERATIONS) { + assert(exhaustive.feerate == simple.feerate); + } + } + } + + // Find a topologically valid subset of transactions to remove from the graph. + auto del_set = ReadTopologicalSet(depgraph, todo, reader); + // If we did not find anything, use found itself, because we should remove something. + if (del_set.None()) del_set = found.transactions; + todo -= del_set; + src_finder.MarkDone(del_set); + smp_finder.MarkDone(del_set); + exh_finder.MarkDone(del_set); + anc_finder.MarkDone(del_set); + } + + assert(src_finder.AllDone()); + assert(smp_finder.AllDone()); + assert(exh_finder.AllDone()); + assert(anc_finder.AllDone()); +} + +FUZZ_TARGET(clusterlin_linearization_chunking) +{ + // Verify the behavior of LinearizationChunking. + + // Retrieve a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + + // Retrieve a topologically-valid subset of depgraph. + auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto subset = SetInfo(depgraph, ReadTopologicalSet(depgraph, todo, reader)); + + // Retrieve a valid linearization for depgraph. + auto linearization = ReadLinearization(depgraph, reader); + + // Construct a LinearizationChunking object, initially for the whole linearization. + LinearizationChunking chunking(depgraph, linearization); + + // Incrementally remove transactions from the chunking object, and check various properties at + // every step. + while (todo.Any()) { + assert(chunking.NumChunksLeft() > 0); + + // Construct linearization with just todo. + std::vector linearization_left; + for (auto i : linearization) { + if (todo[i]) linearization_left.push_back(i); + } + + // Compute the chunking for linearization_left. + auto chunking_left = ChunkLinearization(depgraph, linearization_left); + + // Verify that it matches the feerates of the chunks of chunking. + assert(chunking.NumChunksLeft() == chunking_left.size()); + for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) { + assert(chunking.GetChunk(i).feerate == chunking_left[i]); + } + + // Check consistency of chunking. + TestBitSet combined; + for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) { + const auto& chunk_info = chunking.GetChunk(i); + // Chunks must be non-empty. + assert(chunk_info.transactions.Any()); + // Chunk feerates must be monotonically non-increasing. + if (i > 0) assert(!(chunk_info.feerate >> chunking.GetChunk(i - 1).feerate)); + // Chunks must be a subset of what is left of the linearization. + assert(chunk_info.transactions.IsSubsetOf(todo)); + // Chunks' claimed feerates must match their transactions' aggregate feerate. + assert(depgraph.FeeRate(chunk_info.transactions) == chunk_info.feerate); + // Chunks must be the highest-feerate remaining prefix. + SetInfo accumulator, best; + for (auto j : linearization) { + if (todo[j] && !combined[j]) { + accumulator |= SetInfo(depgraph, j); + if (best.feerate.IsEmpty() || accumulator.feerate > best.feerate) { + best = accumulator; + } + } + } + assert(best.transactions == chunk_info.transactions); + assert(best.feerate == chunk_info.feerate); + // Chunks cannot overlap. + assert(!chunk_info.transactions.Overlaps(combined)); + combined |= chunk_info.transactions; + // Chunks must be topological. + for (auto idx : chunk_info.transactions) { + assert((depgraph.Ancestors(idx) & todo).IsSubsetOf(combined)); + } + } + assert(combined == todo); + + // Verify the expected properties of LinearizationChunking::Intersect: + auto intersect = chunking.Intersect(subset); + // - Intersecting again doesn't change the result. + assert(chunking.Intersect(intersect) == intersect); + // - The intersection is topological. + TestBitSet intersect_anc; + for (auto idx : intersect.transactions) { + intersect_anc |= (depgraph.Ancestors(idx) & todo); + } + assert(intersect.transactions == intersect_anc); + // - The claimed intersection feerate matches its transactions. + assert(intersect.feerate == depgraph.FeeRate(intersect.transactions)); + // - The intersection may only be empty if its input is empty. + assert(intersect.transactions.Any() == subset.transactions.Any()); + // - The intersection feerate must be as high as the input. + assert(intersect.feerate >= subset.feerate); + // - No non-empty intersection between the intersection and a prefix of the chunks of the + // remainder of the linearization may be better than the intersection. + TestBitSet prefix; + for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) { + prefix |= chunking.GetChunk(i).transactions; + auto reintersect = SetInfo(depgraph, prefix & intersect.transactions); + if (!reintersect.feerate.IsEmpty()) { + assert(reintersect.feerate <= intersect.feerate); + } + } + + // Find a subset to remove from linearization. + auto done = ReadTopologicalSet(depgraph, todo, reader); + if (done.None()) { + // We need to remove a non-empty subset, so fall back to the unlinearized ancestors of + // the first transaction in todo if done is empty. + done = depgraph.Ancestors(todo.First()) & todo; + } + todo -= done; + chunking.MarkDone(done); + subset = SetInfo(depgraph, subset.transactions - done); + } + + assert(chunking.NumChunksLeft() == 0); +} + +FUZZ_TARGET(clusterlin_linearize) +{ + // Verify the behavior of Linearize(). + + // Retrieve an RNG seed, an iteration count, and a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + uint64_t rng_seed{0}; + uint64_t iter_count{0}; + try { + reader >> VARINT(iter_count) >> Using(depgraph) >> rng_seed; + } catch (const std::ios_base::failure&) {} + + // Optionally construct an old linearization for it. + std::vector old_linearization; + { + uint8_t have_old_linearization{0}; + try { + reader >> have_old_linearization; + } catch(const std::ios_base::failure&) {} + if (have_old_linearization & 1) { + old_linearization = ReadLinearization(depgraph, reader); + SanityCheck(depgraph, old_linearization); + } + } + + // Invoke Linearize(). + iter_count &= 0x7ffff; + auto [linearization, optimal] = Linearize(depgraph, iter_count, rng_seed, old_linearization); + SanityCheck(depgraph, linearization); + auto chunking = ChunkLinearization(depgraph, linearization); + + // Linearization must always be as good as the old one, if provided. + if (!old_linearization.empty()) { + auto old_chunking = ChunkLinearization(depgraph, old_linearization); + auto cmp = CompareChunks(chunking, old_chunking); + assert(cmp >= 0); + } + + // If the iteration count is sufficiently high, an optimal linearization must be found. + // Each linearization step can use up to 2^k iterations, with steps k=1..n. That sum is + // 2 * (2^n - 1) + const uint64_t n = depgraph.TxCount(); + if (n <= 18 && iter_count > 2U * ((uint64_t{1} << n) - 1U)) { + assert(optimal); + } + + // If Linearize claims optimal result, run quality tests. + if (optimal) { + // It must be as good as SimpleLinearize. + auto [simple_linearization, simple_optimal] = SimpleLinearize(depgraph, MAX_SIMPLE_ITERATIONS); + SanityCheck(depgraph, simple_linearization); + auto simple_chunking = ChunkLinearization(depgraph, simple_linearization); + auto cmp = CompareChunks(chunking, simple_chunking); + assert(cmp >= 0); + // If SimpleLinearize finds the optimal result too, they must be equal (if not, + // SimpleLinearize is broken). + if (simple_optimal) assert(cmp == 0); + + // Only for very small clusters, test every topologically-valid permutation. + if (depgraph.TxCount() <= 7) { + std::vector perm_linearization(depgraph.TxCount()); + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) perm_linearization[i] = i; + // Iterate over all valid permutations. + do { + // Determine whether perm_linearization is topological. + TestBitSet perm_done; + bool perm_is_topo{true}; + for (auto i : perm_linearization) { + perm_done.Set(i); + if (!depgraph.Ancestors(i).IsSubsetOf(perm_done)) { + perm_is_topo = false; + break; + } + } + // If so, verify that the obtained linearization is as good as the permutation. + if (perm_is_topo) { + auto perm_chunking = ChunkLinearization(depgraph, perm_linearization); + auto cmp = CompareChunks(chunking, perm_chunking); + assert(cmp >= 0); + } + } while(std::next_permutation(perm_linearization.begin(), perm_linearization.end())); + } + } +} diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 8f3e357a84..41c971c237 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -27,7 +27,6 @@ #include namespace { -const TestingSetup* g_setup; const Coin EMPTY_COIN{}; bool operator==(const Coin& a, const Coin& b) @@ -39,8 +38,7 @@ bool operator==(const Coin& a, const Coin& b) void initialize_coins_view() { - static const auto testing_setup = MakeNoLogFileContext(); - g_setup = testing_setup.get(); + static const auto testing_setup = MakeNoLogFileContext<>(); } FUZZ_TARGET(coins_view, .init = initialize_coins_view) diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index f67b820d11..bc46863f1d 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -27,8 +27,7 @@ FUZZ_TARGET(hex) assert(ToLower(random_hex_string) == hex_data); } (void)IsHexNumber(random_hex_string); - uint256 result; - (void)ParseHashStr(random_hex_string, result); + (void)uint256::FromHex(random_hex_string); (void)uint256S(random_hex_string); try { (void)HexToPubKey(random_hex_string); diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index a3d6ab6375..cb39b3be83 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -77,7 +77,7 @@ FUZZ_TARGET(parse_univalue, .init = initialize_parse_univalue) } try { FlatSigningProvider provider; - (void)EvalDescriptorStringOrObject(univalue, provider); + if (buffer.size() < 10'000) (void)EvalDescriptorStringOrObject(univalue, provider); } catch (const UniValue&) { } catch (const std::runtime_error&) { } diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index 3248ebc4af..7f73c3706c 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -2,44 +2,41 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include -#include +#include +#include #include +#include