Merge pull request #9434

7cb69fa epee: string_tools: keep full path in cut_off_extension (tobtoht)
c7d4bf4 epee: string_tools: remove dot from get_extension (tobtoht)

.github/workflows/build.yml

@@ -27,16 +27,16 @@ jobs:
     env:
       CCACHE_TEMPDIR: /tmp/.ccache-temp
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: /Users/runner/Library/Caches/ccache
         key: ccache-${{ runner.os }}-build-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-
     - name: install dependencies
-      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache
+      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
@@ -51,15 +51,15 @@ jobs:
       run:
         shell: msys2 {0}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: C:\Users\runneradmin\.ccache
         key: ccache-${{ runner.os }}-build-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-
-    - uses: eine/setup-msys2@v2
+    - uses: msys2/setup-msys2@v2
       with:
         update: true
         install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
@@ -77,12 +77,12 @@ jobs:
       CCACHE_TEMPDIR: /tmp/.ccache-temp
     strategy:
       matrix:
-        os: [ubuntu-latest, ubuntu-18.04]
+        os: [ubuntu-22.04, ubuntu-20.04]
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
@@ -101,14 +101,14 @@ jobs:
         ${{env.BUILD_DEFAULT_LINUX}}
 
   libwallet-ubuntu:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       CCACHE_TEMPDIR: /tmp/.ccache-temp
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
@@ -129,15 +129,15 @@ jobs:
 
   test-ubuntu:
     needs: build-ubuntu
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       CCACHE_TEMPDIR: /tmp/.ccache-temp
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
     - name: ccache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
@@ -151,10 +151,11 @@ jobs:
     - name: install monero dependencies
       run: ${{env.APT_INSTALL_LINUX}}
     - name: install Python dependencies
-      run: pip install requests psutil monotonic
+      run: pip install requests psutil monotonic zmq deepdiff
     - name: tests
       env:
         CTEST_OUTPUT_ON_FAILURE: ON
+        DNS_PUBLIC: tcp://9.9.9.9
       run: |
         ${{env.CCACHE_SETTINGS}}
         ${{env.BUILD_DEFAULT_LINUX}}
@@ -166,8 +167,9 @@ jobs:
   source-archive:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
+        fetch-depth: 0
         submodules: recursive
     - name: archive
       run: |
@@ -176,7 +178,7 @@ jobs:
         export OUTPUT="$VERSION.tar"
         echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV
         /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT"
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
       with:
         name: ${{ env.OUTPUT }}
         path: /home/runner/work/monero/monero/${{ env.OUTPUT }}

.github/workflows/depends.yml

@@ -18,7 +18,7 @@ env:
 
 jobs:
   build-cross:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     env:
       CCACHE_TEMPDIR: /tmp/.ccache-temp
     strategy:
@@ -36,13 +36,13 @@ jobs:
             packages: "python3 gperf g++-aarch64-linux-gnu"
           - name: "i686 Win"
             host: "i686-w64-mingw32"
-            packages: "python3 g++-mingw-w64-i686 qttools5-dev-tools"
+            packages: "python3 g++-mingw-w64-i686"
           - name: "i686 Linux"
             host: "i686-pc-linux-gnu"
             packages: "gperf cmake g++-multilib python3-zmq"
           - name: "Win64"
             host: "x86_64-w64-mingw32"
-            packages: "cmake python3 g++-mingw-w64-x86-64 qttools5-dev-tools"
+            packages: "cmake python3 g++-mingw-w64-x86-64"
           - name: "x86_64 Linux"
             host: "x86_64-unknown-linux-gnu"
             packages: "gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
@@ -57,19 +57,20 @@ jobs:
             packages: "clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
     name: ${{ matrix.toolchain.name }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
+        fetch-depth: 0
         submodules: recursive
 # Most volatile cache
     - name: ccache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/.ccache
         key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }}
         restore-keys: ccache-${{ matrix.toolchain.host }}-
 # Less volatile cache
     - name: depends cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: contrib/depends/built
         key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
@@ -78,7 +79,7 @@ jobs:
           depends-${{ matrix.toolchain.host }}-
 # Static cache
     - name: OSX SDK cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: contrib/depends/sdk-sources
         key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }}
@@ -96,7 +97,7 @@ jobs:
       run: |
         ${{env.CCACHE_SETTINGS}}
         make depends target=${{ matrix.toolchain.host }} -j2
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
       if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }}
       with:
         name: ${{ matrix.toolchain.name }}

.github/workflows/gitian.yml

@@ -0,0 +1,49 @@
+name: ci/gh-actions/gitian
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-gitian:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        operating-system:
+          - name: "Linux"
+            option: "l"
+          - name: "Windows"
+            option: "w"
+          - name: "Android"
+            option: "a"
+          - name: "FreeBSD"
+            option: "f"
+          - name: "macOS"
+            option: "m"
+    name: ${{ matrix.operating-system.name }}
+    steps:
+    - name: prepare
+      run: |
+        sudo apt update
+        curl -O https://raw.githubusercontent.com/monero-project/monero/${{ github.ref_name }}/contrib/gitian/gitian-build.py
+        chmod +x gitian-build.py
+    - name: setup
+      run: |
+        ./gitian-build.py --setup --docker github-actions ${{ github.ref_name }}
+    - name: build
+      run: |
+        ./gitian-build.py --docker --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} github-actions ${{ github.ref_name }}
+    - name: post build
+      run: |
+        cd out/${{ github.ref_name }}
+        shasum -a256 *
+        echo \`\`\` >> $GITHUB_STEP_SUMMARY
+        shasum -a256 * >> $GITHUB_STEP_SUMMARY
+        echo \`\`\` >> $GITHUB_STEP_SUMMARY
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.operating-system.name }}
+        path: |
+          out/${{ github.ref_name }}/*

CMakeLists.txt

@@ -794,7 +794,7 @@ else()
     set(USE_LTO_DEFAULT false)
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--stack,10485760")
     if(NOT BUILD_64)
-      add_definitions(-DWINVER=0x0501 -D_WIN32_WINNT=0x0501)
+      add_definitions(-DWINVER=0x0600 -D_WIN32_WINNT=0x0600)
     endif()
   endif()
   set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes")
@@ -1178,6 +1178,9 @@ find_library(NORM_LIBRARY norm)
 find_library(GSSAPI_LIBRARY gssapi_krb5)
 find_library(PROTOLIB_LIBRARY protolib)
 find_library(SODIUM_LIBRARY sodium)
+find_library(BSD_LIBRARY bsd)
+find_library(MD_LIBRARY md)
+find_library(PROTOKIT_LIBRARY protokit)
 
 if(NOT ZMQ_INCLUDE_PATH)
   message(FATAL_ERROR "Could not find required header zmq.h")
@@ -1200,6 +1203,15 @@ endif()
 if(SODIUM_LIBRARY)
   set(ZMQ_LIB "${ZMQ_LIB};${SODIUM_LIBRARY}")
 endif()
+if(BSD_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${BSD_LIBRARY}")
+endif()
+if(MD_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${MD_LIBRARY}")
+endif()
+if(PROTOKIT_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${PROTOKIT_LIBRARY}")
+endif()
 
 include(external/supercop/functions.cmake) # place after setting flags and before src directory inclusion
 add_subdirectory(contrib)

Makefile

@@ -48,7 +48,7 @@ all: release-all
 
 depends:
 	cd contrib/depends && $(MAKE) HOST=$(target) && cd ../.. && mkdir -p build/$(target)/release
-	cd build/$(target)/release && cmake -DCMAKE_TOOLCHAIN_FILE=$(CURDIR)/contrib/depends/$(target)/share/toolchain.cmake ../../.. && $(MAKE)
+	cd build/$(target)/release && USE_DEVICE_TREZOR_MANDATORY=1 cmake -DCMAKE_TOOLCHAIN_FILE=$(CURDIR)/contrib/depends/$(target)/share/toolchain.cmake ../../.. && $(MAKE)
 
 cmake-debug:
 	mkdir -p $(builddir)/debug
@@ -104,7 +104,7 @@ release-all:
 
 release-static:
 	mkdir -p $(builddir)/release
-	cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
+	cd $(builddir)/release && cmake -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
 
 coverage:
 	mkdir -p $(builddir)/debug

README.md

@@ -138,8 +138,8 @@ Dates are provided in the format YYYY-MM-DD.
 | 1978433                        | 2019-11-30 | v12               | v0.15.0.0              | v0.16.0.0                  | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs
 | 2210000                        | 2020-10-17 | v13               | v0.17.0.0              | v0.17.3.2                  | New CLSAG transaction format
 | 2210720                        | 2020-10-18 | v14               | v0.17.1.1              | v0.17.3.2                  | forbid old MLSAG transaction format
-| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.0.0                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
-| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.0.0                  | forbid old v14 transaction format
+| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.3.4                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
+| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.3.4                  | forbid old v14 transaction format
 | XXXXXXX                        | XXX-XX-XX | XXX                | vX.XX.X.X              | vX.XX.X.X                  | XXX |
 
 X's indicate that these details have not been determined as of commit date.
@@ -178,7 +178,6 @@ library archives (`.a`).
 | libunwind    | any           | NO       | `libunwind8-dev`     | `libunwind`  | `libunwind-devel`  | `libunwind-devel`   | YES      | Stack traces    |
 | liblzma      | any           | NO       | `liblzma-dev`        | `xz`         | `liblzma-devel`    | `xz-devel`          | YES      | For libunwind   |
 | libreadline  | 6.3.0         | NO       | `libreadline6-dev`   | `readline`   | `readline-devel`   | `readline-devel`    | YES      | Input editing   |
-| ldns         | 1.6.17        | NO       | `libldns-dev`        | `ldns`       | `libldns-devel`    | `ldns-devel`        | YES      | SSL toolkit     |
 | expat        | 1.1           | NO       | `libexpat1-dev`      | `expat`      | `expat-devel`      | `expat-devel`       | YES      | XML parsing     |
 | GTest        | 1.5           | YES      | `libgtest-dev`[1]    | `gtest`      | `gtest-devel`      | `gtest-devel`       | YES      | Test suite      |
 | ccache       | any           | NO       | `ccache`             | `ccache`     | `ccache`           | `ccache`            | YES      | Compil. cache   |
@@ -205,23 +204,23 @@ then:
 Install all dependencies at once on Debian/Ubuntu:
 
 ```
-sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz
+sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz
 ```
 
 Install all dependencies at once on Arch:
 ```
-sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline ldns expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd
+sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd
 ```
 
 Install all dependencies at once on Fedora:
 ```
-sudo dnf install gcc gcc-c++ cmake pkgconf boost-devel openssl-devel zeromq-devel openpgm-devel unbound-devel libsodium-devel libunwind-devel xz-devel readline-devel ldns-devel expat-devel gtest-devel ccache doxygen graphviz qt5-linguist hidapi-devel libusbx-devel protobuf-devel protobuf-compiler systemd-devel
+sudo dnf install gcc gcc-c++ cmake pkgconf boost-devel openssl-devel zeromq-devel openpgm-devel unbound-devel libsodium-devel libunwind-devel xz-devel readline-devel expat-devel gtest-devel ccache doxygen graphviz qt5-linguist hidapi-devel libusbx-devel protobuf-devel protobuf-compiler systemd-devel
 ```
 
 Install all dependencies at once on openSUSE:
 
 ```
-sudo zypper ref && sudo zypper in cppzmq-devel ldns-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++
+sudo zypper ref && sudo zypper in cppzmq-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++
 ```
 
 Install all dependencies at once on macOS with the provided Brewfile:
@@ -345,7 +344,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (
     ```bash
     git clone https://github.com/monero-project/monero.git
     cd monero
-    git checkout v0.18.0.0
+    git checkout v0.18.3.4
     ```
 
 * Build:
@@ -464,10 +463,10 @@ application.
     cd monero
     ```
 
-* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.0.0'. If you don't care about the version and just want binaries from master, skip this step:
+* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.3.4'. If you don't care about the version and just want binaries from master, skip this step:
 
     ```bash
-    git checkout v0.18.0.0
+    git checkout v0.18.3.4
     ```
 
 * If you are on a 64-bit system, run:

cmake/FindLibUSB.cmake

@@ -113,7 +113,7 @@ if ( LibUSB_FOUND )
     if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC)
         if (APPLE)
             if(DEPENDS)
-                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
+                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit -framework Security")
             else()
                 find_library(COREFOUNDATION CoreFoundation)
                 find_library(IOKIT IOKit)

contrib/brew/Brewfile

@@ -25,7 +25,6 @@ brew "unbound"
 brew "libsodium"
 brew "miniupnpc"
 brew "readline"
-brew "ldns"
 brew "expat"
 brew "ccache"
 brew "doxygen"

contrib/depends/Makefile

@@ -110,8 +110,7 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
-qt_packages_$(NO_QT) = $(qt_packages) 
-packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_)
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
 
 all_packages = $(packages) $(native_packages)

contrib/depends/config.site.in

@@ -7,27 +7,12 @@ ac_tool_prefix=${host_alias}-
 if test -z $with_boost; then
   with_boost=$depends_prefix
 fi
-if test -z $with_qt_plugindir; then
-  with_qt_plugindir=$depends_prefix/plugins
-fi
-if test -z $with_qt_translationdir; then
-  with_qt_translationdir=$depends_prefix/translations
-fi
 
 if test x@host_os@ = xdarwin; then
   BREW=no
   PORT=no
 fi
 
-if test x@host_os@ = xmingw32; then
-  if test -z $with_qt_incdir; then
-    with_qt_incdir=$depends_prefix/include
-  fi
-  if test -z $with_qt_libdir; then
-    with_qt_libdir=$depends_prefix/lib
-  fi
-fi
-
 PATH=$depends_prefix/native/bin:$PATH
 PKG_CONFIG="`which pkg-config` --static"
 

contrib/depends/funcs.mk

@@ -143,8 +143,11 @@ $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig
 $(1)_config_env+=PATH="$(build_prefix)/bin:$(PATH)"
 $(1)_build_env+=PATH="$(build_prefix)/bin:$(PATH)"
 $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)"
-$(1)_autoconf=./configure --host=$($($(1)_type)_host) --disable-dependency-tracking --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"
+$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"
 
+ifeq ($(filter $(1),libusb unbound),)
+$(1)_autoconf += --disable-dependency-tracking
+endif
 ifneq ($($(1)_nm),)
 $(1)_autoconf += NM="$$($(1)_nm)"
 endif

contrib/depends/packages/expat.mk

@@ -1,12 +1,12 @@
 package=expat
-$(package)_version=2.4.1
-$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_4_1
+$(package)_version=2.6.0
+$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=2f9b6a580b94577b150a7d5617ad4643a4301a6616ff459307df3e225bcfbf40
+$(package)_sha256_hash=ff60e6a6b6ce570ae012dc7b73169c7fdf4b6bf08c12ed0ec6f55736b78d85ba
 
 define $(package)_set_vars
-$(package)_config_opts=--enable-static
-$(package)_config_opts=--disable-shared
+$(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples
+$(package)_config_opts+=--enable-option-checking --without-xmlwf --with-pic
 $(package)_config_opts+=--prefix=$(host_prefix)
 endef
 
@@ -23,6 +23,6 @@ define $(package)_stage_cmds
 endef
 
 define $(package)_postprocess_cmds
-  rm lib/*.la
+  rm -rf share lib/cmake lib/*.la
 endef
 

contrib/depends/packages/hidapi.mk

@@ -1,8 +1,8 @@
 package=hidapi
-$(package)_version=0.11.0
-$(package)_download_path=https://github.com/libusb/hidapi/archive
+$(package)_version=0.13.1
+$(package)_download_path=https://github.com/libusb/hidapi/archive/refs/tags
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=391d8e52f2d6a5cf76e2b0c079cfefe25497ba1d4659131297081fc0cd744632
+$(package)_sha256_hash=476a2c9a4dc7d1fc97dd223b84338dbea3809a84caea2dcd887d9778725490e3
 $(package)_linux_dependencies=libusb eudev
 $(package)_patches=missing_win_include.patch
 

contrib/depends/packages/ldns.mk

@@ -1,34 +0,0 @@
-package=ldns
-$(package)_version=1.7.1
-$(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/
-$(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=8ac84c16bdca60e710eea75782356f3ac3b55680d40e1530d7cea474ac208229
-$(package)_dependencies=openssl
-
-define $(package)_set_vars
-  $(package)_config_opts=--disable-shared --enable-static --with-drill
-  $(package)_config_opts+=--with-ssl=$(host_prefix)
-  $(package)_config_opts_release=--disable-debug-mode
-  $(package)_config_opts_linux=--with-pic
-endef
-
-define $(package)_preprocess_cmds
-   cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub .
-endef
-
-define $(package)_config_cmds
-  $($(package)_autoconf)
-endef
-
-define $(package)_build_cmds
-  $(MAKE)
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install-h install-lib
-endef
-
-define $(package)_postprocess_cmds
-  rm lib/*.la
-endef
-

contrib/depends/packages/libusb.mk

@@ -1,8 +1,8 @@
 package=libusb
-$(package)_version=1.0.22
-$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
+$(package)_version=1.0.26
+$(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version)
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
+$(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5
 
 define $(package)_preprocess_cmds
   autoreconf -i

contrib/depends/packages/openssl.mk

@@ -1,21 +1,19 @@
 package=openssl
-$(package)_version=1.1.1l
+$(package)_version=3.0.13
 $(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
-$(package)_patches=fix_darwin.patch
+$(package)_sha256_hash=88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313
 
 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
-$(package)_config_env_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
-$(package)_build_env_android=ANDROID_NDK_HOME="$(host_prefix)/native"
-$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
+$(package)_config_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_build_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native"
+$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl --libdir=$(host_prefix)/lib
 $(package)_config_opts+=no-capieng
 $(package)_config_opts+=no-dso
 $(package)_config_opts+=no-dtls1
 $(package)_config_opts+=no-ec_nistp_64_gcc_128
 $(package)_config_opts+=no-gost
-$(package)_config_opts+=no-heartbeats
 $(package)_config_opts+=no-md2
 $(package)_config_opts+=no-rc5
 $(package)_config_opts+=no-rdrand
@@ -23,8 +21,8 @@ $(package)_config_opts+=no-rfc3779
 $(package)_config_opts+=no-sctp
 $(package)_config_opts+=no-shared
 $(package)_config_opts+=no-ssl-trace
-$(package)_config_opts+=no-ssl2
 $(package)_config_opts+=no-ssl3
+$(package)_config_opts+=no-tests
 $(package)_config_opts+=no-unit-test
 $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-zlib
@@ -50,8 +48,7 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \
-  patch -p1 < $($(package)_patch_dir)/fix_darwin.patch
+  sed -i.old 's|crypto ssl apps util tools fuzz providers doc|crypto ssl util tools providers|' build.info
 endef
 
 define $(package)_config_cmds

contrib/depends/packages/packages.mk

@@ -1,4 +1,4 @@
-packages:=boost openssl zeromq libiconv expat ldns unbound
+packages:=boost openssl zeromq libiconv expat unbound
 
 # ccache is useless in gitian builds
 ifneq ($(GITIAN),1)
@@ -20,7 +20,6 @@ freebsd_packages = ncurses readline sodium
 
 linux_packages = eudev ncurses readline sodium $(hardware_packages)
 linux_native_packages = $(hardware_native_packages)
-qt_packages = qt
 
 ifeq ($(build_tests),ON)
 packages += gtest

contrib/depends/packages/qt.mk

@@ -1,175 +0,0 @@
-PACKAGE=qt
-$(package)_version=5.15.1
-$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules
-$(package)_suffix=everywhere-src-$($(package)_version).tar.xz
-$(package)_file_name=qtbase-$($(package)_suffix)
-$(package)_sha256_hash=33960404d579675b7210de103ed06a72613bfc4305443e278e2d32a3eb1f3d8c
-$(package)_build_subdir=qtbase
-$(package)_qt_libs=corelib
-$(package)_patches=fix_qt_pkgconfig.patch fix_no_printer.patch fix_rcc_determinism.patch no-xlib.patch
-
-$(package)_qttranslations_file_name=qttranslations-$($(package)_suffix)
-$(package)_qttranslations_sha256_hash=46e0c0e3a511fbcc803a4146204062e47f6ed43b34d98a3c27372a03b8746bd8
-
-$(package)_qttools_file_name=qttools-$($(package)_suffix)
-$(package)_qttools_sha256_hash=c98ee5f0f980bf68cbf0c94d62434816a92441733de50bd9adbe9b9055f03498
-
-$(package)_extra_sources  = $($(package)_qttranslations_file_name)
-$(package)_extra_sources += $($(package)_qttools_file_name)
-
-define $(package)_set_vars
-$(package)_config_opts_release = -release
-$(package)_config_opts_debug = -debug
-$(package)_config_opts += -bindir $(build_prefix)/bin
-$(package)_config_opts += -c++std c++11
-$(package)_config_opts += -confirm-license
-$(package)_config_opts += -dbus-runtime
-$(package)_config_opts += -hostprefix $(build_prefix)
-$(package)_config_opts += -no-compile-examples
-$(package)_config_opts += -no-cups
-$(package)_config_opts += -no-egl
-$(package)_config_opts += -no-eglfs
-$(package)_config_opts += -no-evdev
-$(package)_config_opts += -no-gui
-$(package)_config_opts += -no-freetype
-$(package)_config_opts += -no-gif
-$(package)_config_opts += -no-glib
-$(package)_config_opts += -no-icu
-$(package)_config_opts += -no-ico
-$(package)_config_opts += -no-iconv
-$(package)_config_opts += -no-kms
-$(package)_config_opts += -no-linuxfb
-$(package)_config_opts += -no-libjpeg
-$(package)_config_opts += -no-libudev
-$(package)_config_opts += -no-mtdev
-$(package)_config_opts += -no-openvg
-$(package)_config_opts += -no-reduce-relocations
-$(package)_config_opts += -no-sql-db2
-$(package)_config_opts += -no-sql-ibase
-$(package)_config_opts += -no-sql-oci
-$(package)_config_opts += -no-sql-tds
-$(package)_config_opts += -no-sql-mysql
-$(package)_config_opts += -no-sql-odbc
-$(package)_config_opts += -no-sql-psql
-$(package)_config_opts += -no-sql-sqlite
-$(package)_config_opts += -no-sql-sqlite2
-$(package)_config_opts += -no-use-gold-linker
-$(package)_config_opts += -nomake examples
-$(package)_config_opts += -nomake tests
-$(package)_config_opts += -opensource
-$(package)_config_opts += -no-openssl
-$(package)_config_opts += -optimized-qmake
-$(package)_config_opts += -pch
-$(package)_config_opts += -pkg-config
-$(package)_config_opts += -prefix $(host_prefix)
-$(package)_config_opts += -no-libpng
-$(package)_config_opts += -qt-pcre
-$(package)_config_opts += -qt-harfbuzz
-$(package)_config_opts += -no-zlib
-$(package)_config_opts += -static
-$(package)_config_opts += -silent
-$(package)_config_opts += -v
-$(package)_config_opts += -no-feature-bearermanagement
-$(package)_config_opts += -no-feature-colordialog
-$(package)_config_opts += -no-feature-dial
-$(package)_config_opts += -no-feature-filesystemwatcher
-$(package)_config_opts += -no-feature-fontcombobox
-$(package)_config_opts += -no-feature-ftp
-$(package)_config_opts += -no-feature-image_heuristic_mask
-$(package)_config_opts += -no-feature-keysequenceedit
-$(package)_config_opts += -no-feature-lcdnumber
-$(package)_config_opts += -no-feature-pdf
-$(package)_config_opts += -no-feature-printdialog
-$(package)_config_opts += -no-feature-printer
-$(package)_config_opts += -no-feature-printpreviewdialog
-$(package)_config_opts += -no-feature-printpreviewwidget
-$(package)_config_opts += -no-feature-sessionmanager
-$(package)_config_opts += -no-feature-sql
-$(package)_config_opts += -no-feature-statemachine
-$(package)_config_opts += -no-feature-syntaxhighlighter
-$(package)_config_opts += -no-feature-textbrowser
-$(package)_config_opts += -no-feature-textodfwriter
-$(package)_config_opts += -no-feature-topleveldomain
-$(package)_config_opts += -no-feature-udpsocket
-$(package)_config_opts += -no-feature-undocommand
-$(package)_config_opts += -no-feature-undogroup
-$(package)_config_opts += -no-feature-undostack
-$(package)_config_opts += -no-feature-undoview
-$(package)_config_opts += -no-feature-vnc
-$(package)_config_opts += -no-feature-wizard
-$(package)_config_opts_linux = -no-fontconfig
-$(package)_config_opts_linux += -no-opengl
-$(package)_config_opts_linux += -no-xcb
-$(package)_config_opts_linux += -no-feature-xlib
-endef
-
-define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttranslations_file_name),$($(package)_qttranslations_file_name),$($(package)_qttranslations_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttools_file_name),$($(package)_qttools_file_name),$($(package)_qttools_sha256_hash))
-endef
-
-define $(package)_extract_cmds
-  mkdir -p $($(package)_extract_dir) && \
-  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_qttranslations_sha256_hash)  $($(package)_source_dir)/$($(package)_qttranslations_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_qttools_sha256_hash)  $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  mkdir qtbase && \
-  tar --strip-components=1 -xf $($(package)_source) -C qtbase && \
-  mkdir qttranslations && \
-  tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \
-  mkdir qttools && \
-  tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools
-endef
-
-
-define $(package)_preprocess_cmds
-  sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \
-  sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \
-  sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \
-  sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \
-  cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \
-  sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \
-  patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \
-  patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \
-  echo "!host_build: QMAKE_CFLAGS     += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  echo "!host_build: QMAKE_CXXFLAGS   += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  echo "!host_build: QMAKE_LFLAGS     += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \
-  echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf && \
-  echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "s|QMAKE_CFLAGS           += |!host_build: QMAKE_CFLAGS            = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "s|QMAKE_CXXFLAGS         += |!host_build: QMAKE_CXXFLAGS            = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS            = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf
-endef
-
-define $(package)_config_cmds
-  export PKG_CONFIG_SYSROOT_DIR=/ && \
-  export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \
-  export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig  && \
-  ./configure $($(package)_config_opts) && \
-  echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \
-  $(MAKE) sub-src-clean && \
-  cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \
-  cd translations && ../../qtbase/bin/qmake translations.pro -o Makefile && cd ../.. &&\
-  cd qttools/src/linguist/lrelease/ && ../../../../qtbase/bin/qmake lrelease.pro -o Makefile
-endef
-
-define $(package)_build_cmds
-  $(MAKE) -C src $(addprefix sub-,$($(package)_qt_libs)) && \
-  $(MAKE) -C ../qttools/src/linguist/lrelease && \
-  $(MAKE) -C ../qttranslations
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) -C src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && cd .. &&\
-  $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \
-  $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets
-endef
-
-define $(package)_postprocess_cmds
-  rm -rf native/mkspecs/ native/lib/ lib/cmake/ && \
-  rm -f lib/lib*.la lib/*.prl plugins/*/*.prl
-endef

contrib/depends/packages/unbound.mk

@@ -1,17 +1,21 @@
 package=unbound
-$(package)_version=1.15.0
+$(package)_version=1.19.1
 $(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=a480dc6c8937447b98d161fe911ffc76cfaffa2da18788781314e81339f1126f
-$(package)_dependencies=openssl expat ldns
+$(package)_sha256_hash=bc1d576f3dd846a0739adc41ffaa702404c6767d2b6082deb9f2f97cbb24a3a9
+$(package)_dependencies=openssl expat
 $(package)_patches=disable-glibc-reallocarray.patch
 
 
 define $(package)_set_vars
-  $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only
+  $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix)
+  $(package)_config_opts+=--with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no
+  $(package)_config_opts+=--without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only
   $(package)_config_opts_linux=--with-pic
   $(package)_config_opts_w64=--enable-static-exe --sysconfdir=/etc --prefix=$(host_prefix) --target=$(host_prefix)
+  $(package)_config_opts_x86_64_darwin=ac_cv_func_SHA384_Init=yes
   $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread"
+  $(package)_cflags_mingw32+="-D_WIN32_WINNT=0x600"
 endef
 
 define $(package)_preprocess_cmds
@@ -30,6 +34,3 @@ endef
 define $(package)_stage_cmds
   $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef
-
-define $(package)_postprocess_cmds
-endef

contrib/depends/patches/openssl/fix_darwin.patch

@@ -1,60 +0,0 @@
-From 96ac8f13f4d0ee96baf5724d9f96c44c34b8606c Mon Sep 17 00:00:00 2001
-From: David Carlier <devnexen@gmail.com>
-Date: Tue, 24 Aug 2021 22:40:14 +0100
-Subject: [PATCH] Darwin platform allows to build on releases before
- Yosemite/ios 8.
-
-issue #16407 #16408
-
-Reviewed-by: Paul Dale <pauli@openssl.org>
-Reviewed-by: Tomas Mraz <tomas@openssl.org>
-(Merged from https://github.com/openssl/openssl/pull/16409)
----
- crypto/rand/rand_unix.c |  5 +----
- include/crypto/rand.h   | 10 ++++++++++
- 2 files changed, 11 insertions(+), 4 deletions(-)
-
-diff --git a/crypto/rand/rand_unix.c b/crypto/rand/rand_unix.c
-index 43f1069d151d..0f4525106af7 100644
---- a/crypto/rand/rand_unix.c
-+++ b/crypto/rand/rand_unix.c
-@@ -34,9 +34,6 @@
- #if defined(__OpenBSD__)
- # include <sys/param.h>
- #endif
--#if defined(__APPLE__)
--# include <CommonCrypto/CommonRandom.h>
--#endif
- 
- #if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__)
- # include <sys/types.h>
-@@ -381,7 +378,7 @@ static ssize_t syscall_random(void *buf, size_t buflen)
-         if (errno != ENOSYS)
-             return -1;
-     }
--#  elif defined(__APPLE__)
-+#  elif defined(OPENSSL_APPLE_CRYPTO_RANDOM)
-     if (CCRandomGenerateBytes(buf, buflen) == kCCSuccess)
- 	    return (ssize_t)buflen;
- 
-diff --git a/include/crypto/rand.h b/include/crypto/rand.h
-index 5350d3a93119..674f840fd13c 100644
---- a/include/crypto/rand.h
-+++ b/include/crypto/rand.h
-@@ -20,6 +20,16 @@
- 
- # include <openssl/rand.h>
- 
-+# if defined(__APPLE__) && !defined(OPENSSL_NO_APPLE_CRYPTO_RANDOM)
-+#  include <Availability.h>
-+#  if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101000) || \
-+     (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 80000)
-+#   define OPENSSL_APPLE_CRYPTO_RANDOM 1
-+#   include <CommonCrypto/CommonCryptoError.h>
-+#   include <CommonCrypto/CommonRandom.h>
-+#  endif
-+# endif
-+
- /* forward declaration */
- typedef struct rand_pool_st RAND_POOL;
- 

contrib/depends/patches/qt/fix_no_printer.patch

@@ -1,19 +0,0 @@
---- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
-+++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
-@@ -52,6 +52,7 @@
- //
-
- #include <QtCore/qglobal.h>
-+#include <qpa/qplatformprintdevice.h>
-
- #ifndef QT_NO_PRINTER
-
---- x/qtbase/src/plugins/plugins.pro
-+++ y/qtbase/src/plugins/plugins.pro
-@@ -9,6 +9,3 @@ qtHaveModule(gui) {
-     !android:qtConfig(library): SUBDIRS *= generic
- }
- qtHaveModule(widgets): SUBDIRS += styles
--
--!winrt:qtHaveModule(printsupport): \
--    SUBDIRS += printsupport

contrib/depends/patches/qt/fix_qt_pkgconfig.patch

@@ -1,11 +0,0 @@
---- old/qtbase/mkspecs/features/qt_module.prf
-+++ new/qtbase/mkspecs/features/qt_module.prf
-@@ -269,7 +269,7 @@ load(qt_installs)
- load(qt_targets)
- 
- # this builds on top of qt_common
--!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
-+if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
-     CONFIG += create_pc
-     QMAKE_PKGCONFIG_DESTDIR = pkgconfig
-     host_build: \

contrib/depends/patches/qt/fix_rcc_determinism.patch

@@ -1,15 +0,0 @@
---- old/qtbase/src/tools/rcc/rcc.cpp
-+++ new/qtbase/src/tools/rcc/rcc.cpp
-@@ -207,7 +207,11 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib)
-     if (lib.formatVersion() >= 2) {
-         // last modified time stamp
-         const QDateTime lastModified = m_fileInfo.lastModified();
--        lib.writeNumber8(quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0));
-+        quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0);
-+        static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong();
-+        if (sourceDate != 0)
-+            lastmod = sourceDate;
-+        lib.writeNumber8(lastmod);
-         if (text || pass1)
-             lib.writeChar('\n');
-     }

contrib/depends/patches/qt/no-xlib.patch

@@ -1,69 +0,0 @@
-From 9563cef873ae82e06f60708d706d054717e801ce Mon Sep 17 00:00:00 2001
-From: Carl Dong <contact@carldong.me>
-Date: Thu, 18 Jul 2019 17:22:05 -0400
-Subject: [PATCH] Wrap xlib related code blocks in #if's
-
-They are not necessary to compile QT.
----
- qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp | 8 ++++++++
- 1 file changed, 8 insertions(+)
-
-diff --git a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
-index 7c62c2e2b3..c05c6c0a07 100644
---- a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
-+++ b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
-@@ -49,7 +49,9 @@
- #include <QtGui/QWindow>
- #include <QtGui/QBitmap>
- #include <QtGui/private/qguiapplication_p.h>
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
- #include <X11/cursorfont.h>
-+#endif
- #include <xcb/xfixes.h>
- #include <xcb/xcb_image.h>
- 
-@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window)
-     xcb_flush(xcb_connection());
- }
- 
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
- static int cursorIdForShape(int cshape)
- {
-     int cursorId = 0;
-@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape)
-     }
-     return cursorId;
- }
-+#endif
- 
- xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape)
- {
-@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape)
- xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
- {
-     xcb_connection_t *conn = xcb_connection();
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
-     int cursorId = cursorIdForShape(cshape);
-+#endif
-     xcb_cursor_t cursor = XCB_NONE;
- 
-     // Try Xcursor first
-@@ -586,6 +592,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
-     // Non-standard X11 cursors are created from bitmaps
-     cursor = createNonStandardCursor(cshape);
- 
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
-     // Create a glpyh cursor if everything else failed
-     if (!cursor && cursorId) {
-         cursor = xcb_generate_id(conn);
-@@ -593,6 +600,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
-                                 cursorId, cursorId + 1,
-                                 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0);
-     }
-+#endif
- 
-     if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) {
-         const char *name = cursorNames[cshape].front();
----
-2.22.0
-

contrib/depends/toolchain.cmake.in

@@ -27,8 +27,6 @@ SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a)
 SET(UNBOUND_INCLUDE_DIR @prefix@/include)
 SET(UNBOUND_LIBRARIES @prefix@/lib/libunbound.a)
 
-SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE)
-
 if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android")
 SET(LIBUNWIND_INCLUDE_DIR @prefix@/include)
 SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a)
@@ -146,8 +144,11 @@ elseif(ARCHITECTURE STREQUAL "aarch64")
 endif()
 
 if(ARCHITECTURE STREQUAL "riscv64")
-    set(NO_AES ON)
-    set(ARCH "rv64imafdc")
+  if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    set(BUILD_TAG "linux-riscv64")
+  endif()
+  set(ARCH_ID "riscv64")
+  set(ARCH "rv64gc")
 endif()
 
 if(ARCHITECTURE STREQUAL "i686")

contrib/epee/include/file_io_utils.h

@@ -29,6 +29,7 @@
 
 #include <string>
 #include <ctime>
+#include <cstdint>
 
 namespace epee
 {

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -583,11 +583,8 @@ namespace net_utils
             break;
         }
       }
-      else if (ec.value())
-        terminate();
       else {
-        cancel_timer();
-        on_interrupted();
+        terminate();
       }
     };
     m_strand.post(

contrib/epee/include/span.h

@@ -147,6 +147,16 @@ namespace epee
     return {reinterpret_cast<const std::uint8_t*>(src.data()), src.size_bytes()}; 
   }
 
+  //! \return `span<std::uint8_t>` from a STL compatible `src`.
+  template<typename T>
+  constexpr span<std::uint8_t> to_mut_byte_span(T& src)
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_empty<value_type>(), "empty value types will not work -> sizeof == 1");
+    static_assert(!has_padding<value_type>(), "source value type may have padding");
+    return {reinterpret_cast<std::uint8_t*>(src.data()), src.size() * sizeof(value_type)};
+  }
+
   //! \return `span<const std::uint8_t>` which represents the bytes at `&src`.
   template<typename T>
   span<const std::uint8_t> as_byte_span(const T& src) noexcept

contrib/epee/include/storages/parserse_base_utils.h

@@ -30,6 +30,7 @@
 
 #include <boost/utility/string_ref_fwd.hpp>
 #include <string>
+#include <cstdint>
 
 namespace epee 
 {

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -33,6 +33,9 @@
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #else 

contrib/epee/include/storages/portable_storage_from_json.h

@@ -31,6 +31,9 @@
 #include "parserse_base_utils.h"
 #include "file_io_utils.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100
 
 namespace epee

contrib/epee/include/storages/portable_storage_val_converters.h

@@ -37,6 +37,7 @@
 #include "misc_log_ex.h"
 
 #include <boost/lexical_cast.hpp>
+#include <boost/numeric/conversion/bounds.hpp>
 #include <typeinfo>
 #include <iomanip>
 

contrib/epee/include/string_tools.h

@@ -31,6 +31,7 @@
 #include "mlocker.h"
 
 #include <boost/utility/string_ref.hpp>
+#include <boost/algorithm/string.hpp> 
 #include <sstream>
 #include <string>
 #include <cstdint>
@@ -69,23 +70,19 @@ namespace string_tools
 #ifdef _WIN32
    std::string get_current_module_path();
 #endif
-  bool set_module_name_and_folder(const std::string& path_to_process_);
-  bool trim_left(std::string& str);
-  bool trim_right(std::string& str);
+  void set_module_name_and_folder(const std::string& path_to_process_);
+  void trim_left(std::string& str);
+  void trim_right(std::string& str);
   //----------------------------------------------------------------------------
   inline std::string& trim(std::string& str)
   {
-    trim_left(str);
-    trim_right(str);
+    boost::trim(str);
     return str;
   }
   //----------------------------------------------------------------------------
-  inline std::string trim(const std::string& str_)
+  inline std::string trim(const std::string& str)
   {
-    std::string str = str_;
-    trim_left(str);
-    trim_right(str);
-    return str;
+    return boost::trim_copy(str);
   }
   std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false);
   

contrib/epee/src/file_io_utils.cpp

@@ -29,7 +29,7 @@
 #include <fstream>
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/operations.hpp>
-#ifdef WIN32
+#ifdef _WIN32
 #include <windows.h>
 #include "string_tools.h"
 #endif
@@ -70,7 +70,7 @@ namespace file_io_utils
 
 	bool save_string_to_file(const std::string& path_to_file, const std::string& str)
 	{
-#ifdef WIN32
+#ifdef _WIN32
                 std::wstring wide_path;
                 try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
                 HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
@@ -104,7 +104,7 @@ namespace file_io_utils
 
 	bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size)
 	{
-#ifdef WIN32
+#ifdef _WIN32
                 std::wstring wide_path;
                 try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
                 HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
@@ -153,7 +153,7 @@ namespace file_io_utils
 
 	bool get_file_size(const std::string& path_to_file, uint64_t &size)
 	{
-#ifdef WIN32
+#ifdef _WIN32
                 std::wstring wide_path;
                 try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
                 HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);

contrib/epee/src/mlog.cpp

@@ -176,11 +176,12 @@ void mlog_configure(const std::string &filename_base, bool console, const std::s
       std::vector<boost::filesystem::path> found_files;
       const boost::filesystem::directory_iterator end_itr;
       const boost::filesystem::path filename_base_path(filename_base);
+      const std::string filename_base_name = filename_base_path.filename().string();
       const boost::filesystem::path parent_path = filename_base_path.has_parent_path() ? filename_base_path.parent_path() : ".";
       for (boost::filesystem::directory_iterator iter(parent_path); iter != end_itr; ++iter)
       {
-        const std::string filename = iter->path().string();
-        if (filename.size() >= filename_base.size() && std::memcmp(filename.data(), filename_base.data(), filename_base.size()) == 0)
+        const std::string filename = iter->path().filename().string();
+        if (filename.size() >= filename_base_name.size() && std::memcmp(filename.data(), filename_base_name.data(), filename_base_name.size()) == 0)
         {
           found_files.push_back(iter->path());
         }
@@ -338,11 +339,21 @@ bool is_stdout_a_tty()
   return is_a_tty.load(std::memory_order_relaxed);
 }
 
+static bool is_nocolor()
+{
+  static const char *no_color_var = getenv("NO_COLOR");
+  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
+  return no_color;
+}
+
 void set_console_color(int color, bool bright)
 {
   if (!is_stdout_a_tty())
     return;
 
+  if (is_nocolor())
+    return;
+
   switch(color)
   {
   case console_color_default:
@@ -461,6 +472,9 @@ void reset_console_color() {
   if (!is_stdout_a_tty())
     return;
 
+  if (is_nocolor())
+    return;
+
 #ifdef WIN32
   HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
   SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);

contrib/epee/src/net_ssl.cpp

@@ -496,6 +496,13 @@ void ssl_options_t::configure(
   const std::string& host) const
 {
   socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true));
+  {
+    // in case server is doing "virtual" domains, set hostname
+    SSL* const ssl_ctx = socket.native_handle();
+    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
+      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
+  }
+
 
   /* Using system-wide CA store for client verification is funky - there is
      no expected hostname for server to verify against. If server doesn't have
@@ -513,11 +520,7 @@ void ssl_options_t::configure(
   {
     socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert);
 
-    // in case server is doing "virtual" domains, set hostname
-    SSL* const ssl_ctx = socket.native_handle();
-    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
-      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
-
+    
     socket.set_verify_callback([&](const bool preverified, boost::asio::ssl::verify_context &ctx)
     {
       // preverified means it passed system or user CA check. System CA is never loaded

contrib/epee/src/portable_storage.cpp

@@ -49,7 +49,7 @@ namespace serialization
     byte_stream ss;
     ss.reserve(initial_buffer_size);
     store_to_binary(ss);
-    target = epee::byte_slice{std::move(ss)};
+    target = epee::byte_slice{std::move(ss), false};
     return true;
     CATCH_ENTRY("portable_storage::store_to_binary", false);
   }

contrib/epee/src/readline_buffer.cpp

@@ -1,4 +1,5 @@
 #include "readline_buffer.h"
+#include "string_tools.h"
 #include <readline/readline.h>
 #include <readline/history.h>
 #include <iostream>
@@ -173,7 +174,7 @@ static void handle_line(char* line)
     line_stat = rdln::full;
     the_line = line;
     std::string test_line = line;
-    boost::trim_right(test_line);
+    epee::string_tools::trim_right(test_line);
     if(!test_line.empty())
     {
       if (!same_as_last_line(test_line))
@@ -238,6 +239,10 @@ static char** attempted_completion(const char* text, int start, int end)
 
 static void install_line_handler()
 {
+#if RL_READLINE_VERSION >= 0x0801
+  rl_variable_bind("enable-bracketed-paste", "off");
+#endif
+
   rl_attempted_completion_function = attempted_completion;
   rl_callback_handler_install("", handle_line);
   stifle_history(500);

contrib/epee/src/string_tools.cpp

@@ -38,9 +38,12 @@
 #include <cstdlib>
 #include <string>
 #include <type_traits>
+#include <system_error>
 #include <boost/lexical_cast.hpp>
+#include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/utility/string_ref.hpp>
+#include <boost/filesystem.hpp>
 #include "misc_log_ex.h"
 #include "storages/parserse_base_utils.h"
 #include "hex.h"
@@ -157,46 +160,33 @@ namespace string_tools
     return pname;
   }
 #endif
-	
-    bool set_module_name_and_folder(const std::string& path_to_process_)
-	{
-    std::string path_to_process = path_to_process_;
+
+  void set_module_name_and_folder(const std::string& path_to_process_)
+  {
+    boost::filesystem::path path_to_process = path_to_process_;
+
 #ifdef _WIN32
     path_to_process = get_current_module_path();
 #endif 
-		std::string::size_type a = path_to_process.rfind( '\\' );
-		if(a == std::string::npos )
-		{
-			a = path_to_process.rfind( '/' );
-		}
-		if ( a != std::string::npos )
-		{	
-			get_current_module_name() = path_to_process.substr(a+1, path_to_process.size());
-			get_current_module_folder() = path_to_process.substr(0, a);
-			return true;
-		}else
-			return false;
 
-	}
+    get_current_module_name() = path_to_process.filename().string();
+    get_current_module_folder() = path_to_process.parent_path().string();
+  }
 
 	//----------------------------------------------------------------------------
-	bool trim_left(std::string& str)
-	{
-		for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));)
-			str.erase(str.begin());
-			
-		return true;
-	}
+  void trim_left(std::string& str)
+  {
+    boost::trim_left(str);
+    return;
+  }
+
 	//----------------------------------------------------------------------------
-	bool trim_right(std::string& str)
-	{
+  void trim_right(std::string& str)
+  {
+    boost::trim_right(str);
+    return;
+  }
 
-		for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));)
-			str.erase( --((it++).base()));
-
-		return true;
-	}
-  //----------------------------------------------------------------------------
   std::string pad_string(std::string s, size_t n, char c, bool prepend)
   {
     if (s.size() < n)
@@ -209,28 +199,22 @@ namespace string_tools
     return s;
   }
   
-    std::string get_extension(const std::string& str)
-	{
-		std::string res;
-		std::string::size_type pos = str.rfind('.');
-		if(std::string::npos == pos)
-			return res;
-		
-		res = str.substr(pos+1, str.size()-pos);
-		return res;
-	}
-	//----------------------------------------------------------------------------
-	std::string cut_off_extension(const std::string& str)
-	{
-		std::string res;
-		std::string::size_type pos = str.rfind('.');
-		if(std::string::npos == pos)
-			return str;
+  std::string get_extension(const std::string& str)
+  {
+    std::string ext_with_dot = boost::filesystem::path(str).extension().string();
+
+    if (ext_with_dot.empty())
+      return {};
+
+    return ext_with_dot.erase(0, 1);
+  }
+
+	//----------------------------------------------------------------------------
+  std::string cut_off_extension(const std::string& str)
+  {
+    return boost::filesystem::path(str).replace_extension("").string();
+  }
 
-		res = str.substr(0, pos);
-		return res;
-	}
-  //----------------------------------------------------------------------------
 #ifdef _WIN32
   std::wstring utf8_to_utf16(const std::string& str)
   {

contrib/gitian/DOCKRUN.md

@@ -57,7 +57,8 @@ The dockrun.sh script will do everything to build the binaries. Just specify the
 version to build as its only argument, e.g.
 
 ```bash
-./dockrun.sh v0.17.3.0
+VERSION=v0.18.3.4
+./dockrun.sh $VERSION
 ```
 
 The build should run to completion with no errors, and will display the SHA256 checksums
@@ -78,7 +79,7 @@ e.g.
 
 ```bash
 # Run build processes with 8 threads
-OPT="-j 8" ./dockrun.sh v0.17.3.0
+OPT="-j 8" ./dockrun.sh $VERSION
 ```
 
 Post-build
@@ -98,16 +99,16 @@ more builder/var/install-linux.log
 more builder/var/build-linux.log
 ```
 
-You can find the compiled archives inside of the container at the following directory (be sure to replace `v0.17.3.0` with the version being built):
+You can find the compiled archives inside of the container at the following directory:
 
 ```bash
 docker exec -it gitrun /bin/bash
-ls -la out/v0.17.3.0/
+ls -la out/$VERSION/
 ```
 
-To copy the compiled archives to the local host out of the Docker container, you can run the following (be sure to replace `v0.17.3.0` with the version being built):
+To copy the compiled archives to the local host out of the Docker container, you can run the following:
 
 ```bash
 mkdir out
-docker cp gitrun:/home/ubuntu/out/v0.17.3.0 out
+docker cp gitrun:/home/ubuntu/out/$VERSION out
 ```

contrib/gitian/README.md

@@ -133,7 +133,7 @@ Common setup part:
 su - gitianuser
 
 GH_USER=YOUR_GITHUB_USER_NAME
-VERSION=v0.18.0.0
+VERSION=v0.18.3.4
 ```
 
 Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. 

contrib/gitian/gitian-linux.yml

@@ -21,6 +21,7 @@ packages:
 - "g++-7-arm-linux-gnueabihf"
 - "gcc-arm-linux-gnueabihf"
 - "g++-arm-linux-gnueabihf"
+- "g++-riscv64-linux-gnu"
 - "g++-7-multilib"
 - "gcc-7-multilib"
 - "binutils-arm-linux-gnueabihf"
@@ -43,7 +44,7 @@ files: []
 script: |
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu"
+  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu riscv64-linux-gnu"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="date"
   HOST_CFLAGS="-O2 -g"
@@ -159,7 +160,13 @@ script: |
     fi
     export C_INCLUDE_PATH="$EXTRA_INCLUDES"
     export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
-    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON -DCMAKE_SKIP_RPATH=ON
+    #  glibc only added riscv support in 2.27, disable backwards compatibility
+    if [ "$i" == "riscv64-linux-gnu" ]; then
+      BACKCOMPAT_OPTION=OFF
+    else
+      BACKCOMPAT_OPTION=ON
+    fi
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=${BACKCOMPAT_OPTION} -DCMAKE_SKIP_RPATH=ON
     make ${MAKEOPTS}
     chmod 755 bin/*
     cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin

docs/ANONYMITY_NETWORKS.md

@@ -71,13 +71,13 @@ type, and max connections:
 
 ```
 --anonymous-inbound rveahdfho7wo4b2m.onion:28083,127.0.0.1:28083,25
---anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000,127.0.0.1:30000
+--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p,127.0.0.1:30000
 ```
 
 which tells `monerod` that a max of 25 inbound Tor connections are being
 received at address "rveahdfho7wo4b2m.onion:28083" and forwarded to `monerod`
 localhost port 28083, and a default max I2P connections are being received at
-address "cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000" and
+address "cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p" and
 forwarded to `monerod` localhost port 30000.
 These addresses will be shared with outgoing peers, over the same network type,
 otherwise the peer will not be notified of the peer address by the proxy.

external/easylogging++/easylogging++.cc

@@ -149,6 +149,11 @@ static el::Color colorFromLevel(el::Level level)
 
 static void setConsoleColor(el::Color color, bool bright)
 {
+  static const char *no_color_var = getenv("NO_COLOR");
+  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
+  if (no_color)
+    return;
+
 #if ELPP_OS_WINDOWS
   HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
   switch (color)

external/randomx

@@ -1 +1 @@
-Subproject commit 85c527a62301b7b8be89d941020308b1cb92b75c
+Subproject commit 102f8acf90a7649ada410de5499a7ec62e49e1da

src/blockchain_db/blockchain_db.h

@@ -1883,16 +1883,18 @@ public:
   }
   virtual ~db_txn_guard()
   {
-    if (active)
-      stop();
+    stop();
   }
   void stop()
   {
-    if (readonly)
-      db->block_rtxn_stop();
-    else
-      db->block_wtxn_stop();
-    active = false;
+    if (active)
+    {
+      if (readonly)
+        db->block_rtxn_stop();
+      else
+        db->block_wtxn_stop();
+      active = false;
+    }
   }
   void abort()
   {

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -465,6 +465,32 @@ void mdb_txn_safe::increment_txns(int i)
 	num_active_txns += i;
 }
 
+#define TXN_PREFIX(flags); \
+  mdb_txn_safe auto_txn; \
+  mdb_txn_safe* txn_ptr = &auto_txn; \
+  if (m_batch_active) \
+    txn_ptr = m_write_txn; \
+  else \
+  { \
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
+      throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
+  } \
+
+#define TXN_PREFIX_RDONLY() \
+  MDB_txn *m_txn; \
+  mdb_txn_cursors *m_cursors; \
+  mdb_txn_safe auto_txn; \
+  bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \
+  if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get(); \
+  else auto_txn.uncheck()
+#define TXN_POSTFIX_RDONLY()
+
+#define TXN_POSTFIX_SUCCESS() \
+  do { \
+    if (! m_batch_active) \
+      auto_txn.commit(); \
+  } while(0)
+
 void lmdb_resized(MDB_env *env, int isactive)
 {
   mdb_txn_safe::prevent_new_txns();
@@ -713,21 +739,20 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin
   }
   else
   {
-    MDB_txn *rtxn;
-    mdb_txn_cursors *rcurs;
-    bool my_rtxn = block_rtxn_start(&rtxn, &rcurs);
-    for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
     {
-      // we have access to block weight, which will be greater or equal to block size,
-      // so use this as a proxy. If it's too much off, we might have to check actual size,
-      // which involves reading more data, so is not really wanted
-      size_t block_weight = get_block_weight(block_num);
-      total_block_size += block_weight;
-      // Track number of blocks being totalled here instead of assuming, in case
-      // some blocks were to be skipped for being outliers.
-      ++num_blocks_used;
+      TXN_PREFIX_RDONLY();
+      for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
+      {
+        // we have access to block weight, which will be greater or equal to block size,
+        // so use this as a proxy. If it's too much off, we might have to check actual size,
+        // which involves reading more data, so is not really wanted
+        size_t block_weight = get_block_weight(block_num);
+        total_block_size += block_weight;
+        // Track number of blocks being totalled here instead of assuming, in case
+        // some blocks were to be skipped for being outliers.
+        ++num_blocks_used;
+      }
     }
-    if (my_rtxn) block_rtxn_stop();
     avg_block_size = total_block_size / (num_blocks_used ? num_blocks_used : 1);
     MDEBUG("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
   }
@@ -1678,32 +1703,6 @@ void BlockchainLMDB::unlock()
   check_open();
 }
 
-#define TXN_PREFIX(flags); \
-  mdb_txn_safe auto_txn; \
-  mdb_txn_safe* txn_ptr = &auto_txn; \
-  if (m_batch_active) \
-    txn_ptr = m_write_txn; \
-  else \
-  { \
-    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
-      throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
-  } \
-
-#define TXN_PREFIX_RDONLY() \
-  MDB_txn *m_txn; \
-  mdb_txn_cursors *m_cursors; \
-  mdb_txn_safe auto_txn; \
-  bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \
-  if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get(); \
-  else auto_txn.uncheck()
-#define TXN_POSTFIX_RDONLY()
-
-#define TXN_POSTFIX_SUCCESS() \
-  do { \
-    if (! m_batch_active) \
-      auto_txn.commit(); \
-  } while(0)
-
 
 // The below two macros are for DB access within block add/remove, whether
 // regular batch txn is in use or not. m_write_txn is used as a batch txn, even
@@ -3923,13 +3922,20 @@ void BlockchainLMDB::block_rtxn_stop() const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   mdb_txn_reset(m_tinfo->m_ti_rtxn);
   memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
+  /* cancel out the increment from rtxn_start */
+  mdb_txn_safe::increment_txns(-1);
 }
 
 bool BlockchainLMDB::block_rtxn_start() const
 {
   MDB_txn *mtxn;
   mdb_txn_cursors *mcur;
-  return block_rtxn_start(&mtxn, &mcur);
+  /* auto_txn is only used for the create gate */
+  mdb_txn_safe auto_txn;
+  bool ret = block_rtxn_start(&mtxn, &mcur);
+  if (ret)
+    auto_txn.increment_txns(1); /* remember there is an active readtxn */
+  return ret;
 }
 
 void BlockchainLMDB::block_wtxn_start()

src/blockchain_utilities/blockchain_stats.cpp

@@ -46,6 +46,77 @@ using namespace cryptonote;
 
 static bool stop_requested = false;
 
+static bool do_inputs, do_outputs, do_ringsize, do_hours, do_emission, do_fees, do_diff;
+
+static struct tm prevtm, currtm;
+static uint64_t prevsz, currsz;
+static uint64_t prevtxs, currtxs;
+static uint64_t currblks;
+static uint64_t h;
+static uint64_t totins, totouts, totrings;
+static boost::multiprecision::uint128_t prevemission, prevfees;
+static boost::multiprecision::uint128_t emission, fees;
+static boost::multiprecision::uint128_t totdiff, mindiff, maxdiff;
+
+#define MAX_INOUT	0xffffffff
+#define MAX_RINGS	0xffffffff
+
+static uint32_t minins = MAX_INOUT, maxins;
+static uint32_t minouts = MAX_INOUT, maxouts;
+static uint32_t minrings = MAX_RINGS, maxrings;
+static uint32_t io, tottxs;
+static uint32_t txhr[24];
+
+static void doprint()
+{
+  char timebuf[64];
+
+  strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm);
+  prevtm = currtm;
+  std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
+  prevsz += currsz;
+  currsz = 0;
+  prevtxs += currtxs;
+  currtxs = 0;
+  if (!tottxs)
+    tottxs = 1;
+  if (do_emission) {
+    std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
+    prevemission += emission;
+    emission = 0;
+  }
+  if (do_fees) {
+    std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
+    prevfees += fees;
+    fees = 0;
+  }
+  if (do_diff) {
+    std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
+    mindiff = 0; maxdiff = 0; totdiff = 0;
+  }
+  if (do_inputs) {
+    std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
+    minins = MAX_INOUT; maxins = 0; totins = 0;
+  }
+  if (do_outputs) {
+    std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
+    minouts = MAX_INOUT; maxouts = 0; totouts = 0;
+  }
+  if (do_ringsize) {
+    std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
+    minrings = MAX_RINGS; maxrings = 0; totrings = 0;
+  }
+  if (do_hours) {
+    for (int i=0; i<24; i++) {
+      std::cout << "\t" << txhr[i];
+      txhr[i] = 0;
+    }
+  }
+  currblks = 0;
+  tottxs = 0;
+  std::cout << ENDL;
+}
+
 int main(int argc, char* argv[])
 {
   TRY_ENTRY();
@@ -123,13 +194,13 @@ int main(int argc, char* argv[])
   network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
   block_start = command_line::get_arg(vm, arg_block_start);
   block_stop = command_line::get_arg(vm, arg_block_stop);
-  bool do_inputs = command_line::get_arg(vm, arg_inputs);
-  bool do_outputs = command_line::get_arg(vm, arg_outputs);
-  bool do_ringsize = command_line::get_arg(vm, arg_ringsize);
-  bool do_hours = command_line::get_arg(vm, arg_hours);
-  bool do_emission = command_line::get_arg(vm, arg_emission);
-  bool do_fees = command_line::get_arg(vm, arg_fees);
-  bool do_diff = command_line::get_arg(vm, arg_diff);
+  do_inputs = command_line::get_arg(vm, arg_inputs);
+  do_outputs = command_line::get_arg(vm, arg_outputs);
+  do_ringsize = command_line::get_arg(vm, arg_ringsize);
+  do_hours = command_line::get_arg(vm, arg_hours);
+  do_emission = command_line::get_arg(vm, arg_emission);
+  do_fees = command_line::get_arg(vm, arg_fees);
+  do_diff = command_line::get_arg(vm, arg_diff);
 
   LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
   std::unique_ptr<Blockchain> core_storage;
@@ -211,25 +282,7 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
   }
   std::cout << ENDL;
 
-#define MAX_INOUT	0xffffffff
-#define MAX_RINGS	0xffffffff
-
-  struct tm prevtm = {0}, currtm;
-  uint64_t prevsz = 0, currsz = 0;
-  uint64_t prevtxs = 0, currtxs = 0;
-  uint64_t currblks = 0;
-  uint64_t totins = 0, totouts = 0, totrings = 0;
-  boost::multiprecision::uint128_t prevemission = 0, prevfees = 0;
-  boost::multiprecision::uint128_t emission = 0, fees = 0;
-  boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0;
-  uint32_t minins = MAX_INOUT, maxins = 0;
-  uint32_t minouts = MAX_INOUT, maxouts = 0;
-  uint32_t minrings = MAX_RINGS, maxrings = 0;
-  uint32_t io, tottxs = 0;
-  uint32_t txhr[24] = {0};
-  unsigned int i;
-
-  for (uint64_t h = block_start; h < block_stop; ++h)
+  for (h = block_start; h < block_stop; ++h)
   {
     cryptonote::blobdata bd = db->get_block_blob_from_height(h);
     cryptonote::block blk;
@@ -239,7 +292,6 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
       return 1;
     }
     time_t tt = blk.timestamp;
-    char timebuf[64];
     epee::misc_utils::get_gmt_time(tt, currtm);
     if (!prevtm.tm_year)
       prevtm = currtm;
@@ -247,54 +299,9 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
     if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27))
     {
       // check for timestamp fudging around month ends
-      if (prevtm.tm_mday == 1 && currtm.tm_mday > 27)
-        goto skip;
-      strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm);
-      prevtm = currtm;
-      std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
-      prevsz += currsz;
-      currsz = 0;
-      prevtxs += currtxs;
-      currtxs = 0;
-      if (!tottxs)
-        tottxs = 1;
-      if (do_emission) {
-        std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
-        prevemission += emission;
-        emission = 0;
-      }
-      if (do_fees) {
-        std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
-        prevfees += fees;
-        fees = 0;
-      }
-      if (do_diff) {
-        std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
-        mindiff = 0; maxdiff = 0; totdiff = 0;
-      }
-      if (do_inputs) {
-        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
-        minins = MAX_INOUT; maxins = 0; totins = 0;
-      }
-      if (do_outputs) {
-        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
-        minouts = MAX_INOUT; maxouts = 0; totouts = 0;
-      }
-      if (do_ringsize) {
-        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
-        minrings = MAX_RINGS; maxrings = 0; totrings = 0;
-      }
-      if (do_hours) {
-        for (i=0; i<24; i++) {
-          std::cout << "\t" << txhr[i];
-          txhr[i] = 0;
-        }
-      }
-      currblks = 0;
-      tottxs = 0;
-      std::cout << ENDL;
+      if (!(prevtm.tm_mday == 1 && currtm.tm_mday > 27))
+        doprint();
     }
-skip:
     currsz += bd.size();
     uint64_t coinbase_amount;
     uint64_t tx_fee_amount = 0;
@@ -371,6 +378,8 @@ skip:
     if (stop_requested)
       break;
   }
+  if (currblks)
+    doprint();
 
   core_storage->deinit();
   return 0;

src/checkpoints/checkpoints.cpp

@@ -240,6 +240,17 @@ namespace cryptonote
     ADD_CHECKPOINT2(2092500, "c4e00820c9c7989b49153d5e90ae095a18a11d990e82fcc3be54e6ed785472b5", "0xb4e585a31369cb");
     ADD_CHECKPOINT2(2182500, "0d22b5f81982eff21d094af9e821dc2007e6342069e3b1a37b15d97646353124", "0xead4a874083492");
     ADD_CHECKPOINT2(2661600, "41c9060e8426012238e8a26da26fcb90797436896cc70886a894c2c560bcccf2", "0x2e0d87526ff161f");
+    ADD_CHECKPOINT2(2677000, "1b9fee6246eeb176bd17d637bf252e9af54a4218675f01b4449cc0901867f9eb", "0x2f165bc1a5163ba");
+    ADD_CHECKPOINT2(2706000, "d8eb144c5e1fe6b329ecc900ec95e7792fccff84175fb23a25ed59d7299a511c", "0x310f7d89372f705");
+    ADD_CHECKPOINT2(2720000, "b19fb41dff15bd1016afbee9f8469f05aab715c9e5d1b974466a11fd58ecbb86", "0x3216b5851ddbb61");
+    ADD_CHECKPOINT2(2817000, "39726d19ccaac01d150bec827b877ffae710b516bd633503662036ef4422e577", "0x3900669561954c1");
+    ADD_CHECKPOINT2(2844000, "28fc7b446dfef5b469f5778eb72ddf32a307a5f5a9823d1c394e772349e05d40", "0x3af384ec0e97d12");
+    ADD_CHECKPOINT2(2851000, "5bf0e47fc782263191a33f63a67db6c711781dc2a3c442e17ed901ec401be5c9", "0x3b6cd8a8ed610e8");
+    ADD_CHECKPOINT2(2971000, "3d4cac5ac515eeabd18769ab943af85f36db51d28720def0d0e6effc2c8f5ce3", "0x436e532738b8b5b");
+    ADD_CHECKPOINT2(2985000, "08f5e6b7301c1b6ed88268a28f8677a06e8ff943b3f9e48d3080f71f9c134bfb", "0x444b7b42a633c96");
+    ADD_CHECKPOINT2(3088000, "bddf8ca09110d33d6d497f13a113630c2b6af1c84d4f3a6f35cb1446f2604ade", "0x4aed3615c2f8c3e");
+    ADD_CHECKPOINT2(3102800, "083f4a34f9490403b564286e7f13fd1ed45c52c86fa47195f151594e5bc87504", "0x4bbed52d4da5dfb");
+    ADD_CHECKPOINT2(3198000, "1d685b39be51e4e84e0af69fa78e023c7cb21de7d33acd012d0371d5f78712d5", "0x517d415fee3a816");
     return true;
   }
 

src/common/boost_serialization_helper.h

@@ -35,6 +35,7 @@
 #include <boost/archive/portable_binary_iarchive.hpp>
 #include <boost/filesystem/operations.hpp>
 
+#include "common/util.h"
 
 namespace tools
 {
@@ -110,7 +111,7 @@ namespace tools
     catch(...)
     {
       // if failed, try reading in unportable mode
-      boost::filesystem::copy_file(file_path, file_path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists);
+      tools::copy_file(file_path, file_path + ".unportable");
       data_file.close();
       data_file.open( file_path, std::ios_base::binary | std::ios_base::in);
       if(data_file.fail())

src/common/combinator.h

@@ -34,6 +34,7 @@
 #include <iostream>
 #include <vector>
 #include <stdexcept>
+#include <cstdint>
 
 namespace tools {
 

src/common/data_cache.h

@@ -0,0 +1,65 @@
+// Copyright (c) 2014-2022, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once 
+
+#include <unordered_set>
+#include <mutex>
+
+namespace tools
+{
+  template<typename T, size_t MAX_SIZE>
+  class data_cache
+  {
+  public:
+    void add(const T& value)
+    {
+      std::lock_guard<std::mutex> lock(m);
+      if (data.insert(value).second)
+      {
+        T& old_value = buf[counter++ % MAX_SIZE];
+        data.erase(old_value);
+        old_value = value;
+      }
+    }
+
+    bool has(const T& value) const
+    {
+      std::lock_guard<std::mutex> lock(m);
+      return (data.find(value) != data.end());
+    }
+
+  private:
+    mutable std::mutex m;
+    std::unordered_set<T> data;
+    T buf[MAX_SIZE] = {};
+    size_t counter = 0;
+  };
+}

src/common/dns_utils.cpp

@@ -30,6 +30,8 @@
 // check local first (in the event of static or in-source compilation of libunbound)
 #include "unbound.h"
 
+#include <deque>
+#include <set>
 #include <stdlib.h>
 #include "include_base_utils.h"
 #include "common/threadpool.h"
@@ -327,11 +329,6 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec
   dnssec_available = false;
   dnssec_valid = false;
 
-  if (!check_address_syntax(url.c_str()))
-  {
-    return addresses;
-  }
-
   // destructor takes care of cleanup
   ub_result_ptr result;
 
@@ -414,16 +411,6 @@ DNSResolver DNSResolver::create()
   return DNSResolver();
 }
 
-bool DNSResolver::check_address_syntax(const char *addr) const
-{
-  // if string doesn't contain a dot, we won't consider it a url for now.
-  if (strchr(addr,'.') == NULL)
-  {
-    return false;
-  }
-  return true;
-}
-
 namespace dns_utils
 {
 
@@ -521,7 +508,7 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std
 
   // send all requests in parallel
   std::deque<bool> avail(dns_urls.size(), false), valid(dns_urls.size(), false);
-  tools::threadpool& tpool = tools::threadpool::getInstance();
+  tools::threadpool& tpool = tools::threadpool::getInstanceForIO();
   tools::threadpool::waiter waiter(tpool);
   for (size_t n = 0; n < dns_urls.size(); ++n)
   {

src/common/dns_utils.h

@@ -159,15 +159,6 @@ private:
   // TODO: modify this to accommodate DNSSEC
   std::vector<std::string> get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid);
 
-  /**
-   * @brief Checks a string to see if it looks like a URL
-   *
-   * @param addr the string to be checked
-   *
-   * @return true if it looks enough like a URL, false if not
-   */
-  bool check_address_syntax(const char *addr) const;
-
   DNSResolverData *m_data;
 }; // class DNSResolver
 

src/common/perf_timer.cpp

@@ -62,7 +62,7 @@ namespace tools
     while (1)
     {
       t1 = epee::misc_utils::get_ns_count();
-      if (t1 - t0 > 1*1000000000) break; // work one second
+      if (t1 - t0 > 1*100000000) break; // work 0.1 seconds
     }
 
     uint64_t r1 = get_tick_count();

src/common/threadpool.h

@@ -31,6 +31,7 @@
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/thread.hpp>
 #include <cstddef>
+#include <deque>
 #include <functional>
 #include <utility>
 #include <vector>
@@ -42,10 +43,14 @@ namespace tools
 class threadpool
 {
 public:
-  static threadpool& getInstance() {
+  static threadpool& getInstanceForCompute() {
     static threadpool instance;
     return instance;
   }
+  static threadpool& getInstanceForIO() {
+    static threadpool instance(8);
+    return instance;
+  }
   static threadpool *getNewForUnitTests(unsigned max_threads = 0) {
     return new threadpool(max_threads);
   }

src/common/util.cpp

@@ -115,6 +115,24 @@ static int flock_exnb(int fd)
 
 namespace tools
 {
+
+  void copy_file(const std::string& from, const std::string& to)
+  {
+    using boost::filesystem::path;
+  #if BOOST_VERSION < 107400
+    // Remove this preprocessor if/else when we are bumping the boost version.
+    boost::filesystem::copy_file(
+        path(from),
+        path(to),
+        boost::filesystem::copy_option::overwrite_if_exists);
+  #else
+    boost::filesystem::copy_file(
+        path(from),
+        path(to),
+        boost::filesystem::copy_options::overwrite_existing);
+  #endif
+  }
+
   std::function<void(int)> signal_handler::m_handler;
 
   private_file::private_file() noexcept : m_handle(), m_filename() {}
@@ -122,7 +140,7 @@ namespace tools
   private_file::private_file(std::FILE* handle, std::string&& filename) noexcept
     : m_handle(handle), m_filename(std::move(filename)) {}
 
-  private_file private_file::create(std::string name)
+  private_file private_file::create(std::string name, uint32_t extra_flags)
   {
 #ifdef WIN32
     struct close_handle
@@ -175,7 +193,7 @@ namespace tools
         name.c_str(),
         GENERIC_WRITE, FILE_SHARE_READ,
         std::addressof(attributes),
-        CREATE_NEW, (FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE),
+        CREATE_NEW, (FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE | extra_flags),
         nullptr
       )
     };
@@ -194,7 +212,7 @@ namespace tools
       }
     }
 #else
-    const int fdr = open(name.c_str(), (O_RDONLY | O_CREAT), S_IRUSR);
+    const int fdr = open(name.c_str(), (O_RDONLY | O_CREAT | extra_flags), S_IRUSR);
     if (0 <= fdr)
     {
       struct stat rstats = {};
@@ -225,6 +243,23 @@ namespace tools
     return {};
   }
 
+  private_file private_file::drop_and_recreate(std::string filename)
+  {
+    if (epee::file_io_utils::is_file_exist(filename)) {
+      boost::system::error_code ec{};
+      boost::filesystem::remove(filename, ec);
+      if (ec) {
+        MERROR("Failed to remove " << filename << ": " << ec.message());
+        return {};
+      }
+    }
+#ifdef WIN32
+    return create(filename);
+#else
+    return create(filename, O_EXCL);
+#endif
+  }
+
   private_file::~private_file() noexcept
   {
     try
@@ -882,13 +917,6 @@ std::string get_nix_version_display_string()
 
   bool is_local_address(const std::string &address)
   {
-    // always assume Tor/I2P addresses to be untrusted by default
-    if (is_privacy_preserving_network(address))
-    {
-      MDEBUG("Address '" << address << "' is Tor/I2P, non local");
-      return false;
-    }
-
     // extract host
     epee::net_utils::http::url_content u_c;
     if (!epee::net_utils::parse_url(address, u_c))
@@ -902,20 +930,22 @@ std::string get_nix_version_display_string()
       return false;
     }
 
-    // resolve to IP
-    boost::asio::io_service io_service;
-    boost::asio::ip::tcp::resolver resolver(io_service);
-    boost::asio::ip::tcp::resolver::query query(u_c.host, "");
-    boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query);
-    while (i != boost::asio::ip::tcp::resolver::iterator())
+    if (u_c.host == "localhost" || boost::ends_with(u_c.host, ".localhost")) { // RFC 6761 (6.3)
+      MDEBUG("Address '" << address << "' is local");
+      return true;
+    }
+
+    boost::system::error_code ec;
+    const auto parsed_ip = boost::asio::ip::address::from_string(u_c.host, ec);
+    if (ec) {
+      MDEBUG("Failed to parse '" << address << "' as IP address: " << ec.message() << ". Considering it not local");
+      return false;
+    }
+
+    if (parsed_ip.is_loopback())
     {
-      const boost::asio::ip::tcp::endpoint &ep = *i;
-      if (ep.address().is_loopback())
-      {
-        MDEBUG("Address '" << address << "' is local");
-        return true;
-      }
-      ++i;
+      MDEBUG("Address '" << address << "' is local");
+      return true;
     }
 
     MDEBUG("Address '" << address << "' is not local");

src/common/util.h

@@ -67,6 +67,8 @@ namespace tools
     }
   };
 
+  void copy_file(const std::string& from, const std::string& to);
+
   //! A file restricted to process owner AND process. Deletes file on destruction.
   class private_file {
     std::unique_ptr<std::FILE, close_file> m_handle;
@@ -80,7 +82,11 @@ namespace tools
 
     /*! \return File only readable by owner and only used by this process
       OR `private_file{}` on error. */
-    static private_file create(std::string filename);
+    static private_file create(std::string filename, uint32_t extra_flags = 0);
+
+    /*! \return Drop and create file only readable by owner and only used
+      by this process OR `private_file{}` on error. */
+    static private_file drop_and_recreate(std::string filename);    
 
     private_file(private_file&&) = default;
     private_file& operator=(private_file&&) = default;

src/crypto/c_threads.h

@@ -30,29 +30,41 @@
 #pragma once
 
 #ifdef _WIN32
+
 #include <windows.h>
-#define CTHR_MUTEX_TYPE	HANDLE
-#define CTHR_MUTEX_INIT	NULL
-#define CTHR_MUTEX_LOCK(x)	do { if (x == NULL) { \
-    HANDLE p = CreateMutex(NULL, FALSE, NULL); \
-    if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \
-      CloseHandle(p); \
-  } WaitForSingleObject(x, INFINITE); } while(0)
-#define CTHR_MUTEX_UNLOCK(x)	ReleaseMutex(x)
+
+#define CTHR_RWLOCK_TYPE	SRWLOCK
+#define CTHR_RWLOCK_INIT	SRWLOCK_INIT
+#define CTHR_RWLOCK_LOCK_WRITE(x)	AcquireSRWLockExclusive(&x)
+#define CTHR_RWLOCK_UNLOCK_WRITE(x)	ReleaseSRWLockExclusive(&x)
+#define CTHR_RWLOCK_LOCK_READ(x)	AcquireSRWLockShared(&x)
+#define CTHR_RWLOCK_UNLOCK_READ(x)	ReleaseSRWLockShared(&x)
+#define CTHR_RWLOCK_TRYLOCK_READ(x)	TryAcquireSRWLockShared(&x)
+
 #define CTHR_THREAD_TYPE	HANDLE
-#define CTHR_THREAD_RTYPE	void
-#define CTHR_THREAD_RETURN	return
-#define CTHR_THREAD_CREATE(thr, func, arg)	thr = (HANDLE)_beginthread(func, 0, arg)
-#define CTHR_THREAD_JOIN(thr)			WaitForSingleObject(thr, INFINITE)
+#define CTHR_THREAD_RTYPE	unsigned __stdcall
+#define CTHR_THREAD_RETURN	_endthreadex(0); return 0;
+#define CTHR_THREAD_CREATE(thr, func, arg)	((thr = (HANDLE)_beginthreadex(0, 0, func, arg, 0, 0)) != 0L)
+#define CTHR_THREAD_JOIN(thr)			do { WaitForSingleObject(thr, INFINITE); CloseHandle(thr); } while(0)
+#define CTHR_THREAD_CLOSE(thr)			CloseHandle((HANDLE)thr);
+
 #else
+
 #include <pthread.h>
-#define CTHR_MUTEX_TYPE pthread_mutex_t
-#define CTHR_MUTEX_INIT	PTHREAD_MUTEX_INITIALIZER
-#define CTHR_MUTEX_LOCK(x)	pthread_mutex_lock(&x)
-#define CTHR_MUTEX_UNLOCK(x)	pthread_mutex_unlock(&x)
+
+#define CTHR_RWLOCK_TYPE	pthread_rwlock_t
+#define CTHR_RWLOCK_INIT	PTHREAD_RWLOCK_INITIALIZER
+#define CTHR_RWLOCK_LOCK_WRITE(x)	pthread_rwlock_wrlock(&x)
+#define CTHR_RWLOCK_UNLOCK_WRITE(x)	pthread_rwlock_unlock(&x)
+#define CTHR_RWLOCK_LOCK_READ(x)	pthread_rwlock_rdlock(&x)
+#define CTHR_RWLOCK_UNLOCK_READ(x)	pthread_rwlock_unlock(&x)
+#define CTHR_RWLOCK_TRYLOCK_READ(x)	(pthread_rwlock_tryrdlock(&x) == 0)
+
 #define CTHR_THREAD_TYPE pthread_t
 #define CTHR_THREAD_RTYPE	void *
 #define CTHR_THREAD_RETURN	return NULL
-#define CTHR_THREAD_CREATE(thr, func, arg)	pthread_create(&thr, NULL, func, arg)
+#define CTHR_THREAD_CREATE(thr, func, arg)	(pthread_create(&thr, NULL, func, arg) == 0)
 #define CTHR_THREAD_JOIN(thr)			pthread_join(thr, NULL)
+#define CTHR_THREAD_CLOSE(thr)
+
 #endif

src/crypto/hash-ops.h

@@ -97,5 +97,9 @@ void rx_slow_hash_allocate_state(void);
 void rx_slow_hash_free_state(void);
 uint64_t rx_seedheight(const uint64_t height);
 void rx_seedheights(const uint64_t height, uint64_t *seed_height, uint64_t *next_height);
-void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash, int miners, int is_alt);
-void rx_reorg(const uint64_t split_height);
+
+void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads);
+void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash);
+
+void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads);
+uint32_t rx_get_miner_thread(void);

src/crypto/jh.c

@@ -34,7 +34,7 @@ typedef struct {
 	unsigned long long databitlen;    /*the message size in bits*/
 	unsigned long long datasize_in_buffer;      /*the size of the message remained in buffer; assumed to be multiple of 8bits except for the last partial block at the end of the message*/
 	DATA_ALIGN16(uint64 x[8][2]);     /*the 1024-bit state, ( x[i][0] || x[i][1] ) is the ith row of the state in the pseudocode*/
-	unsigned char buffer[64];         /*the 512-bit message block to be hashed;*/
+	DATA_ALIGN16(unsigned char buffer[64]);         /*the 512-bit message block to be hashed;*/
 } hashState;
 
 
@@ -213,16 +213,24 @@ static void E8(hashState *state)
 /*The compression function F8 */
 static void F8(hashState *state)
 {
-      uint64  i;
+      uint64_t* x = (uint64_t*)state->x;
 
       /*xor the 512-bit message with the fist half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[i >> 1][i & 1] ^= ((uint64*)state->buffer)[i];
+      for (int i = 0; i < 8; ++i) {
+            uint64 b;
+            memcpy(&b, &state->buffer[i << 3], sizeof(b));
+            x[i] ^= b;
+      }
 
       /*the bijective function E8 */
       E8(state);
 
       /*xor the 512-bit message with the second half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[(8+i) >> 1][(8+i) & 1] ^= ((uint64*)state->buffer)[i];
+      for (int i = 0; i < 8; ++i) {
+            uint64 b;
+            memcpy(&b, &state->buffer[i << 3], sizeof(b));
+            x[i + 8] ^= b;
+      }
 }
 
 /*before hashing a message, initialize the hash state as H0 */
@@ -240,6 +248,7 @@ static HashReturn Init(hashState *state, int hashbitlen)
             case 224: memcpy(state->x,JH224_H0,128); break;
             case 256: memcpy(state->x,JH256_H0,128); break;
             case 384: memcpy(state->x,JH384_H0,128); break;
+            default:
             case 512: memcpy(state->x,JH512_H0,128); break;
       }
 

src/crypto/rx-slow-hash.c

@@ -43,32 +43,41 @@
 
 #define RX_LOGCAT	"randomx"
 
+// Report large page allocation failures as debug messages
+#define alloc_err_msg(x) mdebug(RX_LOGCAT, x);
+
+static CTHR_RWLOCK_TYPE main_dataset_lock = CTHR_RWLOCK_INIT;
+static CTHR_RWLOCK_TYPE main_cache_lock = CTHR_RWLOCK_INIT;
+
+static randomx_dataset *main_dataset = NULL;
+static randomx_cache *main_cache = NULL;
+static char main_seedhash[HASH_SIZE];
+static int main_seedhash_set = 0;
+
+static CTHR_RWLOCK_TYPE secondary_cache_lock = CTHR_RWLOCK_INIT;
+
+static randomx_cache *secondary_cache = NULL;
+static char secondary_seedhash[HASH_SIZE];
+static int secondary_seedhash_set = 0;
+
 #if defined(_MSC_VER)
 #define THREADV __declspec(thread)
 #else
 #define THREADV __thread
 #endif
 
-typedef struct rx_state {
-  CTHR_MUTEX_TYPE rs_mutex;
-  char rs_hash[HASH_SIZE];
-  uint64_t  rs_height;
-  randomx_cache *rs_cache;
-} rx_state;
+static THREADV randomx_vm *main_vm_full = NULL;
+static THREADV randomx_vm *main_vm_light = NULL;
+static THREADV randomx_vm *secondary_vm_light = NULL;
 
-static CTHR_MUTEX_TYPE rx_mutex = CTHR_MUTEX_INIT;
-static CTHR_MUTEX_TYPE rx_dataset_mutex = CTHR_MUTEX_INIT;
+static THREADV uint32_t miner_thread = 0;
 
-static rx_state rx_s[2] = {{CTHR_MUTEX_INIT,{0},0,0},{CTHR_MUTEX_INIT,{0},0,0}};
-
-static randomx_dataset *rx_dataset;
-static int rx_dataset_nomem;
-static int rx_dataset_nolp;
-static uint64_t rx_dataset_height;
-static THREADV randomx_vm *rx_vm = NULL;
+static bool is_main(const char* seedhash) { return main_seedhash_set && (memcmp(seedhash, main_seedhash, HASH_SIZE) == 0); }
+static bool is_secondary(const char* seedhash) { return secondary_seedhash_set && (memcmp(seedhash, secondary_seedhash, HASH_SIZE) == 0); }
 
 static void local_abort(const char *msg)
 {
+  merror(RX_LOGCAT, "%s", msg);
   fprintf(stderr, "%s\n", msg);
 #ifdef NDEBUG
   _exit(1);
@@ -77,6 +86,16 @@ static void local_abort(const char *msg)
 #endif
 }
 
+static void hash2hex(const char* hash, char* hex) {
+  const char* d = "0123456789abcdef";
+  for (int i = 0; i < HASH_SIZE; ++i) {
+    const uint8_t b = hash[i];
+    hex[i * 2 + 0] = d[b >> 4];
+    hex[i * 2 + 1] = d[b & 15];
+  }
+  hex[HASH_SIZE * 2] = '\0';
+}
+
 static inline int disabled_flags(void) {
   static int flags = -1;
 
@@ -157,19 +176,6 @@ static unsigned int get_seedhash_epoch_blocks(void)
   return blocks;
 }
 
-void rx_reorg(const uint64_t split_height) {
-  int i;
-  CTHR_MUTEX_LOCK(rx_mutex);
-  for (i=0; i<2; i++) {
-    if (split_height <= rx_s[i].rs_height) {
-      if (rx_s[i].rs_height == rx_dataset_height)
-        rx_dataset_height = 1;
-      rx_s[i].rs_height = 1;	/* set to an invalid seed height */
-    }
-  }
-  CTHR_MUTEX_UNLOCK(rx_mutex);
-}
-
 uint64_t rx_seedheight(const uint64_t height) {
   const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag();
   const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks();
@@ -183,6 +189,103 @@ void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nexth
   *nextheight = rx_seedheight(height + get_seedhash_epoch_lag());
 }
 
+static void rx_alloc_dataset(randomx_flags flags, randomx_dataset** dataset, int ignore_env)
+{
+  if (*dataset) {
+    return;
+  }
+
+  if (disabled_flags() & RANDOMX_FLAG_FULL_MEM) {
+    static int shown = 0;
+    if (!shown) {
+      shown = 1;
+      minfo(RX_LOGCAT, "RandomX dataset is disabled by MONERO_RANDOMX_UMASK environment variable.");
+    }
+    return;
+  }
+
+  if (!ignore_env && !getenv("MONERO_RANDOMX_FULL_MEM")) {
+    static int shown = 0;
+    if (!shown) {
+      shown = 1;
+      minfo(RX_LOGCAT, "RandomX dataset is not enabled by default. Use MONERO_RANDOMX_FULL_MEM environment variable to enable it.");
+    }
+    return;
+  }
+
+  *dataset = randomx_alloc_dataset((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags());
+  if (!*dataset) {
+    alloc_err_msg("Couldn't allocate RandomX dataset using large pages");
+    *dataset = randomx_alloc_dataset(flags & ~disabled_flags());
+    if (!*dataset) {
+      merror(RX_LOGCAT, "Couldn't allocate RandomX dataset");
+    }
+  }
+}
+
+static void rx_alloc_cache(randomx_flags flags, randomx_cache** cache)
+{
+  if (*cache) {
+    return;
+  }
+
+  *cache = randomx_alloc_cache((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags());
+  if (!*cache) {
+    alloc_err_msg("Couldn't allocate RandomX cache using large pages");
+    *cache = randomx_alloc_cache(flags & ~disabled_flags());
+    if (!*cache) local_abort("Couldn't allocate RandomX cache");
+  }
+}
+
+static void rx_init_full_vm(randomx_flags flags, randomx_vm** vm)
+{
+  if (*vm || !main_dataset || (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) {
+    return;
+  }
+
+  if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) {
+    flags |= RANDOMX_FLAG_SECURE;
+  }
+
+  *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset);
+  if (!*vm) {
+    static int shown = 0;
+    if (!shown) {
+        shown = 1;
+        alloc_err_msg("Couldn't allocate RandomX full VM using large pages (will print only once)");
+    }
+    *vm = randomx_create_vm((flags | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset);
+    if (!*vm) {
+      merror(RX_LOGCAT, "Couldn't allocate RandomX full VM");
+    }
+  }
+}
+
+static void rx_init_light_vm(randomx_flags flags, randomx_vm** vm, randomx_cache* cache)
+{
+  if (*vm) {
+    randomx_vm_set_cache(*vm, cache);
+    return;
+  }
+
+  if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) {
+    flags |= RANDOMX_FLAG_SECURE;
+  }
+
+  flags &= ~RANDOMX_FLAG_FULL_MEM;
+
+  *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags(), cache, NULL);
+  if (!*vm) {
+    static int shown = 0;
+    if (!shown) {
+        shown = 1;
+        alloc_err_msg("Couldn't allocate RandomX light VM using large pages (will print only once)");
+    }
+    *vm = randomx_create_vm(flags & ~disabled_flags(), cache, NULL);
+    if (!*vm) local_abort("Couldn't allocate RandomX light VM");
+  }
+}
+
 typedef struct seedinfo {
   randomx_cache *si_cache;
   unsigned long si_start;
@@ -191,187 +294,231 @@ typedef struct seedinfo {
 
 static CTHR_THREAD_RTYPE rx_seedthread(void *arg) {
   seedinfo *si = arg;
-  randomx_init_dataset(rx_dataset, si->si_cache, si->si_start, si->si_count);
+  randomx_init_dataset(main_dataset, si->si_cache, si->si_start, si->si_count);
   CTHR_THREAD_RETURN;
 }
 
-static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_t seedheight) {
-  if (miners > 1) {
-    unsigned long delta = randomx_dataset_item_count() / miners;
-    unsigned long start = 0;
-    int i;
-    seedinfo *si;
-    CTHR_THREAD_TYPE *st;
-    si = malloc(miners * sizeof(seedinfo));
-    if (si == NULL)
-      local_abort("Couldn't allocate RandomX mining threadinfo");
-    st = malloc(miners * sizeof(CTHR_THREAD_TYPE));
-    if (st == NULL) {
-      free(si);
-      local_abort("Couldn't allocate RandomX mining threadlist");
-    }
-    for (i=0; i<miners-1; i++) {
-      si[i].si_cache = rs_cache;
-      si[i].si_start = start;
-      si[i].si_count = delta;
-      start += delta;
-    }
-    si[i].si_cache = rs_cache;
+static void rx_init_dataset(size_t max_threads) {
+  if (!main_dataset) {
+    return;
+  }
+
+  // leave 2 CPU cores for other tasks
+  const size_t num_threads = (max_threads < 4) ? 1 : (max_threads - 2);
+  seedinfo* si = malloc(num_threads * sizeof(seedinfo));
+  if (!si) local_abort("Couldn't allocate RandomX mining threadinfo");
+
+  const uint32_t delta = randomx_dataset_item_count() / num_threads;
+  uint32_t start = 0;
+
+  const size_t n1 = num_threads - 1;
+  for (size_t i = 0; i < n1; ++i) {
+    si[i].si_cache = main_cache;
     si[i].si_start = start;
-    si[i].si_count = randomx_dataset_item_count() - start;
-    for (i=1; i<miners; i++) {
-      CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i]);
-    }
-    randomx_init_dataset(rx_dataset, rs_cache, 0, si[0].si_count);
-    for (i=1; i<miners; i++) {
-      CTHR_THREAD_JOIN(st[i]);
-    }
-    free(st);
-    free(si);
-  } else {
-    randomx_init_dataset(rx_dataset, rs_cache, 0, randomx_dataset_item_count());
+    si[i].si_count = delta;
+    start += delta;
   }
-  rx_dataset_height = seedheight;
+
+  si[n1].si_cache = main_cache;
+  si[n1].si_start = start;
+  si[n1].si_count = randomx_dataset_item_count() - start;
+
+  CTHR_THREAD_TYPE *st = malloc(num_threads * sizeof(CTHR_THREAD_TYPE));
+  if (!st) local_abort("Couldn't allocate RandomX mining threadlist");
+
+  CTHR_RWLOCK_LOCK_READ(main_cache_lock);
+  for (size_t i = 0; i < n1; ++i) {
+    if (!CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i])) {
+      local_abort("Couldn't start RandomX seed thread");
+    }
+  }
+  randomx_init_dataset(main_dataset, si[n1].si_cache, si[n1].si_start, si[n1].si_count);
+  for (size_t i = 0; i < n1; ++i) CTHR_THREAD_JOIN(st[i]);
+  CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
+
+  free(st);
+  free(si);
+
+  minfo(RX_LOGCAT, "RandomX dataset initialized");
 }
 
-void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length,
-  char *hash, int miners, int is_alt) {
-  uint64_t s_height = rx_seedheight(mainheight);
-  int toggle = (s_height & get_seedhash_epoch_blocks()) != 0;
-  randomx_flags flags = enabled_flags() & ~disabled_flags();
-  rx_state *rx_sp;
-  randomx_cache *cache;
+typedef struct thread_info {
+  char seedhash[HASH_SIZE];
+  size_t max_threads;
+} thread_info;
 
-  CTHR_MUTEX_LOCK(rx_mutex);
+static CTHR_THREAD_RTYPE rx_set_main_seedhash_thread(void *arg) {
+  thread_info* info = arg;
 
-  /* if alt block but with same seed as mainchain, no need for alt cache */
-  if (is_alt) {
-    if (s_height == seedheight && !memcmp(rx_s[toggle].rs_hash, seedhash, HASH_SIZE))
-      is_alt = 0;
-  } else {
-  /* RPC could request an earlier block on mainchain */
-    if (s_height > seedheight)
-      is_alt = 1;
-    /* miner can be ahead of mainchain */
-    else if (s_height < seedheight)
-      toggle ^= 1;
+  CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock);
+  CTHR_RWLOCK_LOCK_WRITE(main_cache_lock);
+
+  // Double check that seedhash wasn't already updated
+  if (is_main(info->seedhash)) {
+    CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock);
+    CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+    free(info);
+    CTHR_THREAD_RETURN;
+  }
+  memcpy(main_seedhash, info->seedhash, HASH_SIZE);
+  main_seedhash_set = 1;
+
+  char buf[HASH_SIZE * 2 + 1];
+  hash2hex(main_seedhash, buf);
+  minfo(RX_LOGCAT, "RandomX new main seed hash is %s", buf);
+
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  rx_alloc_dataset(flags, &main_dataset, 0);
+  rx_alloc_cache(flags, &main_cache);
+
+  randomx_init_cache(main_cache, info->seedhash, HASH_SIZE);
+  minfo(RX_LOGCAT, "RandomX main cache initialized");
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock);
+
+  // From this point, rx_slow_hash can calculate hashes in light mode, but dataset is not initialized yet
+  rx_init_dataset(info->max_threads);
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+
+  free(info);
+  CTHR_THREAD_RETURN;
+}
+
+void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads) {
+  // Early out if seedhash didn't change
+  if (is_main(seedhash)) {
+    return;
   }
 
-  toggle ^= (is_alt != 0);
+  // Update main cache and dataset in the background
+  thread_info* info = malloc(sizeof(thread_info));
+  if (!info) local_abort("Couldn't allocate RandomX mining threadinfo");
 
-  rx_sp = &rx_s[toggle];
-  CTHR_MUTEX_LOCK(rx_sp->rs_mutex);
-  CTHR_MUTEX_UNLOCK(rx_mutex);
+  memcpy(info->seedhash, seedhash, HASH_SIZE);
+  info->max_threads = max_dataset_init_threads;
 
-  cache = rx_sp->rs_cache;
-  if (cache == NULL) {
-    if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) {
-      cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES);
-      if (cache == NULL) {
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache");
-      }
-    }
-    if (cache == NULL) {
-      cache = randomx_alloc_cache(flags);
-      if (cache == NULL)
-        local_abort("Couldn't allocate RandomX cache");
-    }
+  CTHR_THREAD_TYPE t;
+  if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) {
+    local_abort("Couldn't start RandomX seed thread");
   }
-  if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) {
-    randomx_init_cache(cache, seedhash, HASH_SIZE);
-    rx_sp->rs_cache = cache;
-    rx_sp->rs_height = seedheight;
-    memcpy(rx_sp->rs_hash, seedhash, HASH_SIZE);
-  }
-  if (rx_vm == NULL) {
-    if ((flags & RANDOMX_FLAG_JIT) && !miners) {
-        flags |= RANDOMX_FLAG_SECURE & ~disabled_flags();
-    }
-    if (miners && (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) {
-      miners = 0;
-    }
-    if (miners) {
-      CTHR_MUTEX_LOCK(rx_dataset_mutex);
-      if (!rx_dataset_nomem) {
-        if (rx_dataset == NULL) {
-          if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) {
-            rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES);
-            if (rx_dataset == NULL) {
-              mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset");
-            }
-          }
-          if (rx_dataset == NULL)
-            rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT);
-          if (rx_dataset != NULL)
-            rx_initdata(rx_sp->rs_cache, miners, seedheight);
+  CTHR_THREAD_CLOSE(t);
+}
+
+void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) {
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  int success = 0;
+
+  // Fast path (seedhash == main_seedhash)
+  // Multiple threads can run in parallel in fast or light mode, 1-2 ms or 10-15 ms per hash per thread
+  if (is_main(seedhash)) {
+    // If CTHR_RWLOCK_TRYLOCK_READ fails it means dataset is being initialized now, so use the light mode
+    if (main_dataset && CTHR_RWLOCK_TRYLOCK_READ(main_dataset_lock)) {
+      // Double check that main_seedhash didn't change
+      if (is_main(seedhash)) {
+        rx_init_full_vm(flags, &main_vm_full);
+        if (main_vm_full) {
+          randomx_calculate_hash(main_vm_full, data, length, result_hash);
+          success = 1;
         }
       }
-      if (rx_dataset != NULL)
-        flags |= RANDOMX_FLAG_FULL_MEM;
-      else {
-        miners = 0;
-        if (!rx_dataset_nomem) {
-          rx_dataset_nomem = 1;
-          mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner");
-        }
+      CTHR_RWLOCK_UNLOCK_READ(main_dataset_lock);
+    } else {
+      CTHR_RWLOCK_LOCK_READ(main_cache_lock);
+      // Double check that main_seedhash didn't change
+      if (is_main(seedhash)) {
+        rx_init_light_vm(flags, &main_vm_light, main_cache);
+        randomx_calculate_hash(main_vm_light, data, length, result_hash);
+        success = 1;
       }
-      CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
+      CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
     }
-    if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES) && !rx_dataset_nolp) {
-      rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset);
-      if(rx_vm == NULL) { //large pages failed
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM");
-        rx_dataset_nolp = 1;
-      }
-    }
-    if (rx_vm == NULL)
-      rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
-    if(rx_vm == NULL) {//fallback if everything fails
-      flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0);
-      rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
-    }
-    if (rx_vm == NULL)
-      local_abort("Couldn't allocate RandomX VM");
-  } else if (miners) {
-    CTHR_MUTEX_LOCK(rx_dataset_mutex);
-    if (rx_dataset != NULL && rx_dataset_height != seedheight)
-      rx_initdata(cache, miners, seedheight);
-    else if (rx_dataset == NULL) {
-      /* this is a no-op if the cache hasn't changed */
-      randomx_vm_set_cache(rx_vm, rx_sp->rs_cache);
-    }
-    CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
-  } else {
-    /* this is a no-op if the cache hasn't changed */
-    randomx_vm_set_cache(rx_vm, rx_sp->rs_cache);
   }
-  /* mainchain users can run in parallel */
-  if (!is_alt)
-    CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex);
-  randomx_calculate_hash(rx_vm, data, length, hash);
-  /* altchain slot users always get fully serialized */
-  if (is_alt)
-    CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex);
+
+  if (success) {
+    return;
+  }
+
+  char buf[HASH_SIZE * 2 + 1];
+
+  // Slow path (seedhash != main_seedhash, but seedhash == secondary_seedhash)
+  // Multiple threads can run in parallel in light mode, 10-15 ms per hash per thread
+  if (!secondary_cache) {
+    CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock);
+    if (!secondary_cache) {
+      hash2hex(seedhash, buf);
+      minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf);
+
+      rx_alloc_cache(flags, &secondary_cache);
+      randomx_init_cache(secondary_cache, seedhash, HASH_SIZE);
+      minfo(RX_LOGCAT, "RandomX secondary cache updated");
+      memcpy(secondary_seedhash, seedhash, HASH_SIZE);
+      secondary_seedhash_set = 1;
+    }
+    CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock);
+  }
+
+  CTHR_RWLOCK_LOCK_READ(secondary_cache_lock);
+  if (is_secondary(seedhash)) {
+    rx_init_light_vm(flags, &secondary_vm_light, secondary_cache);
+    randomx_calculate_hash(secondary_vm_light, data, length, result_hash);
+    success = 1;
+  }
+  CTHR_RWLOCK_UNLOCK_READ(secondary_cache_lock);
+
+  if (success) {
+    return;
+  }
+
+  // Slowest path (seedhash != main_seedhash, seedhash != secondary_seedhash)
+  // Only one thread runs at a time and updates secondary_seedhash if needed, up to 200-500 ms per hash
+  CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock);
+  if (!is_secondary(seedhash)) {
+    hash2hex(seedhash, buf);
+    minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf);
+
+    randomx_init_cache(secondary_cache, seedhash, HASH_SIZE);
+    minfo(RX_LOGCAT, "RandomX secondary cache updated");
+    memcpy(secondary_seedhash, seedhash, HASH_SIZE);
+    secondary_seedhash_set = 1;
+  }
+  rx_init_light_vm(flags, &secondary_vm_light, secondary_cache);
+  randomx_calculate_hash(secondary_vm_light, data, length, result_hash);
+  CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock);
 }
 
-void rx_slow_hash_allocate_state(void) {
+void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads) {
+  miner_thread = value;
+
+  // If dataset is not allocated yet, try to allocate and initialize it
+  CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock);
+  if (main_dataset) {
+    CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+    return;
+  }
+
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  rx_alloc_dataset(flags, &main_dataset, 1);
+  rx_init_dataset(max_dataset_init_threads);
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
 }
 
-void rx_slow_hash_free_state(void) {
-  if (rx_vm != NULL) {
-    randomx_destroy_vm(rx_vm);
-    rx_vm = NULL;
+uint32_t rx_get_miner_thread() {
+  return miner_thread;
+}
+
+void rx_slow_hash_allocate_state() {}
+
+static void rx_destroy_vm(randomx_vm** vm) {
+  if (*vm) {
+    randomx_destroy_vm(*vm);
+    *vm = NULL;
   }
 }
 
-void rx_stop_mining(void) {
-  CTHR_MUTEX_LOCK(rx_dataset_mutex);
-  if (rx_dataset != NULL) {
-    randomx_dataset *rd = rx_dataset;
-    rx_dataset = NULL;
-    randomx_release_dataset(rd);
-  }
-  rx_dataset_nomem = 0;
-  rx_dataset_nolp = 0;
-  CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
+void rx_slow_hash_free_state() {
+  rx_destroy_vm(&main_vm_full);
+  rx_destroy_vm(&main_vm_light);
+  rx_destroy_vm(&secondary_vm_light);
 }

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -989,7 +989,7 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
-  bool out_can_be_to_acc(const boost::optional<crypto::view_tag>& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index)
+  bool out_can_be_to_acc(const boost::optional<crypto::view_tag>& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index, hw::device* hwdev)
   {
     // If there is no view tag to check, the output can possibly belong to the account.
     // Will need to derive the output pub key to be certain whether or not the output belongs to the account.
@@ -1002,7 +1002,15 @@ namespace cryptonote
     // Therefore can fail out early to avoid expensive crypto ops needlessly deriving output public key to
     // determine if output belongs to the account.
     crypto::view_tag derived_view_tag;
-    crypto::derive_view_tag(derivation, output_index, derived_view_tag);
+    if (hwdev != nullptr)
+    {
+      bool r = hwdev->derive_view_tag(derivation, output_index, derived_view_tag);
+      CHECK_AND_ASSERT_MES(r, false, "Failed to derive view tag");
+    }
+    else
+    {
+      crypto::derive_view_tag(derivation, output_index, derived_view_tag);
+    }
     return view_tag == derived_view_tag;
   }
   //---------------------------------------------------------------
@@ -1012,7 +1020,7 @@ namespace cryptonote
     bool r = acc.get_device().generate_key_derivation(tx_pub_key, acc.m_view_secret_key, derivation);
     CHECK_AND_ASSERT_MES(r, false, "Failed to generate key derivation");
     crypto::public_key pk;
-    if (out_can_be_to_acc(view_tag_opt, derivation, output_index))
+    if (out_can_be_to_acc(view_tag_opt, derivation, output_index, &acc.get_device()))
     {
       r = acc.get_device().derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk);
       CHECK_AND_ASSERT_MES(r, false, "Failed to derive public key");
@@ -1026,7 +1034,7 @@ namespace cryptonote
       CHECK_AND_ASSERT_MES(output_index < additional_tx_pub_keys.size(), false, "wrong number of additional tx pubkeys");
       r = acc.get_device().generate_key_derivation(additional_tx_pub_keys[output_index], acc.m_view_secret_key, derivation);
       CHECK_AND_ASSERT_MES(r, false, "Failed to generate key derivation");
-      if (out_can_be_to_acc(view_tag_opt, derivation, output_index))
+      if (out_can_be_to_acc(view_tag_opt, derivation, output_index, &acc.get_device()))
       {
         r = acc.get_device().derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk);
         CHECK_AND_ASSERT_MES(r, false, "Failed to derive public key");
@@ -1040,7 +1048,7 @@ namespace cryptonote
   {
     // try the shared tx pubkey
     crypto::public_key subaddress_spendkey;
-    if (out_can_be_to_acc(view_tag_opt, derivation, output_index))
+    if (out_can_be_to_acc(view_tag_opt, derivation, output_index, &hwdev))
     {
       CHECK_AND_ASSERT_MES(hwdev.derive_subaddress_public_key(out_key, derivation, output_index, subaddress_spendkey), boost::none, "Failed to derive subaddress public key");
       auto found = subaddresses.find(subaddress_spendkey);
@@ -1052,7 +1060,7 @@ namespace cryptonote
     if (!additional_derivations.empty())
     {
       CHECK_AND_ASSERT_MES(output_index < additional_derivations.size(), boost::none, "wrong number of additional derivations");
-      if (out_can_be_to_acc(view_tag_opt, additional_derivations[output_index], output_index))
+      if (out_can_be_to_acc(view_tag_opt, additional_derivations[output_index], output_index, &hwdev))
       {
         CHECK_AND_ASSERT_MES(hwdev.derive_subaddress_public_key(out_key, additional_derivations[output_index], output_index, subaddress_spendkey), boost::none, "Failed to derive subaddress public key");
         auto found = subaddresses.find(subaddress_spendkey);
@@ -1221,7 +1229,7 @@ namespace cryptonote
     char *end = NULL;
     errno = 0;
     const unsigned long long ull = strtoull(buf, &end, 10);
-    CHECK_AND_ASSERT_THROW_MES(ull != ULONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
+    CHECK_AND_ASSERT_THROW_MES(ull != ULLONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
     CHECK_AND_ASSERT_THROW_MES(ull != 0 || amount == 0, "Overflow in rounding");
     return ull;
   }

src/cryptonote_basic/cryptonote_format_utils.h

@@ -91,7 +91,7 @@ namespace cryptonote
   bool get_encrypted_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash8& payment_id);
   void set_tx_out(const uint64_t amount, const crypto::public_key& output_public_key, const bool use_view_tags, const crypto::view_tag& view_tag, tx_out& out);
   bool check_output_types(const transaction& tx, const uint8_t hf_version);
-  bool out_can_be_to_acc(const boost::optional<crypto::view_tag>& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index);
+  bool out_can_be_to_acc(const boost::optional<crypto::view_tag>& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index, hw::device *hwdev = nullptr);
   bool is_out_to_acc(const account_keys& acc, const crypto::public_key& output_public_key, const crypto::public_key& tx_pub_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t output_index, const boost::optional<crypto::view_tag>& view_tag_opt = boost::optional<crypto::view_tag>());
   struct subaddress_receive_info
   {

src/cryptonote_basic/hardfork.h

@@ -231,6 +231,11 @@ namespace cryptonote
      */
     uint64_t get_window_size() const { return window_size; }
 
+    /**
+     * @brief returns info for all known hard forks
+     */
+    const std::vector<hardfork_t>& get_hardforks() const { return heights; }
+
   private:
 
     uint8_t get_block_version(uint64_t height) const;

src/cryptonote_basic/miner.cpp

@@ -82,6 +82,7 @@
 using namespace epee;
 
 #include "miner.h"
+#include "crypto/hash.h"
 
 
 extern "C" void slow_hash_allocate_state();
@@ -436,7 +437,6 @@ namespace cryptonote
   {
     m_stop = true;
   }
-  extern "C" void rx_stop_mining(void);
   //-----------------------------------------------------------------------------------------------------
   bool miner::stop()
   {
@@ -469,7 +469,6 @@ namespace cryptonote
     MINFO("Mining has been stopped, " << m_threads.size() << " finished" );
     m_threads.clear();
     m_threads_autodetect.clear();
-    rx_stop_mining();
     return true;
   }
   //-----------------------------------------------------------------------------------------------------
@@ -524,6 +523,8 @@ namespace cryptonote
   bool miner::worker_thread()
   {
     const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment
+    bool rx_set = false;
+
     MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]");
     MGINFO("Miner thread was started ["<< th_local_index << "]");
     uint32_t nonce = m_starter_nonce + th_local_index;
@@ -574,6 +575,13 @@ namespace cryptonote
 
       b.nonce = nonce;
       crypto::hash h;
+
+      if ((b.major_version >= RX_BLOCK_VERSION) && !rx_set)
+      {
+        crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency());
+        rx_set = true;
+      }
+
       m_gbh(b, height, NULL, tools::get_max_concurrency(), h);
 
       if(check_hash(h, local_diff))

src/cryptonote_basic/tx_extra.h

@@ -52,7 +52,7 @@ namespace cryptonote
 
     // load
     template <template <bool> class Archive>
-    bool do_serialize(Archive<false>& ar)
+    bool member_do_serialize(Archive<false>& ar)
     {
       // size - 1 - because of variant tag
       for (size = 1; size <= TX_EXTRA_PADDING_MAX_COUNT; ++size)
@@ -73,7 +73,7 @@ namespace cryptonote
 
     // store
     template <template <bool> class Archive>
-    bool do_serialize(Archive<true>& ar)
+    bool member_do_serialize(Archive<true>& ar)
     {
       if(TX_EXTRA_PADDING_MAX_COUNT < size)
         return false;
@@ -129,7 +129,7 @@ namespace cryptonote
 
     // load
     template <template <bool> class Archive>
-    bool do_serialize(Archive<false>& ar)
+    bool member_do_serialize(Archive<false>& ar)
     {
       std::string field;
       if(!::do_serialize(ar, field))
@@ -142,7 +142,7 @@ namespace cryptonote
 
     // store
     template <template <bool> class Archive>
-    bool do_serialize(Archive<true>& ar)
+    bool member_do_serialize(Archive<true>& ar)
     {
       std::ostringstream oss;
       binary_archive<true> oar(oss);

src/cryptonote_basic/verification_context.h

@@ -42,7 +42,12 @@ namespace cryptonote
     static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none");
 
     relay_method m_relay; // gives indication on how tx should be relayed (if at all)
-    bool m_verifivation_failed; //bad tx, should drop connection
+    bool m_verifivation_failed; //bad tx, tx should not enter mempool and connection should be dropped unless m_no_drop_offense
+    // Do not add to mempool, do not relay, but also do not punish the peer for sending or drop
+    // connections to them. Used for low fees, tx_extra too big, "relay-only rules". Not to be
+    // confused with breaking soft fork rules, because tx could be later added to the chain if mined
+    // because it does not violate consensus rules.
+    bool m_no_drop_offense;
     bool m_verifivation_impossible; //the transaction is related with an alternative blockchain
     bool m_added_to_pool; 
     bool m_low_mixin;
@@ -53,6 +58,8 @@ namespace cryptonote
     bool m_overspend;
     bool m_fee_too_low;
     bool m_too_few_outputs;
+    bool m_tx_extra_too_big;
+    bool m_nonzero_unlock_time;
   };
 
   struct block_verification_context

src/cryptonote_config.h

@@ -30,6 +30,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <stdexcept>
 #include <string>
 #include <boost/uuid/uuid.hpp>
@@ -206,6 +207,11 @@
 
 #define DNS_BLOCKLIST_LIFETIME (86400 * 8)
 
+//The limit is enough for the mandatory transaction content with 16 outputs (547 bytes),
+//a custom tag (1 byte) and up to 32 bytes of custom data for each recipient.
+// (1+32) + (1+1+16*32) + (1+16*32) = 1060
+#define MAX_TX_EXTRA_SIZE                       1060
+
 // New constants are intended to go here
 namespace config
 {
@@ -248,6 +254,7 @@ namespace config
   const unsigned char HASH_KEY_MM_SLOT = 'm';
   const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS_SEED[] = "multisig_tx_privkeys_seed";
   const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS[] = "multisig_tx_privkeys";
+  const constexpr char HASH_KEY_TXHASH_AND_MIXRING[] = "txhash_and_mixring";
 
   // Multisig
   const uint32_t MULTISIG_MAX_SIGNERS{16};

src/cryptonote_core/CMakeLists.txt

@@ -31,7 +31,9 @@ set(cryptonote_core_sources
   cryptonote_core.cpp
   tx_pool.cpp
   tx_sanity_check.cpp
-  cryptonote_tx_utils.cpp)
+  cryptonote_tx_utils.cpp
+  tx_verification_utils.cpp
+)
 
 set(cryptonote_core_headers)
 

src/cryptonote_core/blockchain.cpp

@@ -57,6 +57,7 @@
 #include "common/notify.h"
 #include "common/varint.h"
 #include "common/pruning.h"
+#include "common/data_cache.h"
 #include "time_helper.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -98,7 +99,8 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) :
   m_difficulty_for_next_block(1),
   m_btc_valid(false),
   m_batch_success(true),
-  m_prepare_height(0)
+  m_prepare_height(0),
+  m_rct_ver_cache()
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
 }
@@ -456,6 +458,14 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline
     if (!update_next_cumulative_weight_limit())
       return false;
   }
+
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+  {
+    const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height()));
+    if (seedhash != crypto::null_hash)
+      rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+  }
+
   return true;
 }
 //------------------------------------------------------------------
@@ -570,6 +580,12 @@ void Blockchain::pop_blocks(uint64_t nblocks)
 
   if (stop_batch)
     m_db->batch_stop();
+
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+  {
+    const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height()));
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+  }
 }
 //------------------------------------------------------------------
 // This function tells BlockchainDB to remove the top block from the
@@ -1239,18 +1255,20 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
   }
 
   m_hardfork->reorganize_from_chain_height(split_height);
-  get_block_longhash_reorg(split_height);
 
   std::shared_ptr<tools::Notify> reorg_notify = m_reorg_notify;
   if (reorg_notify)
     reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(),
         "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL);
 
+  const uint64_t new_height = m_db->height();
+  const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height));
+
   crypto::hash prev_id;
   if (!get_block_hash(alt_chain.back().bl, prev_id))
     MERROR("Failed to get block hash of an alternative chain's tip");
   else
-    send_miner_notifications(prev_id, alt_chain.back().already_generated_coins);
+    send_miner_notifications(new_height, seedhash, prev_id, alt_chain.back().already_generated_coins);
 
   for (const auto& notifier : m_block_notifiers)
   {
@@ -1262,6 +1280,9 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
     }
   }
 
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+
   MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height());
   return true;
 }
@@ -2001,7 +2022,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
       {
         seedhash = get_block_id_by_height(seedheight);
       }
-      get_altblock_longhash(bei.bl, proof_of_work, get_current_blockchain_height(), bei.height, seedheight, seedhash);
+      get_altblock_longhash(bei.bl, proof_of_work, seedhash);
     } else
     {
       get_block_longhash(this, bei.bl, proof_of_work, bei.height, 0);
@@ -2044,7 +2065,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
       cryptonote::blobdata blob;
       if (m_tx_pool.have_tx(txid, relay_category::legacy))
       {
-        if (m_tx_pool.get_transaction_info(txid, td))
+        if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/))
         {
           bei.block_cumulative_weight += td.weight;
         }
@@ -3192,7 +3213,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const
   }
   return false;
 }
-bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const
+bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys)
 {
   PERF_TIMER(expand_transaction_2);
   CHECK_AND_ASSERT_MES(tx.version == 2, false, "Transaction version is not 2");
@@ -3434,7 +3455,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
   std::vector < uint64_t > results;
   results.resize(tx.vin.size(), 0);
 
-  tools::threadpool& tpool = tools::threadpool::getInstance();
+  tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
   tools::threadpool::waiter waiter(tpool);
   int threads = tpool.get_max_concurrency();
 
@@ -3515,6 +3536,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
         false, "Transaction spends at least one output which is too young");
   }
 
+  // Warn that new RCT types are present, and thus the cache is not being used effectively
+  static constexpr const std::uint8_t RCT_CACHE_TYPE = rct::RCTTypeBulletproofPlus;
+  if (tx.rct_signatures.type > RCT_CACHE_TYPE)
+  {
+    MWARNING("RCT cache is not caching new verification results. Please update RCT_CACHE_TYPE!");
+  }
+
   if (tx.version == 1)
   {
     if (threads > 1)
@@ -3536,12 +3564,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
   }
   else
   {
-    if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys))
-    {
-      MERROR_VER("Failed to expand rct signatures!");
-      return false;
-    }
-
     // from version 2, check ringct signatures
     // obviously, the original and simple rct APIs use a mixRing that's indexes
     // in opposite orders, because it'd be too simple otherwise...
@@ -3559,61 +3581,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
     case rct::RCTTypeCLSAG:
     case rct::RCTTypeBulletproofPlus:
     {
-      // check all this, either reconstructed (so should really pass), or not
-      {
-        if (pubkeys.size() != rv.mixRing.size())
-        {
-          MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
-          return false;
-        }
-        for (size_t i = 0; i < pubkeys.size(); ++i)
-        {
-          if (pubkeys[i].size() != rv.mixRing[i].size())
-          {
-            MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
-            return false;
-          }
-        }
-
-        for (size_t n = 0; n < pubkeys.size(); ++n)
-        {
-          for (size_t m = 0; m < pubkeys[n].size(); ++m)
-          {
-            if (pubkeys[n][m].dest != rct::rct2pk(rv.mixRing[n][m].dest))
-            {
-              MERROR_VER("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m);
-              return false;
-            }
-            if (pubkeys[n][m].mask != rct::rct2pk(rv.mixRing[n][m].mask))
-            {
-              MERROR_VER("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m);
-              return false;
-            }
-          }
-        }
-      }
-
-      const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size();
-      if (n_sigs != tx.vin.size())
-      {
-        MERROR_VER("Failed to check ringct signatures: mismatched MGs/vin sizes");
-        return false;
-      }
-      for (size_t n = 0; n < tx.vin.size(); ++n)
-      {
-        bool error;
-        if (rct::is_rct_clsag(rv.type))
-          error = memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.CLSAGs[n].I, 32);
-        else
-          error = rv.p.MGs[n].II.empty() || memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.MGs[n].II[0], 32);
-        if (error)
-        {
-          MERROR_VER("Failed to check ringct signatures: mismatched key image");
-          return false;
-        }
-      }
-
-      if (!rct::verRctNonSemanticsSimple(rv))
+      if (!ver_rct_non_semantics_simple_cached(tx, pubkeys, m_rct_ver_cache, RCT_CACHE_TYPE))
       {
         MERROR_VER("Failed to check ringct signatures!");
         return false;
@@ -3622,6 +3590,12 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
     }
     case rct::RCTTypeFull:
     {
+      if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys))
+      {
+        MERROR_VER("Failed to expand rct signatures!");
+        return false;
+      }
+
       // check all this, either reconstructed (so should really pass), or not
       {
         bool size_matches = true;
@@ -3736,7 +3710,7 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b
       div128_64(hi, lo, median_block_weight, &hi, &lo, NULL, NULL);
       assert(hi == 0);
       lo -= lo / 20;
-      return lo;
+      return lo == 0 ? 1 : lo;
     }
     else
     {
@@ -3858,6 +3832,8 @@ void Blockchain::get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_block
   const uint8_t version = get_current_hard_fork_version();
   const uint64_t db_height = m_db->height();
 
+  CHECK_AND_ASSERT_THROW_MES(grace_blocks <= CRYPTONOTE_REWARD_BLOCKS_WINDOW, "Grace blocks invalid In 2021 fee scaling estimate.");
+
   // we want Mlw = median of max((min(Mbw, 1.7 * Ml), Zm), Ml / 1.7)
   // Mbw: block weight for the last 99990 blocks, 0 for the next 10
   // Ml: penalty free zone (dynamic), aka long_term_median, aka median of max((min(Mb, 1.7 * Ml), Zm), Ml / 1.7)
@@ -3871,7 +3847,6 @@ void Blockchain::get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_block
   const uint64_t Mlw_penalty_free_zone_for_wallet = std::max<uint64_t>(rm.median(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5);
 
   // Msw: median over [100 - grace blocks] past + [grace blocks] future blocks
-  CHECK_AND_ASSERT_THROW_MES(grace_blocks <= 100, "Grace blocks invalid In 2021 fee scaling estimate.");
   std::vector<uint64_t> weights;
   get_last_n_blocks_weights(weights, 100 - grace_blocks);
   weights.reserve(100);
@@ -4552,11 +4527,15 @@ leave:
     }
   }
 
-  send_miner_notifications(id, already_generated_coins);
+  const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height));
+  send_miner_notifications(new_height, seedhash, id, already_generated_coins);
 
   for (const auto& notifier: m_block_notifiers)
     notifier(new_height - 1, {std::addressof(bl), 1});
 
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+
   return true;
 }
 //------------------------------------------------------------------
@@ -4638,40 +4617,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
   }
   else
   {
-    const uint64_t block_weight = m_db->get_block_weight(db_height - 1);
+    const uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
+    const uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks);
 
-    uint64_t long_term_median;
-    if (db_height == 1)
-    {
-      long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5;
-    }
-    else
-    {
-      uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
-      if (nblocks == db_height)
-        --nblocks;
-      long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks);
-    }
-
-    m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
-
-    uint64_t short_term_constraint = m_long_term_effective_median_block_weight;
-    if (hf_version >= HF_VERSION_2021_SCALING)
-      short_term_constraint += m_long_term_effective_median_block_weight * 7 / 10;
-    else
-      short_term_constraint += m_long_term_effective_median_block_weight * 2 / 5;
-    uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint);
-
-    if (db_height == 1)
-    {
-      long_term_median = long_term_block_weight;
-    }
-    else
-    {
-      m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1);
-      m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight);
-      long_term_median = m_long_term_block_weights_cache_rolling_median.median();
-    }
     m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
 
     std::vector<uint64_t> weights;
@@ -5137,7 +5085,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
     return true;
 
   bool blocks_exist = false;
-  tools::threadpool& tpool = tools::threadpool::getInstance();
+  tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
   unsigned threads = tpool.get_max_concurrency();
   blocks.resize(blocks_entry.size());
 
@@ -5604,7 +5552,7 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "e9371004b9f6be59921b27bc81e28b4715845ade1c6d16891d5c455f72e21365";
+static const char expected_block_hashes_hash[] = "8ada865350270fd008397684d978dac75ea4029a8a1ffcaa9975c43be119ec19";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
   if (get_checkpoints == nullptr || !m_fast_sync)
@@ -5761,24 +5709,15 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_
   m_btc_valid = true;
 }
 
-void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins)
+void Blockchain::send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins)
 {
   if (m_miner_notifiers.empty())
     return;
 
-  const uint64_t height = m_db->height();
   const uint8_t major_version = m_hardfork->get_ideal_version(height);
   const difficulty_type diff = get_difficulty_for_next_block();
   const uint64_t median_weight = m_current_block_cumul_weight_median;
 
-  crypto::hash seed_hash = crypto::null_hash;
-  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
-  {
-    uint64_t seed_height, next_height;
-    crypto::rx_seedheights(height, &seed_height, &next_height);
-    seed_hash = get_block_id_by_height(seed_height);
-  }
-
   std::vector<tx_block_template_backlog_entry> tx_backlog;
   m_tx_pool.get_block_template_backlog(tx_backlog);
 

src/cryptonote_core/blockchain.h

@@ -57,6 +57,7 @@
 #include "rpc/core_rpc_server_commands_defs.h"
 #include "cryptonote_basic/difficulty.h"
 #include "cryptonote_tx_utils.h"
+#include "tx_verification_utils.h"
 #include "cryptonote_basic/verification_context.h"
 #include "crypto/hash.h"
 #include "checkpoints/checkpoints.h"
@@ -158,6 +159,13 @@ namespace cryptonote
      */
     bool deinit();
 
+    /**
+     * @brief get a set of blockchain checkpoint hashes
+     *
+     * @return set of blockchain checkpoint hashes
+     */
+    const checkpoints& get_checkpoints() const { return m_checkpoints; }
+
     /**
      * @brief assign a set of blockchain checkpoint hashes
      *
@@ -589,6 +597,15 @@ namespace cryptonote
      */
     bool store_blockchain();
 
+    /**
+     * @brief expands v2 transaction data from blockchain
+     *
+     * RingCT transactions do not transmit some of their data if it
+     * can be reconstituted by the receiver. This function expands
+     * that implicit data.
+     */
+    static bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys);
+
     /**
      * @brief validates a transaction's inputs
      *
@@ -892,6 +909,13 @@ namespace cryptonote
      */
     uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return m_hardfork->get_earliest_ideal_height_for_version(version); }
 
+    /**
+     * @brief returns info for all known hard forks
+     *
+     * @return the hardforks
+     */
+    const std::vector<hardfork_t>& get_hardforks() const { return m_hardfork->get_hardforks(); }
+
     /**
      * @brief get information about hardfork voting for a version
      *
@@ -1208,6 +1232,9 @@ namespace cryptonote
     uint64_t m_prepare_nblocks;
     std::vector<block> *m_prepare_blocks;
 
+    // cache for verifying transaction RCT non semantics
+    mutable rct_ver_cache_t m_rct_ver_cache;
+
     /**
      * @brief collects the keys for all outputs being "spent" as an input
      *
@@ -1560,15 +1587,6 @@ namespace cryptonote
      */
     void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints);
 
-    /**
-     * @brief expands v2 transaction data from blockchain
-     *
-     * RingCT transactions do not transmit some of their data if it
-     * can be reconstituted by the receiver. This function expands
-     * that implicit data.
-     */
-    bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const;
-
     /**
      * @brief invalidates any cached block template
      */
@@ -1584,9 +1602,11 @@ namespace cryptonote
     /**
      * @brief sends new block notifications to ZMQ `miner_data` subscribers
      *
+     * @param height current blockchain height
+     * @param seed_hash seed hash to use for mining
      * @param prev_id hash of new blockchain tip
      * @param already_generated_coins total coins mined by the network so far
      */
-    void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins);
+    void send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins);
   };
 }  // namespace cryptonote

src/cryptonote_core/cryptonote_core.cpp

@@ -252,6 +252,10 @@ namespace cryptonote
       m_pprotocol = &m_protocol_stub;
   }
   //-----------------------------------------------------------------------------------
+  const checkpoints& core::get_checkpoints() const
+  {
+    return m_blockchain_storage.get_checkpoints();
+  }
   void core::set_checkpoints(checkpoints&& chk_pts)
   {
     m_blockchain_storage.set_checkpoints(std::move(chk_pts));
@@ -1009,7 +1013,7 @@ namespace cryptonote
 
     CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
 
-    tools::threadpool& tpool = tools::threadpool::getInstance();
+    tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
     tools::threadpool::waiter waiter(tpool);
     epee::span<tx_blob_entry>::const_iterator it = tx_blobs.begin();
     for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
@@ -1095,7 +1099,7 @@ namespace cryptonote
       else if(tvc[i].m_verifivation_impossible)
       {MERROR_VER("Transaction verification impossible: " << results[i].hash);}
 
-      if(tvc[i].m_added_to_pool)
+      if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE)
       {
         MDEBUG("tx added: " << results[i].hash);
         valid_events = true;
@@ -1406,21 +1410,66 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::notify_txpool_event(const epee::span<const cryptonote::blobdata> tx_blobs, epee::span<const crypto::hash> tx_hashes, epee::span<const cryptonote::transaction> txs, const std::vector<bool> &just_broadcasted) const
+  {
+    if (!m_zmq_pub)
+      return true;
+
+    if (tx_blobs.size() != tx_hashes.size() || tx_blobs.size() != txs.size() || tx_blobs.size() != just_broadcasted.size())
+      return false;
+
+    /* Publish txs via ZMQ that are "just broadcasted" by the daemon. This is
+       done here in addition to `handle_incoming_txs` in order to guarantee txs
+       are pub'd via ZMQ when we know the daemon has/will broadcast to other
+       nodes & *after* the tx is visible in the pool. This should get called
+       when the user submits a tx to a daemon in the "fluff" epoch relaying txs
+       via a public network. */
+    if (std::count(just_broadcasted.begin(), just_broadcasted.end(), true) == 0)
+      return true;
+
+    std::vector<txpool_event> results{};
+    results.resize(tx_blobs.size());
+    for (std::size_t i = 0; i < results.size(); ++i)
+    {
+      results[i].tx = std::move(txs[i]);
+      results[i].hash = std::move(tx_hashes[i]);
+      results[i].blob_size = tx_blobs[i].size();
+      results[i].weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, results[i].blob_size);
+      results[i].res = just_broadcasted[i];
+    }
+
+    m_zmq_pub(std::move(results));
+
+    return true;
+  }
+  //-----------------------------------------------------------------------------------------------
   void core::on_transactions_relayed(const epee::span<const cryptonote::blobdata> tx_blobs, const relay_method tx_relay)
   {
+    // lock ensures duplicate txs aren't pub'd via zmq
+    CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
+
     std::vector<crypto::hash> tx_hashes{};
     tx_hashes.resize(tx_blobs.size());
 
+    std::vector<cryptonote::transaction> txs{};
+    txs.resize(tx_blobs.size());
+
     for (std::size_t i = 0; i < tx_blobs.size(); ++i)
     {
-      cryptonote::transaction tx{};
-      if (!parse_and_validate_tx_from_blob(tx_blobs[i], tx, tx_hashes[i]))
+      if (!parse_and_validate_tx_from_blob(tx_blobs[i], txs[i], tx_hashes[i]))
       {
         LOG_ERROR("Failed to parse relayed transaction");
         return;
       }
     }
-    m_mempool.set_relayed(epee::to_span(tx_hashes), tx_relay);
+
+    std::vector<bool> just_broadcasted{};
+    just_broadcasted.reserve(tx_hashes.size());
+
+    m_mempool.set_relayed(epee::to_span(tx_hashes), tx_relay, just_broadcasted);
+
+    if (m_zmq_pub && matches_category(tx_relay, relay_category::legacy))
+      notify_txpool_event(tx_blobs, epee::to_span(tx_hashes), epee::to_span(txs), just_broadcasted);
   }
   //-----------------------------------------------------------------------------------------------
   bool core::get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash)
@@ -1607,10 +1656,6 @@ namespace cryptonote
     if (((size_t)-1) <= 0xffffffff && block_blob.size() >= 0x3fffffff)
       MWARNING("This block's size is " << block_blob.size() << ", closing on the 32 bit limit");
 
-    // load json & DNS checkpoints every 10min/hour respectively,
-    // and verify them with respect to what blocks we already have
-    CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
-
     block lb;
     if (!b)
     {
@@ -1682,6 +1727,11 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const
+  {
+    return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const
   {
     m_mempool.get_transactions(txs, include_sensitive_data);
@@ -1694,6 +1744,11 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const
   {
     m_mempool.get_transaction_stats(stats, include_sensitive_data);

src/cryptonote_core/cryptonote_core.h

@@ -436,6 +436,13 @@ namespace cryptonote
       */
      void set_cryptonote_protocol(i_cryptonote_protocol* pprotocol);
 
+     /**
+      * @copydoc Blockchain::get_checkpoints
+      *
+      * @note see Blockchain::get_checkpoints()
+      */
+     const checkpoints& get_checkpoints() const;
+
      /**
       * @copydoc Blockchain::set_checkpoints
       *
@@ -503,6 +510,23 @@ namespace cryptonote
      bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const;
 
      /**
+      * @copydoc tx_memory_pool::get_pool_transactions_info
+      * @param include_sensitive_txes include private transactions
+      *
+      * @note see tx_memory_pool::get_pool_transactions_info
+      */
+     bool get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes = false) const;
+
+     /**
+      * @copydoc tx_memory_pool::get_pool_info
+      * @param include_sensitive_txes include private transactions
+      * @param max_tx_count max allowed added_txs in response
+      *
+      * @note see tx_memory_pool::get_pool_info
+      */
+     bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
+    /**
       * @copydoc tx_memory_pool::get_transactions
       * @param include_sensitive_txes include private transactions
       *
@@ -1035,6 +1059,13 @@ namespace cryptonote
       */
      bool relay_txpool_transactions();
 
+     /**
+      * @brief sends notification of txpool events to subscribers
+      *
+      * @return true on success, false otherwise
+      */
+     bool notify_txpool_event(const epee::span<const cryptonote::blobdata> tx_blobs, epee::span<const crypto::hash> tx_hashes, epee::span<const cryptonote::transaction> txs, const std::vector<bool> &just_broadcasted) const;
+
      /**
       * @brief checks DNS versions
       *

src/cryptonote_core/cryptonote_tx_utils.cpp

@@ -203,7 +203,7 @@ namespace cryptonote
     return addr.m_view_public_key;
   }
   //---------------------------------------------------------------
-  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool shuffle_outs, bool use_view_tags)
+  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool shuffle_outs, bool use_view_tags)
   {
     hw::device &hwdev = sender_account_keys.get_device();
 
@@ -218,7 +218,7 @@ namespace cryptonote
     amount_keys.clear();
 
     tx.version = rct ? 2 : 1;
-    tx.unlock_time = unlock_time;
+    tx.unlock_time = 0;
 
     tx.extra = extra;
     crypto::public_key txkey_pub;
@@ -437,6 +437,8 @@ namespace cryptonote
     if (!sort_tx_extra(tx.extra, tx.extra))
       return false;
 
+    CHECK_AND_ASSERT_MES(tx.extra.size() <= MAX_TX_EXTRA_SIZE, false, "TX extra size (" << tx.extra.size() << ") is greater than max allowed (" << MAX_TX_EXTRA_SIZE << ")");
+
     //check money
     if(summary_outs_money > summary_inputs_money )
     {
@@ -604,7 +606,7 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
-  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool use_view_tags)
+  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool use_view_tags)
   {
     hw::device &hwdev = sender_account_keys.get_device();
     hwdev.open_tx(tx_key);
@@ -625,7 +627,7 @@ namespace cryptonote
       }
 
       bool shuffle_outs = true;
-      bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, shuffle_outs, use_view_tags);
+      bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, tx_key, additional_tx_keys, rct, rct_config, shuffle_outs, use_view_tags);
       hwdev.close_tx();
       return r;
     } catch(...) {
@@ -634,14 +636,14 @@ namespace cryptonote
     }
   }
   //---------------------------------------------------------------
-  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry>& sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time)
+  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry>& sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx)
   {
      std::unordered_map<crypto::public_key, cryptonote::subaddress_index> subaddresses;
      subaddresses[sender_account_keys.m_account_address.m_spend_public_key] = {0,0};
      crypto::secret_key tx_key;
      std::vector<crypto::secret_key> additional_tx_keys;
      std::vector<tx_destination_entry> destinations_copy = destinations;
-     return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0});
+     return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0});
   }
   //---------------------------------------------------------------
   bool generate_genesis_block(
@@ -669,10 +671,10 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
-  void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, const uint64_t seed_height, const crypto::hash& seed_hash)
+  void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash)
   {
     blobdata bd = get_block_hashing_blob(b);
-    rx_slow_hash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data, 0, 1);
+    rx_slow_hash(seed_hash.data, bd.data(), bd.size(), res.data);
   }
 
   bool get_block_longhash(const Blockchain *pbc, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners)
@@ -686,20 +688,16 @@ namespace cryptonote
     }
     if (major_version >= RX_BLOCK_VERSION)
     {
-      uint64_t seed_height, main_height;
       crypto::hash hash;
       if (pbc != NULL)
       {
-        seed_height = rx_seedheight(height);
+        const uint64_t seed_height = rx_seedheight(height);
         hash = seed_hash ? *seed_hash : pbc->get_pending_block_id_by_height(seed_height);
-        main_height = pbc->get_current_blockchain_height();
       } else
       {
         memset(&hash, 0, sizeof(hash));  // only happens when generating genesis block
-        seed_height = 0;
-        main_height = 0;
       }
-      rx_slow_hash(main_height, seed_height, hash.data, bd.data(), bd.size(), res.data, seed_hash ? 0 : miners, !!seed_hash);
+      rx_slow_hash(hash.data, bd.data(), bd.size(), res.data);
     } else {
       const int pow_variant = major_version >= 7 ? major_version - 6 : 0;
       crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height);
@@ -713,20 +711,10 @@ namespace cryptonote
 	return get_block_longhash(pbc, bd, res, height, b.major_version, seed_hash, miners);
   }
 
-  bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners)
-  {
-    return get_block_longhash(pbc, b, res, height, NULL, miners);
-  }
-
-  crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const int miners)
+  crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const crypto::hash *seed_hash, const int miners)
   {
     crypto::hash p = crypto::null_hash;
-    get_block_longhash(pbc, b, p, height, miners);
+    get_block_longhash(pbc, b, p, height, seed_hash, miners);
     return p;
   }
-
-  void get_block_longhash_reorg(const uint64_t split_height)
-  {
-    rx_reorg(split_height);
-  }
 }

src/cryptonote_core/cryptonote_tx_utils.h

@@ -118,9 +118,9 @@ namespace cryptonote
 
   //---------------------------------------------------------------
   crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr);
-  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time);
-  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool shuffle_outs = true, bool use_view_tags = false);
-  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool use_view_tags = false);
+  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx);
+  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool shuffle_outs = true, bool use_view_tags = false);
+  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool use_view_tags = false);
   bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub,  const crypto::secret_key &tx_key,
                                       const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index,
                                       const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys,
@@ -144,14 +144,10 @@ namespace cryptonote
     );
 
   class Blockchain;
-  bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height,
-    const int major_version, const crypto::hash *seed_hash, const int miners);
-  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners);
-  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners);
-  void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height,
-    const uint64_t seed_height, const crypto::hash& seed_hash);
-  crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const int miners);
-  void get_block_longhash_reorg(const uint64_t split_height);
+  bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners = 0);
+  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0);
+  crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0);
+  void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash);
 
 }
 

src/cryptonote_core/tx_pool.cpp

@@ -133,6 +133,12 @@ namespace cryptonote
     // class code expects unsigned values throughout
     if (m_next_check < time_t(0))
       throw std::runtime_error{"Unexpected time_t (system clock) value"};
+
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_start_time = (time_t)0;
+    // We don't set these to "now" already here as we don't know how long it takes from construction
+    // of the pool until it "goes to work". It's safer to set when the first actual txs enter the
+    // corresponding lists.
   }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
@@ -207,6 +213,7 @@ namespace cryptonote
     {
       tvc.m_verifivation_failed = true;
       tvc.m_fee_too_low = true;
+      tvc.m_no_drop_offense = true;
       return false;
     }
 
@@ -219,6 +226,25 @@ namespace cryptonote
       return false;
     }
 
+    size_t tx_extra_size = tx.extra.size();
+    if (!kept_by_block && tx_extra_size > MAX_TX_EXTRA_SIZE)
+    {
+      LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE);
+      tvc.m_verifivation_failed = true;
+      tvc.m_tx_extra_too_big = true;
+      tvc.m_no_drop_offense = true;
+      return false;
+    }
+
+    if (!kept_by_block && tx.unlock_time)
+    {
+      LOG_PRINT_L1("transaction unlock time is not zero: " << tx.unlock_time);
+      tvc.m_verifivation_failed = true;
+      tvc.m_nonzero_unlock_time = true;
+      tvc.m_no_drop_offense = true;
+      return false;
+    }
+
     // if the transaction came from a block popped from the chain,
     // don't check if we have its key images as spent.
     // TODO: Investigate why not?
@@ -230,6 +256,7 @@ namespace cryptonote
         LOG_PRINT_L1("Transaction with id= "<< id << " used already spent key images");
         tvc.m_verifivation_failed = true;
         tvc.m_double_spend = true;
+        tvc.m_no_drop_offense = true;
         return false;
       }
     }
@@ -281,7 +308,7 @@ namespace cryptonote
             return false;
 
           m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time);
           lock.commit();
         }
         catch (const std::exception &e)
@@ -352,7 +379,7 @@ namespace cryptonote
 
           m_blockchain.remove_txpool_tx(id);
           m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time);
         }
         lock.commit();
       }
@@ -373,7 +400,7 @@ namespace cryptonote
 
     ++m_cookie;
 
-    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)));
+    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size());
 
     prune(m_txpool_max_weight);
 
@@ -402,11 +429,30 @@ namespace cryptonote
     m_txpool_max_weight = bytes;
   }
   //---------------------------------------------------------------------------------
+  void tx_memory_pool::reduce_txpool_weight(size_t weight)
+  {
+    if (weight > m_txpool_weight)
+    {
+      MERROR("Underflow in txpool weight");
+      m_txpool_weight = 0;
+    }
+    else
+    {
+      m_txpool_weight -= weight;
+    }
+  }
+  //---------------------------------------------------------------------------------
   void tx_memory_pool::prune(size_t bytes)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
+
+    // Nothing to do if already empty
+    if (m_txs_by_fee_and_receive_time.empty())
+      return;
+
     if (bytes == 0)
       bytes = m_txpool_max_weight;
+
     CRITICAL_REGION_LOCAL1(m_blockchain);
     LockedTXN lock(m_blockchain.get_db());
     bool changed = false;
@@ -423,8 +469,14 @@ namespace cryptonote
         txpool_tx_meta_t meta;
         if (!m_blockchain.get_txpool_tx_meta(txid, meta))
         {
-          MERROR("Failed to find tx_meta in txpool");
-          return;
+          static bool warned = false;
+          if (!warned)
+          {
+            MERROR("Failed to find tx_meta in txpool (will only print once)");
+            warned = true;
+          }
+          --it;
+          continue;
         }
         // don't prune the kept_by_block ones, they're likely added because we're adding a block with those
         if (meta.kept_by_block)
@@ -442,10 +494,16 @@ namespace cryptonote
         // remove first, in case this throws, so key images aren't removed
         MINFO("Pruning tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first);
         m_blockchain.remove_txpool_tx(txid);
-        m_txpool_weight -= meta.weight;
+        reduce_txpool_weight(meta.weight);
         remove_transaction_keyimages(tx, txid);
         MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first);
-        m_txs_by_fee_and_receive_time.erase(it--);
+
+        auto it_prev = it;
+        --it_prev;
+
+        remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted));
+        it = it_prev;
+
         changed = true;
       }
       catch (const std::exception &e)
@@ -527,8 +585,7 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
 
-    auto sorted_it = find_tx_in_sorted_container(id);
-
+    bool sensitive = false;
     try
     {
       LockedTXN lock(m_blockchain.get_db());
@@ -559,10 +616,11 @@ namespace cryptonote
       do_not_relay = meta.do_not_relay;
       double_spend_seen = meta.double_spend_seen;
       pruned = meta.pruned;
+      sensitive = !meta.matches(relay_category::broadcasted);
 
       // remove first, in case this throws, so key images aren't removed
       m_blockchain.remove_txpool_tx(id);
-      m_txpool_weight -= tx_weight;
+      reduce_txpool_weight(tx_weight);
       remove_transaction_keyimages(tx, id);
       lock.commit();
     }
@@ -572,13 +630,12 @@ namespace cryptonote
       return false;
     }
 
-    if (sorted_it != m_txs_by_fee_and_receive_time.end())
-      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive);
     ++m_cookie;
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const
+  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const
   {
     PERF_TIMER(get_transaction_info);
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -590,7 +647,12 @@ namespace cryptonote
       txpool_tx_meta_t meta;
       if (!m_blockchain.get_txpool_tx_meta(txid, meta))
       {
-        MERROR("Failed to find tx in txpool");
+        LOG_PRINT_L2("Failed to find tx in txpool: " << txid);
+        return false;
+      }
+      if (!include_sensitive_data && !meta.matches(relay_category::broadcasted))
+      {
+        // We don't want sensitive data && the tx is sensitive, so no need to return it
         return false;
       }
       cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
@@ -616,11 +678,13 @@ namespace cryptonote
       td.kept_by_block = meta.kept_by_block;
       td.last_failed_height = meta.last_failed_height;
       td.last_failed_id = meta.last_failed_id;
-      td.receive_time = meta.receive_time;
-      td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time;
+      td.receive_time = include_sensitive_data ? meta.receive_time : 0;
+      td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0;
       td.relayed = meta.relayed;
       td.do_not_relay = meta.do_not_relay;
       td.double_spend_seen = meta.double_spend_seen;
+      if (include_blob)
+        td.tx_blob = std::move(txblob);
     }
     catch (const std::exception &e)
     {
@@ -630,6 +694,25 @@ namespace cryptonote
 
     return true;
   }
+  //------------------------------------------------------------------
+  bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    txs.clear();
+
+    for (const auto &it: txids)
+    {
+      tx_details details;
+      bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/);
+      if (success)
+      {
+        txs.push_back(std::make_pair(it, std::move(details)));
+      }
+    }
+    return true;
+  }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const
   {
@@ -691,15 +774,7 @@ namespace cryptonote
          (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) )
       {
         LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age );
-        auto sorted_it = find_tx_in_sorted_container(txid);
-        if (sorted_it == m_txs_by_fee_and_receive_time.end())
-        {
-          LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
-        }
-        else
-        {
-          m_txs_by_fee_and_receive_time.erase(sorted_it);
-        }
+        remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted));
         m_timed_out_transactions.insert(txid);
         remove.push_back(std::make_pair(txid, meta.weight));
       }
@@ -725,7 +800,7 @@ namespace cryptonote
           {
             // remove first, so we only remove key images if the tx removal succeeds
             m_blockchain.remove_txpool_tx(txid);
-            m_txpool_weight -= entry.second;
+            reduce_txpool_weight(entry.second);
             remove_transaction_keyimages(tx, txid);
           }
         }
@@ -820,8 +895,10 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------------------------
-  void tx_memory_pool::set_relayed(const epee::span<const crypto::hash> hashes, const relay_method method)
+  void tx_memory_pool::set_relayed(const epee::span<const crypto::hash> hashes, const relay_method method, std::vector<bool> &just_broadcasted)
   {
+    just_broadcasted.clear();
+
     crypto::random_poisson_seconds embargo_duration{dandelionpp_embargo_average};
     const auto now = std::chrono::system_clock::now();
     uint64_t next_relay = uint64_t{std::numeric_limits<time_t>::max()};
@@ -831,12 +908,14 @@ namespace cryptonote
     LockedTXN lock(m_blockchain.get_db());
     for (const auto& hash : hashes)
     {
+      bool was_just_broadcasted = false;
       try
       {
         txpool_tx_meta_t meta;
         if (m_blockchain.get_txpool_tx_meta(hash, meta))
         {
           // txes can be received as "stem" or "fluff" in either order
+          const bool already_broadcasted = meta.matches(relay_category::broadcasted);
           meta.upgrade_relay_method(method);
           meta.relayed = true;
 
@@ -849,6 +928,12 @@ namespace cryptonote
             meta.last_relayed_time = std::chrono::system_clock::to_time_t(now);
 
           m_blockchain.update_txpool_tx(hash, meta);
+          // wait until db update succeeds to ensure tx is visible in the pool
+          was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted);
+
+          if (was_just_broadcasted)
+            // Make sure the tx gets re-added with an updated time
+            add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now));
         }
       }
       catch (const std::exception &e)
@@ -856,6 +941,7 @@ namespace cryptonote
         MERROR("Failed to update txpool transaction metadata: " << e.what());
         // continue
       }
+      just_broadcasted.emplace_back(was_just_broadcasted);
     }
     lock.commit();
     set_if_less(m_next_check, time_t(next_relay));
@@ -900,6 +986,81 @@ namespace cryptonote
     }, false, category);
   }
   //------------------------------------------------------------------
+  bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    incremental = true;
+    if (start_time == (time_t)0)
+    {
+      // Giving no start time means give back whole pool
+      incremental = false;
+    }
+    else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0))
+    {
+      if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time))
+      {
+        // If either of the two lists do not go back far enough it's not possible to
+        // deliver incremental pool info
+        incremental = false;
+      }
+      // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time
+    }
+    else
+    {
+      // Some incremental info still missing completely
+      incremental = false;
+    }
+
+    added_txs.clear();
+    remaining_added_txids.clear();
+    removed_txs.clear();
+
+    std::vector<crypto::hash> txids;
+    if (!incremental)
+    {
+      LOG_PRINT_L2("Giving back the whole pool");
+      // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the
+      // anonymous method somehow results in an LMDB error with transactions we have to build a list of
+      // ids first and get the full info afterwards
+      get_transaction_hashes(txids, include_sensitive);
+      if (txids.size() > max_tx_count)
+      {
+        remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end());
+        txids.erase(txids.begin() + max_tx_count, txids.end());
+      }
+      get_transactions_info(txids, added_txs, include_sensitive);
+      return true;
+    }
+
+    // Give back incrementally, based on time of entry into the map
+    for (const auto &pit : m_added_txs_by_id)
+    {
+      if (pit.second >= start_time)
+        txids.push_back(pit.first);
+    }
+    get_transactions_info(txids, added_txs, include_sensitive);
+    if (added_txs.size() > max_tx_count)
+    {
+      remaining_added_txids.reserve(added_txs.size() - max_tx_count);
+      for (size_t i = max_tx_count; i < added_txs.size(); ++i)
+        remaining_added_txids.push_back(added_txs[i].first);
+      added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end());
+    }
+
+    std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time);
+    while (rit != m_removed_txs_by_time.end())
+    {
+      if (include_sensitive || !rit->second.sensitive)
+      {
+        removed_txs.push_back(rit->second.txid);
+      }
+      ++rit;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------
   void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -917,26 +1078,61 @@ namespace cryptonote
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
-    const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted;
-    backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive));
-    txpool_tx_meta_t tmp_meta;
-    m_blockchain.for_all_txpool_txes([this, &backlog, &tmp_meta](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){
-      transaction tx;
-      if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx)))
-      {
-        MERROR("Failed to parse tx from txpool");
-        // continue
-        return true;
-      }
-      tx.set_hash(txid);
 
-      tmp_meta = meta;
-
-      if (is_transaction_ready_to_go(tmp_meta, txid, *bd, tx))
-        backlog.push_back({txid, meta.weight, meta.fee});
+    std::vector<tx_block_template_backlog_entry> tmp;
+    uint64_t total_weight = 0;
 
+    // First get everything from the mempool, filter it later
+    m_blockchain.for_all_txpool_txes([&tmp, &total_weight](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*){
+      tmp.emplace_back(tx_block_template_backlog_entry{txid, meta.weight, meta.fee});
+      total_weight += meta.weight;
       return true;
-    }, true, category);
+    }, false, include_sensitive ? relay_category::all : relay_category::broadcasted);
+
+    // Limit backlog to 112.5% of current median weight. This is enough to mine a full block with the optimal block reward
+    const uint64_t median_weight = m_blockchain.get_current_cumulative_block_weight_median();
+    const uint64_t max_backlog_weight = median_weight + (median_weight / 8);
+
+    // If the total weight is too high, choose the best paying transactions
+    if (total_weight > max_backlog_weight)
+      std::stable_sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b){ return a.fee * b.weight > b.fee * a.weight; });
+
+    backlog.clear();
+    uint64_t w = 0;
+
+    std::unordered_set<crypto::key_image> k_images;
+
+    for (const tx_block_template_backlog_entry& e : tmp)
+    {
+      try
+      {
+        txpool_tx_meta_t meta;
+        if (!m_blockchain.get_txpool_tx_meta(e.id, meta))
+          continue;
+
+        cryptonote::blobdata txblob;
+        if (!m_blockchain.get_txpool_tx_blob(e.id, txblob, relay_category::all))
+          continue;
+
+        cryptonote::transaction tx;
+        if (is_transaction_ready_to_go(meta, e.id, txblob, tx))
+        {
+          if (have_key_images(k_images, tx))
+            continue;
+          append_key_images(k_images, tx);
+
+          backlog.push_back(e);
+          w += e.weight;
+          if (w > max_backlog_weight)
+            break;
+        }
+      }
+      catch (const std::exception &e)
+      {
+        MERROR("Failed to check transaction readiness: " << e.what());
+        // continue, not fatal
+      }
+    }
   }
   //------------------------------------------------------------------
   void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const
@@ -1569,6 +1765,12 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
 
+    // Simply throw away incremental info, too difficult to update
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
+
     MINFO("Validating txpool contents for v" << (unsigned)version);
 
     LockedTXN lock(m_blockchain.get_db());
@@ -1626,6 +1828,106 @@ namespace cryptonote
     return n_removed;
   }
   //---------------------------------------------------------------------------------
+  void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time)
+  {
+
+    time_t now = time(NULL);
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it == m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.insert(std::make_pair(txid, now));
+    }
+    else
+    {
+      // This tx was already added to the map earlier, probably because then it was in the "stem"
+      // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set
+      // a new time for clients that are not allowed to see sensitive txs to make sure they will
+      // see it now if they query incrementally
+      it->second = now;
+
+      auto sorted_it = find_tx_in_sorted_container(txid);
+      if (sorted_it == m_txs_by_fee_and_receive_time.end())
+      {
+        MDEBUG("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container");
+      }
+      else
+      {
+        m_txs_by_fee_and_receive_time.erase(sorted_it);
+      }
+    }
+    m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(fee, receive_time), txid);
+
+    // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time'
+    // whether we have that txid there and if yes remove it; this results in possible duplicates
+    // where we return certain txids as deleted AND in the pool at the same time which requires
+    // clients to process deleted ones BEFORE processing pool txs
+    if (m_added_txs_start_time == (time_t)0)
+    {
+      m_added_txs_start_time = now;
+    }
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive)
+  {
+    if (sorted_it == m_txs_by_fee_and_receive_time.end())
+    {
+      LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
+    }
+    else
+    {
+      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    }
+
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it != m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.erase(it);
+    }
+    else
+    {
+      MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs");
+    }
+    track_removed_tx(txid, sensitive);
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive)
+  {
+    time_t now = time(NULL);
+    m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive}));
+    MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size());
+    if (m_removed_txs_start_time == (time_t)0)
+    {
+      m_removed_txs_start_time = now;
+    }
+
+    // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set
+    // an absolute size limit plus delete entries that are x minutes old (which is ok because clients
+    // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back)
+    const int MAX_REMOVED = 20000;
+    if (m_removed_txs_by_time.size() > MAX_REMOVED)
+    {
+      auto erase_it = m_removed_txs_by_time.begin();
+      std::advance(erase_it, MAX_REMOVED / 4 + 1);
+      m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it);
+      m_removed_txs_start_time = m_removed_txs_by_time.begin()->first;
+      MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size());
+    }
+    else
+    {
+      time_t earliest = now - (30 * 60);  // 30 minutes
+      std::map<time_t, removed_tx_info>::iterator from, to;
+      from = m_removed_txs_by_time.begin();
+      to = m_removed_txs_by_time.lower_bound(earliest);
+      int distance = std::distance(from, to);
+      if (distance > 0)
+      {
+        m_removed_txs_by_time.erase(from, to);
+        m_removed_txs_start_time = earliest;
+        MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size());
+      }
+    }
+  }
+  //---------------------------------------------------------------------------------
   bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -1633,6 +1935,10 @@ namespace cryptonote
 
     m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT;
     m_txs_by_fee_and_receive_time.clear();
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
     m_spent_key_images.clear();
     m_txpool_weight = 0;
     std::vector<crypto::hash> remove;
@@ -1657,7 +1963,7 @@ namespace cryptonote
           MFATAL("Failed to insert key images from txpool tx");
           return false;
         }
-        m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid);
+        add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time);
         m_txpool_weight += meta.weight;
         return true;
       }, true, relay_category::all);

src/cryptonote_core/tx_pool.h

@@ -69,11 +69,12 @@ namespace cryptonote
     {
       // sort by greatest first, not least
       if (a.first.first > b.first.first) return true;
-      else if (a.first.first < b.first.first) return false;
-      else if (a.first.second < b.first.second) return true;
-      else if (a.first.second > b.first.second) return false;
-      else if (a.second != b.second) return true;
-      else return false;
+      if (a.first.first < b.first.first) return false;
+
+      if (a.first.second < b.first.second) return true;
+      if (a.first.second > b.first.second) return false;
+
+      return memcmp(a.second.data, b.second.data, sizeof(crypto::hash)) < 0;
     }
   };
 
@@ -266,7 +267,11 @@ namespace cryptonote
     void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive = false) const;
 
     /**
-     * @brief get (hash, weight, fee) for all transactions in the pool - the minimum required information to create a block template
+     * @brief get (hash, weight, fee) for transactions in the pool - the minimum required information to create a block template
+     *
+     * Not all transactions in the pool will be returned for performance reasons
+     * If there are too many transactions in the pool, only the highest-paying transactions
+     * will be returned - but enough for the miner to create a full block
      *
      * @param backlog return-by-reference that data
      * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes
@@ -353,8 +358,10 @@ namespace cryptonote
      *
      * @param hashes list of tx hashes that are about to be relayed
      * @param tx_relay update how the tx left this node
+     * @param just_broadcasted true if a tx was just broadcasted
+     *
      */
-    void set_relayed(epee::span<const crypto::hash> hashes, relay_method tx_relay);
+    void set_relayed(epee::span<const crypto::hash> hashes, relay_method tx_relay, std::vector<bool> &just_broadcasted);
 
     /**
      * @brief get the total number of transactions in the pool
@@ -406,6 +413,13 @@ namespace cryptonote
      */
     void set_txpool_max_weight(size_t bytes);
 
+    /**
+     * @brief reduce the cumulative txpool weight by the weight provided
+     *
+     * @param weight the weight to reduce the total txpool weight by
+     */
+    void reduce_txpool_weight(size_t weight);
+
 #define CURRENT_MEMPOOL_ARCHIVE_VER    11
 #define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER    13
 
@@ -415,6 +429,7 @@ namespace cryptonote
     struct tx_details
     {
       transaction tx;  //!< the transaction
+      cryptonote::blobdata tx_blob; //!< the transaction's binary blob
       size_t blob_size;  //!< the transaction's size
       size_t weight;  //!< the transaction's weight
       uint64_t fee;  //!< the transaction's fee amount
@@ -453,13 +468,25 @@ namespace cryptonote
     /**
      * @brief get infornation about a single transaction
      */
-    bool get_transaction_info(const crypto::hash &txid, tx_details &td) const;
+    bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const;
+
+    /**
+     * @brief get information about multiple transactions
+     */
+    bool get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive_data = false) const;
 
     /**
      * @brief get transactions not in the passed set
      */
     bool get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const;
 
+    /**
+     * @brief get info necessary for update of pool-related info in a wallet, preferably incremental
+     *
+     * @return true on success, false on error
+     */
+    bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
   private:
 
     /**
@@ -564,6 +591,10 @@ namespace cryptonote
      */
     void prune(size_t bytes = 0);
 
+    void add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time);
+    void remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive);
+    void track_removed_tx(const crypto::hash& txid, bool sensitive);
+
     //TODO: confirm the below comments and investigate whether or not this
     //      is the desired behavior
     //! map key images to transactions which spent them
@@ -596,6 +627,26 @@ private:
 
     std::atomic<uint64_t> m_cookie; //!< incremented at each change
 
+    // Info when transactions entered the pool, accessible by txid
+    std::unordered_map<crypto::hash, time_t> m_added_txs_by_id;
+
+    // Info at what time the pool started to track the adding of transactions
+    time_t m_added_txs_start_time;
+
+    struct removed_tx_info
+    {
+      crypto::hash txid;
+      bool sensitive;
+    };
+
+    // Info about transactions that were removed from the pool, ordered by the time
+    // of deletion
+    std::multimap<time_t, removed_tx_info> m_removed_txs_by_time;
+
+    // Info how far back in time the list of removed tx ids currently reaches
+    // (it gets shorted periodically to prevent overflow)
+    time_t m_removed_txs_start_time;
+
     /**
      * @brief get an iterator to a transaction in the sorted container
      *

src/cryptonote_core/tx_verification_utils.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "cryptonote_core/blockchain.h"
+#include "cryptonote_core/tx_verification_utils.h"
+#include "ringct/rctSigs.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "blockchain"
+
+#define VER_ASSERT(cond, msgexpr) CHECK_AND_ASSERT_MES(cond, false, msgexpr)
+
+using namespace cryptonote;
+
+// Do RCT expansion, then do post-expansion sanity checks, then do full non-semantics verification.
+static bool expand_tx_and_ver_rct_non_sem(transaction& tx, const rct::ctkeyM& mix_ring)
+{
+    // Pruned transactions can not be expanded and verified because they are missing RCT data
+    VER_ASSERT(!tx.pruned, "Pruned transaction will not pass verRctNonSemanticsSimple");
+
+    // Calculate prefix hash
+    const crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
+
+    // Expand mixring, tx inputs, tx key images, prefix hash message, etc into the RCT sig
+    const bool exp_res = Blockchain::expand_transaction_2(tx, tx_prefix_hash, mix_ring);
+    VER_ASSERT(exp_res, "Failed to expand rct signatures!");
+
+    const rct::rctSig& rv = tx.rct_signatures;
+
+    // Check that expanded RCT mixring == input mixring
+    VER_ASSERT(rv.mixRing == mix_ring, "Failed to check ringct signatures: mismatched pubkeys/mixRing");
+
+    // Check CLSAG/MLSAG size against transaction input
+    const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size();
+    VER_ASSERT(n_sigs == tx.vin.size(), "Failed to check ringct signatures: mismatched input sigs/vin sizes");
+
+    // For each input, check that the key images were copied into the expanded RCT sig correctly
+    for (size_t n = 0; n < n_sigs; ++n)
+    {
+        const crypto::key_image& nth_vin_image = boost::get<txin_to_key>(tx.vin[n]).k_image;
+
+        if (rct::is_rct_clsag(rv.type))
+        {
+            const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.CLSAGs[n].I, 32);
+            VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched CLSAG key image");
+        }
+        else
+        {
+            const bool mg_nonempty = !rv.p.MGs[n].II.empty();
+            VER_ASSERT(mg_nonempty, "Failed to check ringct signatures: missing MLSAG key image");
+            const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.MGs[n].II[0], 32);
+            VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched MLSAG key image");
+        }
+    }
+
+    // Mix ring data is now known to be correctly incorporated into the RCT sig inside tx.
+    return rct::verRctNonSemanticsSimple(rv);
+}
+
+// Create a unique identifier for pair of tx blob + mix ring
+static crypto::hash calc_tx_mixring_hash(const transaction& tx, const rct::ctkeyM& mix_ring)
+{
+    std::stringstream ss;
+
+    // Start with domain seperation
+    ss << config::HASH_KEY_TXHASH_AND_MIXRING;
+
+    // Then add TX hash
+    const crypto::hash tx_hash = get_transaction_hash(tx);
+    ss.write(tx_hash.data, sizeof(crypto::hash));
+
+    // Then serialize mix ring
+    binary_archive<true> ar(ss);
+    ::do_serialize(ar, const_cast<rct::ctkeyM&>(mix_ring));
+
+    // Calculate hash of TX hash and mix ring blob
+    crypto::hash tx_and_mixring_hash;
+    get_blob_hash(ss.str(), tx_and_mixring_hash);
+
+    return tx_and_mixring_hash;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace cryptonote
+{
+
+bool ver_rct_non_semantics_simple_cached
+(
+    transaction& tx,
+    const rct::ctkeyM& mix_ring,
+    rct_ver_cache_t& cache,
+    const std::uint8_t rct_type_to_cache
+)
+{
+    // Hello future Monero dev! If you got this assert, read the following carefully:
+    //
+    // For this version of RCT, the way we guaranteed that verification caches do not generate false
+    // positives (and thus possibly enabling double spends) is we take a hash of two things. One,
+    // we use get_transaction_hash() which gives us a (cryptographically secure) unique
+    // representation of all "knobs" controlled by the possibly malicious constructor of the
+    // transaction. Two, we take a hash of all *previously validated* blockchain data referenced by
+    // this transaction which is required to validate the ring signature. In our case, this is the
+    // mixring. Future versions of the protocol may differ in this regard, but if this assumptions
+    // holds true in the future, enable the verification hash by modifying the `untested_tx`
+    // condition below.
+    const bool untested_tx = tx.version > 2 || tx.rct_signatures.type > rct::RCTTypeBulletproofPlus;
+    VER_ASSERT(!untested_tx, "Unknown TX type. Make sure RCT cache works correctly with this type and then enable it in the code here.");
+
+    // Don't cache older (or newer) rctSig types
+    // This cache only makes sense when it caches data from mempool first,
+    // so only "current fork version-enabled" RCT types need to be cached
+    if (tx.rct_signatures.type != rct_type_to_cache)
+    {
+        MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " skipped");
+        return expand_tx_and_ver_rct_non_sem(tx, mix_ring);
+    }
+
+    // Generate unique hash for tx+mix_ring pair
+    const crypto::hash tx_mixring_hash = calc_tx_mixring_hash(tx, mix_ring);
+
+    // Search cache for successful verification of same TX + mix ring combination
+    if (cache.has(tx_mixring_hash))
+    {
+        MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " hit");
+        return true;
+    }
+
+    // We had a cache miss, so now we must expand the mix ring and do full verification
+    MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " missed");
+    if (!expand_tx_and_ver_rct_non_sem(tx, mix_ring))
+    {
+        return false;
+    }
+
+    // At this point, the TX RCT verified successfully, so add it to the cache and return true
+    cache.add(tx_mixring_hash);
+
+    return true;
+}
+
+} // namespace cryptonote

src/cryptonote_core/tx_verification_utils.h

@@ -0,0 +1,78 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "common/data_cache.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+
+namespace cryptonote
+{
+
+// Modifying this value should not affect consensus. You can adjust it for performance needs
+static constexpr const size_t RCT_VER_CACHE_SIZE = 8192;
+
+using rct_ver_cache_t = ::tools::data_cache<::crypto::hash, RCT_VER_CACHE_SIZE>;
+
+/**
+ * @brief Cached version of rct::verRctNonSemanticsSimple
+ *
+ * This function will not affect how the transaction is serialized and it will never modify the
+ * transaction prefix.
+ *
+ * The reference to tx is mutable since the transaction's ring signatures may be expanded by
+ * Blockchain::expand_transaction_2. However, on cache hits, the transaction will not be
+ * expanded. This means that the caller does not need to call expand_transaction_2 on this
+ * transaction before passing it; the transaction will not successfully verify with "old" RCT data
+ * if the transaction has been otherwise modified since the last verification.
+ *
+ * But, if cryptonote::get_transaction_hash(tx) returns a "stale" hash, this function is not
+ * guaranteed to work. So make sure that the cryptonote::transaction passed has not had
+ * modifications to it since the last time its hash was fetched without properly invalidating the
+ * hashes.
+ *
+ * rct_type_to_cache can be any RCT version value as long as rct::verRctNonSemanticsSimple works for
+ * this RCT version, but for most applications, it doesn't make sense to not make this version
+ * the "current" RCT version (i.e. the version that transactions in the mempool are).
+ *
+ * @param tx transaction which contains RCT signature to verify
+ * @param mix_ring mixring referenced by this tx. THIS DATA MUST BE PREVIOUSLY VALIDATED
+ * @param cache saves tx+mixring hashes used to cache calls
+ * @param rct_type_to_cache Only RCT sigs with version (e.g. RCTTypeBulletproofPlus) will be cached
+ * @return true when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return true
+ * @return false when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return false
+ */
+bool ver_rct_non_semantics_simple_cached
+(
+    transaction& tx,
+    const rct::ctkeyM& mix_ring,
+    rct_ver_cache_t& cache,
+    std::uint8_t rct_type_to_cache
+);
+
+} // namespace cryptonote

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -537,6 +537,10 @@ namespace cryptonote
       MLOG_PEER_STATE("requesting chain");
     }
 
+    // load json & DNS checkpoints every 10min/hour respectively,
+    // and verify them with respect to what blocks we already have
+    CHECK_AND_ASSERT_MES(m_core.update_checkpoints(), 1, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
+
     return 1;
   }
   //------------------------------------------------------------------------------------------------------------------------
@@ -819,6 +823,10 @@ namespace cryptonote
           post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
           MLOG_PEER_STATE("requesting chain");
         }            
+
+        // load json & DNS checkpoints every 10min/hour respectively,
+        // and verify them with respect to what blocks we already have
+        CHECK_AND_ASSERT_MES(m_core.update_checkpoints(), 1, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
       }
     } 
     else
@@ -971,8 +979,18 @@ namespace cryptonote
   int t_cryptonote_protocol_handler<t_core>::handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& context)
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_NEW_TRANSACTIONS (" << arg.txs.size() << " txes)");
+    std::unordered_set<blobdata> seen;
     for (const auto &blob: arg.txs)
+    {
       MLOGIF_P2P_MESSAGE(cryptonote::transaction tx; crypto::hash hash; bool ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx, hash);, ret, "Including transaction " << hash);
+      if (seen.find(blob) != seen.end())
+      {
+        LOG_PRINT_CCONTEXT_L1("Duplicate transaction in notification, dropping connection");
+        drop_connection(context, false, false);
+        return 1;
+      }
+      seen.insert(blob);
+    }
 
     if(context.m_state != cryptonote_connection_context::state_normal)
       return 1;
@@ -1012,7 +1030,7 @@ namespace cryptonote
     for (auto& tx : arg.txs)
     {
       tx_verification_context tvc{};
-      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true))
+      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true) && !tvc.m_no_drop_offense)
       {
         LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection");
         drop_connection(context, false, false);

src/cryptonote_protocol/levin_notify.cpp

@@ -396,6 +396,8 @@ namespace levin
         for (auto& connection : connections)
         {
           std::sort(connection.first.begin(), connection.first.end()); // don't leak receive order
+          connection.first.erase(std::unique(connection.first.begin(), connection.first.end()),
+                                  connection.first.end());
           make_payload_send_txs(*zone_->p2p, std::move(connection.first), connection.second, zone_->pad_txs, true);
         }
 
@@ -542,6 +544,7 @@ namespace levin
       i_core_events* core_;
       std::vector<blobdata> txs_;
       boost::uuids::uuid source_;
+      relay_method tx_relay;
 
       //! \pre Called in `zone_->strand`
       void operator()()
@@ -549,7 +552,7 @@ namespace levin
         if (!zone_ || !core_ || txs_.empty())
           return;
 
-        if (!zone_->fluffing)
+        if (!zone_->fluffing || tx_relay == relay_method::local)
         {
           core_->on_transactions_relayed(epee::to_span(txs_), relay_method::stem);
           for (int tries = 2; 0 < tries; tries--)
@@ -589,7 +592,7 @@ namespace levin
 
       change_channels(change_channels&&) = default;
       change_channels(const change_channels& source)
-        : zone_(source.zone_), map_(source.map_.clone())
+        : zone_(source.zone_), map_(source.map_.clone()), fluffing_(source.fluffing_)
       {}
 
       //! \pre Called within `zone_->strand`.
@@ -740,9 +743,14 @@ namespace levin
   notify::status notify::get_status() const noexcept
   {
     if (!zone_)
-      return {false, false};
+      return {false, false, false};
 
-    return {!zone_->noise.empty(), CRYPTONOTE_NOISE_CHANNELS <= zone_->connection_count};
+    // `connection_count` is only set when `!noise.empty()`.
+    const std::size_t connection_count = zone_->connection_count;
+    bool has_outgoing = connection_count;
+    if (zone_->noise.empty())
+      has_outgoing = zone_->p2p->get_out_connections_count();
+    return {!zone_->noise.empty(), CRYPTONOTE_NOISE_CHANNELS <= connection_count, has_outgoing};
   }
 
   void notify::new_out_connection()
@@ -871,7 +879,7 @@ namespace levin
           {
             // this will change a local/forward tx to stem or fluff ...
             zone_->strand.dispatch(
-              dandelionpp_notify{zone_, core_, std::move(txs), source}
+              dandelionpp_notify{zone_, core_, std::move(txs), source, tx_relay}
             );
             break;
           }

src/cryptonote_protocol/levin_notify.h

@@ -75,7 +75,8 @@ namespace levin
     struct status
     {
       bool has_noise;
-      bool connections_filled;
+      bool connections_filled; //!< True when has zone has `CRYPTONOTE_NOISE_CHANNELS` outgoing noise channels
+      bool has_outgoing; //!< True when zone has outgoing connections
     };
 
     //! Construct an instance that cannot notify.

src/daemon/main.cpp

@@ -219,6 +219,19 @@ int main(int argc, char const * argv[])
       {
         po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), core_settings), vm);
       }
+      catch (const po::unknown_option &e)
+      {
+        std::string unrecognized_option = e.get_option_name();
+        if (all_options.find_nothrow(unrecognized_option, false))
+        {
+          std::cerr << "Option '" << unrecognized_option << "' is not allowed in the config file, please use it as a command line flag." << std::endl;
+        }
+        else
+        {
+          std::cerr << "Unrecognized option '" << unrecognized_option << "' in config file." << std::endl;
+        }
+        return 1;
+      }
       catch (const std::exception &e)
       {
         // log system isn't initialized yet

src/device/device.hpp

@@ -177,6 +177,7 @@ namespace hw {
         virtual bool  derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub,  crypto::public_key &derived_pub) = 0;
         virtual bool  secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) = 0;
         virtual bool  generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image) = 0;
+        virtual bool  derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag) = 0;
 
         // alternative prototypes available in libringct
         rct::key scalarmultKey(const rct::key &P, const rct::key &a)

src/device/device_default.hpp

@@ -101,7 +101,7 @@ namespace hw {
             bool  derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub,  crypto::public_key &derived_pub) override;
             bool  secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) override;
             bool  generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image) override;
-            bool  derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag);
+            bool  derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag) override;
 
 
             /* ======================================================================= */

src/device/device_ledger.cpp

@@ -266,6 +266,7 @@ namespace hw {
     #define INS_DERIVE_PUBLIC_KEY               0x36
     #define INS_DERIVE_SECRET_KEY               0x38
     #define INS_GEN_KEY_IMAGE                   0x3A
+    #define INS_DERIVE_VIEW_TAG                 0x3B
     #define INS_SECRET_KEY_ADD                  0x3C
     #define INS_SECRET_KEY_SUB                  0x3E
     #define INS_GENERATE_KEYPAIR                0x40
@@ -525,6 +526,8 @@ namespace hw {
         {0x2c97, 0x0001, 0, 0xffa0}, 
         {0x2c97, 0x0004, 0, 0xffa0},       
         {0x2c97, 0x0005, 0, 0xffa0},
+        {0x2c97, 0x0006, 0, 0xffa0},
+        {0x2c97, 0x0007, 0, 0xffa0},
     };
 
     bool device_ledger::connect(void) {
@@ -1308,6 +1311,54 @@ namespace hw {
         return true;
     }
 
+    bool device_ledger::derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag){
+      #ifdef DEBUG_HWDEVICE
+      crypto::key_derivation derivation_x;
+      if ((this->mode == TRANSACTION_PARSE) && has_view_key) {
+        derivation_x = derivation;
+      } else {
+        derivation_x = hw::ledger::decrypt(derivation);
+      }
+      const std::size_t            output_index_x = output_index;
+      crypto::view_tag             view_tag_x;
+      log_hexbuffer("derive_view_tag: [[IN]]  derivation  ", derivation_x.data, 32);
+      log_message  ("derive_view_tag: [[IN]]  output_index", std::to_string(output_index_x));
+      this->controle_device->derive_view_tag(derivation_x, output_index_x, view_tag_x);
+      log_hexbuffer("derive_view_tag: [[OUT]] view_tag ", &view_tag_x.data, 1);
+      #endif
+
+      if ((this->mode == TRANSACTION_PARSE) && has_view_key) {
+        //If we are in TRANSACTION_PARSE, the given derivation has been retrieved uncrypted (wihtout the help
+        //of the device), so continue that way.
+        MDEBUG( "derive_view_tag  : PARSE mode with known viewkey");
+        crypto::derive_view_tag(derivation, output_index, view_tag);
+      } else {
+        AUTO_LOCK_CMD();
+        int offset = set_command_header_noopt(INS_DERIVE_VIEW_TAG);
+        //derivation
+        this->send_secret((unsigned char*)derivation.data, offset);
+        //index
+        this->buffer_send[offset+0] = output_index>>24;
+        this->buffer_send[offset+1] = output_index>>16;
+        this->buffer_send[offset+2] = output_index>>8;
+        this->buffer_send[offset+3] = output_index>>0;
+        offset += 4;
+
+        this->buffer_send[4] = offset-5;
+        this->length_send = offset;
+        this->exchange();
+
+        //view tag
+        memmove(&view_tag.data, &this->buffer_recv[0], 1);
+      }
+
+      #ifdef DEBUG_HWDEVICE
+      hw::ledger::check1("derive_view_tag", "view_tag", &view_tag_x.data, &view_tag.data);
+      #endif
+
+      return true;
+    }
+
     /* ======================================================================= */
     /*                               TRANSACTION                               */
     /* ======================================================================= */
@@ -1548,7 +1599,6 @@ namespace hw {
       const size_t                             output_index_x                 = output_index;
       const bool                               need_additional_txkeys_x       = need_additional_txkeys;
       const bool                               use_view_tags_x                = use_view_tags;
-      const crypto::view_tag                   view_tag_x                     = view_tag;
       
       std::vector<crypto::secret_key>    additional_tx_keys_x;
       for (const auto &k: additional_tx_keys) {
@@ -1558,6 +1608,7 @@ namespace hw {
       std::vector<crypto::public_key>          additional_tx_public_keys_x;
       std::vector<rct::key>                    amount_keys_x;
       crypto::public_key                       out_eph_public_key_x;
+      crypto::view_tag                         view_tag_x;
 
       log_message  ("generate_output_ephemeral_keys: [[IN]] tx_version", std::to_string(tx_version_x));
       //log_hexbuffer("generate_output_ephemeral_keys: [[IN]] sender_account_keys.view", sender_account_keys.m_sview_secret_key.data, 32);
@@ -1575,11 +1626,15 @@ namespace hw {
       if(need_additional_txkeys_x) {
         log_hexbuffer("generate_output_ephemeral_keys: [[IN]] additional_tx_keys[oi]", additional_tx_keys_x[output_index].data, 32);
       }
+      log_message  ("generate_output_ephemeral_keys: [[IN]] use_view_tags",  std::to_string(use_view_tags_x));
       this->controle_device->generate_output_ephemeral_keys(tx_version_x, sender_account_keys_x, txkey_pub_x, tx_key_x, dst_entr_x, change_addr_x, output_index_x, need_additional_txkeys_x,  additional_tx_keys_x,
                                                             additional_tx_public_keys_x, amount_keys_x, out_eph_public_key_x, use_view_tags_x, view_tag_x);
       if(need_additional_txkeys_x) {
         log_hexbuffer("additional_tx_public_keys_x: [[OUT]] additional_tx_public_keys_x", additional_tx_public_keys_x.back().data, 32);
       }
+      if(use_view_tags_x) {
+        log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] view_tag", &view_tag_x.data, 1);
+      }
       log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] amount_keys ", (char*)amount_keys_x.back().bytes, 32);
       log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] out_eph_public_key ", out_eph_public_key_x.data, 32);
       #endif
@@ -1633,6 +1688,9 @@ namespace hw {
         memset(&this->buffer_send[offset], 0, 32);
         offset += 32;
       }
+      //use_view_tags
+      this->buffer_send[offset] = use_view_tags;
+      offset++;
 
       this->buffer_send[4] = offset-5;
       this->length_send = offset;
@@ -1663,6 +1721,14 @@ namespace hw {
         recv_len -= 32;
       }
 
+      if (use_view_tags)
+      {
+        ASSERT_X(recv_len>=1, "Not enough data from device");
+        memmove(&view_tag.data, &this->buffer_recv[offset], 1);
+        offset++;
+        recv_len -= 1;
+      }
+
       // add ABPkeys
       this->add_output_key_mapping(dst_entr.addr.m_view_public_key, dst_entr.addr.m_spend_public_key, dst_entr.is_subaddress, is_change,
                                    need_additional_txkeys, output_index,
@@ -1675,6 +1741,9 @@ namespace hw {
         hw::ledger::check32("generate_output_ephemeral_keys", "additional_tx_key", additional_tx_public_keys_x.back().data, additional_tx_public_keys.back().data);
       }
       hw::ledger::check32("generate_output_ephemeral_keys", "out_eph_public_key", out_eph_public_key_x.data, out_eph_public_key.data);
+      if (use_view_tags) {
+        hw::ledger::check1("generate_output_ephemeral_keys", "view_tag", &view_tag_x.data, &view_tag.data);
+      }
       #endif
 
       return true;
@@ -1860,7 +1929,7 @@ namespace hw {
 
         // ======  Aout, Bout, AKout, C, v, k ======
         kv_offset = data_offset;
-        if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG) {
+        if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG || type==rct::RCTTypeBulletproofPlus) {
           C_offset = kv_offset+ (8)*outputs_size;
         } else {
           C_offset = kv_offset+ (32+32)*outputs_size;
@@ -1877,7 +1946,7 @@ namespace hw {
           offset = set_command_header(INS_VALIDATE, 0x02, i+1);
           //options
           this->buffer_send[offset] = (i==outputs_size-1)? 0x00:0x80 ;
-          this->buffer_send[offset] |= (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG)?0x02:0x00;
+          this->buffer_send[offset] |= (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG || type==rct::RCTTypeBulletproofPlus)?0x02:0x00;
           offset += 1;
           //is_subaddress
           this->buffer_send[offset] = outKeys.is_subaddress;
@@ -1898,7 +1967,7 @@ namespace hw {
           memmove(this->buffer_send+offset, data+C_offset,32);
           offset += 32;
           C_offset += 32;
-          if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG) {
+          if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG || type==rct::RCTTypeBulletproofPlus) {
             //k
             memset(this->buffer_send+offset, 0, 32);
             offset += 32;

src/device/device_ledger.hpp

@@ -44,7 +44,7 @@ namespace hw {
 
     /* Minimal supported version */
     #define MINIMAL_APP_VERSION_MAJOR    1
-    #define MINIMAL_APP_VERSION_MINOR    6
+    #define MINIMAL_APP_VERSION_MINOR    8
     #define MINIMAL_APP_VERSION_MICRO    0
 
     #define VERSION(M,m,u)       ((M)<<16|(m)<<8|(u))
@@ -249,6 +249,7 @@ namespace hw {
         bool  derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub,  crypto::public_key &derived_pub) override;
         bool  secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) override;
         bool  generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image) override;
+        bool  derive_view_tag(const crypto::key_derivation &derivation, const size_t output_index, crypto::view_tag &view_tag) override;
 
         /* ======================================================================= */
         /*                               TRANSACTION                               */

src/device/log.cpp

@@ -165,6 +165,10 @@ namespace hw {
     void check8(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) {
       check(msg, info, h, d, 8, crypted);
     }
+
+    void check1(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) {
+      check(msg, info, h, d, 1, crypted);
+    }
     #endif
 
   }

src/device/log.hpp

@@ -75,6 +75,7 @@ namespace hw {
 
         void check32(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false);
         void check8(const std::string &msg, const std::string &info, const char *h, const char *d,  bool crypted=false);
+        void check1(const std::string &msg, const std::string &info, const char *h, const char *d,  bool crypted=false);
 
         void set_check_verbose(bool verbose);
         #endif

src/device_trezor/device_trezor.cpp

@@ -324,8 +324,8 @@ namespace trezor {
 
       std::vector<protocol::ki::MoneroTransferDetails> mtds;
       std::vector<protocol::ki::MoneroExportedKeyImage> kis;
-      protocol::ki::key_image_data(wallet, transfers, mtds, client_version() <= 1);
-      protocol::ki::generate_commitment(mtds, transfers, req, client_version() <= 1);
+      protocol::ki::key_image_data(wallet, transfers, mtds);
+      protocol::ki::generate_commitment(mtds, transfers, req);
 
       EVENT_PROGRESS(0.);
       this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get());
@@ -511,7 +511,7 @@ namespace trezor {
                                 tools::wallet2::signed_tx_set & signed_tx,
                                 hw::tx_aux_data & aux_data)
     {
-      CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset");
+      CHECK_AND_ASSERT_THROW_MES(std::get<0>(unsigned_tx.transfers) == 0, "Unsuported non zero offset");
 
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
@@ -522,7 +522,7 @@ namespace trezor {
       const size_t num_tx = unsigned_tx.txes.size();
       m_num_transations_to_sign = num_tx;
       signed_tx.key_images.clear();
-      signed_tx.key_images.resize(unsigned_tx.transfers.second.size());
+      signed_tx.key_images.resize(std::get<2>(unsigned_tx.transfers).size());
 
       for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) {
         std::shared_ptr<protocol::tx::Signer> signer;
@@ -566,8 +566,8 @@ namespace trezor {
         cpend.key_images = key_images;
 
         // KI sync
-        for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){
-          signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image;
+        for(size_t cidx=0, trans_max=std::get<2>(unsigned_tx.transfers).size(); cidx < trans_max; ++cidx){
+          signed_tx.key_images[cidx] = std::get<2>(unsigned_tx.transfers)[cidx].m_key_image;
         }
 
         size_t num_sources = cdata.tx_data.sources.size();
@@ -579,9 +579,9 @@ namespace trezor {
           CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped");
 
           size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped];
-          CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset");
+          CHECK_AND_ASSERT_THROW_MES(idx_map_src >= std::get<0>(unsigned_tx.transfers), "Invalid offset");
 
-          idx_map_src -= unsigned_tx.transfers.first;
+          idx_map_src -= std::get<0>(unsigned_tx.transfers);
           CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index");
 
           const auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]);
@@ -635,11 +635,7 @@ namespace trezor {
       }
 
       // Step: sort
-      auto perm_req = signer->step_permutation();
-      if (perm_req){
-        auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req);
-        signer->step_permutation_ack(perm_ack);
-      }
+      signer->sort_ki();
       EVENT_PROGRESS(3, 1, 1);
 
       // Step: input_vini
@@ -697,13 +693,13 @@ namespace trezor {
     unsigned device_trezor::client_version()
     {
       auto trezor_version = get_version();
-      if (trezor_version <= pack_version(2, 0, 10)){
-        throw exc::TrezorException("Trezor firmware 2.0.10 and lower are not supported. Please update.");
+      if (trezor_version < pack_version(2, 4, 3)){
+        throw exc::TrezorException("Minimal Trezor firmware version is 2.4.3. Please update.");
       }
 
-      unsigned client_version = 1;
-      if (trezor_version >= pack_version(2, 3, 1)){
-        client_version = 3;
+      unsigned client_version = 3;
+      if (trezor_version >= pack_version(2, 5, 2)){
+        client_version = 4;
       }
 
 #ifdef WITH_TREZOR_DEBUGGING
@@ -739,14 +735,6 @@ namespace trezor {
       CHECK_AND_ASSERT_THROW_MES(init_msg, "TransactionInitRequest is empty");
       CHECK_AND_ASSERT_THROW_MES(init_msg->has_tsx_data(), "TransactionInitRequest has no transaction data");
       CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized");  // make sure the caller did not reset features
-      const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0;
-
-      if (nonce_required && init_msg->tsx_data().payment_id().size() == 8){
-        // Versions 2.0.9 and lower do not support payment ID
-        if (get_version() <= pack_version(2, 0, 9)) {
-          throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update.");
-        }
-      }
     }
 
     void device_trezor::transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data)

src/device_trezor/trezor/protocol.cpp

@@ -38,6 +38,7 @@
 #include <crypto/hmac-keccak.h>
 #include <ringct/rctSigs.h>
 #include <ringct/bulletproofs.h>
+#include <ringct/bulletproofs_plus.h>
 #include "cryptonote_config.h"
 #include <sodium.h>
 #include <sodium/crypto_verify_32.h>
@@ -145,8 +146,7 @@ namespace ki {
 
   bool key_image_data(wallet_shim * wallet,
                       const std::vector<tools::wallet2::transfer_details> & transfers,
-                      std::vector<MoneroTransferDetails> & res,
-                      bool need_all_additionals)
+                      std::vector<MoneroTransferDetails> & res)
   {
     for(auto & td : transfers){
       ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td);
@@ -159,11 +159,7 @@ namespace ki {
       cres.set_internal_output_index(td.m_internal_output_index);
       cres.set_sub_addr_major(td.m_subaddr_index.major);
       cres.set_sub_addr_minor(td.m_subaddr_index.minor);
-      if (need_all_additionals) {
-        for (auto &aux : additional_tx_pub_keys) {
-          cres.add_additional_tx_pub_keys(key_to_string(aux));
-        }
-      } else if (!additional_tx_pub_keys.empty() && additional_tx_pub_keys.size() > td.m_internal_output_index) {
+      if (!additional_tx_pub_keys.empty() && additional_tx_pub_keys.size() > td.m_internal_output_index) {
         cres.add_additional_tx_pub_keys(key_to_string(additional_tx_pub_keys[td.m_internal_output_index]));
       }
     }
@@ -194,8 +190,7 @@ namespace ki {
 
   void generate_commitment(std::vector<MoneroTransferDetails> & mtds,
                            const std::vector<tools::wallet2::transfer_details> & transfers,
-                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req,
-                           bool need_subaddr_indices)
+                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req)
   {
     req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>();
 
@@ -219,16 +214,6 @@ namespace ki {
       auto & st = search.first->second;
       st.insert(cur.m_subaddr_index.minor);
     }
-
-    if (need_subaddr_indices) {
-      for (auto &x: sub_indices) {
-        auto subs = req->add_subs();
-        subs->set_account(x.first);
-        for (auto minor : x.second) {
-          subs->add_minor_indices(minor);
-        }
-      }
-    }
   }
 
   void live_refresh_ack(const ::crypto::secret_key & view_key_priv,
@@ -399,7 +384,7 @@ namespace tx {
     m_tx_idx = tx_idx;
     m_ct.tx_data = cur_src_tx();
     m_multisig = false;
-    m_client_version = 1;
+    m_client_version = 3;
   }
 
   void Signer::extract_payment_id(){
@@ -474,25 +459,19 @@ namespace tx {
       auto & cur = src.outputs[i];
       auto out = dst->add_outputs();
 
-      if (i == src.real_output || need_ring_indices || client_version() <= 1) {
+      if (i == src.real_output || need_ring_indices) {
         out->set_idx(cur.first);
       }
-      if (i == src.real_output || need_ring_keys || client_version() <= 1) {
+      if (i == src.real_output || need_ring_keys) {
         translate_rct_key(out->mutable_key(), &(cur.second));
       }
     }
 
     dst->set_real_out_tx_key(key_to_string(src.real_out_tx_key));
     dst->set_real_output_in_tx_index(src.real_output_in_tx_index);
-
-    if (client_version() <= 1) {
-      for (auto &cur : src.real_out_additional_tx_keys) {
-        dst->add_real_out_additional_tx_keys(key_to_string(cur));
-      }
-    } else if (!src.real_out_additional_tx_keys.empty()) {
+    if (!src.real_out_additional_tx_keys.empty()) {
       dst->add_real_out_additional_tx_keys(key_to_string(src.real_out_additional_tx_keys.at(src.real_output_in_tx_index)));
     }
-
     dst->set_amount(src.amount);
     dst->set_rct(src.rct);
     dst->set_mask(key_to_string(src.mask));
@@ -532,7 +511,7 @@ namespace tx {
 
     m_ct.tx.version = 2;
     m_ct.tx.unlock_time = tx.unlock_time;
-    m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 1);
+    m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 3);
 
     tsx_data.set_version(1);
     tsx_data.set_client_version(client_version());
@@ -543,18 +522,13 @@ namespace tx {
     tsx_data.set_monero_version(std::string(MONERO_VERSION) + "|" + MONERO_VERSION_TAG);
     tsx_data.set_hard_fork(m_aux_data->hard_fork ? m_aux_data->hard_fork.get() : 0);
 
-    if (client_version() <= 1){
-      assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end());
-    }
-
     // Rsig decision
     auto rsig_data = tsx_data.mutable_rsig_data();
     m_ct.rsig_type = get_rsig_type(tx.rct_config, tx.splitted_dsts.size());
     rsig_data->set_rsig_type(m_ct.rsig_type);
-    if (tx.rct_config.range_proof_type != rct::RangeProofBorromean){
-      m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1);
-      rsig_data->set_bp_version((uint32_t) m_ct.bp_version);
-    }
+    CHECK_AND_ASSERT_THROW_MES(tx.rct_config.range_proof_type != rct::RangeProofBorromean, "Borromean rsig not supported");
+    m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1);
+    rsig_data->set_bp_version((uint32_t) m_ct.bp_version);
 
     generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size());
     assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end());
@@ -652,22 +626,6 @@ namespace tx {
     });
   }
 
-  std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){
-    sort_ki();
-    if (client_version() >= 2){
-      return nullptr;
-    }
-
-    auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>();
-    assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end());
-
-    return res;
-  }
-
-  void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){
-
-  }
-
   std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index");
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index");
@@ -711,8 +669,10 @@ namespace tx {
   }
 
   void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){
+    CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported");
     cryptonote::tx_out tx_out;
     rct::Bulletproof bproof{};
+    rct::BulletproofPlus bproof_plus{};
     rct::ctkey out_pk{};
     rct::ecdhTuple ecdh{};
 
@@ -727,7 +687,7 @@ namespace tx {
         rsig_buff = rsig_data.rsig();
       }
 
-      if (client_version() >= 1 && rsig_data.has_mask()){
+      if (rsig_data.has_mask()){
         rct::key cmask{};
         string_to_key(cmask, rsig_data.mask());
         m_ct.rsig_gamma.emplace_back(cmask);
@@ -751,22 +711,32 @@ namespace tx {
       memcpy(ecdh.amount.bytes, ack->ecdh_info().data(), 8);
     }
 
-    if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){
-      throw exc::ProtocolException("Cannot deserialize bulletproof rangesig");
-    }
-
     m_ct.tx.vout.emplace_back(tx_out);
     m_ct.tx_out_hmacs.push_back(ack->vouti_hmac());
     m_ct.tx_out_pk.emplace_back(out_pk);
     m_ct.tx_out_ecdh.emplace_back(ecdh);
 
-    // ClientV0, if no rsig was generated on Trezor, do not continue.
-    // ClientV1+ generates BP after all masks in the current batch are generated
-    if (!has_rsig || (client_version() >= 1 && is_offloading())){
+    rsig_v bp_obj{};
+    if (has_rsig) {
+      bool deserialize_success;
+      if (is_req_bulletproof_plus()) {
+        deserialize_success = cn_deserialize(rsig_buff, bproof_plus);
+        bp_obj = bproof_plus;
+      } else {
+        deserialize_success = cn_deserialize(rsig_buff, bproof);
+        bp_obj = bproof;
+      }
+      if (!deserialize_success) {
+        throw exc::ProtocolException("Cannot deserialize bulletproof rangesig");
+      }
+    }
+
+    // Generates BP after all masks in the current batch are generated
+    if (!has_rsig || is_offloading()){
       return;
     }
 
-    process_bproof(bproof);
+    process_bproof(bp_obj);
     m_ct.cur_batch_idx += 1;
     m_ct.cur_output_in_batch_idx = 0;
   }
@@ -791,13 +761,21 @@ namespace tx {
       masks.push_back(m_ct.rsig_gamma[bidx]);
     }
 
-    auto bp = bulletproof_PROVE(amounts, masks);
-    auto serRsig = cn_serialize(bp);
-    m_ct.tx_out_rsigs.emplace_back(bp);
+    std::string serRsig;
+    if (is_req_bulletproof_plus()) {
+      auto bp = bulletproof_plus_PROVE(amounts, masks);
+      serRsig = cn_serialize(bp);
+      m_ct.tx_out_rsigs.emplace_back(bp);
+    } else {
+      auto bp = bulletproof_PROVE(amounts, masks);
+      serRsig = cn_serialize(bp);
+      m_ct.tx_out_rsigs.emplace_back(bp);
+    }
+
     rsig_data.set_rsig(serRsig);
   }
 
-  void Signer::process_bproof(rct::Bulletproof & bproof){
+  void Signer::process_bproof(rsig_v & bproof){
     CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index");
     auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx];
     for (size_t i = 0; i < batch_size; ++i){
@@ -806,12 +784,22 @@ namespace tx {
 
       rct::key commitment = m_ct.tx_out_pk[bidx].mask;
       commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT);
-      bproof.V.push_back(commitment);
+      if (is_req_bulletproof_plus()) {
+        boost::get<rct::BulletproofPlus>(bproof).V.push_back(commitment);
+      } else {
+        boost::get<rct::Bulletproof>(bproof).V.push_back(commitment);
+      }
     }
 
     m_ct.tx_out_rsigs.emplace_back(bproof);
-    if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) {
-      throw exc::ProtocolException("Returned range signature is invalid");
+    if (is_req_bulletproof_plus()) {
+      if (!rct::bulletproof_plus_VERIFY(boost::get<rct::BulletproofPlus>(m_ct.tx_out_rsigs.back()))) {
+        throw exc::ProtocolException("Returned range signature is invalid");
+      }
+    } else {
+      if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) {
+        throw exc::ProtocolException("Returned range signature is invalid");
+      }
     }
   }
 
@@ -840,6 +828,7 @@ namespace tx {
   }
 
   void Signer::step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev){
+    CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported");
     m_ct.rv = std::make_shared<rct::rctSig>();
     m_ct.rv->txnFee = ack->rv().txn_fee();
     m_ct.rv->type = static_cast<uint8_t>(ack->rv().rv_type());
@@ -864,24 +853,15 @@ namespace tx {
 
     // RctSig
     auto num_sources = m_ct.tx_data.sources.size();
-    if (is_simple() || is_req_bulletproof()){
-      auto dst = &m_ct.rv->pseudoOuts;
-      if (is_bulletproof()){
-        dst = &m_ct.rv->p.pseudoOuts;
-      }
-
-      dst->clear();
-      for (const auto &pseudo_out : m_ct.pseudo_outs) {
-        dst->emplace_back();
-        string_to_key(dst->back(), pseudo_out);
-      }
-
-      m_ct.rv->mixRing.resize(num_sources);
-    } else {
-      m_ct.rv->mixRing.resize(m_ct.tsx_data.mixin());
-      m_ct.rv->mixRing[0].resize(num_sources);
+    auto dst = &m_ct.rv->p.pseudoOuts;
+    dst->clear();
+    for (const auto &pseudo_out : m_ct.pseudo_outs) {
+      dst->emplace_back();
+      string_to_key(dst->back(), pseudo_out);
     }
 
+    m_ct.rv->mixRing.resize(num_sources);
+
     CHECK_AND_ASSERT_THROW_MES(m_ct.tx_out_pk.size() == m_ct.tx_out_ecdh.size(), "Invalid vector sizes");
     for(size_t i = 0; i < m_ct.tx_out_ecdh.size(); ++i){
       m_ct.rv->outPk.push_back(m_ct.tx_out_pk[i]);
@@ -889,10 +869,10 @@ namespace tx {
     }
 
     for(size_t i = 0; i < m_ct.tx_out_rsigs.size(); ++i){
-      if (is_bulletproof()){
-        m_ct.rv->p.bulletproofs.push_back(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs[i]));
+      if (is_req_bulletproof_plus()) {
+        m_ct.rv->p.bulletproofs_plus.push_back(boost::get<rct::BulletproofPlus>(m_ct.tx_out_rsigs[i]));
       } else {
-        m_ct.rv->p.rangeSigs.push_back(boost::get<rct::rangeSig>(m_ct.tx_out_rsigs[i]));
+        m_ct.rv->p.bulletproofs.push_back(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs[i]));
       }
     }
 
@@ -936,8 +916,8 @@ namespace tx {
   void Signer::step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack){
     m_ct.signatures.push_back(ack->signature());
 
-    // Sync updated pseudo_outputs, client_version>=1, HF10+
-    if (client_version() >= 1 && ack->has_pseudo_out()){
+    // Sync updated pseudo_outputs
+    if (ack->has_pseudo_out()){
       CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.pseudo_outs.size(), "Invalid pseudo-out index");
       m_ct.pseudo_outs[m_ct.cur_input_idx] = ack->pseudo_out();
       if (is_bulletproof()){
@@ -955,6 +935,8 @@ namespace tx {
   }
 
   void Signer::step_final_ack(std::shared_ptr<const messages::monero::MoneroTransactionFinalAck> ack){
+    CHECK_AND_ASSERT_THROW_MES(is_clsag(), "Only CLSAGs signatures are supported");
+
     if (m_multisig){
       auto & cout_key = ack->cout_key();
       for(auto & cur : m_ct.couts){
@@ -975,47 +957,34 @@ namespace tx {
     m_ct.enc_keys = ack->tx_enc_keys();
 
     // Opening the sealed signatures
-    if (client_version() >= 3){
-      if(!ack->has_opening_key()){
-        throw exc::ProtocolException("Client version 3+ requires sealed signatures");
-      }
-
-      for(size_t i = 0; i < m_ct.signatures.size(); ++i){
-        CHECK_AND_ASSERT_THROW_MES(m_ct.signatures[i].size() > crypto::chacha::TAG_SIZE, "Invalid signature size");
-        std::string nonce = compute_sealing_key(ack->opening_key(), i, true);
-        std::string key = compute_sealing_key(ack->opening_key(), i, false);
-        size_t plen = m_ct.signatures[i].size() - crypto::chacha::TAG_SIZE;
-        std::unique_ptr<uint8_t[]> plaintext(new uint8_t[plen]);
-        uint8_t * buff = plaintext.get();
-
-        protocol::crypto::chacha::decrypt(
-            m_ct.signatures[i].data(),
-            m_ct.signatures[i].size(),
-            reinterpret_cast<const uint8_t *>(key.data()),
-            reinterpret_cast<const uint8_t *>(nonce.data()),
-            reinterpret_cast<char *>(buff), &plen);
-        m_ct.signatures[i].assign(reinterpret_cast<const char *>(buff), plen);
-      }
+    if(!ack->has_opening_key()){
+      throw exc::ProtocolException("Client version 3+ requires sealed signatures");
     }
 
-    if (m_ct.rv->type == rct::RCTTypeCLSAG){
-      m_ct.rv->p.CLSAGs.reserve(m_ct.signatures.size());
-      for (size_t i = 0; i < m_ct.signatures.size(); ++i) {
-        rct::clsag clsag;
-        if (!cn_deserialize(m_ct.signatures[i], clsag)) {
-          throw exc::ProtocolException("Cannot deserialize clsag[i]");
-        }
-        m_ct.rv->p.CLSAGs.push_back(clsag);
-      }
-    } else {
-      m_ct.rv->p.MGs.reserve(m_ct.signatures.size());
-      for (size_t i = 0; i < m_ct.signatures.size(); ++i) {
-        rct::mgSig mg;
-        if (!cn_deserialize(m_ct.signatures[i], mg)) {
-          throw exc::ProtocolException("Cannot deserialize mg[i]");
-        }
-        m_ct.rv->p.MGs.push_back(mg);
+    for(size_t i = 0; i < m_ct.signatures.size(); ++i){
+      CHECK_AND_ASSERT_THROW_MES(m_ct.signatures[i].size() > crypto::chacha::TAG_SIZE, "Invalid signature size");
+      std::string nonce = compute_sealing_key(ack->opening_key(), i, true);
+      std::string key = compute_sealing_key(ack->opening_key(), i, false);
+      size_t plen = m_ct.signatures[i].size() - crypto::chacha::TAG_SIZE;
+      std::unique_ptr<uint8_t[]> plaintext(new uint8_t[plen]);
+      uint8_t * buff = plaintext.get();
+
+      protocol::crypto::chacha::decrypt(
+          m_ct.signatures[i].data(),
+          m_ct.signatures[i].size(),
+          reinterpret_cast<const uint8_t *>(key.data()),
+          reinterpret_cast<const uint8_t *>(nonce.data()),
+          reinterpret_cast<char *>(buff), &plen);
+      m_ct.signatures[i].assign(reinterpret_cast<const char *>(buff), plen);
+    }
+
+    m_ct.rv->p.CLSAGs.reserve(m_ct.signatures.size());
+    for (size_t i = 0; i < m_ct.signatures.size(); ++i) {
+      rct::clsag clsag;
+      if (!cn_deserialize(m_ct.signatures[i], clsag)) {
+        throw exc::ProtocolException("Cannot deserialize clsag[i]");
       }
+      m_ct.rv->p.CLSAGs.push_back(clsag);
     }
 
     m_ct.tx.rct_signatures = *(m_ct.rv);

src/device_trezor/trezor/protocol.hpp

@@ -116,8 +116,7 @@ namespace ki {
    */
   bool key_image_data(wallet_shim * wallet,
                       const std::vector<tools::wallet2::transfer_details> & transfers,
-                      std::vector<MoneroTransferDetails> & res,
-                      bool need_all_additionals=false);
+                      std::vector<MoneroTransferDetails> & res);
 
   /**
    * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync.
@@ -129,8 +128,7 @@ namespace ki {
    */
   void generate_commitment(std::vector<MoneroTransferDetails> & mtds,
                            const std::vector<tools::wallet2::transfer_details> & transfers,
-                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req,
-                           bool need_subaddr_indices=false);
+                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req);
 
   /**
    * Processes Live refresh step response, parses KI, checks the signature
@@ -166,7 +164,7 @@ namespace tx {
   ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt);
   std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv=false);
 
-  typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v;
+  typedef boost::variant<rct::Bulletproof, rct::BulletproofPlus> rsig_v;
 
   /**
    * Transaction signer state holder.
@@ -232,8 +230,8 @@ namespace tx {
     }
 
     const tools::wallet2::transfer_details & get_transfer(size_t idx) const {
-      CHECK_AND_ASSERT_THROW_MES(idx < m_unsigned_tx->transfers.second.size() + m_unsigned_tx->transfers.first && idx >= m_unsigned_tx->transfers.first, "Invalid transfer index");
-      return m_unsigned_tx->transfers.second[idx - m_unsigned_tx->transfers.first];
+      CHECK_AND_ASSERT_THROW_MES(idx < std::get<2>(m_unsigned_tx->transfers).size() + std::get<0>(m_unsigned_tx->transfers) && idx >= std::get<0>(m_unsigned_tx->transfers), "Invalid transfer index");
+      return std::get<2>(m_unsigned_tx->transfers)[idx - std::get<0>(m_unsigned_tx->transfers)];
     }
 
     const tools::wallet2::transfer_details & get_source_transfer(size_t idx) const {
@@ -247,7 +245,7 @@ namespace tx {
     void compute_integrated_indices(TsxData * tsx_data);
     bool should_compute_bp_now() const;
     void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data);
-    void process_bproof(rct::Bulletproof & bproof);
+    void process_bproof(rsig_v & bproof);
     void set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys=false, bool need_ring_indices=false);
 
   public:
@@ -260,8 +258,6 @@ namespace tx {
     void step_set_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetInputAck> ack);
 
     void sort_ki();
-    std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> step_permutation();
-    void step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack);
 
     std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> step_set_vini_input(size_t idx);
     void step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack);
@@ -290,11 +286,15 @@ namespace tx {
       return m_client_version;
     }
 
-    bool is_simple() const {
+    uint8_t get_rv_type() const {
       if (!m_ct.rv){
         throw std::invalid_argument("RV not initialized");
       }
-      auto tp = m_ct.rv->type;
+      return m_ct.rv->type;
+    }
+
+    bool is_simple() const {
+      auto tp = get_rv_type();
       return tp == rct::RCTTypeSimple;
     }
 
@@ -302,12 +302,27 @@ namespace tx {
       return m_ct.tx_data.rct_config.range_proof_type != rct::RangeProofBorromean;
     }
 
+    bool is_req_clsag() const {
+      return is_req_bulletproof() && m_ct.tx_data.rct_config.bp_version >= 3;
+    }
+
+    bool is_req_bulletproof_plus() const {
+      return is_req_bulletproof() && m_ct.tx_data.rct_config.bp_version == 4;  // rct::genRctSimple
+    }
+
     bool is_bulletproof() const {
-      if (!m_ct.rv){
-        throw std::invalid_argument("RV not initialized");
-      }
-      auto tp = m_ct.rv->type;
-      return tp == rct::RCTTypeBulletproof || tp == rct::RCTTypeBulletproof2 || tp == rct::RCTTypeCLSAG;
+      auto tp = get_rv_type();
+      return rct::is_rct_bulletproof(tp) || rct::is_rct_bulletproof_plus(tp);
+    }
+
+    bool is_bulletproof_plus() const {
+      auto tp = get_rv_type();
+      return rct::is_rct_bulletproof_plus(tp);
+    }
+
+    bool is_clsag() const {
+      auto tp = get_rv_type();
+      return rct::is_rct_clsag(tp);
     }
 
     bool is_offloading() const {

src/gen_multisig/gen_multisig.cpp

@@ -50,7 +50,6 @@
 using namespace std;
 using namespace epee;
 using namespace cryptonote;
-using boost::lexical_cast;
 namespace po = boost::program_options;
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -84,6 +83,9 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str
 
   try
   {
+    if (total == 0)
+      throw std::runtime_error("Signer group of size 0 is not allowed.");
+
     // create M wallets first
     std::vector<boost::shared_ptr<tools::wallet2>> wallets(total);
     for (size_t n = 0; n < total; ++n)
@@ -118,13 +120,17 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str
       ss << "  " << name << std::endl;
     }
 
-    //exchange keys unless exchange_multisig_keys returns no extra info
-    while (!kex_msgs_intermediate[0].empty())
+    // exchange keys until the wallets are done
+    bool ready{false};
+    wallets[0]->multisig(&ready);
+    while (!ready)
     {
       for (size_t n = 0; n < total; ++n)
       {
           kex_msgs_intermediate[n] = wallets[n]->exchange_multisig_keys(pwd_container->password(), kex_msgs_intermediate);
       }
+
+      wallets[0]->multisig(&ready);
     }
 
     std::string address = wallets[0]->get_account().get_public_address_str(wallets[0]->nettype());

src/multisig/multisig_account.cpp

@@ -127,7 +127,7 @@ namespace multisig
   bool multisig_account::multisig_is_ready() const
   {
     if (main_kex_rounds_done())
-      return m_kex_rounds_complete >= multisig_kex_rounds_required(m_signers.size(), m_threshold) + 1;
+      return m_kex_rounds_complete >= multisig_setup_rounds_required(m_signers.size(), m_threshold);
     else
       return false;
   }
@@ -175,19 +175,20 @@ namespace multisig
     // only mutate account if update succeeds
     multisig_account temp_account{*this};
     temp_account.set_multisig_config(threshold, std::move(signers));
-    temp_account.kex_update_impl(expanded_msgs_rnd1);
+    temp_account.kex_update_impl(expanded_msgs_rnd1, false);
     *this = std::move(temp_account);
   }
   //----------------------------------------------------------------------------------------------------------------------
   // multisig_account: EXTERNAL
   //----------------------------------------------------------------------------------------------------------------------
-  void multisig_account::kex_update(const std::vector<multisig_kex_msg> &expanded_msgs)
+  void multisig_account::kex_update(const std::vector<multisig_kex_msg> &expanded_msgs,
+    const bool force_update_use_with_caution /*= false*/)
   {
     CHECK_AND_ASSERT_THROW_MES(account_is_active(), "multisig account: tried to update kex, but kex isn't initialized yet.");
     CHECK_AND_ASSERT_THROW_MES(!multisig_is_ready(), "multisig account: tried to update kex, but kex is already complete.");
 
     multisig_account temp_account{*this};
-    temp_account.kex_update_impl(expanded_msgs);
+    temp_account.kex_update_impl(expanded_msgs, force_update_use_with_caution);
     *this = std::move(temp_account);
   }
   //----------------------------------------------------------------------------------------------------------------------
@@ -200,4 +201,11 @@ namespace multisig
     return num_signers - threshold + 1;
   }
   //----------------------------------------------------------------------------------------------------------------------
+  // EXTERNAL
+  //----------------------------------------------------------------------------------------------------------------------
+  std::uint32_t multisig_setup_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold)
+  {
+    return multisig_kex_rounds_required(num_signers, threshold) + 1;
+  }
+  //----------------------------------------------------------------------------------------------------------------------
 } //namespace multisig

src/multisig/multisig_account.h

@@ -169,12 +169,20 @@ namespace multisig
     *    - The main interface for multisig key exchange, this handles all the work of processing input messages,
     *      creating new messages for new rounds, and finalizing the multisig shared public key when kex is complete.
     * param: expanded_msgs - kex messages corresponding to the account's 'in progress' round
+    * param: force_update_use_with_caution - try to force the account to update with messages from an incomplete signer set.
+    *    - If this is the post-kex verification round, only require one input message.
+    *      - Force updating here should only be done if we can safely assume an honest signer subgroup of size 'threshold'
+    *        will complete the account.
+    *    - If this is an intermediate round, only require messages from 'num signers - 1 - (round - 1)' other signers.
+    *      - If force updating with maliciously-crafted messages, the resulting account will be invalid (either unable
+    *        to complete signatures, or a 'hostage' to the malicious signer [i.e. can't sign without his participation]).
     */
-    void kex_update(const std::vector<multisig_kex_msg> &expanded_msgs);
+    void kex_update(const std::vector<multisig_kex_msg> &expanded_msgs,
+      const bool force_update_use_with_caution = false);
 
   private:
     // implementation of kex_update() (non-transactional)
-    void kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs);
+    void kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs, const bool incomplete_signer_set);
     /**
     * brief: initialize_kex_update - Helper for kex_update_impl()
     *    - Collect the local signer's shared keys to ignore in incoming messages, build the aggregate ancillary key
@@ -245,4 +253,13 @@ namespace multisig
   * return: number of kex rounds required
   */
   std::uint32_t multisig_kex_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold);
+
+  /**
+  * brief: multisig_setup_rounds_required - The number of setup rounds required to produce an M-of-N shared key.
+  *    - A participant must complete all kex rounds and 1 initialization round.
+  * param: num_signers - number of participants in multisig (N)
+  * param: threshold - threshold of multisig (M)
+  * return: number of setup rounds required
+  */
+  std::uint32_t multisig_setup_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold);
 } //namespace multisig

src/multisig/multisig_account_kex_impl.cpp

@@ -74,7 +74,7 @@ namespace multisig
       "Multisig threshold may not be larger than number of signers.");
     CHECK_AND_ASSERT_THROW_MES(threshold > 0, "Multisig threshold must be > 0.");
     CHECK_AND_ASSERT_THROW_MES(round > 0, "Multisig kex round must be > 0.");
-    CHECK_AND_ASSERT_THROW_MES(round <= multisig_kex_rounds_required(num_signers, threshold) + 1,
+    CHECK_AND_ASSERT_THROW_MES(round <= multisig_setup_rounds_required(num_signers, threshold),
       "Trying to process multisig kex for an invalid round.");
   }
   //----------------------------------------------------------------------------------------------------------------------
@@ -181,7 +181,8 @@ namespace multisig
   *    Key aggregation via aggregation coefficients prevents key cancellation attacks.
   *    See: https://www.getmonero.org/resources/research-lab/pubs/MRL-0009.pdf
   * param: final_keys - address components (public keys) obtained from other participants (not shared with local)
-  * param: privkeys_inout - private keys of address components known by local; each key will be multiplied by an aggregation coefficient (return by reference)
+  * param: privkeys_inout - private keys of address components known by local; each key will be multiplied by an aggregation
+  *                         coefficient (return by reference)
   * return: final multisig public spend key for the account
   */
   //----------------------------------------------------------------------------------------------------------------------
@@ -199,7 +200,8 @@ namespace multisig
     for (std::size_t multisig_keys_index{0}; multisig_keys_index < privkeys_inout.size(); ++multisig_keys_index)
     {
       crypto::public_key pubkey;
-      CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(privkeys_inout[multisig_keys_index], pubkey), "Failed to derive public key");
+      CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(privkeys_inout[multisig_keys_index], pubkey),
+        "Failed to derive public key");
 
       own_keys_mapping[pubkey] = multisig_keys_index;
 
@@ -307,8 +309,7 @@ namespace multisig
   * INTERNAL
   *
   * brief: multisig_kex_msgs_sanitize_pubkeys - Sanitize multisig kex messages.
-  *    - Removes duplicates from msg pubkeys, ignores pubkeys equal to the local account's signing key,
-  *      ignores messages signed by the local account, ignores keys found in input 'exclusion set',
+  *    - Removes duplicates from msg pubkeys, ignores keys found in input 'exclusion set',
   *      constructs map of pubkey:origins.
   *    - Requires that all input msgs have the same round number.
   *
@@ -316,15 +317,13 @@ namespace multisig
   *
   *    - If the messages' round numbers are all '1', then only the message signing pubkey is considered
   *      'recommended'. Furthermore, the 'exclusion set' is ignored.
-  * param: own_pubkey - local account's signing key (key used to sign multisig messages)
   * param: expanded_msgs - set of multisig kex messages to process
   * param: exclude_pubkeys - pubkeys to exclude from output set
   * outparam: sanitized_pubkeys_out - processed pubkeys obtained from msgs, mapped to their origins
   * return: round number shared by all input msgs
   */
   //----------------------------------------------------------------------------------------------------------------------
-  static std::uint32_t multisig_kex_msgs_sanitize_pubkeys(const crypto::public_key &own_pubkey,
-    const std::vector<multisig_kex_msg> &expanded_msgs,
+  static std::uint32_t multisig_kex_msgs_sanitize_pubkeys(const std::vector<multisig_kex_msg> &expanded_msgs,
     const std::vector<crypto::public_key> &exclude_pubkeys,
     multisig_keyset_map_memsafe_t &sanitized_pubkeys_out)
   {
@@ -339,10 +338,6 @@ namespace multisig
     // - origins = all the signing pubkeys that recommended a given msg pubkey
     for (const auto &expanded_msg : expanded_msgs)
     {
-      // ignore messages from self
-      if (expanded_msg.get_signing_pubkey() == own_pubkey)
-        continue;
-
       // in round 1, only the signing pubkey is treated as a msg pubkey
       if (round == 1)
       {
@@ -355,10 +350,6 @@ namespace multisig
         // copy all pubkeys from message into list
         for (const auto &pubkey : expanded_msg.get_msg_pubkeys())
         {
-          // ignore own pubkey
-          if (pubkey == own_pubkey)
-            continue;
-
           // ignore pubkeys in 'ignore' set
           if (std::find(exclude_pubkeys.begin(), exclude_pubkeys.end(), pubkey) != exclude_pubkeys.end())
             continue;
@@ -375,6 +366,31 @@ namespace multisig
   /**
   * INTERNAL
   *
+  * brief: remove_key_from_mapped_sets - Remove a specified key from the mapped sets in a multisig keyset map.
+  * param: key_to_remove - specified key to remove
+  * inoutparam: keyset_inout - keyset to update
+  */
+  //----------------------------------------------------------------------------------------------------------------------
+  static void remove_key_from_mapped_sets(const crypto::public_key &key_to_remove,
+    multisig_keyset_map_memsafe_t &keyset_inout)
+  {
+    // remove specified key from each mapped set
+    for (auto keyset_it = keyset_inout.begin(); keyset_it != keyset_inout.end();)
+    {
+      // remove specified key from this set
+      keyset_it->second.erase(key_to_remove);
+
+      // remove empty keyset positions or increment iterator
+      if (keyset_it->second.size() == 0)
+        keyset_it = keyset_inout.erase(keyset_it);
+      else
+        ++keyset_it;
+    }
+  }
+  //----------------------------------------------------------------------------------------------------------------------
+  /**
+  * INTERNAL
+  *
   * brief: evaluate_multisig_kex_round_msgs - Evaluate pubkeys from a kex round in order to prepare for the next round.
   *    - Sanitizes input msgs.
   *    - Require uniqueness in: 'exclude_pubkeys'.
@@ -392,6 +408,8 @@ namespace multisig
   * param: signers - expected participants in multisig kex
   * param: expanded_msgs - set of multisig kex messages to process
   * param: exclude_pubkeys - derivations held by the local account corresponding to round 'expected_round'
+  * param: incomplete_signer_set - only require the minimum number of signers to complete this round
+  *                                minimum = num_signers - (round num - 1)   (including local signer)
   * return: fully sanitized and validated pubkey:origins map for building the account's next kex round message
   */
   //----------------------------------------------------------------------------------------------------------------------
@@ -400,7 +418,8 @@ namespace multisig
     const std::uint32_t expected_round,
     const std::vector<crypto::public_key> &signers,
     const std::vector<multisig_kex_msg> &expanded_msgs,
-    const std::vector<crypto::public_key> &exclude_pubkeys)
+    const std::vector<crypto::public_key> &exclude_pubkeys,
+    const bool incomplete_signer_set)
   {
     // exclude_pubkeys should all be unique
     for (auto it = exclude_pubkeys.begin(); it != exclude_pubkeys.end(); ++it)
@@ -410,21 +429,31 @@ namespace multisig
     }
 
     // sanitize input messages
-    multisig_keyset_map_memsafe_t pubkey_origins_map;
-    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(base_pubkey, expanded_msgs, exclude_pubkeys, pubkey_origins_map);
+    multisig_keyset_map_memsafe_t pubkey_origins_map;  //map: [pubkey : [origins]]
+    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(expanded_msgs, exclude_pubkeys, pubkey_origins_map);
     CHECK_AND_ASSERT_THROW_MES(round == expected_round,
       "Kex messages were for round [" << round << "], but expected round is [" << expected_round << "]");
 
+    // remove the local signer from each origins set in the sanitized pubkey map
+    // note: intermediate kex rounds only need keys from other signers to make progress (keys from self are useless)
+    remove_key_from_mapped_sets(base_pubkey, pubkey_origins_map);
+
     // evaluate pubkeys collected
-    std::unordered_map<crypto::public_key, std::unordered_set<crypto::public_key>> origin_pubkeys_map;
+    std::unordered_map<crypto::public_key, std::unordered_set<crypto::public_key>> origin_pubkeys_map;  //map: [origin: [pubkeys]]
 
     // 1. each pubkey should be recommended by a precise number of signers
+    const std::size_t num_recommendations_per_pubkey_required{
+        incomplete_signer_set
+        ? 1
+        : round
+      };
+
     for (const auto &pubkey_and_origins : pubkey_origins_map)
     {
       // expected amount = round_num
       // With each successive round, pubkeys are shared by incrementally larger groups,
       //  starting at 1 in round 1 (i.e. the local multisig key to start kex with).
-      CHECK_AND_ASSERT_THROW_MES(pubkey_and_origins.second.size() == round,
+      CHECK_AND_ASSERT_THROW_MES(pubkey_and_origins.second.size() >= num_recommendations_per_pubkey_required,
         "A pubkey recommended by multisig kex messages had an unexpected number of recommendations.");
 
       // map (sanitized) pubkeys back to origins
@@ -433,8 +462,18 @@ namespace multisig
     }
 
     // 2. the number of unique signers recommending pubkeys should equal the number of signers passed in (minus the local signer)
-    CHECK_AND_ASSERT_THROW_MES(origin_pubkeys_map.size() == signers.size() - 1,
-      "Number of unique other signers does not equal number of other signers that recommended pubkeys.");
+    // - if an incomplete set is allowed, then we need at least one signer to represent each subgroup in this round that
+    //   doesn't include the local signer
+    const std::size_t num_signers_required{
+        incomplete_signer_set
+        ? signers.size() - 1 - (round - 1)
+        : signers.size() - 1
+      };
+
+    CHECK_AND_ASSERT_THROW_MES(origin_pubkeys_map.size() >= num_signers_required,
+      "Number of unique other signers recommending pubkeys does not equal number of required other signers "
+      "(kex round: " << round << ", num signers found: " << origin_pubkeys_map.size() << ", num signers required: " <<
+      num_signers_required << ").");
 
     // 3. each origin should recommend a precise number of pubkeys
 
@@ -461,19 +500,20 @@ namespace multisig
 
     // other signers: (N - 2) choose (msg_round_num - 1)
       // - Each signer recommends keys they share with other signers.
-      // - In each round, a signer shares a key with 'round num - 1' other signers.
-      // - Since 'origins pubkey map' excludes keys shared with the local account,
-      //   only keys shared with participants 'other than local and self' will be in the map (e.g. N - 2 signers).
-      // - So other signers will recommend (N - 2) choose (msg_round_num - 1) pubkeys (after removing keys shared with local).
-      // - Each origin should have a shared key with each group of size 'round - 1'.
-      // Note: Keys shared with local are ignored to facilitate kex round boosting, where one or more signers may
+      // - In each round, every group of size 'round num' will have a key. From a single signer's perspective,
+      //   they will share a key with every group of size 'round num - 1' of other signers.
+      // - Since 'origins pubkey map' excludes keys shared with the local account, only keys shared with participants
+      //   'other than local and self' will be in the map (e.g. N - 2 signers).
+      // - Other signers will recommend (N - 2) choose (msg_round_num - 1) pubkeys (after removing keys shared with local).
+      // Note: Keys shared with local are filtered out to facilitate kex round boosting, where one or more signers may
       //       have boosted the local signer (implying they didn't have access to the local signer's previous round msg).
     const std::uint32_t expected_recommendations_others = n_choose_k_f(signers.size() - 2, round - 1);
 
     // local: (N - 1) choose (msg_round_num - 1)
     const std::uint32_t expected_recommendations_self = n_choose_k_f(signers.size() - 1, round - 1);
 
-    // note: expected_recommendations_others would be 0 in the last round of 1-of-N, but we return early for that case
+    // note: expected_recommendations_others would be 0 in the last round of 1-of-N, but we don't call this function for
+    //       that case
     CHECK_AND_ASSERT_THROW_MES(expected_recommendations_self > 0 && expected_recommendations_others > 0,
       "Bad num signers or round num (possibly numerical limits exceeded).");
 
@@ -485,7 +525,7 @@ namespace multisig
     for (const auto &origin_and_pubkeys : origin_pubkeys_map)
     {
       CHECK_AND_ASSERT_THROW_MES(origin_and_pubkeys.second.size() == expected_recommendations_others,
-        "A pubkey recommended by multisig kex messages had an unexpected number of recommendations.");
+        "A multisig signer recommended an unexpected number of pubkeys.");
 
       // 2 (continued). only expected signers should be recommending keys
       CHECK_AND_ASSERT_THROW_MES(std::find(signers.begin(), signers.end(), origin_and_pubkeys.first) != signers.end(),
@@ -507,6 +547,7 @@ namespace multisig
   * param: expected_round - expected kex round of input messages
   * param: signers - expected participants in multisig kex
   * param: expanded_msgs - set of multisig kex messages to process
+  * param: incomplete_signer_set - only require the minimum amount of messages to complete this round (1 message)
   * return: sanitized and validated pubkey:origins map
   */
   //----------------------------------------------------------------------------------------------------------------------
@@ -514,15 +555,20 @@ namespace multisig
     const crypto::public_key &base_pubkey,
     const std::uint32_t expected_round,
     const std::vector<crypto::public_key> &signers,
-    const std::vector<multisig_kex_msg> &expanded_msgs)
+    const std::vector<multisig_kex_msg> &expanded_msgs,
+    const bool incomplete_signer_set)
   {
     // sanitize input messages
     const std::vector<crypto::public_key> dummy;
-    multisig_keyset_map_memsafe_t pubkey_origins_map;
-    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(base_pubkey, expanded_msgs, dummy, pubkey_origins_map);
+    multisig_keyset_map_memsafe_t pubkey_origins_map;  //map: [pubkey : [origins]]
+    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(expanded_msgs, dummy, pubkey_origins_map);
     CHECK_AND_ASSERT_THROW_MES(round == expected_round,
       "Kex messages were for round [" << round << "], but expected round is [" << expected_round << "]");
 
+    // note: do NOT remove the local signer from the pubkey origins map, since the post-kex round can be force-updated with
+    //       just the local signer's post-kex message (if the local signer were removed, then the post-kex message's pubkeys
+    //       would be completely lost)
+
     // evaluate pubkeys collected
 
     // 1) there should only be two pubkeys
@@ -533,17 +579,26 @@ namespace multisig
     CHECK_AND_ASSERT_THROW_MES(pubkey_origins_map.begin()->second == (++(pubkey_origins_map.begin()))->second,
       "Multisig post-kex round messages from other signers did not all recommend the same pubkey pair.");
 
-    // 3) all signers should be present in the recommendation list
+    // 3) all signers should be present in the recommendation list (unless an incomplete list is permitted)
     auto origins = pubkey_origins_map.begin()->second;
-    origins.insert(base_pubkey);  //add self
+    origins.insert(base_pubkey);  //add self if missing
 
-    CHECK_AND_ASSERT_THROW_MES(origins.size() == signers.size(),
-      "Multisig post-kex round message origins don't line up with multisig signer set.");
+    const std::size_t num_signers_required{
+        incomplete_signer_set
+        ? 1
+        : signers.size()
+      };
 
-    for (const crypto::public_key &signer : signers)
+    CHECK_AND_ASSERT_THROW_MES(origins.size() >= num_signers_required,
+      "Multisig post-kex round message origins don't line up with multisig signer set "
+      "(num signers found: " << origins.size() << ", num signers required: " << num_signers_required << ").");
+
+    for (const crypto::public_key &origin : origins)
     {
-      CHECK_AND_ASSERT_THROW_MES(origins.find(signer) != origins.end(),
-        "Could not find an expected signer in multisig post-kex round messages (all signers expected).");
+      // note: if num_signers_required == signers.size(), then this test will ensure all signers are present in 'origins',
+      //       which contains only unique pubkeys
+      CHECK_AND_ASSERT_THROW_MES(std::find(signers.begin(), signers.end(), origin) != signers.end(),
+        "An unknown origin recommended a multisig post-kex verification messsage.");
     }
 
     return pubkey_origins_map;
@@ -564,6 +619,7 @@ namespace multisig
   * param: expanded_msgs - set of multisig kex messages to process
   * param: exclude_pubkeys - keys held by the local account corresponding to round 'current_round'
   *    - If 'current_round' is the final round, these are the local account's shares of the final aggregate key.
+  * param: incomplete_signer_set - allow messages from an incomplete signer set
   * outparam: keys_to_origins_map_out - map between round keys and identity keys
   *    - If in the final round, these are key shares recommended by other signers for the final aggregate key.
   *    - Otherwise, these are the local account's DH derivations for the next round.
@@ -578,6 +634,7 @@ namespace multisig
     const std::vector<crypto::public_key> &signers,
     const std::vector<multisig_kex_msg> &expanded_msgs,
     const std::vector<crypto::public_key> &exclude_pubkeys,
+    const bool incomplete_signer_set,
     multisig_keyset_map_memsafe_t &keys_to_origins_map_out)
   {
     check_multisig_config(current_round, threshold, signers.size());
@@ -598,7 +655,8 @@ namespace multisig
         current_round,
         signers,
         expanded_msgs,
-        exclude_pubkeys);
+        exclude_pubkeys,
+        incomplete_signer_set);
     }
     else //(current_round == kex_rounds_required + 1)
     {
@@ -606,7 +664,8 @@ namespace multisig
       evaluated_pubkeys = evaluate_multisig_post_kex_round_msgs(base_pubkey,
         current_round,
         signers,
-        expanded_msgs);
+        expanded_msgs,
+        incomplete_signer_set);
     }
 
     // prepare keys-to-origins map for updating the multisig account
@@ -693,9 +752,9 @@ namespace multisig
     {
       // post-kex verification round: check that the multisig pubkey and common pubkey were recommended by other signers
       CHECK_AND_ASSERT_THROW_MES(result_keys_to_origins_map.count(m_multisig_pubkey) > 0,
-        "Multisig post-kex round: expected multisig pubkey wasn't found in other signers' messages.");
+        "Multisig post-kex round: expected multisig pubkey wasn't found in input messages.");
       CHECK_AND_ASSERT_THROW_MES(result_keys_to_origins_map.count(m_common_pubkey) > 0,
-        "Multisig post-kex round: expected common pubkey wasn't found in other signers' messages.");
+        "Multisig post-kex round: expected common pubkey wasn't found in input messages.");
 
       // save keys that should be recommended to other signers
       // - for convenience, re-recommend the post-kex verification message once an account is complete
@@ -790,7 +849,8 @@ namespace multisig
   //----------------------------------------------------------------------------------------------------------------------
   // multisig_account: INTERNAL
   //----------------------------------------------------------------------------------------------------------------------
-  void multisig_account::kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs)
+  void multisig_account::kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs,
+    const bool incomplete_signer_set)
   {
     // check messages are for the expected kex round
     check_messages_round(expanded_msgs, m_kex_rounds_complete + 1);
@@ -816,6 +876,7 @@ namespace multisig
       m_signers,
       expanded_msgs,
       exclude_pubkeys,
+      incomplete_signer_set,
       result_keys_to_origins_map);
 
     // finish account update

src/multisig/multisig_kex_msg.cpp

@@ -206,8 +206,13 @@ namespace multisig
   //----------------------------------------------------------------------------------------------------------------------
   void multisig_kex_msg::parse_and_validate_msg()
   {
+    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(),
+      "Multisig kex msg magic inconsistency.");
+    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() >= MULTISIG_KEX_V1_MAGIC.size(),
+      "Multisig kex msg magic inconsistency.");
+
     // check message type
-    CHECK_AND_ASSERT_THROW_MES(m_msg.size() > 0, "Kex message unexpectedly empty.");
+    CHECK_AND_ASSERT_THROW_MES(m_msg.size() >= MULTISIG_KEX_MSG_V2_MAGIC_1.size(), "Kex message unexpectedly small.");
     CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_V1_MAGIC.size()) != MULTISIG_KEX_V1_MAGIC,
       "V1 multisig kex messages are deprecated (unsafe).");
     CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_MSG_V1_MAGIC.size()) != MULTISIG_KEX_MSG_V1_MAGIC,
@@ -215,8 +220,6 @@ namespace multisig
 
     // deserialize the message
     std::string msg_no_magic;
-    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(),
-      "Multisig kex msg magic inconsistency.");
     CHECK_AND_ASSERT_THROW_MES(tools::base58::decode(m_msg.substr(MULTISIG_KEX_MSG_V2_MAGIC_1.size()), msg_no_magic),
       "Multisig kex msg decoding error.");
     binary_archive<false> b_archive{epee::strspan<std::uint8_t>(msg_no_magic)};

src/multisig/multisig_tx_builder_ringct.cpp

@@ -820,7 +820,6 @@ tx_builder_ringct_t::~tx_builder_ringct_t()
 bool tx_builder_ringct_t::init(
   const cryptonote::account_keys& account_keys,
   const std::vector<std::uint8_t>& extra,
-  const std::uint64_t unlock_time,
   const std::uint32_t subaddr_account,
   const std::set<std::uint32_t>& subaddr_minor_indices,
   std::vector<cryptonote::tx_source_entry>& sources,
@@ -854,7 +853,7 @@ bool tx_builder_ringct_t::init(
 
   // misc. fields
   unsigned_tx.version = 2;  //rct = 2
-  unsigned_tx.unlock_time = unlock_time;
+  unsigned_tx.unlock_time = 0;
 
   // sort inputs
   sort_sources(sources);