Berlin update

.gitignore

@@ -101,3 +101,4 @@ cache_*_db
 
 # log aggregation logs folder variable file, no sense to track it
 test/logs_aggregator/.env
+test/logs_aggregator/loki

.hmy/multibls/s0fn/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key

@@ -0,0 +1 @@
+dd11eabd17b257220f9523efd6524f12b7c1356f21667a87fe1b8c630bdf9f7d247087f85a8f661a7ef8ccde6e6287d54361aa2287fdf258321a1cc1d885b45a3b636a23e40aec0357dbd699c82a36e07fff9ca6fd85f79652320113

.hmy/multibls/s1fn/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key

@@ -0,0 +1 @@
+837cdffcc049ba89931d570e4be38bfef513df65d8dd277b0da8e62adf8c23b3c5229a2a4b36aa2c90c17d490dd5f95869365ebc4429d1efc784d25a64db246870d4bb4831d98ec02317ea88b76491a7fdfd2fa5e10f267e8f3989d9

Makefile

@@ -12,14 +12,14 @@ RPMBUILD=$(HOME)/rpmbuild
 DEBBUILD=$(HOME)/debbuild
 SHELL := bash
 
-.PHONY: all help libs exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker
+.PHONY: all help libs exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker go-vet go-test docker build_localnet_validator protofiles travis_go_checker travis_rpc_checker travis_rosetta_checker debug-start-log debug-stop-log debug-restart-log debug-delete-log
 
 all: libs
 	bash ./scripts/go_executable_build.sh -S
 
 help:
 	@echo "all - build the harmony binary & bootnode along with the MCL & BLS libs (if necessary)"
-	@echo "libs - build only the MCL & BLS libs (if necessary) "
+	@echo "libs - build only the MCL & BLS libs (if necessary)"
 	@echo "exe - build the harmony binary & bootnode"
 	@echo "race - build the harmony binary & bootnode with race condition checks"
 	@echo "trace-pointer - build the harmony binary & bootnode with pointer analysis"
@@ -55,10 +55,15 @@ help:
 	@echo "travis_rpc_checker - run the Travis RPC checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH"
 	@echo "travis_rosetta_checker - run the Travis Rosetta checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH"
 	@echo "debug_external - cleans up environment, rebuilds the binary, and deploys with external nodes"
+	@echo "debug-multi-bls - cleans up environment, rebuilds the binary, and deploys with external nodes in configuration 1 harmony process -> 2 validators"
 	@echo "build_localnet_validator - imports validator keys, funds validator accounts, waits for the epoch, and creates external validators on a local network"
-	@echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, needs docker compose and started localnet"
+	@echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, creates"\
+		"persistent volume to store parsed logs between localnet runs, needs docker compose and started localnet"
 	@echo "debug-stop-log - stops a docker compose Promtail->Loki->Grafana stack"
 	@echo "debug-restart-log - restart a docker compose Promtail->Loki->Grafana stack"
+	@echo "debug-delete-log - removes persistent volume for the Loki and host folder for it"
+	@echo "debug-multi-bls-multi-ext-node - start a localnet with multiple external nodes and multi-BLS configuration"
+	@echo "protofiles - generate Go code from protobuf files"
 
 libs:
 	make -C $(TOP)/mcl -j8
@@ -85,6 +90,7 @@ debug:
 debug-kill:
 	bash ./test/kill_node.sh
 	pkill -9 -f debug.sh
+	pkill -9 -f deploy.sh
 
 debug-ext:
 	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
@@ -101,7 +107,7 @@ debug-multi-bls:
 	echo sleep 10s before creating the external validator
 	sleep 10
 	bash ./test/build-localnet-validator.sh
-	
+
 debug-multi-bls-with-terminal:
 	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
 	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
@@ -111,7 +117,7 @@ debug-multi-bls-with-terminal:
 	bash ./test/build-localnet-validator.sh
 	screen -r localnet
 
-debug-multi-bls-multi-ext-node: pre-external
+debug-multi-bls-multi-ext-node:
 	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
 	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
 	./test/debug.sh ./test/configs/local-multi-bls-multi-ext-node.txt &
@@ -120,7 +126,7 @@ debug-multi-bls-multi-ext-node: pre-external
 	bash ./test/build-localnet-validator.sh
 
 clean:
-	rm -rf ./tmp_log*
+	rm -rf ./tmp_log/*
 	rm -rf ./.dht*
 	rm -rf ./db-*
 	rm -rf ./latest
@@ -232,9 +238,6 @@ travis_rpc_checker:
 travis_rosetta_checker:
 	bash ./scripts/travis_rosetta_checker.sh
 
-debug_external: clean
-	bash test/debug-external.sh
-
 build_localnet_validator:
 	bash test/build-localnet-validator.sh
 
@@ -248,3 +251,8 @@ debug-stop-log:
 	bash ./test/logs_aggregator/stop_log_aggregator.sh
 
 debug-restart-log: debug-stop-log debug-start-log
+
+debug-delete-log:
+	docker volume rm logs_aggregator_loki_data
+	@echo "[WARN] - it needs sudo to remove folder created with loki docker image user"
+	sudo rm -rf test/logs_aggregator/loki

README.md

@@ -25,14 +25,15 @@ brew install gmp
 brew install openssl
 sudo ln -sf /usr/local/opt/[email protected] /usr/local/opt/openssl
 ```
-On Linux (Ubuntu)
+On Linux (Ubuntu):
 ```bash
-sudo apt install libgmp-dev  libssl-dev  make gcc g++
+sudo apt install libgmp-dev libssl-dev make gcc g++
 ```
-On Linux (Cent OS / Amazon Linux 2)
+On Linux (Cent OS / Amazon Linux 2):
 ```bash
 sudo yum install glibc-static gmp-devel gmp-static openssl-libs openssl-static gcc-c++
 ```
+
 ### **Docker** (for testing)
 
 On macOS:
@@ -41,13 +42,55 @@ brew install --cask docker
 open /Applications/Docker.app
 ```
 On Linux, reference official documentation [here](https://docs.docker.com/engine/install/).
+
 ### **Bash 4+**
 
 For macOS, you can reference this [guide](http://tldrdevnotes.com/bash-upgrade-3-4-macos). For Linux, you can reference this [guide](https://fossbytes.com/installing-gnu-bash-4-4-linux-distros/).
 
+### **Developer Tools for macOS**
+
+On macOS, make sure you have the **Xcode Command Line Tools** installed. This includes essential tools like `git`, `make`, and other development utilities:
+```bash
+xcode-select --install
+```
+
+## Setting Up MCL & BLS Libraries on macOS
+
+The Harmony project depends on the MCL (multi-curve library) and BLS (Boneh-Lynn-Shacham) cryptographic libraries. These need to be installed and configured before building the project.
+
+### Clone and Build MCL & BLS Repositories
+First, clone the MCL and BLS repositories:
+```bash
+git clone https://github.com/harmony-one/mcl.git
+git clone https://github.com/harmony-one/bls.git
+```
+
+### Update `.zshrc` for MCL and BLS Paths
+
+To ensure the libraries are correctly located when building the project, you need to add the MCL and BLS library paths to your `.zshrc` file.
+
+Add the following lines to your `.zshrc` (or `.bash_profile` for bash users):
+```bash
+# MCL & BLS paths for Harmony
+export MCL_PATH=$GOPATH/src/github.com/harmony-one/mcl
+export BLS_PATH=$GOPATH/src/github.com/harmony-one/bls
+
+# Add library paths for MCL and BLS
+export CGO_CFLAGS="-I$MCL_PATH/include -I$BLS_PATH/include -I/opt/homebrew/opt/[email protected]/include"
+export CGO_LDFLAGS="-L$MCL_PATH/lib -L$BLS_PATH/lib -L/opt/homebrew/opt/[email protected]/lib"
+export LD_LIBRARY_PATH=$MCL_PATH/lib:$BLS_PATH/lib:/opt/homebrew/opt/[email protected]/lib
+export LIBRARY_PATH=$LD_LIBRARY_PATH
+export DYLD_FALLBACK_LIBRARY_PATH=$LD_LIBRARY_PATH
+```
+
+Then, apply the changes by running:
+```bash
+source ~/.zshrc
+```
+
 ## Dev Environment
 
-**Most repos from [harmony-one](https://github.com/harmony-one) assumes the GOPATH convention. More information [here](https://github.com/golang/go/wiki/GOPATH).**
+**Most repos from [harmony-one](https://github.com/harmony-one) assume the GOPATH convention. More information [here](https://github.com/golang/go/wiki/GOPATH).**
 
 ### First Install
 Clone and set up all of the repos with the following set of commands:
@@ -77,9 +120,7 @@ make
 
 ## Dev Docker Image
 
-Included in this repo is a Dockerfile that has a full harmony development environment and
-comes with emacs, vim, ag, tig and other creature comforts. Most importantly, it already has the go environment
-with our C/C++ based library dependencies (`libbls` and `mcl`) set up correctly for you.
+Included in this repo is a Dockerfile that has a full harmony development environment and comes with emacs, vim, ag, tig, and other creature comforts. Most importantly, it already has the go environment with our C/C++ based library dependencies (`libbls` and `mcl`) set up correctly for you.
 
 You can build the docker image for yourself with the following commands:
 ```bash
@@ -93,8 +134,6 @@ docker build -t harmony .
 > ```
 > Learn more about the `--platform` arg and multi-CPU architecture support, [here](https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope) and [here](https://docs.docker.com/desktop/multi-arch/).
 
-
-
 Then you can start your docker container with the following command:
 ```bash
 docker run --rm --name harmony -it -v "$(go env GOPATH)/src/github.com/harmony-one/harmony:/root/go/src/github.com/harmony-one/harmony" harmony /bin/bash
@@ -204,13 +243,9 @@ make test-rosetta-attach
 
 ## License
 
-Harmony is licensed under GNU Lesser General Public License v3.0. See [`LICENSE`](LICENSE) file for
-the terms and conditions.
+Harmony is licensed under GNU Lesser General Public License v3.0. See [`LICENSE`](LICENSE) file for the terms and conditions.
 
-Harmony includes third-party open-source code. In general, a source subtree
-with a `LICENSE` or `COPYRIGHT` file is from a third party, and our
-modifications thereto are licensed under the same third-party open source
-license.
+Harmony includes third-party open-source code. In general, a source subtree with a `LICENSE` or `COPYRIGHT` file is from a third party, and our modifications thereto are licensed under the same third-party open source license.
 
 Also please see [our Fiduciary License Agreement](FLA.md) if you are
 contributing to the project. By your submission of your contribution to us, you

api/service/stagedstreamsync/adapter.go

@@ -19,6 +19,7 @@ type syncProtocol interface {
 	GetRawBlocksByNumber(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([][]byte, [][]byte, sttypes.StreamID, error)
 	GetBlockHashes(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([]common.Hash, sttypes.StreamID, error)
 	GetBlocksByHashes(ctx context.Context, hs []common.Hash, opts ...syncproto.Option) ([]*types.Block, sttypes.StreamID, error)
+	GetRawBlocksByHashes(ctx context.Context, hs []common.Hash, opts ...syncproto.Option) ([][]byte, [][]byte, sttypes.StreamID, error)
 	GetReceipts(ctx context.Context, hs []common.Hash, opts ...syncproto.Option) (receipts []types.Receipts, stid sttypes.StreamID, err error)
 	GetNodeData(ctx context.Context, hs []common.Hash, opts ...syncproto.Option) (data [][]byte, stid sttypes.StreamID, err error)
 	GetAccountRange(ctx context.Context, root common.Hash, origin common.Hash, limit common.Hash, bytes uint64, opts ...syncproto.Option) (accounts []*message.AccountData, proof [][]byte, stid sttypes.StreamID, err error)

api/service/stagedstreamsync/beacon_helper.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/harmony-one/harmony/core"
 	"github.com/harmony-one/harmony/core/types"
-	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/rs/zerolog"
 )
 
@@ -38,16 +37,15 @@ type (
 	}
 )
 
-func newBeaconHelper(bc blockChain, blockC <-chan *types.Block, insertHook func()) *beaconHelper {
+func newBeaconHelper(bc blockChain, logger zerolog.Logger, blockC <-chan *types.Block, insertHook func()) *beaconHelper {
 	return &beaconHelper{
 		bc:            bc,
 		blockC:        blockC,
 		insertHook:    insertHook,
 		lastMileCache: newBlocksByNumber(lastMileCap),
 		insertC:       make(chan insertTask, 1),
 		closeC:        make(chan struct{}),
-		logger: utils.Logger().With().
-			Str("module", "staged stream sync").
+		logger: logger.With().
 			Str("sub-module", "beacon helper").
 			Logger(),
 	}