Fix/blobber reg (#643)

Author: cnlangzi

.github/workflows/tests.yml

@@ -17,10 +17,10 @@ jobs:
     name: Lints
     runs-on: [self-hosted,docker-builds]
     steps:
-      - name: Setup go 1.17
+      - name: Setup go 1.18
         uses: actions/setup-go@v2
         with:
-          go-version: '1.17' # The Go version to download (if necessary) and use.
+          go-version: '1.18' # The Go version to download (if necessary) and use.
 
       - name: Clone blobber
         uses: actions/checkout@v2
@@ -41,10 +41,10 @@ jobs:
     name: Unit Tests
     runs-on: [self-hosted,docker-builds]
     steps:
-      - name: Setup go 1.17
+      - name: Setup go 1.18
         uses: actions/setup-go@v2
         with:
-          go-version: '1.17' # The Go version to download (if necessary) and use.
+          go-version: '1.18' # The Go version to download (if necessary) and use.
 
       - name: Clone blobber
         uses: actions/checkout@v2
@@ -66,10 +66,10 @@ jobs:
     runs-on: [ubuntu-20.04]
     timeout-minutes: 40
     steps:
-      - name: Setup go 1.17
+      - name: Setup go 1.18
         uses: actions/setup-go@v2
         with:
-          go-version: '1.17' # The Go version to download (if necessary) and use.
+          go-version: '1.18' # The Go version to download (if necessary) and use.
 
       # - uses: actions/cache@v2
       #   with:
@@ -101,6 +101,6 @@ jobs:
 
           echo =========================[ run tests ]=========================
           #sudo make integration-tests 
-          go17=$(which go)
+          go=$(which go)
           root=$(pwd)
-          sudo CGO_ENABLED=1 root=$root integration=1 $go17 test -tags bn256  ./...
+          sudo CGO_ENABLED=1 root=$root integration=1 $go test -tags bn256  ./...

Makefile

@@ -33,9 +33,9 @@ local-init:
 	#[ -d ./dev.local/data/blobber/config ] && rm -rf ./dev.local/data/blobber/config
 	cp -r ./config ./dev.local/data/blobber/ 
 ifeq ($(UNAME_OS),Darwin)
-	cd ./dev.local/data/blobber/config/ && find . -name "*.yaml" -exec sed -i '' "s/postgres/127.0.0.1/g" {} \;
+	cd ./dev.local/data/blobber/config/ && find . -name "*.yaml" -exec sed -i '' "s/host: postgres/host: 127.0.0.1/g" {} \;
 else
-	cd ./dev.local/data/blobber/config/ && sed -i "s/postgres/127.0.0.1/g" ./0chain_blobber.yaml
+	cd ./dev.local/data/blobber/config/ && sed -i "s/host: postgres/host: 127.0.0.1/g" ./0chain_blobber.yaml
 endif
 	cd ./dev.local/data/blobber && [ -d files ] || mkdir files 
 	cd ./dev.local/data/blobber && [ -d data ] || mkdir data 

code/go/0chain.net/blobber/config.go

@@ -1,24 +1,25 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"time"
 
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 	"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
 	"github.com/spf13/viper"
 )
 
-func setupConfig() {
-	fmt.Print("[2/12] load config")
+func setupConfig(configDir string, deploymentMode int) {
+	fmt.Print("> load config")
 	// setup default
 	config.SetupDefaultConfig()
 
 	// setup config file
 	config.SetupConfig(configDir)
 
-	// load config
 	config.Configuration.DeploymentMode = byte(deploymentMode)
 	config.Configuration.ChainID = viper.GetString("server_chain.id")
 	config.Configuration.SignatureScheme = viper.GetString("server_chain.signature_scheme")
@@ -50,8 +51,6 @@ func setupConfig() {
 	config.Configuration.MinioWorkerFreq = viper.GetInt64("minio.worker_frequency")
 	config.Configuration.MinioUseSSL = viper.GetBool("minio.use_ssl")
 
-	config.Configuration.Capacity = viper.GetInt64("capacity")
-
 	config.Configuration.DBAutoMigrate = viper.GetBool("db.automigrate")
 	config.Configuration.PGUserName = viper.GetString("pg.user")
 	config.Configuration.PGPassword = viper.GetString("pg.password")
@@ -66,13 +65,7 @@ func setupConfig() {
 	config.Configuration.DBPassword = viper.GetString("db.password")
 	config.Configuration.DBTablesToKeep = viper.GetStringSlice("db.keep_tables")
 
-	config.Configuration.Capacity = viper.GetInt64("capacity")
-	config.Configuration.ReadPrice = viper.GetFloat64("read_price")
-	config.Configuration.WritePrice = viper.GetFloat64("write_price")
 	config.Configuration.PriceInUSD = viper.GetBool("price_in_usd")
-	config.Configuration.MinLockDemand = viper.GetFloat64("min_lock_demand")
-	config.Configuration.MaxOfferDuration = viper.GetDuration("max_offer_duration")
-	config.Configuration.ChallengeCompletionTime = viper.GetDuration("challenge_completion_time")
 
 	config.Configuration.ReadLockTimeout = int64(
 		viper.GetDuration("read_lock_timeout") / time.Second,
@@ -90,10 +83,6 @@ func setupConfig() {
 	if w := config.Configuration.DelegateWallet; len(w) != 64 {
 		log.Fatal("invalid delegate wallet:", w)
 	}
-	config.Configuration.MinStake = int64(viper.GetFloat64("min_stake") * 1e10)
-	config.Configuration.MaxStake = int64(viper.GetFloat64("max_stake") * 1e10)
-	config.Configuration.NumDelegates = viper.GetInt("num_delegates")
-	config.Configuration.ServiceCharge = viper.GetFloat64("service_charge")
 
 	config.Configuration.MinSubmit = viper.GetInt("min_submit")
 	if config.Configuration.MinSubmit < 1 {
@@ -117,3 +106,36 @@ func setupConfig() {
 
 	fmt.Print("		[OK]\n")
 }
+
+func reloadConfig() error {
+	fmt.Print("> reload config")
+
+	db := datastore.GetStore().GetDB()
+
+	s, ok := config.Get(context.TODO(), db)
+	if ok {
+		if err := s.CopyTo(&config.Configuration); err != nil {
+			return err
+		}
+		fmt.Print("		[OK]\n")
+		return nil
+	}
+
+	config.Configuration.Capacity = viper.GetInt64("capacity")
+	config.Configuration.ChallengeCompletionTime = viper.GetDuration("challenge_completion_time")
+	config.Configuration.MaxOfferDuration = viper.GetDuration("max_offer_duration")
+	config.Configuration.MaxStake = int64(viper.GetFloat64("max_stake") * 1e10)
+	config.Configuration.MinLockDemand = viper.GetFloat64("min_lock_demand")
+	config.Configuration.MinStake = int64(viper.GetFloat64("min_stake") * 1e10)
+	config.Configuration.NumDelegates = viper.GetInt("num_delegates")
+	config.Configuration.ReadPrice = viper.GetFloat64("read_price")
+	config.Configuration.ServiceCharge = viper.GetFloat64("service_charge")
+	config.Configuration.WritePrice = viper.GetFloat64("write_price")
+
+	if err := config.Update(context.TODO(), db); err != nil {
+		return err
+	}
+
+	fmt.Print("		[OK]\n")
+	return nil
+}

code/go/0chain.net/blobber/datastore.go

@@ -12,36 +12,55 @@ import (
 )
 
 func setupDatabase() error {
-	fmt.Print("\r[7/12] connect data store")
+	fmt.Print("\r> connect data store")
 	// check for database connection
 	var pgDB *gorm.DB
 	var err error
 	for i := 0; i < 600; i++ {
-		fmt.Printf("\r[7/12] connect(%v) data store", i)
+		if i > 0 {
+			fmt.Printf("\r connect(%v) data store", i)
+		}
 
 		pgDB, err = datastore.GetStore().GetPgDB()
-		if err == nil && pgDB != nil {
+
+		if err == nil {
 			break
 		}
 
-		if i == 599 {
+		if i == 599 { // no more attempts
 			logging.Logger.Error("Failed to connect to the database. Shutting the server down")
-			return fmt.Errorf("could not get postgres db connection. Error: %v", err)
+			return err
 		}
 
 		time.Sleep(1 * time.Second)
-		fmt.Print("	[OK]\n")
-
 	}
 
-	if !config.Configuration.DBAutoMigrate {
-		logging.Logger.Info("Automigration is skipped")
-		return nil
+	if config.Configuration.DBAutoMigrate {
+		if err := automigration.AutoMigrate(pgDB); err != nil {
+			return fmt.Errorf("error while migrating schema: %v", err)
+		}
 	}
 
-	fmt.Println("\r[8/12] auto migrate datastore")
-	if err := automigration.AutoMigrate(pgDB); err != nil {
-		return fmt.Errorf("error while migrating schema: %v", err)
+	// check for database connection
+	for i := 0; i < 600; i++ {
+		if i > 0 {
+			fmt.Printf("\r connect(%v) data store", i)
+		}
+
+		err = datastore.GetStore().Open()
+
+		if err == nil {
+
+			fmt.Print("	[OK]\n")
+			break
+		}
+
+		if i == 599 { // no more attempts
+			logging.Logger.Error("Failed to connect to the database. Shutting the server down")
+			return err
+		}
+
+		time.Sleep(1 * time.Second)
 	}
 
 	return nil

code/go/0chain.net/blobber/filestore.go

@@ -9,7 +9,7 @@ import (
 var fsStore filestore.FileStore //nolint:unused // global which might be needed somewhere
 
 func setupFileStore() (err error) {
-	fmt.Print("[9/12] setup file store")
+	fmt.Print("> setup file store")
 
 	fsStore, err = filestore.SetupFSStore(filesDir + "/files")
 

code/go/0chain.net/blobber/flags.go

@@ -44,7 +44,7 @@ func init() {
 }
 
 func parseFlags() {
-	fmt.Print("[1/12] load flags")
+	fmt.Print("> load flags")
 	flag.Parse()
 
 	if filesDir == "" {

code/go/0chain.net/blobber/grpc.go

@@ -39,7 +39,7 @@ func startGRPCServer() {
 		panic(err)
 	}
 
-	fmt.Println("[11/12] starting grpc server	[OK]")
+	fmt.Print("> starting grpc server	[OK]\n")
 	go func() {
 		log.Fatal(grpcServer.Serve(lis))
 	}()

code/go/0chain.net/blobber/http.go

@@ -38,7 +38,7 @@ func startHttpServer() {
 	go startServer(&wg, r, mode, httpsPort, true)
 
 	logging.Logger.Info("Ready to listen to the requests")
-	fmt.Println("[12/12] start http server	[OK]")
+	fmt.Print("> start http server	[OK]\n")
 
 	wg.Wait()
 }

code/go/0chain.net/blobber/logging.go

@@ -9,7 +9,7 @@ import (
 )
 
 func setupLogging() {
-	fmt.Print("[3/12] init logging")
+	fmt.Print("> init logging")
 
 	if config.Development() {
 		logging.InitLogging("development", logDir, "0chainBlobber.log")

code/go/0chain.net/blobber/main.go

@@ -6,12 +6,23 @@ import (
 )
 
 func main() {
+
 	parseFlags()
 
-	setupConfig()
+	setupConfig(configDir, deploymentMode)
 
 	setupLogging()
 
+	if err := setupDatabase(); err != nil {
+		logging.Logger.Error("Error setting up data store" + err.Error())
+		panic(err)
+	}
+
+	if err := reloadConfig(); err != nil {
+		logging.Logger.Error("Error reloading config" + err.Error())
+		panic(err)
+	}
+
 	if err := setupMinio(); err != nil {
 		logging.Logger.Error("Error setting up minio " + err.Error())
 		panic(err)
@@ -27,11 +38,6 @@ func main() {
 		panic(err)
 	}
 
-	if err := setupDatabase(); err != nil {
-		logging.Logger.Error("Error setting up data store" + err.Error())
-		panic(err)
-	}
-
 	// Initialize after server chain is setup.
 	if err := setupFileStore(); err != nil {
 		logging.Logger.Error("Error setting up file store" + err.Error())