[server] Add COUCHBASE_BUCKET_RAM_QUOTA

Resolves #317

Makefile

@@ -35,6 +35,7 @@ test-dynamodb:
 	docker rm -vf dynamodb-local-test
 test-couchbase:
 	docker run -d --name couchbase-local-test  -p 8091-8097:8091-8097 -p 11210:11210 -p 11207:11207 -p 18091-18095:18091-18095 -p 18096:18096 -p 18097:18097 couchbase:latest
+	sh scripts/couchbase-test.sh
 	cd server && go clean --testcache && TEST_DBS="couchbase" go test -p 1 -v ./test
 	docker rm -vf couchbase-local-test
 test-all-db:
@@ -44,11 +45,12 @@ test-all-db:
 	docker run -d --name authorizer_arangodb -p 8529:8529 -e ARANGO_NO_AUTH=1 arangodb/arangodb:3.8.4
 	docker run -d --name dynamodb-local-test  -p 8000:8000 amazon/dynamodb-local:latest
 	docker run -d --name couchbase-local-test  -p 8091-8097:8091-8097 -p 11210:11210 -p 11207:11207 -p 18091-18095:18091-18095 -p 18096:18096 -p 18097:18097 couchbase:latest
-	cd server && go clean --testcache && TEST_DBS="sqlite,mongodb,arangodb,scylladb,dynamodb,couchbase" go test -p 1 -v ./test
+	sh scripts/couchbase-test.sh
+	cd server && go clean --testcache && TEST_DBS="sqlite,mongodb,arangodb,scylladb,dynamodb" go test -p 1 -v ./test
 	docker rm -vf authorizer_scylla_db
 	docker rm -vf authorizer_mongodb_db
 	docker rm -vf authorizer_arangodb
 	docker rm -vf dynamodb-local-test
-	docker rm -vf couchbase-local-test
+	# docker rm -vf couchbase-local-test
 generate:
 	cd server && go run github.com/99designs/gqlgen generate && go mod tidy

scripts/couchbase-test.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -x
+set -m
+
+sleep 15
+
+# Setup index and memory quota
+# curl -v -X POST http://127.0.0.1:8091/pools/default -d memoryQuota=300 -d indexMemoryQuota=300
+
+# Setup services
+curl -v http://127.0.0.1:8091/node/controller/setupServices -d services=kv%2Cn1ql%2Cindex
+
+# Setup credentials
+curl -v http://127.0.0.1:8091/settings/web -d port=8091 -d username=Administrator -d password=password
+
+# Setup Memory Optimized Indexes
+curl -i -u Administrator:password -X POST http://127.0.0.1:8091/settings/indexes -d 'storageMode=memory_optimized'
+
+# Load travel-sample bucket
+#curl -v -u Administrator:password -X POST http://127.0.0.1:8091/sampleBuckets/install -d '["travel-sample"]'
+
+echo "Type: $TYPE"
+
+if [ "$TYPE" = "WORKER" ]; then
+  echo "Sleeping ..."
+  sleep 15
+
+  #IP=`hostname -s`
+  IP=`hostname -I | cut -d ' ' -f1`
+  echo "IP: " $IP
+
+  echo "Auto Rebalance: $AUTO_REBALANCE"
+  if [ "$AUTO_REBALANCE" = "true" ]; then
+    couchbase-cli rebalance --cluster=$COUCHBASE_MASTER:8091 --user=Administrator --password=password --server-add=$IP --server-add-username=Administrator --server-add-password=password
+  else
+    couchbase-cli server-add --cluster=$COUCHBASE_MASTER:8091 --user=Administrator --password=password --server-add=$IP --server-add-username=Administrator --server-add-password=password
+  fi;
+fi;

server/constants/env.go

@@ -45,6 +45,9 @@ const (
 	EnvKeyDatabaseCACert = "DATABASE_CA_CERT"
 	// EnvCouchbaseBucket key for env variable COUCHBASE_BUCKET
 	EnvCouchbaseBucket = "COUCHBASE_BUCKET"
+	// EnvCouchbaseBucketRAMQuotaMB key for env variable COUCHBASE_BUCKET_RAM_QUOTA
+	// This value should be parsed as number
+	EnvCouchbaseBucketRAMQuotaMB = "COUCHBASE_BUCKET_RAM_QUOTA"
 	// EnvCouchbaseBucket key for env variable COUCHBASE_SCOPE
 	EnvCouchbaseScope = "COUCHBASE_SCOPE"
 	// EnvKeySmtpHost key for env variable SMTP_HOST

server/db/db.go

@@ -7,6 +7,7 @@ import (
 	"github.com/authorizerdev/authorizer/server/db/providers"
 	"github.com/authorizerdev/authorizer/server/db/providers/arangodb"
 	"github.com/authorizerdev/authorizer/server/db/providers/cassandradb"
+	"github.com/authorizerdev/authorizer/server/db/providers/couchbase"
 	"github.com/authorizerdev/authorizer/server/db/providers/dynamodb"
 	"github.com/authorizerdev/authorizer/server/db/providers/mongodb"
 	"github.com/authorizerdev/authorizer/server/db/providers/sql"
@@ -26,6 +27,7 @@ func InitDB() error {
 	isMongoDB := envs.DatabaseType == constants.DbTypeMongodb
 	isCassandra := envs.DatabaseType == constants.DbTypeCassandraDB || envs.DatabaseType == constants.DbTypeScyllaDB
 	isDynamoDB := envs.DatabaseType == constants.DbTypeDynamoDB
+	isCouchbaseDB := envs.DatabaseType == constants.DbTypeCouchbaseDB
 
 	if isSQL {
 		log.Info("Initializing SQL Driver for: ", envs.DatabaseType)
@@ -72,5 +74,14 @@ func InitDB() error {
 		}
 	}
 
+	if isCouchbaseDB {
+		log.Info("Initializing CouchbaseDB Driver for: ", envs.DatabaseType)
+		Provider, err = couchbase.NewProvider()
+		if err != nil {
+			log.Fatal("Failed to initialize Couchbase driver: ", err)
+			return err
+		}
+	}
+
 	return nil
 }

server/db/models/env.go

@@ -4,10 +4,11 @@ package models
 
 // Env model for db
 type Env struct {
-	Key       string `json:"_key,omitempty" bson:"_key,omitempty" cql:"_key,omitempty" dynamo:"key,omitempty"` // for arangodb
-	ID        string `gorm:"primaryKey;type:char(36)" json:"_id" bson:"_id" cql:"id" dynamo:"id,hash"`
-	EnvData   string `json:"env" bson:"env" cql:"env" dynamo:"env"`
-	Hash      string `json:"hash" bson:"hash" cql:"hash" dynamo:"hash"`
-	UpdatedAt int64  `json:"updated_at" bson:"updated_at" cql:"updated_at" dynamo:"updated_at"`
-	CreatedAt int64  `json:"created_at" bson:"created_at" cql:"created_at" dynamo:"created_at"`
+	Key           string `json:"_key,omitempty" bson:"_key,omitempty" cql:"_key,omitempty" dynamo:"key,omitempty"` // for arangodb
+	ID            string `gorm:"primaryKey;type:char(36)" json:"_id" bson:"_id" cql:"id" dynamo:"id,hash"`
+	EnvData       string `json:"env" bson:"env" cql:"env" dynamo:"env"`
+	Hash          string `json:"hash" bson:"hash" cql:"hash" dynamo:"hash"`
+	EncryptionKey string `json:"encryption_key" bson:"encryption_key" cql:"encryption_key" dynamo:"encryption_key"` // couchbase has "hash" as reserved keyword so we cannot use it. This will be empty for other dbs.
+	UpdatedAt     int64  `json:"updated_at" bson:"updated_at" cql:"updated_at" dynamo:"updated_at"`
+	CreatedAt     int64  `json:"created_at" bson:"created_at" cql:"created_at" dynamo:"created_at"`
 }

server/db/providers/couchbase/env.go

@@ -18,6 +18,7 @@ func (p *provider) AddEnv(ctx context.Context, env models.Env) (models.Env, erro
 	env.CreatedAt = time.Now().Unix()
 	env.UpdatedAt = time.Now().Unix()
 	env.Key = env.ID
+	env.EncryptionKey = env.Hash
 
 	insertOpt := gocb.InsertOptions{
 		Context: ctx,
@@ -32,6 +33,7 @@ func (p *provider) AddEnv(ctx context.Context, env models.Env) (models.Env, erro
 // UpdateEnv to update environment information in database
 func (p *provider) UpdateEnv(ctx context.Context, env models.Env) (models.Env, error) {
 	env.UpdatedAt = time.Now().Unix()
+	env.EncryptionKey = env.Hash
 
 	updateEnvQuery := fmt.Sprintf("UPDATE %s.%s SET env = $1, updated_at = $2 WHERE _id = $3", p.scopeName, models.Collections.Env)
 	_, err := p.db.Query(updateEnvQuery, &gocb.QueryOptions{
@@ -50,7 +52,7 @@ func (p *provider) UpdateEnv(ctx context.Context, env models.Env) (models.Env, e
 func (p *provider) GetEnv(ctx context.Context) (models.Env, error) {
 	var env models.Env
 
-	query := fmt.Sprintf("SELECT _id, env, created_at, updated_at FROM %s.%s LIMIT 1", p.scopeName, models.Collections.Env)
+	query := fmt.Sprintf("SELECT _id, env, encryption_key, created_at, updated_at FROM %s.%s LIMIT 1", p.scopeName, models.Collections.Env)
 	q, err := p.db.Query(query, &gocb.QueryOptions{
 		Context:         ctx,
 		ScanConsistency: gocb.QueryScanConsistencyRequestPlus,
@@ -63,5 +65,6 @@ func (p *provider) GetEnv(ctx context.Context) (models.Env, error) {
 	if err != nil {
 		return env, err
 	}
+	env.Hash = env.EncryptionKey
 	return env, nil
 }

server/db/providers/couchbase/provider.go

@@ -4,70 +4,77 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
 	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/couchbase/gocb/v2"
 
-	"github.com/authorizerdev/authorizer/server/constants"
 	"github.com/authorizerdev/authorizer/server/db/models"
 	"github.com/authorizerdev/authorizer/server/memorystore"
-	"github.com/couchbase/gocb/v2"
 )
 
-// TODO change following provider to new db provider
+const (
+	defaultBucketName = "authorizer"
+	defaultScope      = "_default"
+)
+
 type provider struct {
 	db        *gocb.Scope
 	scopeName string
 }
 
-// NewProvider returns a new SQL provider
-// TODO change following provider to new db provider
+// NewProvider returns a new Couchbase provider
 func NewProvider() (*provider, error) {
-	// scopeName := os.Getenv(constants.EnvCouchbaseScope)
-	bucketName := os.Getenv(constants.EnvCouchbaseBucket)
-	scopeName := os.Getenv(constants.EnvCouchbaseScope)
-
+	bucketName := memorystore.RequiredEnvStoreObj.GetRequiredEnv().CouchbaseBucket
+	scopeName := memorystore.RequiredEnvStoreObj.GetRequiredEnv().CouchbaseScope
 	dbURL := memorystore.RequiredEnvStoreObj.GetRequiredEnv().DatabaseURL
 	userName := memorystore.RequiredEnvStoreObj.GetRequiredEnv().DatabaseUsername
 	password := memorystore.RequiredEnvStoreObj.GetRequiredEnv().DatabasePassword
-
 	opts := gocb.ClusterOptions{
 		Username: userName,
 		Password: password,
 	}
-
+	if bucketName == "" {
+		bucketName = defaultBucketName
+	}
+	if scopeName == "" {
+		scopeName = defaultScope
+	}
 	cluster, err := gocb.Connect(dbURL, opts)
-
 	if err != nil {
 		return nil, err
 	}
-
 	// To create the bucket and scope if not exist
 	bucket, err := CreateBucketAndScope(cluster, bucketName, scopeName)
-
 	if err != nil {
 		return nil, err
 	}
-
 	scope := bucket.Scope(scopeName)
-
 	scopeIdentifier := fmt.Sprintf("%s.%s", bucketName, scopeName)
-
 	v := reflect.ValueOf(models.Collections)
 	for i := 0; i < v.NumField(); i++ {
-		field := v.Field(i)
+		collectionName := v.Field(i)
 		user := gocb.CollectionSpec{
-			Name:      field.String(),
+			Name:      collectionName.String(),
 			ScopeName: scopeName,
 		}
 		collectionOpts := gocb.CreateCollectionOptions{
 			Context: context.TODO(),
 		}
-		_ = bucket.Collections().CreateCollection(user, &collectionOpts)
-		// if err != nil && !errors.Is(err, gocb.ErrCollectionExists) {
-		// 	return nil, err
-		// }
-		indexQuery := fmt.Sprintf("CREATE PRIMARY INDEX ON %s.%s", scopeIdentifier, field.String())
-		scope.Query(indexQuery, nil)
+		err = bucket.Collections().CreateCollection(user, &collectionOpts)
+		if err != nil && !errors.Is(err, gocb.ErrCollectionExists) {
+			return nil, err
+		}
+		// TODO: find how to fix this sleep time.
+		// Add wait time for successful collection creation
+		time.Sleep(5 * time.Second)
+		indexQuery := fmt.Sprintf("CREATE PRIMARY INDEX ON %s.%s", scopeIdentifier, collectionName.String())
+		_, err = scope.Query(indexQuery, nil)
+		if err != nil && !strings.Contains(err.Error(), "The index #primary already exists") {
+			return nil, err
+		}
 	}
 
 	indices := GetIndex(scopeIdentifier)
@@ -84,33 +91,36 @@ func NewProvider() (*provider, error) {
 }
 
 func CreateBucketAndScope(cluster *gocb.Cluster, bucketName string, scopeName string) (*gocb.Bucket, error) {
+	bucketRAMQuotaMB := memorystore.RequiredEnvStoreObj.GetRequiredEnv().CouchbaseBucketRAMQuotaMB
+	if bucketRAMQuotaMB == "" {
+		bucketRAMQuotaMB = "1000"
+	}
+	bucketRAMQuota, err := strconv.ParseInt(bucketRAMQuotaMB, 10, 64)
+	if err != nil {
+		return nil, err
+	}
 	settings := gocb.BucketSettings{
 		Name:            bucketName,
-		RAMQuotaMB:      1000,
-		NumReplicas:     1,
+		RAMQuotaMB:      uint64(bucketRAMQuota),
 		BucketType:      gocb.CouchbaseBucketType,
 		EvictionPolicy:  gocb.EvictionPolicyTypeValueOnly,
 		FlushEnabled:    true,
 		CompressionMode: gocb.CompressionModeActive,
 	}
-
-	err := cluster.Buckets().CreateBucket(gocb.CreateBucketSettings{
+	err = cluster.Buckets().CreateBucket(gocb.CreateBucketSettings{
 		BucketSettings:         settings,
 		ConflictResolutionType: gocb.ConflictResolutionTypeSequenceNumber,
 	}, nil)
-
 	bucket := cluster.Bucket(bucketName)
-
 	if err != nil && !errors.Is(err, gocb.ErrBucketExists) {
 		return bucket, err
 	}
-
-	err = bucket.Collections().CreateScope(scopeName, nil)
-
-	if err != nil && !errors.Is(err, gocb.ErrScopeExists) {
-		return bucket, err
+	if scopeName != defaultScope {
+		err = bucket.Collections().CreateScope(scopeName, nil)
+		if err != nil && !errors.Is(err, gocb.ErrScopeExists) {
+			return bucket, err
+		}
 	}
-
 	return bucket, nil
 }
 

server/env/env.go

@@ -83,6 +83,7 @@ func InitAllEnv() error {
 	osAwsSecretKey := os.Getenv(constants.EnvAwsSecretAccessKey)
 	osCouchbaseBucket := os.Getenv(constants.EnvCouchbaseBucket)
 	osCouchbaseScope := os.Getenv(constants.EnvCouchbaseScope)
+	osCouchbaseBucketRAMQuotaMB := os.Getenv(constants.EnvCouchbaseBucketRAMQuotaMB)
 
 	// os bool vars
 	osAppCookieSecure := os.Getenv(constants.EnvKeyAppCookieSecure)
@@ -154,6 +155,13 @@ func InitAllEnv() error {
 		envData[constants.EnvCouchbaseBucket] = osCouchbaseBucket
 	}
 
+	if val, ok := envData[constants.EnvCouchbaseBucketRAMQuotaMB]; !ok || val == "" {
+		envData[constants.EnvCouchbaseBucketRAMQuotaMB] = osCouchbaseBucketRAMQuotaMB
+	}
+	if osCouchbaseBucketRAMQuotaMB != "" && envData[constants.EnvCouchbaseBucketRAMQuotaMB] != osCouchbaseBucketRAMQuotaMB {
+		envData[constants.EnvCouchbaseBucketRAMQuotaMB] = osCouchbaseBucketRAMQuotaMB
+	}
+
 	if val, ok := envData[constants.EnvCouchbaseScope]; !ok || val == "" {
 		envData[constants.EnvCouchbaseScope] = osCouchbaseScope
 	}

server/memorystore/required_env_store.go

@@ -33,8 +33,9 @@ type RequiredEnv struct {
 	AwsAccessKeyID     string `json:"AWS_ACCESS_KEY_ID"`
 	AwsSecretAccessKey string `json:"AWS_SECRET_ACCESS_KEY"`
 	// Couchbase related envs
-	CouchbaseBucket string `json:"COUCHBASE_BUCKET"`
-	CouchbaseScope  string `json:"COUCHBASE_SCOPE"`
+	CouchbaseBucket           string `json:"COUCHBASE_BUCKET"`
+	CouchbaseScope            string `json:"COUCHBASE_SCOPE"`
+	CouchbaseBucketRAMQuotaMB string `json:"COUCHBASE_BUCKET_RAM_QUOTA"`
 }
 
 // RequiredEnvObj is a simple in-memory store for sessions.
@@ -98,6 +99,7 @@ func InitRequiredEnv() error {
 	awsSecretAccessKey := os.Getenv(constants.EnvAwsSecretAccessKey)
 	couchbaseBucket := os.Getenv(constants.EnvCouchbaseBucket)
 	couchbaseScope := os.Getenv(constants.EnvCouchbaseScope)
+	couchbaseBucketRAMQuotaMB := os.Getenv(constants.EnvCouchbaseBucketRAMQuotaMB)
 
 	if strings.TrimSpace(redisURL) == "" {
 		if cli.ARG_REDIS_URL != nil && *cli.ARG_REDIS_URL != "" {
@@ -140,24 +142,25 @@ func InitRequiredEnv() error {
 	}
 
 	requiredEnv := RequiredEnv{
-		EnvPath:            envPath,
-		DatabaseURL:        dbURL,
-		DatabaseType:       dbType,
-		DatabaseName:       dbName,
-		DatabaseHost:       dbHost,
-		DatabasePort:       dbPort,
-		DatabaseUsername:   dbUsername,
-		DatabasePassword:   dbPassword,
-		DatabaseCert:       dbCert,
-		DatabaseCertKey:    dbCertKey,
-		DatabaseCACert:     dbCACert,
-		RedisURL:           redisURL,
-		DisableRedisForEnv: disableRedisForEnv,
-		AwsRegion:          awsRegion,
-		AwsAccessKeyID:     awsAccessKeyID,
-		AwsSecretAccessKey: awsSecretAccessKey,
-		CouchbaseBucket:    couchbaseBucket,
-		CouchbaseScope:     couchbaseScope,
+		EnvPath:                   envPath,
+		DatabaseURL:               dbURL,
+		DatabaseType:              dbType,
+		DatabaseName:              dbName,
+		DatabaseHost:              dbHost,
+		DatabasePort:              dbPort,
+		DatabaseUsername:          dbUsername,
+		DatabasePassword:          dbPassword,
+		DatabaseCert:              dbCert,
+		DatabaseCertKey:           dbCertKey,
+		DatabaseCACert:            dbCACert,
+		RedisURL:                  redisURL,
+		DisableRedisForEnv:        disableRedisForEnv,
+		AwsRegion:                 awsRegion,
+		AwsAccessKeyID:            awsAccessKeyID,
+		AwsSecretAccessKey:        awsSecretAccessKey,
+		CouchbaseBucket:           couchbaseBucket,
+		CouchbaseScope:            couchbaseScope,
+		CouchbaseBucketRAMQuotaMB: couchbaseBucketRAMQuotaMB,
 	}
 
 	RequiredEnvStoreObj = &RequiredEnvStore{

server/test/admin_signup_test.go

@@ -25,7 +25,6 @@ func adminSignupTests(t *testing.T, s TestSetup) {
 		_, err = resolvers.AdminSignupResolver(ctx, model.AdminSignupInput{
 			AdminSecret: "admin123",
 		})
-
 		assert.Nil(t, err)
 	})
 }

server/test/resolvers_test.go

@@ -58,6 +58,12 @@ func TestResolvers(t *testing.T) {
 			memorystore.Provider.UpdateEnvVariable(constants.EnvAwsRegion, "ap-south-1")
 			os.Setenv(constants.EnvAwsRegion, "ap-south-1")
 		}
+		if dbType == constants.DbTypeCouchbaseDB {
+			memorystore.Provider.UpdateEnvVariable(constants.EnvKeyDatabaseUsername, "Administrator")
+			os.Setenv(constants.EnvKeyDatabaseUsername, "Administrator")
+			memorystore.Provider.UpdateEnvVariable(constants.EnvKeyDatabasePassword, "password")
+			os.Setenv(constants.EnvKeyDatabasePassword, "password")
+		}
 
 		memorystore.InitRequiredEnv()