Merge branch '172-observation-api' into 'master'

feat: create an API to provide automatic checks of DB migrations (#172)

# Description
* CLI, API initiate the processes that is being executed on DLE side, async
* `observer-start` - starts dblab session and starts observing
  * create session on the platform
* `observer-finish` - finished dblab session, makes summary, reports to the Platform
  * change clone status (OK -> EXPORTING -> OK)
  * update session on the platform (upload meta)
  * upload logs (single request)
* do not idle delete clone in EXPORTING status
* `database-lab`: `statement_timeout`
  * add the ability to configure Postgres configuration per clone: `dblab clone create ... --extra-config key:value`
  * add error check to the checklist: parse logs to count errors/fatals
* format: switch logs format from text to CSV (+ update idle detector)
  * idle detector
  * gather logs from Docker (interval from start to finish)

# Related issue
#172

# Examples
<!-- Provide examples for reviewers and users: new CLI output, screenshots, etc -->

# Checklist
<!-- Before merging, ensure that all checkboxes are marked: -->
- [x] the MR description has been reviewed
- [x] this MR contains text changes and they have been reviewed OR there are no texts changes
- [x] this MR contains GUI/CLI changes and they have been reviewed OR there are no GUI/CLI changes
- [x] this MR contains API changes, specifications reflect those changes and they have been reviewed OR there are no API changes

Closes #172

See merge request postgres-ai/database-lab!179

Author: agneum

api/swagger-spec/dblab_server_swagger.yaml

@@ -1,7 +1,7 @@
 swagger: "2.0"
 info:
   description: "This is a Database Lab sample server."
-  version: "0.1.0"
+  version: "2.0.0"
   title: "Database Lab"
   contact:
     email: "[email protected]"
@@ -10,11 +10,11 @@ info:
     url: "https://gitlab.com/postgres-ai/database-lab/blob/master/LICENSE"
 basePath: "/"
 tags:
-  - name: "Database Lab"
+  - name: "Database Lab Engine"
     description: "API Reference"
     externalDocs:
-      description: "Database Lab Docs"
-      url: "https://postgres.ai/docs/database-lab/what_is_database_lab"
+      description: "Database Lab Engine Docs"
+      url: "https://postgres.ai/docs/database-lab"
 schemes:
   - "https"
   - "http"

cmd/cli/commands/clone/actions.go

@@ -9,7 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
-	"os"
+	"strings"
 	"sync"
 
 	"github.com/urfave/cli/v2"
@@ -18,11 +18,6 @@ import (
 	"gitlab.com/postgres-ai/database-lab/pkg/client/dblabapi/types"
 	"gitlab.com/postgres-ai/database-lab/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/pkg/models"
-	"gitlab.com/postgres-ai/database-lab/pkg/observer"
-)
-
-const (
-	errorExitStatus = 1
 )
 
 // list runs a request to list clones of an instance.
@@ -50,48 +45,47 @@ func list() func(*cli.Context) error {
 }
 
 // create runs a request to create a new clone.
-func create() func(*cli.Context) error {
-	return func(cliCtx *cli.Context) error {
-		dblabClient, err := commands.ClientByCLIContext(cliCtx)
-		if err != nil {
-			return err
-		}
+func create(cliCtx *cli.Context) error {
+	dblabClient, err := commands.ClientByCLIContext(cliCtx)
+	if err != nil {
+		return err
+	}
 
-		cloneRequest := types.CloneCreateRequest{
-			ID:        cliCtx.String("id"),
-			Project:   cliCtx.String("project"),
-			Protected: cliCtx.Bool("protected"),
-			DB: &types.DatabaseRequest{
-				Username: cliCtx.String("username"),
-				Password: cliCtx.String("password"),
-			},
-		}
+	cloneRequest := types.CloneCreateRequest{
+		ID:        cliCtx.String("id"),
+		Protected: cliCtx.Bool("protected"),
+		DB: &types.DatabaseRequest{
+			Username: cliCtx.String("username"),
+			Password: cliCtx.String("password"),
+		},
+	}
 
-		if cliCtx.IsSet("snapshot-id") {
-			cloneRequest.Snapshot = &types.SnapshotCloneFieldRequest{ID: cliCtx.String("snapshot-id")}
-		}
+	if cliCtx.IsSet("snapshot-id") {
+		cloneRequest.Snapshot = &types.SnapshotCloneFieldRequest{ID: cliCtx.String("snapshot-id")}
+	}
 
-		var clone *models.Clone
+	cloneRequest.ExtraConf = splitFlags(cliCtx.StringSlice("extra-config"))
 
-		if cliCtx.Bool("async") {
-			clone, err = dblabClient.CreateCloneAsync(cliCtx.Context, cloneRequest)
-		} else {
-			clone, err = dblabClient.CreateClone(cliCtx.Context, cloneRequest)
-		}
+	var clone *models.Clone
 
-		if err != nil {
-			return err
-		}
-
-		commandResponse, err := json.MarshalIndent(clone, "", "    ")
-		if err != nil {
-			return err
-		}
+	if cliCtx.Bool("async") {
+		clone, err = dblabClient.CreateCloneAsync(cliCtx.Context, cloneRequest)
+	} else {
+		clone, err = dblabClient.CreateClone(cliCtx.Context, cloneRequest)
+	}
 
-		_, err = fmt.Fprintln(cliCtx.App.Writer, string(commandResponse))
+	if err != nil {
+		return err
+	}
 
+	commandResponse, err := json.MarshalIndent(clone, "", "    ")
+	if err != nil {
 		return err
 	}
+
+	_, err = fmt.Fprintln(cliCtx.App.Writer, string(commandResponse))
+
+	return err
 }
 
 // status runs a request to get clone info.
@@ -200,58 +194,64 @@ func destroy() func(*cli.Context) error {
 	}
 }
 
-// observe runs a request to observe clone.
-func observe() func(*cli.Context) error {
-	return func(cliCtx *cli.Context) error {
-		dblabClient, err := commands.ClientByCLIContext(cliCtx)
-		if err != nil {
-			return err
-		}
-
-		cloneID := cliCtx.Args().First()
+// startObservation runs a request to startObservation clone.
+func startObservation(cliCtx *cli.Context) error {
+	dblabClient, err := commands.ClientByCLIContext(cliCtx)
+	if err != nil {
+		return err
+	}
 
-		clone, err := dblabClient.GetClone(cliCtx.Context, cloneID)
-		if err != nil {
-			return err
-		}
+	cloneID := cliCtx.Args().First()
 
-		obsConfig := observer.Config{
-			Follow:                 cliCtx.Bool("follow"),
-			IntervalSeconds:        cliCtx.Uint64("interval-seconds"),
-			MaxLockDurationSeconds: cliCtx.Uint64("max-lock-duration-seconds"),
-			MaxDurationSeconds:     cliCtx.Uint64("max-duration-seconds"),
-			SSLMode:                cliCtx.String("sslmode"),
-		}
+	observationConfig := types.Config{
+		ObservationInterval: cliCtx.Uint64("observation-interval"),
+		MaxLockDuration:     cliCtx.Uint64("max-lock-duration"),
+		MaxDuration:         cliCtx.Uint64("max-duration"),
+	}
 
-		obs := observer.NewObserver(obsConfig, cliCtx.App.Writer)
+	start := types.StartObservationRequest{
+		CloneID: cloneID,
+		Config:  observationConfig,
+		Tags:    splitFlags(cliCtx.StringSlice("tags")),
+	}
 
-		clone.DB.Password = cliCtx.String("password")
+	session, err := dblabClient.StartObservation(cliCtx.Context, start)
+	if err != nil {
+		return err
+	}
 
-		return obs.Start(clone)
+	commandResponse, err := json.MarshalIndent(session, "", "    ")
+	if err != nil {
+		return err
 	}
-}
 
-// observeSummary shows observing summary and check satisfaction of performance requirements.
-func observeSummary() func(*cli.Context) error {
-	return func(cliCtx *cli.Context) error {
-		obs := observer.NewObserver(observer.Config{}, cliCtx.App.Writer)
+	_, err = fmt.Fprintln(cliCtx.App.Writer, string(commandResponse))
 
-		if err := obs.LoadObserverState(); err != nil {
-			return err
-		}
+	return err
+}
 
-		if err := obs.PrintSummary(); err != nil {
-			return err
-		}
+// stopObservation shows observing summary and check satisfaction of performance requirements.
+func stopObservation(cliCtx *cli.Context) error {
+	dblabClient, err := commands.ClientByCLIContext(cliCtx)
+	if err != nil {
+		return err
+	}
 
-		if err := obs.CheckPerformanceRequirements(); err != nil {
-			log.Err("Performance check error:", err)
-			// Exit with error status without printing additional error logs.
-			os.Exit(errorExitStatus)
-		}
+	cloneID := cliCtx.Args().First()
 
-		return nil
+	result, err := dblabClient.StopObservation(cliCtx.Context, types.StopObservationRequest{CloneID: cloneID})
+	if err != nil {
+		return err
+	}
+
+	commandResponse, err := json.MarshalIndent(result, "", "    ")
+	if err != nil {
+		return err
 	}
+
+	_, err = fmt.Fprintln(cliCtx.App.Writer, string(commandResponse))
+
+	return err
 }
 
 func forward(cliCtx *cli.Context) error {
@@ -335,3 +335,20 @@ func retrieveClonePort(cliCtx *cli.Context, wg *sync.WaitGroup, remoteHost *url.
 
 	return clone.DB.Port, nil
 }
+
+func splitFlags(flags []string) map[string]string {
+	const maxSplitParts = 2
+
+	extraConfig := make(map[string]string, len(flags))
+
+	if len(flags) == 0 {
+		return extraConfig
+	}
+
+	for _, cfg := range flags {
+		parsed := strings.SplitN(cfg, "=", maxSplitParts)
+		extraConfig[parsed[0]] = parsed[1]
+	}
+
+	return extraConfig
+}

cmd/cli/commands/clone/command_list.go

@@ -31,7 +31,7 @@ func CommandList() []*cli.Command {
 			{
 				Name:   "create",
 				Usage:  "create new clone",
-				Action: create(),
+				Action: create,
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:     "username",
@@ -51,10 +51,6 @@ func CommandList() []*cli.Command {
 						Name:  "snapshot-id",
 						Usage: "snapshot ID (optional)",
 					},
-					&cli.StringFlag{
-						Name:  "project",
-						Usage: "project name (optional)",
-					},
 					&cli.BoolFlag{
 						Name:    "protected",
 						Usage:   "mark instance as protected from deletion",
@@ -65,6 +61,10 @@ func CommandList() []*cli.Command {
 						Usage:   "run the command asynchronously",
 						Aliases: []string{"a"},
 					},
+					&cli.StringSliceFlag{
+						Name:  "extra-config",
+						Usage: "set an extra database configuration for the clone. An example: statement_timeout='1s'",
+					},
 				},
 			},
 			{
@@ -110,50 +110,39 @@ func CommandList() []*cli.Command {
 				},
 			},
 			{
-				Name:      "observe",
-				Usage:     "[EXPERIMENTAL] monitor clone state",
+				Name:      "start-observation",
+				Usage:     "[EXPERIMENTAL] start clone state monitoring",
 				ArgsUsage: "CLONE_ID",
 				Before:    checkCloneIDBefore,
-				Action:    observe(),
+				Action:    startObservation,
 				Flags: []cli.Flag{
-					&cli.StringFlag{
-						Name:     "password",
-						Usage:    "clone database password",
-						EnvVars:  []string{"CLONE_PASSWORD"},
-						Required: true,
-					},
-					&cli.StringFlag{
-						Name:    "sslmode",
-						Usage:   "connection SSL mode",
-						EnvVars: []string{"SSLMODE"},
-						Value:   "disable",
-					},
-					&cli.BoolFlag{
-						Name:    "follow",
-						Usage:   "follow state monitor output",
-						Aliases: []string{"f"},
-					},
 					&cli.IntFlag{
-						Name:    "interval-seconds",
-						Usage:   "interval of metric gathering and output",
-						EnvVars: []string{"DBLAB_INTERVAL_SECONDS"},
+						Name:    "observation-interval",
+						Usage:   "interval of metric gathering and output (in seconds)",
+						EnvVars: []string{"DBLAB_OBSERVATION_INTERVAL"},
 					},
 					&cli.IntFlag{
-						Name:    "max-lock-duration-seconds",
-						Usage:   "maximum allowed duration for locks",
-						EnvVars: []string{"DBLAB_MAX_LOCK_DURATION_SECONDS"},
+						Name:    "max-lock-duration",
+						Usage:   "maximum allowed duration for locks (in seconds)",
+						EnvVars: []string{"DBLAB_MAX_LOCK_DURATION"},
 					},
 					&cli.IntFlag{
-						Name:    "max-duration-seconds",
-						Usage:   "maximum allowed duration for operation",
-						EnvVars: []string{"DBLAB_MAX_DURATION_SECONDS"},
+						Name:    "max-duration",
+						Usage:   "maximum allowed duration for observation (in seconds)",
+						EnvVars: []string{"DBLAB_MAX_DURATION"},
+					},
+					&cli.StringSliceFlag{
+						Name:  "tags",
+						Usage: "set tags for the observation session. An example: branch=patch-1",
 					},
 				},
 			},
 			{
-				Name:   "observe-summary",
-				Usage:  "[EXPERIMENTAL] summarize clone monitoring and check results",
-				Action: observeSummary(),
+				Name:      "stop-observation",
+				Usage:     "[EXPERIMENTAL] summarize clone monitoring and check results",
+				ArgsUsage: "CLONE_ID",
+				Before:    checkCloneIDBefore,
+				Action:    stopObservation,
 			},
 			{
 				Name:  "port-forward",

cmd/database-lab/main.go

@@ -21,6 +21,7 @@ import (
 
 	"gitlab.com/postgres-ai/database-lab/pkg/config"
 	"gitlab.com/postgres-ai/database-lab/pkg/log"
+	"gitlab.com/postgres-ai/database-lab/pkg/observer"
 	"gitlab.com/postgres-ai/database-lab/pkg/retrieval"
 	"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/tools/cont"
 	"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
@@ -122,8 +123,13 @@ func main() {
 		cfg.Server.VerificationToken = opts.VerificationToken
 	}
 
+	observerCfg := &observer.Config{
+		CloneDir:  cfg.Provision.Options.ClonesMountDir,
+		SocketDir: cfg.Provision.Options.UnixSocketDir,
+	}
+
 	// Start the Database Lab.
-	server := srv.NewServer(&cfg.Server, cloningSvc, platformSvc)
+	server := srv.NewServer(&cfg.Server, observerCfg, cloningSvc, platformSvc, dockerCLI)
 	if err = server.Run(); err != nil {
 		log.Fatalf(err)
 	}

configs/config.example.logical_generic.yml

@@ -255,7 +255,7 @@ cloning:
 #
 #  # Token for authorization in Platform API. This token can be obtained on
 #  # the Postgres.ai Console: https://postgres.ai/console/YOUR_ORG_NAME/tokens
-#  # This token needs to be kept in secret, known only to the administrtor.
+#  # This token needs to be kept in secret, known only to the administrator.
 #  accessToken: "platform_access_token"
 #
 #  # Enable authorization with personal tokens of the organization's members.

configs/config.example.logical_rds_iam.yml

@@ -256,7 +256,7 @@ cloning:
 #
 #  # Token for authorization in Platform API. This token can be obtained on
 #  # the Postgres.ai Console: https://postgres.ai/console/YOUR_ORG_NAME/tokens
-#  # This token needs to be kept in secret, known only to the administrtor.
+#  # This token needs to be kept in secret, known only to the administrator.
 #  accessToken: "platform_access_token"
 #
 #  # Enable authorization with personal tokens of the organization's members.