dbqctl v0.4.0

Author: sholokhov

checks.yaml

@@ -7,11 +7,11 @@ validations:
     checks:
       - id: row_count > 0
         description: "data should be present" # optional
-        on_fail: error # optional (error, warn), default "error"
+        on_error: alert # optional (ignore, alert), default "alert"
 
       - id: row_count between 100 and 30000
         description: "expected rows count"
-        on_fail: warn
+        on_error: ignore
 
       - id: null_count(pickup_ntaname) == 0
         description: "no nulls are allowed in column: pickup_ntaname"
@@ -40,11 +40,11 @@ validations:
     checks:
       - id: row_count > 0
         description: "data should be present"
-        on_fail: error
+        on_error: alert
 
       - id: row_count between 200000 and 300000
         description: "expected rows count"
-        on_fail: warn
+        on_error: ignore
 
       - id: min(price) > 0
         description: "min(price) should be greater than zero"
@@ -60,7 +60,7 @@ validations:
     checks:
       - id: row_count > 0
         description: "data should be present"
-        on_fail: error
+        on_error: alert
 
       - id: min(salary) > 0
         description: "min(salary) should be greater than zero"

cmd/check.go

@@ -16,12 +16,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/DataBridgeTech/dbqcore"
-	"github.com/DataBridgeTech/dbqctl/internal"
 	"log/slog"
 	"os"
 	"strings"
 
+	"github.com/DataBridgeTech/dbqcore"
+	"github.com/DataBridgeTech/dbqctl/internal"
+
 	"github.com/spf13/cobra"
 )
 
@@ -82,7 +83,7 @@ By automating these checks, you can proactively identify and address data qualit
 							})
 						}
 
-						if !pass && strGetOrDefault(string(check.OnFail), string(dbqcore.OnFailActionError)) == string(dbqcore.OnFailActionError) {
+						if !pass && strGetOrDefault(string(check.OnError), string(dbqcore.OnErrorActionAlert)) == string(dbqcore.OnErrorActionAlert) {
 							exitCode = 1
 						}
 					}

cmd/version.go

@@ -22,7 +22,7 @@ import (
 )
 
 const (
-	DbqCtlVersion = "v0.3.0"
+	DbqCtlVersion = "v0.4.0"
 )
 
 func NewVersionCommand() *cobra.Command {

go.mod

@@ -3,7 +3,7 @@ module github.com/DataBridgeTech/dbqctl
 go 1.24.5
 
 require (
-	github.com/DataBridgeTech/dbqcore v0.3.0
+	github.com/DataBridgeTech/dbqcore v0.4.0
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/pflag v1.0.7
 	github.com/spf13/viper v1.20.1
@@ -29,16 +29,16 @@ require (
 	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
-	github.com/sagikazarmark/locafero v0.9.0 // indirect
+	github.com/sagikazarmark/locafero v0.10.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.14.0 // indirect
 	github.com/spf13/cast v1.9.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.opentelemetry.io/otel v1.37.0 // indirect
 	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sys v0.34.0 // indirect
-	golang.org/x/text v0.27.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 )

go.sum

@@ -5,6 +5,8 @@ github.com/ClickHouse/ch-go v0.67.0/go.mod h1:2MSAeyVmgt+9a2k2SQPPG1b4qbTPzdGDpf
 github.com/ClickHouse/clickhouse-go/v2 v2.39.0 h1:spDlvQPW4d2EIOmzxeoRdeUPQ5j9zFryEx6L+XjfGoM=
 github.com/ClickHouse/clickhouse-go/v2 v2.39.0/go.mod h1:m13KylpdcPzpIjznlfXp53IpdgZ7plTxOSCZnKphYZ8=
 github.com/DataBridgeTech/dbqcore v0.3.0/go.mod h1:frSX2soCK0jiZMu1rcPPbvHar2KMtKL0/Hh2fHiQJ6s=
+github.com/DataBridgeTech/dbqcore v0.4.0 h1:GEoH9o6ByltB6VUil02Wu6/vA2qarVcFd7j18h5Eeiw=
+github.com/DataBridgeTech/dbqcore v0.4.0/go.mod h1:frSX2soCK0jiZMu1rcPPbvHar2KMtKL0/Hh2fHiQJ6s=
 github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
 github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -64,12 +66,16 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k=
 github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk=
+github.com/sagikazarmark/locafero v0.10.0 h1:FM8Cv6j2KqIhM2ZK7HZjm4mpj9NBktLgowT1aN9q5Cc=
+github.com/sagikazarmark/locafero v0.10.0/go.mod h1:Ieo3EUsjifvQu4NZwV5sPd4dwvu0OCgEQV7vjc9yDjw=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
 github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
 github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA=
 github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo=
 github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE=
@@ -126,13 +132,17 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
 golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
 golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=

internal/app.go

@@ -91,7 +91,7 @@ func (app *DbqAppImpl) ProfileDataset(srcId string, dataset string, sample bool,
 		return nil, err
 	}
 
-	return dbqProfiler.ProfileDataset(context.Background(), dataset, sample, maxConcurrent) // todo: ctx propagation
+	return dbqProfiler.ProfileDataset(context.Background(), dataset, sample, maxConcurrent, true) // todo: ctx propagation
 }
 
 func (app *DbqAppImpl) GetDbqConfig() *dbqcore.DbqConfig {

readme.md

@@ -77,17 +77,17 @@ Refer to [checks.yaml](./checks.yaml) example for full configuration overview.
 version: "1"
 validations:
   # https://clickhouse.com/docs/getting-started/example-datasets/nyc-taxi
-  - dataset: ch@[nyc_taxi.trips_small]
+  - dataset: ch@[nyc_taxi.trips_small, nyc_taxi.trips_full]
     # common pre-filter for every check, e.g. to run daily check only for yesterday
     where: "pickup_datetime > '2014-01-01'"
     checks:
       - id: row_count > 0
         description: "data should be present" # optional
-        on_fail: error # optional (error, warn), default "error"
+        on_error: alert # optional (ignore, alert), default "alert"
 
       - id: row_count between 100 and 30000
         description: "expected rows count"
-        on_fail: warn
+        on_error: ignore
 
       - id: null_count(pickup_ntaname) == 0
         description: "no nulls are allowed in column: pickup_ntaname"
@@ -108,35 +108,28 @@ validations:
         description: "raw query quality test"
         query: |
           select countIf(trip_distance == 0) > 0 from {{table}} where 1=1
-          
+
   # https://wiki.postgresql.org/wiki/Sample_Databases
   - dataset: pg@[public.land_registry_price_paid_uk]
     # exclude January for example
     where: "transfer_date >= '2025-02-01 00:00:00.000000'"
     checks:
       - id: row_count > 0
         description: "data should be present"
-        on_fail: error
-        
-      - id: row_count between 200000 and 300000
-        description: "expected rows count"
-        on_fail: warn
-        
+        on_error: alert
+
       - id: min(price) > 0
         description: "min(price) should be greater than zero"
-        
+
       - id: max(price) < 100000000
         description: "max(price) should be less than 100_000_000"
-        
-      - id: stddev_pop(price) < 500000
-        description: "price stddev"
-        
+
   # https://github.com/datacharmer/test_db
   - dataset: mysql@[employees.salaries]
     checks:
       - id: row_count > 0
         description: "data should be present"
-        on_fail: error
+        on_error: alert
 ```
 
 ### Commands
@@ -168,18 +161,21 @@ Use "dbqctl [command] --help" for more information about a command.
 
 ### Quick usage examples
 ```bash
-# check connection to datasource
-$ dqbctl ping cnn-id
+# check connection for datasource
+$ dqbctl ping -d cnn-id
+
+# check connection for all configured datasources
+$ dqbctl ping
 
 # automatically import datasets from datasource with applied filter and in-place update config file 
-$ dbqctl import cnn-id --filter "reporting.*" --cfg checks.yaml --update-cfg
+$ dbqctl import -d cnn-id --filter "reporting" --update-config
 
 # run checks from checks.yaml file
-$ dbqctl check --checks checks.yaml
+$ dbqctl check --checks ./checks.yaml
 
 # override default dbqctl config file
 $ dbqctl --config /path/to/dbq.yaml import
 
-# run dataset profile to collect general stats
-$ dbqctl profile --datasource cnn-id --dataset table_name
+# run dataset profile to collect general stats (limit concurrent jobs to 8)
+$ dbqctl profile -d cnn-id --dataset table_name -j 8
 ```