Skip to content

Commit ec1d4c2

Browse files
kaidaguerrekaidaguerre
committed
Merge branch 'v0.21.x'
# Conflicts: # CHANGELOG.md
2 parents 9648a01 + f7622b4 commit ec1d4c2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+381
-275
lines changed

‎CHANGELOG.md

+11-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,21 @@
1-
## v0.21.3 [2024-01-05]
1+
## v0.21.4 [2024-01-23]
2+
_Bug fixes_
3+
* Fixed schema clone function failing if table has an LTREE column. ([#4079](https://github.com/turbot/steampipe/issues/4079))
4+
* Maintain the order of execution when running multiple queries in batch mode. ([#3728](https://github.com/turbot/steampipe/issues/3728))
5+
* Fixes issue where using any meta-command would load connection state even if not required. ([#3614](https://github.com/turbot/steampipe/issues/3614))
6+
* Fixes issue where plugin version backfilling would write versions.json to cwd if the plugin folder is not found. ([#4073](https://github.com/turbot/steampipe/issues/4073))
7+
* Simplifies and fix available port check. ([#4030](https://github.com/turbot/steampipe/issues/4030))
8+
9+
10+
## v0.21.3 [2023-12-22]
211
_Whats new_
312
* Allow using pprof on FDW when STEAMPIPE_FDW_PPROF environment variable is set. ([#368](https://github.com/turbot/steampipe-postgres-fdw/issues/368))
413

514
_Bug fixes_
615
* Set connection state to error if plugin load fails. ([#4043](https://github.com/turbot/steampipe/issues/4043))
716
* Fixes incorrect row count in timing output for aggregator connections. ([#402](https://github.com/turbot/steampipe-postgres-fdw/issues/402))
817
* OpenTelemetry metric names must only contain [A-Za-z0-9_.-]. ([#369](https://github.com/turbot/steampipe-postgres-fdw/issues/369))
18+
* Maintain the order of execution when running multiple queries in batch mode. ([#3728](https://github.com/turbot/steampipe/issues/3728))
919

1020
## v0.21.2 [2023-12-12]
1121
_Whats new_

‎cmd/list.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,13 @@ func getRunListSubCmd(opts listSubCmdOptions) func(cmd *cobra.Command, args []st
4343
}
4444

4545
return func(cmd *cobra.Command, _ []string) {
46+
ctx := cmd.Context()
4647
workspacePath := viper.GetString(constants.ArgModLocation)
4748

48-
w, errAndWarnings := workspace.Load(cmd.Context(), workspacePath)
49+
w, errAndWarnings := workspace.Load(ctx, workspacePath)
4950
error_helpers.FailOnError(errAndWarnings.GetError())
5051

51-
modResources, depResources, err := listResourcesInMod(cmd.Context(), w.Mod, cmd)
52+
modResources, depResources, err := listResourcesInMod(ctx, w.Mod, cmd)
5253
error_helpers.FailOnErrorWithMessage(err, "could not list resources")
5354
if len(modResources)+len(depResources) == 0 {
5455
fmt.Println("No resources available to execute.")

‎cmd/mod.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ func runModListCmd(cmd *cobra.Command, _ []string) {
285285
}
286286

287287
opts := modinstaller.NewInstallOpts(workspaceMod)
288-
installer, err := modinstaller.NewModInstaller(opts)
288+
installer, err := modinstaller.NewModInstaller(ctx, opts)
289289
error_helpers.FailOnError(err)
290290

291291
treeString := installer.GetModList()

‎cmd/plugin.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,10 @@ Examples:
7272
7373
# Uninstall a plugin
7474
steampipe plugin uninstall aws`,
75-
PersistentPostRun: func(_ *cobra.Command, args []string) {
75+
PersistentPostRun: func(cmd *cobra.Command, args []string) {
7676
utils.LogTime("cmd.plugin.PersistentPostRun start")
7777
defer utils.LogTime("cmd.plugin.PersistentPostRun end")
78-
plugin.CleanupOldTmpDirs()
78+
plugin.CleanupOldTmpDirs(cmd.Context())
7979
},
8080
}
8181
cmd.AddCommand(pluginInstallCmd())
@@ -297,7 +297,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
297297

298298
// reload the config, since an installation should have created a new config file
299299
var cmd = viper.Get(constants.ConfigKeyActiveCommand).(*cobra.Command)
300-
config, errorsAndWarnings := steampipeconfig.LoadSteampipeConfig(viper.GetString(constants.ArgModLocation), cmd.Name())
300+
config, errorsAndWarnings := steampipeconfig.LoadSteampipeConfig(ctx, viper.GetString(constants.ArgModLocation), cmd.Name())
301301
if errorsAndWarnings.GetError() != nil {
302302
error_helpers.ShowWarning(fmt.Sprintf("Failed to reload config - install report may be incomplete (%s)", errorsAndWarnings.GetError()))
303303
} else {
@@ -315,7 +315,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
315315
func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string, wg *sync.WaitGroup, returnChannel chan *display.PluginInstallReport) {
316316
var report *display.PluginInstallReport
317317

318-
pluginAlreadyInstalled, _ := plugin.Exists(pluginName)
318+
pluginAlreadyInstalled, _ := plugin.Exists(ctx, pluginName)
319319
if pluginAlreadyInstalled {
320320
// set the bar to MAX
321321
//nolint:golint,errcheck // the error happens if we set this over the max value
@@ -393,7 +393,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
393393
}
394394

395395
// load up the version file data
396-
versionData, err := versionfile.LoadPluginVersionFile()
396+
versionData, err := versionfile.LoadPluginVersionFile(ctx)
397397
if err != nil {
398398
error_helpers.ShowError(ctx, fmt.Errorf("error loading current plugin data"))
399399
exitCode = constants.ExitCodePluginLoadingError
@@ -419,7 +419,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
419419
// get the args and retrieve the installed versions
420420
for _, p := range plugins {
421421
ref := ociinstaller.NewSteampipeImageRef(p)
422-
isExists, _ := plugin.Exists(p)
422+
isExists, _ := plugin.Exists(ctx, p)
423423
if isExists {
424424
if strings.HasPrefix(ref.DisplayImageRef(), constants.SteampipeHubOCIBase) {
425425
runUpdatesFor = append(runUpdatesFor, versionData.Plugins[ref.DisplayImageRef()])
@@ -634,7 +634,7 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) {
634634

635635
err := showPluginListOutput(pluginList, failedPluginMap, missingPluginMap, res, outputFormat)
636636
if err != nil {
637-
error_helpers.ShowError(cmd.Context(), err)
637+
error_helpers.ShowError(ctx, err)
638638
}
639639

640640
}
@@ -814,7 +814,7 @@ func getPluginList(ctx context.Context) (pluginList []plugin.PluginListItem, fai
814814
// TODO do we really need to look at installed plugins - can't we just use the plugin connection map
815815
// get a list of the installed plugins by inspecting the install location
816816
// pass pluginConnectionMap so we can populate the connections for each plugin
817-
pluginList, err := plugin.List(pluginConnectionMap)
817+
pluginList, err := plugin.List(ctx, pluginConnectionMap)
818818
if err != nil {
819819
res.Error = err
820820
return nil, nil, nil, res

‎cmd/plugin_manager.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,12 @@ func doRunPluginManager(cmd *cobra.Command) error {
7373
}
7474

7575
func createPluginManager(cmd *cobra.Command) (*pluginmanager_service.PluginManager, error) {
76+
ctx := cmd.Context()
7677
logger := createPluginManagerLog()
7778

7879
log.Printf("[INFO] starting plugin manager")
7980
// build config map
80-
steampipeConfig, errorsAndWarnings := steampipeconfig.LoadConnectionConfig()
81+
steampipeConfig, errorsAndWarnings := steampipeconfig.LoadConnectionConfig(ctx)
8182
if errorsAndWarnings.GetError() != nil {
8283
log.Printf("[WARN] failed to load connection config: %v", errorsAndWarnings.GetError())
8384
return nil, errorsAndWarnings.Error
@@ -97,7 +98,7 @@ func createPluginManager(cmd *cobra.Command) (*pluginmanager_service.PluginManag
9798
configMap := connection.NewConnectionConfigMap(steampipeConfig.Connections)
9899
log.Printf("[TRACE] loaded config map: %s", strings.Join(steampipeConfig.ConnectionNames(), ","))
99100

100-
pluginManager, err := pluginmanager_service.NewPluginManager(cmd.Context(), configMap, steampipeConfig.PluginsInstances, logger)
101+
pluginManager, err := pluginmanager_service.NewPluginManager(ctx, configMap, steampipeConfig.PluginsInstances, logger)
101102
if err != nil {
102103
log.Printf("[WARN] failed to create plugin manager: %s", err.Error())
103104
return nil, err

‎cmd/query.go

+48-55
Original file line numberDiff line numberDiff line change
@@ -220,67 +220,60 @@ func executeSnapshotQuery(initData *query.InitData, ctx context.Context) int {
220220
}
221221
}
222222

223-
// build ordered list of queries
224-
// (ordered for testing repeatability)
225-
var queryNames = utils.SortedMapKeys(initData.Queries)
226-
227-
if len(queryNames) > 0 {
228-
for _, name := range queryNames {
229-
resolvedQuery := initData.Queries[name]
230-
// if a manual query is being run (i.e. not a named query), convert into a query and add to workspace
231-
// this is to allow us to use existing dashboard execution code
232-
queryProvider, existingResource := ensureSnapshotQueryResource(name, resolvedQuery, initData.Workspace)
233-
234-
// we need to pass the embedded initData to GenerateSnapshot
235-
baseInitData := &initData.InitData
236-
237-
// so a dashboard name was specified - just call GenerateSnapshot
238-
snap, err := dashboardexecute.GenerateSnapshot(ctx, queryProvider.Name(), baseInitData, nil)
239-
if err != nil {
240-
exitCode = constants.ExitCodeSnapshotCreationFailed
241-
error_helpers.FailOnError(err)
242-
}
223+
for _, resolvedQuery := range initData.Queries {
224+
// if a manual query is being run (i.e. not a named query), convert into a query and add to workspace
225+
// this is to allow us to use existing dashboard execution code
226+
queryProvider, existingResource := ensureSnapshotQueryResource(resolvedQuery.Name, resolvedQuery, initData.Workspace)
227+
228+
// we need to pass the embedded initData to GenerateSnapshot
229+
baseInitData := &initData.InitData
230+
231+
// so a dashboard name was specified - just call GenerateSnapshot
232+
snap, err := dashboardexecute.GenerateSnapshot(ctx, queryProvider.Name(), baseInitData, nil)
233+
if err != nil {
234+
exitCode = constants.ExitCodeSnapshotCreationFailed
235+
error_helpers.FailOnError(err)
236+
}
243237

244-
// set the filename root for the snapshot (in case needed)
245-
if !existingResource {
246-
snap.FileNameRoot = "query"
247-
}
238+
// set the filename root for the snapshot (in case needed)
239+
if !existingResource {
240+
snap.FileNameRoot = "query"
241+
}
248242

249-
// display the result
250-
switch viper.GetString(constants.ArgOutput) {
251-
case constants.OutputFormatNone:
252-
// do nothing
253-
case constants.OutputFormatSnapshot, constants.OutputFormatSnapshotShort:
254-
// if the format is snapshot, just dump it out
255-
jsonOutput, err := json.MarshalIndent(snap, "", " ")
256-
if err != nil {
257-
error_helpers.FailOnErrorWithMessage(err, "failed to display result as snapshot")
258-
}
259-
fmt.Println(string(jsonOutput))
260-
default:
261-
// otherwise convert the snapshot into a query result
262-
result, err := snapshotToQueryResult(snap)
243+
// display the result
244+
switch viper.GetString(constants.ArgOutput) {
245+
case constants.OutputFormatNone:
246+
// do nothing
247+
case constants.OutputFormatSnapshot, constants.OutputFormatSnapshotShort:
248+
// if the format is snapshot, just dump it out
249+
jsonOutput, err := json.MarshalIndent(snap, "", " ")
250+
if err != nil {
263251
error_helpers.FailOnErrorWithMessage(err, "failed to display result as snapshot")
264-
display.ShowOutput(ctx, result, display.WithTimingDisabled())
265252
}
253+
fmt.Println(string(jsonOutput))
254+
default:
255+
// otherwise convert the snapshot into a query result
256+
result, err := snapshotToQueryResult(snap)
257+
error_helpers.FailOnErrorWithMessage(err, "failed to display result as snapshot")
258+
display.ShowOutput(ctx, result, display.WithTimingDisabled())
259+
}
266260

267-
// share the snapshot if necessary
268-
err = publishSnapshotIfNeeded(ctx, snap)
269-
if err != nil {
270-
exitCode = constants.ExitCodeSnapshotUploadFailed
271-
error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("failed to publish snapshot to %s", viper.GetString(constants.ArgSnapshotLocation)))
272-
}
261+
// share the snapshot if necessary
262+
err = publishSnapshotIfNeeded(ctx, snap)
263+
if err != nil {
264+
exitCode = constants.ExitCodeSnapshotUploadFailed
265+
error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("failed to publish snapshot to %s", viper.GetString(constants.ArgSnapshotLocation)))
266+
}
273267

274-
// export the result if necessary
275-
exportArgs := viper.GetStringSlice(constants.ArgExport)
276-
exportMsg, err := initData.ExportManager.DoExport(ctx, snap.FileNameRoot, snap, exportArgs)
277-
error_helpers.FailOnErrorWithMessage(err, "failed to export snapshot")
278-
// print the location where the file is exported
279-
if len(exportMsg) > 0 && viper.GetBool(constants.ArgProgress) {
280-
fmt.Printf("\n")
281-
fmt.Println(strings.Join(exportMsg, "\n"))
282-
fmt.Printf("\n")
283-
}
268+
// export the result if necessary
269+
exportArgs := viper.GetStringSlice(constants.ArgExport)
270+
exportMsg, err := initData.ExportManager.DoExport(ctx, snap.FileNameRoot, snap, exportArgs)
271+
error_helpers.FailOnErrorWithMessage(err, "failed to export snapshot")
272+
// print the location where the file is exported
273+
if len(exportMsg) > 0 && viper.GetBool(constants.ArgProgress) {
274+
fmt.Printf("\n")
275+
fmt.Println(strings.Join(exportMsg, "\n"))
276+
fmt.Printf("\n")
284277
}
285278
}
286279
return 0

‎cmd/service.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ func runServiceStartCmd(cmd *cobra.Command, _ []string) {
162162
}
163163
}()
164164

165-
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt, os.Kill)
165+
ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, os.Kill)
166166
defer cancel()
167167

168168
listenAddresses := db_local.StartListenType(viper.GetString(constants.ArgDatabaseListenAddresses)).ToListenAddresses()
@@ -484,7 +484,7 @@ func runServiceStatusCmd(cmd *cobra.Command, _ []string) {
484484
}
485485

486486
if viper.GetBool(constants.ArgAll) {
487-
showAllStatus(cmd.Context())
487+
showAllStatus(ctx)
488488
} else {
489489
dbState, dbStateErr := db_local.GetState()
490490
pmState, pmStateErr := pluginmanager.LoadState()
@@ -578,7 +578,7 @@ func runServiceStopCmd(cmd *cobra.Command, _ []string) {
578578
}
579579

580580
// check if there are any connected clients to the service
581-
connectedClients, err := db_local.GetClientCount(cmd.Context())
581+
connectedClients, err := db_local.GetClientCount(ctx)
582582
if err != nil {
583583
exitCode = constants.ExitCodeServiceStopFailure
584584
error_helpers.FailOnErrorWithMessage(err, "service stop failed")

‎pkg/cmdconfig/cmd_hooks.go

+12-7
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ func preRunHook(cmd *cobra.Command, args []string) {
5959
utils.LogTime("cmdhook.preRunHook start")
6060
defer utils.LogTime("cmdhook.preRunHook end")
6161

62+
ctx := cmd.Context()
63+
6264
viper.Set(constants.ConfigKeyActiveCommand, cmd)
6365
viper.Set(constants.ConfigKeyActiveCommandArgs, args)
6466
viper.Set(constants.ConfigKeyIsTerminalTTY, isatty.IsTerminal(os.Stdout.Fd()))
@@ -102,13 +104,14 @@ func preRunHook(cmd *cobra.Command, args []string) {
102104
createLogger(logBuffer, cmd)
103105

104106
// runScheduledTasks skips running tasks if this instance is the plugin manager
105-
waitForTasksChannel = runScheduledTasks(cmd.Context(), cmd, args, ew)
107+
waitForTasksChannel = runScheduledTasks(ctx, cmd, args, ew)
106108

107109
// ensure all plugin installation directories have a version.json file
108110
// (this is to handle the case of migrating an existing installation from v0.20.x)
109111
// no point doing this for the plugin-manager since that would have been done by the initiating CLI process
110112
if !task.IsPluginManagerCmd(cmd) {
111-
versionfile.EnsureVersionFilesInPluginDirectories()
113+
err := versionfile.EnsureVersionFilesInPluginDirectories(ctx)
114+
error_helpers.FailOnError(sperr.WrapWithMessage(err, "failed to ensure version files in plugin directories"))
112115
}
113116

114117
// set the max memory if specified
@@ -185,16 +188,18 @@ func initGlobalConfig() *error_helpers.ErrorAndWarnings {
185188
utils.LogTime("cmdconfig.initGlobalConfig start")
186189
defer utils.LogTime("cmdconfig.initGlobalConfig end")
187190

191+
var cmd = viper.Get(constants.ConfigKeyActiveCommand).(*cobra.Command)
192+
ctx := cmd.Context()
193+
188194
// load workspace profile from the configured install dir
189-
loader, err := getWorkspaceProfileLoader()
195+
loader, err := getWorkspaceProfileLoader(ctx)
190196
if err != nil {
191197
return error_helpers.NewErrorsAndWarning(err)
192198
}
193199

194200
// set global workspace profile
195201
steampipeconfig.GlobalWorkspaceProfile = loader.GetActiveWorkspaceProfile()
196202

197-
var cmd = viper.Get(constants.ConfigKeyActiveCommand).(*cobra.Command)
198203
// set-up viper with defaults from the env and default workspace profile
199204
err = bootstrapViper(loader, cmd)
200205
if err != nil {
@@ -205,7 +210,7 @@ func initGlobalConfig() *error_helpers.ErrorAndWarnings {
205210
ensureInstallDir(viper.GetString(constants.ArgInstallDir))
206211

207212
// load the connection config and HCL options
208-
config, loadConfigErrorsAndWarnings := steampipeconfig.LoadSteampipeConfig(viper.GetString(constants.ArgModLocation), cmd.Name())
213+
config, loadConfigErrorsAndWarnings := steampipeconfig.LoadSteampipeConfig(ctx, viper.GetString(constants.ArgModLocation), cmd.Name())
209214
if loadConfigErrorsAndWarnings.Error != nil {
210215
return loadConfigErrorsAndWarnings
211216
}
@@ -275,7 +280,7 @@ func setCloudTokenDefault(loader *steampipeconfig.WorkspaceProfileLoader) error
275280
return nil
276281
}
277282

278-
func getWorkspaceProfileLoader() (*steampipeconfig.WorkspaceProfileLoader, error) {
283+
func getWorkspaceProfileLoader(ctx context.Context) (*steampipeconfig.WorkspaceProfileLoader, error) {
279284
// set viper default for workspace profile, using EnvWorkspaceProfile env var
280285
SetDefaultFromEnv(constants.EnvWorkspaceProfile, constants.ArgWorkspaceProfile, String)
281286
// set viper default for install dir, using EnvInstallDir env var
@@ -293,7 +298,7 @@ func getWorkspaceProfileLoader() (*steampipeconfig.WorkspaceProfileLoader, error
293298
}
294299

295300
// create loader
296-
loader, err := steampipeconfig.NewWorkspaceProfileLoader(workspaceProfileDir)
301+
loader, err := steampipeconfig.NewWorkspaceProfileLoader(ctx, workspaceProfileDir)
297302
if err != nil {
298303
return nil, err
299304
}

‎pkg/connection/connection_watcher.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package connection
22

33
import (
44
"context"
5+
"log"
6+
57
"github.com/fsnotify/fsnotify"
68
filehelpers "github.com/turbot/go-kit/files"
79
"github.com/turbot/go-kit/filewatcher"
@@ -10,7 +12,6 @@ import (
1012
"github.com/turbot/steampipe/pkg/constants"
1113
"github.com/turbot/steampipe/pkg/filepaths"
1214
"github.com/turbot/steampipe/pkg/steampipeconfig"
13-
"log"
1415
)
1516

1617
type ConnectionWatcher struct {
@@ -62,7 +63,7 @@ func (w *ConnectionWatcher) handleFileWatcherEvent([]fsnotify.Event) {
6263
ctx := context.Background()
6364

6465
log.Printf("[INFO] ConnectionWatcher handleFileWatcherEvent")
65-
config, errorsAndWarnings := steampipeconfig.LoadConnectionConfig()
66+
config, errorsAndWarnings := steampipeconfig.LoadConnectionConfig(context.Background())
6667
// send notification if there were any errors or warnings
6768
if !errorsAndWarnings.Empty() {
6869
w.pluginManager.SendPostgresErrorsAndWarningsNotification(ctx, errorsAndWarnings)

0 commit comments

Comments
 (0)