Make the log level of Go services configurable (#2413)

Summary: Make the log level of Go services configurable

Every Go service in Pixie Cloud and Vizier calls
`services.SetupServiceLogging()`, which hard coded the logrus level to
`info`. As discussed in the issue, there was no way to reduce log volume
or turn on debug logging without rebuilding.

Following the approach suggested in the issue, this PR adds a common
`--log_level` flag to `service_flags.go` and uses it in
`SetupServiceLogging()`. Like the other service flags, it can also be
set with the `PL_LOG_LEVEL` environment variable. It accepts any logrus
level (`panic`, `fatal`, `error`, `warn`, `info`, `debug`, `trace`, case
insensitive) and defaults to `info`, so current deployments behave
exactly as before. An invalid value is a fatal configuration error, the
same way `CheckServiceFlags()` handles other invalid flags.

The level is applied after flags are parsed, so the `Starting service`
line that `SetupService()` logs before parsing is still printed at info
level.

This only affects the Go logrus logger. The C++ (glog) logging in PEM,
Kelvin and the query broker already has its own runtime flags.

Relevant Issues: Fixes #2280

Type of change: /kind feature

Test Plan: Added `//src/shared/services:services_test`, which checks the
default level, explicit levels, case insensitivity, and that an invalid
level is fatal. The explicit level cases fail on `main`. I also built
`scriptmgr` and ran it locally. With no flag it logs at info. With
`--log_level=error`, only error logs are printed after startup. With
`PL_LOG_LEVEL=debug`, debug logs are printed. With `--log_level=bogus`,
it exits with `Flag --log_level or ENV PL_LOG_LEVEL is invalid`. `go
vet` and golangci-lint v2.3.1 with the repo's `.golangci.yaml` are
clean.

Changelog Message:
```release-note
Go services in Pixie Cloud and Vizier now accept a `--log_level` flag (or `PL_LOG_LEVEL` environment variable) to configure their log level. The default remains `info`.
```

Signed-off-by: NopeCharon <115794254+NopeCharon@users.noreply.github.com>

Author: NopeCharon

src/shared/services/BUILD.bazel

@@ -15,6 +15,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//bazel:pl_build_system.bzl", "pl_go_test")
 
 go_library(
     name = "services",
@@ -50,3 +51,16 @@ go_library(
         "@org_golang_google_grpc//status",
     ],
 )
+
+pl_go_test(
+    name = "services_test",
+    srcs = ["logging_test.go"],
+    embed = [":services"],
+    deps = [
+        "@com_github_sirupsen_logrus//:logrus",
+        "@com_github_spf13_pflag//:pflag",
+        "@com_github_spf13_viper//:viper",
+        "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
+    ],
+)

src/shared/services/logging.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
 	"github.com/zenazn/goji/web/mutil"
 
 	version "px.dev/pixie/src/shared/goversion"
@@ -39,7 +40,11 @@ func init() {
 func SetupServiceLogging() {
 	// Setup logging.
 	log.SetOutput(os.Stdout)
-	log.SetLevel(log.InfoLevel)
+	level, err := log.ParseLevel(viper.GetString("log_level"))
+	if err != nil {
+		log.WithError(err).Fatal("Flag --log_level or ENV PL_LOG_LEVEL is invalid")
+	}
+	log.SetLevel(level)
 }
 
 // HTTPLoggingMiddleware is a middleware function used for logging HTTP requests.

src/shared/services/logging_test.go

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2018- The Pixie Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package services
+
+import (
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetupServiceLogging(t *testing.T) {
+	SetupCommonFlags()
+
+	tests := []struct {
+		name          string
+		logLevel      string
+		expectedLevel log.Level
+	}{
+		{
+			name:          "default",
+			logLevel:      "",
+			expectedLevel: log.InfoLevel,
+		},
+		{
+			name:          "debug",
+			logLevel:      "debug",
+			expectedLevel: log.DebugLevel,
+		},
+		{
+			name:          "error",
+			logLevel:      "error",
+			expectedLevel: log.ErrorLevel,
+		},
+		{
+			name:          "case insensitive",
+			logLevel:      "WARN",
+			expectedLevel: log.WarnLevel,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			origLevel := log.GetLevel()
+			t.Cleanup(func() {
+				viper.Reset()
+				log.SetLevel(origLevel)
+			})
+
+			require.NoError(t, viper.BindPFlag("log_level", pflag.Lookup("log_level")))
+			if test.logLevel != "" {
+				viper.Set("log_level", test.logLevel)
+			}
+
+			SetupServiceLogging()
+			assert.Equal(t, test.expectedLevel, log.GetLevel())
+		})
+	}
+}
+
+func TestSetupServiceLogging_InvalidLevel(t *testing.T) {
+	origLevel := log.GetLevel()
+	origExitFunc := log.StandardLogger().ExitFunc
+	t.Cleanup(func() {
+		viper.Reset()
+		log.SetLevel(origLevel)
+		log.StandardLogger().ExitFunc = origExitFunc
+	})
+
+	exitCode := 0
+	log.StandardLogger().ExitFunc = func(code int) {
+		exitCode = code
+	}
+
+	viper.Set("log_level", "verbose")
+	SetupServiceLogging()
+	assert.Equal(t, 1, exitCode)
+}

src/shared/services/service_flags.go

@@ -51,6 +51,7 @@ func setupCommonFlags() {
 	pflag.String("jwt_signing_key", "", "The signing key used for JWTs")
 	pflag.String("pod_name", "<unknown>", "The pod name")
 	pflag.Bool("version", false, "Print the version and quit.")
+	pflag.String("log_level", log.InfoLevel.String(), "The log level. One of: panic, fatal, error, warn, info, debug, trace.")
 }
 
 // SetupCommonFlags sets flags that are used by every service, even non GRPC servers.