Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Admin server - part 2 #10482

Merged
merged 5 commits into from
Jan 21, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions projects/gateway2/admin/krt_snapshot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package admin

import (
"net/http"

"istio.io/istio/pkg/kube/krt"
)

func addKrtSnapshotHandler(path string, mux *http.ServeMux, profiles map[string]dynamicProfileDescription, dbg *krt.DebugHandler) {
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, dbg, r)
})
profiles[path] = func() string { return "KRT Snapshot" }
}
61 changes: 61 additions & 0 deletions projects/gateway2/admin/logging.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package admin

import (
"fmt"
"net/http"

"github.com/solo-io/go-utils/contextutils"
)

// The logging handler is an AtomicLevel that supports dynamically changing the log level at runtime.
func addLoggingHandler(path string, mux *http.ServeMux, profiles map[string]dynamicProfileDescription) {
mux.Handle(path, contextutils.GetLogHandler())
profiles[path] = getLoggingDescription
}

// Gets a string representation of the current log level.
func getLogLevel() string {
return contextutils.GetLogLevel().String()
}

// Gets the html/js to display in the UI for the logging endpoint.
func getLoggingDescription() string {
currentLogLevel := getLogLevel()

// build the options selector, with the current log level selected by default
selectorText := `<select id="loglevelselector">`
supportedLogLevels := []string{"debug", "info", "warn", "error"}
for _, level := range supportedLogLevels {
if level == currentLogLevel {
selectorText += fmt.Sprintf(`<option value="%s" selected>%s</option>`, level, level)
} else {
selectorText += fmt.Sprintf(`<option value="%s">%s</option>`, level, level)
}
}
selectorText += `</select>`

return `View or change the log level of the program. Note: does not persist across pod restarts.<br/>

Log level:
` + selectorText + `

<button onclick="setlevel(document.getElementById('loglevelselector').value)">Submit</button>

<script>
function setlevel(l) {
var xhr = new XMLHttpRequest();
xhr.open('PUT', '/logging', true);
xhr.setRequestHeader("Content-Type", "application/json");

xhr.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
var resp = JSON.parse(this.responseText);
alert("log level set to: " + resp["level"]);
}
};

xhr.send('{"level":"' + l + '"}');
}
</script>
`
}
20 changes: 20 additions & 0 deletions projects/gateway2/admin/pprof.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package admin

import (
"net/http"
"net/http/pprof"
)

func addPprofHandler(path string, mux *http.ServeMux, profiles map[string]dynamicProfileDescription) {
mux.HandleFunc(path, pprof.Index)
mux.HandleFunc(path+"cmdline", pprof.Cmdline)
mux.HandleFunc(path+"profile", pprof.Profile)
mux.HandleFunc(path+"symbol", pprof.Symbol)
mux.HandleFunc(path+"trace", pprof.Trace)

profiles[path] = func() string {
return `PProf related things:<br/>
<a href="` + path + `goroutine?debug=2">full goroutine stack dump</a>
`
}
}
77 changes: 25 additions & 52 deletions projects/gateway2/admin/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,38 +6,41 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"sort"

"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
"github.com/rotisserie/eris"
"github.com/solo-io/gloo/projects/gateway2/controller"
"github.com/solo-io/go-utils/contextutils"
"github.com/solo-io/go-utils/stats"
"istio.io/istio/pkg/kube/krt"
)

const (
AdminPort = 9095
AdminPort = 9091
)

func RunAdminServer(ctx context.Context, setupOpts *controller.SetupOpts) error {
// serverHandlers defines the custom handlers that the Admin Server will support
serverHandlers := getServerHandlers(ctx, setupOpts.KrtDebugger, setupOpts.Cache)

stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), func(mux *http.ServeMux, profiles map[string]string) {
// let people know these moved
profiles[fmt.Sprintf("http://localhost:%d/snapshots/", AdminPort)] = fmt.Sprintf("To see snapshots, port forward to port %d", AdminPort)
})
// initialize the atomic log level
if envLogLevel := os.Getenv(contextutils.LogLevelEnvName); envLogLevel != "" {
contextutils.SetLogLevelFromString(envLogLevel)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is definitely a good reminder that we should pull contextutils into this repo to work towards not needing the go-utils dependency

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah i noticed we use it everywhere for logging. do we no longer want the go-utils dep, even for logging stuff?
if so, i can work on removing it completely from the repo (in a separate PR)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think so but its definitely a different pr.

I wonder if we also just fork it from go-utils and put it in a smaller kgateway-dev repo? Thats the hard question: where should it live.

Lets bring it to the community meeting today.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should remove the contextutils/go-utils dep entirely but i dont think it should factor into this work

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agreed. This is more of an interesting point that we still want something that allows this toggle to change logging holistically from a single end point

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i agree that this is out of scope for this change, but for context, we probably want to change they way we do logging - having them attached to the context is annoying:

  • we have to pass context even to util functions that dont do IO
  • we can't have log topics and enable debug on specific topics

}

startHandlers(ctx, serverHandlers)

return nil
}

// use a function for the profile descriptions so that every time the admin page is displayed, it can show
// up-to-date info in the description (e.g. the current log level)
type dynamicProfileDescription func() string

// getServerHandlers returns the custom handlers for the Admin Server, which will be bound to the http.ServeMux
// These endpoints serve as the basis for an Admin Interface for the Control Plane (https://github.com/solo-io/gloo/issues/6494)
func getServerHandlers(ctx context.Context, dbg *krt.DebugHandler, cache envoycache.SnapshotCache) func(mux *http.ServeMux, profiles map[string]string) {
return func(m *http.ServeMux, profiles map[string]string) {
// These endpoints serve as the basis for an Admin Interface for the Control Plane (https://github.com/kgateway-dev/kgateway/issues/6494)
func getServerHandlers(ctx context.Context, dbg *krt.DebugHandler, cache envoycache.SnapshotCache) func(mux *http.ServeMux, profiles map[string]dynamicProfileDescription) {
return func(m *http.ServeMux, profiles map[string]dynamicProfileDescription) {

/*
// The Input Snapshot is intended to return a list of resources that are persisted in the Kubernetes DB, etcD
Expand All @@ -57,18 +60,13 @@ func getServerHandlers(ctx context.Context, dbg *krt.DebugHandler, cache envoyca
profiles["/snapshots/proxies"] = "Proxy Snapshot"
*/

// The xDS Snapshot is intended to return the full in-memory xDS cache that the Control Plane manages
// and serves up to running proxies.
m.HandleFunc("/snapshots/xds", func(w http.ResponseWriter, r *http.Request) {
response := getXdsSnapshotDataFromCache(cache)
writeJSON(w, response, r)
})
profiles["/snapshots/xds"] = "XDS Snapshot"
addXdsSnapshotHandler("/snapshots/xds", m, profiles, cache)

m.HandleFunc("/snapshots/krt", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, dbg, r)
})
profiles["/snapshots/krt"] = "KRT Snapshot"
addKrtSnapshotHandler("/snapshots/krt", m, profiles, dbg)

addLoggingHandler("/logging", m, profiles)

addPprofHandler("/debug/pprof/", m, profiles)
}
}

Expand Down Expand Up @@ -116,9 +114,9 @@ func writeJSON(w http.ResponseWriter, obj any, req *http.Request) {
}
}

func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error {
func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]dynamicProfileDescription)) error {
mux := new(http.ServeMux)
profileDescriptions := map[string]string{}
profileDescriptions := map[string]dynamicProfileDescription{}
for _, addHandler := range addHandlers {
addHandler(mux, profileDescriptions)
}
Expand Down Expand Up @@ -150,7 +148,7 @@ func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux,
return nil
}

func index(profileDescriptions map[string]string) func(w http.ResponseWriter, r *http.Request) {
func index(profileDescriptions map[string]dynamicProfileDescription) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {

type profile struct {
Expand All @@ -159,11 +157,11 @@ func index(profileDescriptions map[string]string) func(w http.ResponseWriter, r
Desc string
}
var profiles []profile
for href, desc := range profileDescriptions {
for href, descFunc := range profileDescriptions {
profiles = append(profiles, profile{
Name: href,
Href: href,
Desc: desc,
Desc: descFunc(),
})
}

Expand All @@ -181,28 +179,3 @@ func index(profileDescriptions map[string]string) func(w http.ResponseWriter, r
w.Write(buf.Bytes())
}
}

func getXdsSnapshotDataFromCache(xdsCache cache.SnapshotCache) SnapshotResponseData {
cacheKeys := xdsCache.GetStatusKeys()
cacheEntries := make(map[string]interface{}, len(cacheKeys))

for _, k := range cacheKeys {
xdsSnapshot, err := getXdsSnapshot(xdsCache, k)
if err != nil {
cacheEntries[k] = err.Error()
} else {
cacheEntries[k] = xdsSnapshot
}
}

return completeSnapshotResponse(cacheEntries)
}

func getXdsSnapshot(xdsCache cache.SnapshotCache, k string) (cache cache.ResourceSnapshot, err error) {
defer func() {
if r := recover(); r != nil {
err = eris.New(fmt.Sprintf("panic occurred while getting xds snapshot: %v", r))
}
}()
return xdsCache.GetSnapshot(k)
}
45 changes: 45 additions & 0 deletions projects/gateway2/admin/xds_snapshot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package admin

import (
"fmt"
"net/http"

"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
"github.com/rotisserie/eris"
)

// The xDS Snapshot is intended to return the full in-memory xDS cache that the Control Plane manages
// and serves up to running proxies.
func addXdsSnapshotHandler(path string, mux *http.ServeMux, profiles map[string]dynamicProfileDescription, cache envoycache.SnapshotCache) {
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
response := getXdsSnapshotDataFromCache(cache)
writeJSON(w, response, r)
})
profiles[path] = func() string { return "XDS Snapshot" }
}

func getXdsSnapshotDataFromCache(xdsCache cache.SnapshotCache) SnapshotResponseData {
cacheKeys := xdsCache.GetStatusKeys()
cacheEntries := make(map[string]interface{}, len(cacheKeys))

for _, k := range cacheKeys {
xdsSnapshot, err := getXdsSnapshot(xdsCache, k)
if err != nil {
cacheEntries[k] = err.Error()
} else {
cacheEntries[k] = xdsSnapshot
}
}

return completeSnapshotResponse(cacheEntries)
}

func getXdsSnapshot(xdsCache cache.SnapshotCache, k string) (cache cache.ResourceSnapshot, err error) {
defer func() {
if r := recover(); r != nil {
err = eris.New(fmt.Sprintf("panic occurred while getting xds snapshot: %v", r))
}
}()
return xdsCache.GetSnapshot(k)
}
6 changes: 3 additions & 3 deletions test/ginkgo/parallel/ports.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,9 @@ func portInUseListen(proposedPort uint32) error {
}

var denyListPorts = map[uint32]struct{}{
// See gloo/pkg/servers/admin/server.go
// See https://github.com/solo-io/solo-projects/issues/7307 for more details
9095: {},
// See projects/gateway2/admin/server.go
// This port is reserved for the admin server
9091: {},
}

func portInDenyList(proposedPort uint32) error {
Expand Down
Loading