main.go

package main

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	kingpin "github.com/alecthomas/kingpin/v2"
	clientconfigv2 "github.com/gophercloud/utils/v2/openstack/clientconfig"
	"github.com/jpillora/backoff"
	"gopkg.in/yaml.v3"

	"github.com/hashicorp/vault-client-go"
	"github.com/hashicorp/vault-client-go/schema"
	"github.com/openstack-exporter/openstack-exporter/cache"
	"github.com/openstack-exporter/openstack-exporter/exporters"
	"github.com/openstack-exporter/openstack-exporter/utils"
	"github.com/prometheus/client_golang/prometheus"
	pver "github.com/prometheus/client_golang/prometheus/collectors/version"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/promslog"
	"github.com/prometheus/common/promslog/flag"
	"github.com/prometheus/common/version"
	"github.com/prometheus/exporter-toolkit/web"
	webflag "github.com/prometheus/exporter-toolkit/web/kingpinflag"
)

const DEFAULT_OS_CLIENT_CONFIG = "/etc/openstack/clouds.yaml"

const (
	enableExporterMaxAttempts  = 10
	enableExporterRetryMinWait = 500 * time.Millisecond
	enableExporterRetryMaxWait = 5 * time.Second
)

type serviceState int

const (
	serviceAuto serviceState = iota
	serviceEnabled
	serviceDisabled
)

var (
	metrics                  = kingpin.Flag("web.telemetry-path", "uri path to expose metrics").Default("/metrics").String()
	osClientConfig           = kingpin.Flag("os-client-config", "Path to the cloud configuration file").Default(DEFAULT_OS_CLIENT_CONFIG).String()
	prefix                   = kingpin.Flag("prefix", "Prefix for metrics").Default("openstack").String()
	endpointType             = kingpin.Flag("endpoint-type", "openstack endpoint type to use (i.e: public, internal, admin)").Default("public").String()
	collectTime              = kingpin.Flag("collect-metric-time", "time spent collecting each metric").Default("false").Bool()
	disabledMetrics          = kingpin.Flag("disable-metric", "multiple --disable-metric can be specified in the format: service-metric (i.e: cinder-snapshots)").Default("").Short('d').Strings()
	disableSlowMetrics       = kingpin.Flag("disable-slow-metrics", "Disable slow metrics for performance reasons").Default("false").Bool()
	disableDeprecatedMetrics = kingpin.Flag("disable-deprecated-metrics", "Disable deprecated metrics").Default("false").Bool()
	disableCinderAgentUUID   = kingpin.Flag("disable-cinder-agent-uuid", "Disable UUID generation for Cinder agents").Default("false").Bool()
	cloud                    = kingpin.Arg("cloud", "name or id of the cloud to gather metrics from").String()
	multiCloud               = kingpin.Flag("multi-cloud", "Toggle the multiple cloud scraping mode under /probe?cloud=").Default("false").Bool()
	domainID                 = kingpin.Flag("domain-id", "Gather metrics only for the given Domain ID (defaults to all domains)").String()
	cacheEnable              = kingpin.Flag("cache", "Enable Cache mechanism globally").Default("false").Bool()
	cacheTTL                 = kingpin.Flag("cache-ttl", "TTL duration for cache expiry(eg. 10s, 11m, 1h)").Default("300s").Duration()
	tenantID                 = kingpin.Flag("project-id", "Gather metrics only for the given Project ID (defaults to all projects)").String()
	disableServiceAutodetect = kingpin.Flag("disable-service-autodetect", "Disable single-cloud service autodetection and use only explicit service flags").Default("false").Bool()
	novaMetadataMapping      = utils.LabelMapping(kingpin.Flag("nova.metadata-extra-labels", "Map provided server metadata keys to labels in openstack_nova_server_status metric").PlaceHolder("LABEL=KEY,KEY").Default(""))
	dnsConcurrentCount       = kingpin.Flag("dns-concurrent-count", "Number of concurrent requests for DNS recordset collection").Default("10").Int()
)

func main() {

	serviceStates := make(map[string]serviceState, len(exporters.SupportedExporters))

	for _, service := range exporters.SupportedExporters {
		serviceStates[service] = serviceAuto
		disableFlagName := fmt.Sprintf("disable-service.%s", service)
		disableFlagHelp := fmt.Sprintf("Disable the %s service exporter in strict mode", service)
		serviceName := service
		disableValue := false
		kingpin.Flag(disableFlagName, disableFlagHelp).Default("false").Action(func(*kingpin.ParseContext) error {
			if disableValue {
				serviceStates[serviceName] = serviceDisabled
			} else {
				serviceStates[serviceName] = serviceEnabled
			}
			return nil
		}).BoolVar(&disableValue)
	}
	toolkitFlags := webflag.AddFlags(kingpin.CommandLine, ":9180")

	promlogConfig := &promslog.Config{}
	flag.AddFlags(kingpin.CommandLine, promlogConfig)
	kingpin.Version(version.Print("openstack-exporter"))
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()
	logger := promslog.New(promlogConfig)
	logger.Info("Build Version", "version_info", version.Info(), "build_context", version.BuildContext())

	if *cloud == "" && !*multiCloud {
		logger.Error("openstack-exporter: error: required argument 'cloud' or flag --multi-cloud not provided, try --help")
	}

	if *osClientConfig != DEFAULT_OS_CLIENT_CONFIG {
		logger.Debug("Setting Env var OS_CLIENT_CONFIG_FILE", "os_client_config_file", *osClientConfig)
		os.Setenv("OS_CLIENT_CONFIG_FILE", *osClientConfig)
	}

	SetPasswordIfVaultIsUsed(logger)

	if _, err := os.Stat(*osClientConfig); err != nil {
		logger.Error("Could not read config file", "error", err)
		os.Exit(1)
	}

	services, err := resolveServiceConfig(ResolveServiceConfigParams{
		MultiCloud:        *multiCloud,
		Cloud:             *cloud,
		DisableAutodetect: *disableServiceAutodetect,
		ServiceStates:     serviceStates,
	}, logger)
	if err != nil {
		logger.Error("Failed to resolve service configuration", "error", err)
		os.Exit(1)
	}

	ctx1, cancel1 := context.WithCancelCause(context.Background())
	defer cancel1(nil)

	ctx2, cancel2 := signal.NotifyContext(ctx1, syscall.SIGINT, syscall.SIGTERM)
	defer cancel2()

	// Start the backend service.
	if *cacheEnable {
		go cacheBackgroundService(ctx2, services, cancel1, logger)
	}

	// Start the HTTP server.
	go startHTTPServer(ctx2, StartHTTPServerParams{
		Services:     services,
		ToolkitFlags: toolkitFlags,
	}, cancel1, logger)

	<-ctx2.Done()
	if err := context.Cause(ctx2); err != nil && !errors.Is(err, context.Canceled) {
		logger.Error("Shutting down due to error", "error", err)
		os.Exit(1)
	} else {
		logger.Info("Termination signal received. Shutting down...")
	}
}

// ResolveServiceConfigParams holds the arguments for resolveServiceConfig.
type ResolveServiceConfigParams struct {
	MultiCloud        bool
	Cloud             string
	DisableAutodetect bool
	ServiceStates     map[string]serviceState
}

func resolveServiceConfig(params ResolveServiceConfigParams, logger *slog.Logger) ([]string, error) {
	if params.MultiCloud || params.DisableAutodetect {
		setAutoServicesState(params.ServiceStates, serviceEnabled)
		if params.DisableAutodetect && !params.MultiCloud {
			logger.Info("Service autodetection is disabled for single-cloud mode")
		}
		enabledServices := getEnabledServicesFromStates(params.ServiceStates)
		if len(enabledServices) == 0 {
			return nil, errors.New("no services enabled by explicit flags")
		}
		return enabledServices, nil
	}

	logger.Info("Autodetecting available services")
	detectedServices, err := autodetectServices(params.Cloud, logger)
	if err != nil {
		return nil, err
	}
	logger.Info("Autodetected services", "detected_services", detectedServices)

	applyAutodetection(params.ServiceStates, detectedServices)
	enabledServices := getEnabledServicesFromStates(params.ServiceStates)
	logger.Info("Final enabled services", "enabled_services", enabledServices)
	if len(enabledServices) == 0 {
		return nil, errors.New("no services enabled after autodetection and flag filtering")
	}

	return enabledServices, nil
}

func setAutoServicesState(serviceStates map[string]serviceState, state serviceState) {
	for _, service := range exporters.SupportedExporters {
		if serviceStates[service] == serviceAuto {
			serviceStates[service] = state
		}
	}
}

func applyAutodetection(serviceStates map[string]serviceState, detectedServices []string) {
	for _, service := range detectedServices {
		if serviceStates[service] == serviceAuto {
			serviceStates[service] = serviceEnabled
		}
	}
	setAutoServicesState(serviceStates, serviceDisabled)
}

func getEnabledServicesFromStates(serviceStates map[string]serviceState) []string {
	enabledServices := []string{}
	for _, service := range exporters.SupportedExporters {
		if serviceStates[service] == serviceEnabled {
			enabledServices = append(enabledServices, service)
		}
	}
	return enabledServices
}

func autodetectServices(cloud string, logger *slog.Logger) ([]string, error) {
	opts := &clientconfigv2.ClientOpts{Cloud: cloud}
	services, err := exporters.AutodetectServicesFromCatalog(opts, nil, *endpointType)
	if err != nil {
		return nil, err
	}
	return services, nil
}

// cacheBackgroundService runs a background service to collect the metrics and stores in the cache.
// It collects data every cache-ttl/2 time and flush every cache-ttl time.
// The cache data will be read by the Prometheus HandleFunc.
func cacheBackgroundService(ctx context.Context, services []string, cancel context.CancelCauseFunc, logger *slog.Logger) {
	logger.Info("Start cache background service")
	collectTicker := time.NewTicker(*cacheTTL / 2)
	defer collectTicker.Stop()
	ttlTicker := time.NewTicker(*cacheTTL)
	defer ttlTicker.Stop()

	// Collect cache data in the beginning.
	if err := cache.CollectCache(exporters.EnableExporter, *multiCloud, services, *prefix, *cloud, *disabledMetrics, *endpointType, *collectTime, *disableSlowMetrics, *disableDeprecatedMetrics, *disableCinderAgentUUID, *domainID, *tenantID, novaMetadataMapping, *dnsConcurrentCount, nil, logger); err != nil {
		logger.Error("Failed to collect from cache", "err", err)
		cancel(err)
		return
	}

	for {
		select {
		case <-collectTicker.C:
			if err := cache.CollectCache(exporters.EnableExporter, *multiCloud, services, *prefix, *cloud, *disabledMetrics, *endpointType, *collectTime, *disableSlowMetrics, *disableDeprecatedMetrics, *disableCinderAgentUUID, *domainID, *tenantID, novaMetadataMapping, *dnsConcurrentCount, nil, logger); err != nil {
				cancel(err)
				return
			}
		case <-ttlTicker.C:
			cache.FlushExpiredCloudCaches(*cacheTTL)
			logger.Info("Cache TTL flush")
		case <-ctx.Done():
			logger.Info("Backend service is stopping")
			return
		}
	}
}

// StartHTTPServerParams holds the arguments for startHTTPServer.
type StartHTTPServerParams struct {
	Services     []string
	ToolkitFlags *web.FlagConfig
}

func startHTTPServer(ctx context.Context, params StartHTTPServerParams, cancel context.CancelCauseFunc, logger *slog.Logger) {
	links := []web.LandingLinks{}

	if *multiCloud {
		http.HandleFunc("/probe", probeHandler(params.Services, logger))
		http.Handle(*metrics, promhttp.Handler())
		logger.Info("openstack exporter started in multi cloud mode (/probe?cloud=)")
		links = append(links, web.LandingLinks{
			Address: *metrics,
			Text:    "Metrics",
		}, web.LandingLinks{
			Address: "/probe",
			Text:    "Probes",
		})
	} else {
		logger.Info("openstack exporter started in legacy mode")
		http.HandleFunc(*metrics, metricHandler(params.Services, logger))
		links = append(links, web.LandingLinks{
			Address: *metrics,
			Text:    "Metrics",
		})
	}

	if *metrics != "/" && *metrics != "" {
		landingConfig := web.LandingConfig{
			Name:        "openstack_exporter",
			Description: "Prometheus Exporter for openstack",
			Version:     version.Info(),
			Links:       links,
		}

		landingPage, err := web.NewLandingPage(landingConfig)
		if err != nil {
			logger.Error("Failed to create landing page", "error", err)
			cancel(err)
			return
		}
		http.Handle("/", landingPage)
	}

	if *domainID != "" {
		logger.Info("Gathering metrics for configured domain ID", "domain_id", *domainID)
	}

	if *tenantID != "" {
		logger.Info("Gathering metrics for configured project ID", "project_id", *tenantID)
	}

	srv := &http.Server{
		BaseContext: func(net.Listener) context.Context {
			return ctx
		},
	}
	go func() {
		if err := web.ListenAndServe(srv, params.ToolkitFlags, logger); err != nil {
			logger.Error("Failed to start webserver", "error", err)
			cancel(err)
		}
	}()

	<-ctx.Done()
	if err := srv.Shutdown(context.Background()); err != nil {
		logger.Error("HTTP server shutdown error", "error", err)
	}
}

func probeHandler(configuredServices []string, logger *slog.Logger) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx, cancel := context.WithCancel(r.Context())
		defer cancel()
		r = r.WithContext(ctx)

		cloud := r.URL.Query().Get("cloud")
		if cloud == "" {
			http.Error(w, "'cloud' parameter is missing", http.StatusBadRequest)
			return
		}

		enabledServices, err := selectServicesForRequest(configuredServices, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		logger.Info("Enabled services", "enabled_services", enabledServices)

		// Get data from cache
		if *cacheEnable {
			if err := cache.WriteCacheToResponse(w, r, cloud, enabledServices, logger); err != nil {
				logger.Error("Write cache to response failed", "error", err)
			}
			return
		}

		registry := prometheus.NewPedanticRegistry()
		for _, service := range enabledServices {
			exp, err := exporters.EnableExporter(service, *prefix, cloud, *disabledMetrics, *endpointType, *collectTime, *disableSlowMetrics, *disableDeprecatedMetrics, *disableCinderAgentUUID, *domainID, *tenantID, novaMetadataMapping, *dnsConcurrentCount, nil, logger)
			if err != nil {
				logger.Error("Enabling exporter for service failed", "service", service, "error", err)
				continue
			}
			registry.MustRegister(*exp)
			logger.Info("Enabled exporter for service", "service", service)
		}

		h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
		h.ServeHTTP(w, r)
	}
}

// metricHandler builds the exporters and their registry once at startup and
// reuses them across every scrape, instead of re-authenticating to Keystone
// and reconstructing every exporter per request.
func metricHandler(configuredServices []string, logger *slog.Logger) http.HandlerFunc {
	logger.Info("Starting openstack exporter version for cloud", "version", version.Info(), "cloud", *cloud)
	logger.Info("Build context", "build_context", version.BuildContext())

	if *osClientConfig != DEFAULT_OS_CLIENT_CONFIG {
		logger.Debug("Setting Env var OS_CLIENT_CONFIG_FILE", "os_client_config_file", *osClientConfig)
		os.Setenv("OS_CLIENT_CONFIG_FILE", *osClientConfig)
	}

	enabledServices := configuredServices

	// Get data from cache
	if *cacheEnable {
		return func(w http.ResponseWriter, r *http.Request) {
			if err := cache.WriteCacheToResponse(w, r, *cloud, enabledServices, logger); err != nil {
				logger.Error("Write cache to response failed", "error", err)
			}
		}
	}

	registry := prometheus.NewPedanticRegistry()
	enabledExporters := 0
	for _, service := range enabledServices {
		exp, err := enableExporterWithRetry(service, logger)
		if err != nil {
			// Log error and continue with enabling other exporters
			logger.Error("enabling exporter for service failed after retries", "service", service, "error", err)
			continue
		}
		registry.MustRegister(*exp)
		logger.Info("Enabled exporter for service", "service", service)
		enabledExporters++
	}

	if enabledExporters == 0 {
		logger.Error("No exporter has been enabled, exiting")
		os.Exit(-1)
	}

	// expose program version
	registry.MustRegister(pver.NewCollector("openstack_exporter"))

	h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
	return h.ServeHTTP
}

// enableExporterWithRetry retries EnableExporter with exponential backoff so a
// service that is transiently unavailable at startup (e.g. a brief 503 during
// discovery) does not stay permanently disabled for the life of the process.
func enableExporterWithRetry(service string, logger *slog.Logger) (*exporters.OpenStackExporter, error) {
	b := &backoff.Backoff{Min: enableExporterRetryMinWait, Max: enableExporterRetryMaxWait, Factor: 2, Jitter: true}

	var lastErr error
	for attempt := 1; attempt <= enableExporterMaxAttempts; attempt++ {
		exp, err := exporters.EnableExporter(service, *prefix, *cloud, *disabledMetrics, *endpointType, *collectTime, *disableSlowMetrics, *disableDeprecatedMetrics, *disableCinderAgentUUID, *domainID, *tenantID, novaMetadataMapping, *dnsConcurrentCount, nil, logger)
		if err == nil {
			return exp, nil
		}
		lastErr = err
		if attempt == enableExporterMaxAttempts {
			break
		}
		logger.Warn("enabling exporter for service failed, retrying", "service", service, "attempt", attempt, "error", err)
		time.Sleep(b.Duration())
	}
	return nil, lastErr
}

func selectServicesForRequest(configuredServices []string, r *http.Request) ([]string, error) {
	enabledServices := configuredServices

	includeServices := r.URL.Query().Get("include_services")
	if includeServices != "" {
		includeList := parseServiceList(includeServices)
		invalid := invalidExporterNames(includeList)
		if len(invalid) > 0 {
			return nil, fmt.Errorf("invalid include_services: %s", strings.Join(invalid, ","))
		}
		enabledServices = includeList
	}

	excludeList := parseServiceList(r.URL.Query().Get("exclude_services"))
	invalid := invalidExporterNames(excludeList)
	if len(invalid) > 0 {
		return nil, fmt.Errorf("invalid exclude_services: %s", strings.Join(invalid, ","))
	}
	return utils.UniqueElements(utils.RemoveElements(enabledServices, excludeList)), nil
}

func invalidExporterNames(services []string) []string {
	invalid := make([]string, 0, len(services))
	for _, service := range services {
		if !exporters.IsExporterNameValid(service) {
			invalid = append(invalid, service)
		}
	}
	return invalid
}

func parseServiceList(raw string) []string {
	if raw == "" {
		return nil
	}

	services := []string{}
	for _, service := range strings.Split(raw, ",") {
		service = strings.TrimSpace(service)
		if service == "" {
			continue
		}
		services = append(services, service)
	}

	return services
}

func SetPasswordIfVaultIsUsed(logger *slog.Logger) {
	configFileData, err := os.ReadFile(*osClientConfig)
	if err != nil {
		logger.Error("failed to read config file", "err", err)
		os.Exit(1)
	}

	type VaultConfig struct {
		UseVault                    bool   `yaml:"use_vault"`
		VaultAddress                string `yaml:"vault_address"`
		VaultRoleID                 string `yaml:"vault_role_id"`
		VaultSecretID               string `yaml:"vault_secret_id"`
		VaultSecretPath             string `yaml:"vault_secret_path"`
		VaultSecretMountPath        string `yaml:"vault_secret_mount_path"`
		CredentialNameInVaultSecret string `yaml:"credential_name_in_vault_secret"`
	}

	var vaultConfig VaultConfig

	err = yaml.Unmarshal(configFileData, &vaultConfig)
	if err != nil {
		logger.Error("failed to parse config data", "err", err)
		os.Exit(1)
	}

	if !vaultConfig.UseVault {
		return
	}
	client, err := vault.New(vault.WithAddress(vaultConfig.VaultAddress))
	if err != nil {
		logger.Error("failed to create Vault client", "err", err)
		os.Exit(1)
	}
	ctx := context.Background()
	resp, err := client.Auth.AppRoleLogin(
		ctx,
		schema.AppRoleLoginRequest{
			RoleId:   vaultConfig.VaultRoleID,
			SecretId: vaultConfig.VaultSecretID,
		},
	)
	if err != nil {
		logger.Error("failed to login to Vault", "err", err)
		os.Exit(1)
	}
	if err := client.SetToken(resp.Auth.ClientToken); err != nil {
		logger.Error("failed to set Vault token", "err", err)
		os.Exit(1)
	}
	secret, err := client.Secrets.KvV2Read(
		ctx,
		vaultConfig.VaultSecretPath,
		vault.WithMountPath(vaultConfig.VaultSecretMountPath),
	)
	if err != nil {
		logger.Error("failed to get secret from Vault", "err", err)
		os.Exit(1)
	}

	os.Setenv("OS_PASSWORD", secret.Data.Data[vaultConfig.CredentialNameInVaultSecret].(string))
}