Configuration management forms the foundation of reliable cloud-native applications. When I build Go applications for cloud environments, I focus on creating flexible, secure, and dynamic configuration systems. These strategies have proven essential across dozens of production deployments.
Environment-Based Configuration
Go applications deployed in cloud environments need to adapt to different settings without rebuilding. I’ve found that using environment variables provides the simplest path to configuration in containerized deployments.
package main
import (
"fmt"
"os"
"strconv"
"time"
)
type DatabaseConfig struct {
Host string
Port int
User string
Password string
Name string
Timeout time.Duration
}
func loadDatabaseConfigFromEnv() DatabaseConfig {
port, _ := strconv.Atoi(getEnvWithDefault("DB_PORT", "5432"))
timeoutSec, _ := strconv.Atoi(getEnvWithDefault("DB_TIMEOUT_SEC", "30"))
return DatabaseConfig{
Host: getEnvWithDefault("DB_HOST", "localhost"),
Port: port,
User: getEnvWithDefault("DB_USER", "postgres"),
Password: os.Getenv("DB_PASSWORD"),
Name: getEnvWithDefault("DB_NAME", "myapp"),
Timeout: time.Duration(timeoutSec) * time.Second,
}
}
func getEnvWithDefault(key, defaultValue string) string {
value, exists := os.LookupEnv(key)
if !exists {
return defaultValue
}
return value
}
This pattern ensures applications run with sensible defaults while allowing override through environment variables, aligning perfectly with container orchestration platforms like Kubernetes.
Layered Configuration with Viper
For more complex applications, I’ve had great success using Viper to implement a layered configuration approach. This allows blending configuration from files, environment variables, and remote sources.
package main
import (
"fmt"
"log"
"strings"
"github.com/spf13/viper"
)
type Config struct {
Server struct {
Port int
Timeout int
Debug bool
}
Database struct {
DSN string
MaxConnections int
ConnMaxLifetime int
}
Features map[string]bool
}
func LoadConfig(configPath string) (*Config, error) {
// Set defaults
viper.SetDefault("server.port", 8080)
viper.SetDefault("server.timeout", 30)
viper.SetDefault("server.debug", false)
viper.SetDefault("database.maxconnections", 10)
viper.SetDefault("database.connmaxlifetime", 3600)
// Read from file
viper.SetConfigName("config")
viper.SetConfigType("yaml")
viper.AddConfigPath(configPath)
viper.AddConfigPath(".")
// Enable environment variables
viper.SetEnvPrefix("APP")
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
viper.AutomaticEnv()
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
return nil, fmt.Errorf("error reading config file: %s", err)
}
log.Println("No config file found, using defaults and environment variables")
}
var config Config
if err := viper.Unmarshal(&config); err != nil {
return nil, fmt.Errorf("unable to decode config into struct: %s", err)
}
return &config, nil
}
This approach gives applications clear defaults, loads configuration from files when available, and allows environment variables to override settings—essential for cloud-native deployments.
Kubernetes-Native Configuration
When deploying Go applications on Kubernetes, I leverage ConfigMaps and Secrets for configuration management. This approach keeps configuration separate from container images, enabling updates without rebuilds.
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"gopkg.in/yaml.v2"
)
const (
configMountPath = "/etc/config"
secretMountPath = "/etc/secrets"
)
type AppConfig struct {
APIVersion string `yaml:"apiVersion"`
LogLevel string `yaml:"logLevel"`
Features struct {
RealtimeUpdates bool `yaml:"realtimeUpdates"`
Analytics bool `yaml:"analytics"`
} `yaml:"features"`
Database struct {
Host string `yaml:"host"`
Port int `yaml:"port"`
Name string `yaml:"name"`
User string `yaml:"user"`
} `yaml:"database"`
}
func loadKubernetesConfig() (*AppConfig, error) {
// Load main config from ConfigMap
configFile := filepath.Join(configMountPath, "config.yaml")
data, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
var config AppConfig
if err := yaml.Unmarshal(data, &config); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
// Load database password from Secret
passwordFile := filepath.Join(secretMountPath, "db-password")
if _, err := os.Stat(passwordFile); err == nil {
password, err := ioutil.ReadFile(passwordFile)
if err != nil {
log.Printf("Warning: Failed to read database password: %v", err)
} else {
// In a real implementation, you'd set this on your config
// config.Database.Password = string(password)
log.Printf("Loaded database password, length: %d", len(password))
}
}
return &config, nil
}
This approach separates configuration concerns, storing non-sensitive settings in ConfigMaps and credentials in Secrets, both mounted as files in the container.
Secure Secrets Management
Protecting sensitive information is critical for cloud applications. I’ve implemented various approaches depending on deployment requirements.
package main
import (
"context"
"fmt"
"log"
"time"
secretmanager "cloud.google.com/go/secretmanager/apiv1"
secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
)
type Secrets struct {
DBPassword string
APIKeys map[string]string
JWTSigningKey []byte
EncryptionKeys map[string][]byte
}
func loadSecretsFromGCP(projectID string) (*Secrets, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
client, err := secretmanager.NewClient(ctx)
if err != nil {
return nil, fmt.Errorf("failed to create secret manager client: %w", err)
}
defer client.Close()
secrets := &Secrets{
APIKeys: make(map[string]string),
EncryptionKeys: make(map[string][]byte),
}
// Load database password
dbPassword, err := accessSecret(ctx, client, projectID, "db-password", "latest")
if err != nil {
return nil, fmt.Errorf("failed to access db password: %w", err)
}
secrets.DBPassword = dbPassword
// Load JWT signing key
jwtKey, err := accessSecret(ctx, client, projectID, "jwt-signing-key", "latest")
if err != nil {
return nil, fmt.Errorf("failed to access JWT key: %w", err)
}
secrets.JWTSigningKey = []byte(jwtKey)
// Load API keys
services := []string{"payment-gateway", "email-service", "analytics"}
for _, service := range services {
secretName := fmt.Sprintf("%s-api-key", service)
key, err := accessSecret(ctx, client, projectID, secretName, "latest")
if err != nil {
log.Printf("Warning: Failed to load %s: %v", secretName, err)
continue
}
secrets.APIKeys[service] = key
}
return secrets, nil
}
func accessSecret(ctx context.Context, client *secretmanager.Client, projectID, secretID, version string) (string, error) {
name := fmt.Sprintf("projects/%s/secrets/%s/versions/%s", projectID, secretID, version)
req := &secretmanagerpb.AccessSecretVersionRequest{
Name: name,
}
result, err := client.AccessSecretVersion(ctx, req)
if err != nil {
return "", fmt.Errorf("failed to access secret version: %w", err)
}
return string(result.Payload.Data), nil
}
This code demonstrates accessing secrets from Google Cloud Secret Manager, but similar patterns work with HashiCorp Vault, AWS Secrets Manager, or Azure Key Vault.
Feature Flags for Controlled Rollouts
Feature flags enable separating deployment from feature activation, which I’ve found valuable for cloud applications that need controlled rollouts.
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"sync"
"time"
)
type FeatureFlags struct {
mu sync.RWMutex
flags map[string]bool
refresh time.Duration
url string
}
func NewFeatureFlags(url string, refreshInterval time.Duration) *FeatureFlags {
ff := &FeatureFlags{
flags: make(map[string]bool),
refresh: refreshInterval,
url: url,
}
// Set initial defaults
ff.flags["new_dashboard"] = false
ff.flags["advanced_search"] = false
ff.flags["beta_api"] = false
// Start background refresh
go ff.refreshLoop()
return ff
}
func (ff *FeatureFlags) IsEnabled(feature string) bool {
ff.mu.RLock()
defer ff.mu.RUnlock()
enabled, exists := ff.flags[feature]
if !exists {
return false
}
return enabled
}
func (ff *FeatureFlags) refreshLoop() {
for {
if err := ff.fetchFlags(); err != nil {
log.Printf("Error refreshing feature flags: %v", err)
}
time.Sleep(ff.refresh)
}
}
func (ff *FeatureFlags) fetchFlags() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", ff.url, nil)
if err != nil {
return fmt.Errorf("creating request: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("fetching flags: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad status code: %d", resp.StatusCode)
}
var newFlags map[string]bool
if err := json.NewDecoder(resp.Body).Decode(&newFlags); err != nil {
return fmt.Errorf("decoding response: %w", err)
}
ff.mu.Lock()
defer ff.mu.Unlock()
// Update flags
for k, v := range newFlags {
ff.flags[k] = v
}
return nil
}
// Example usage
func featureFlagsExample() {
flags := NewFeatureFlags("https://config.example.com/features", 1*time.Minute)
// In your application logic
if flags.IsEnabled("new_dashboard") {
// Serve new dashboard
} else {
// Serve old dashboard
}
}
This implementation periodically refreshes flags from a central service, allowing teams to control feature activation independently from deployment.
Centralized Configuration with etcd
For distributed systems, I often use etcd to provide centralized configuration accessible to multiple services.
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
)
type ServiceConfig struct {
Port int `json:"port"`
LogLevel string `json:"logLevel"`
RateLimits map[string]int `json:"rateLimits"`
AllowedOrigins []string `json:"allowedOrigins"`
MaxRequestSize int `json:"maxRequestSize"`
}
type ConfigWatcher struct {
client *clientv3.Client
prefix string
config ServiceConfig
updates chan ServiceConfig
}
func NewConfigWatcher(endpoints []string, prefix string) (*ConfigWatcher, error) {
client, err := clientv3.New(clientv3.Config{
Endpoints: endpoints,
DialTimeout: 5 * time.Second,
})
if err != nil {
return nil, fmt.Errorf("failed to connect to etcd: %w", err)
}
watcher := &ConfigWatcher{
client: client,
prefix: prefix,
updates: make(chan ServiceConfig, 1),
}
// Load initial config
if err := watcher.loadConfig(); err != nil {
return nil, err
}
// Start watching for changes
go watcher.watchChanges()
return watcher, nil
}
func (w *ConfigWatcher) loadConfig() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
resp, err := w.client.Get(ctx, w.prefix)
if err != nil {
return fmt.Errorf("failed to get config from etcd: %w", err)
}
if len(resp.Kvs) == 0 {
log.Printf("No configuration found at %s, using defaults", w.prefix)
w.config = ServiceConfig{
Port: 8080,
LogLevel: "info",
RateLimits: map[string]int{"default": 100},
AllowedOrigins: []string{"*"},
MaxRequestSize: 1048576,
}
return nil
}
if err := json.Unmarshal(resp.Kvs[0].Value, &w.config); err != nil {
return fmt.Errorf("failed to parse config: %w", err)
}
return nil
}
func (w *ConfigWatcher) watchChanges() {
watchChan := w.client.Watch(context.Background(), w.prefix)
for response := range watchChan {
for _, event := range response.Events {
if event.Type == clientv3.EventTypePut {
var newConfig ServiceConfig
if err := json.Unmarshal(event.Kv.Value, &newConfig); err != nil {
log.Printf("Failed to parse updated config: %v", err)
continue
}
log.Printf("Configuration updated: %+v", newConfig)
w.config = newConfig
// Notify subscribers
select {
case w.updates <- newConfig:
default:
// Don't block if no one is listening
}
}
}
}
}
func (w *ConfigWatcher) GetConfig() ServiceConfig {
return w.config
}
func (w *ConfigWatcher) SubscribeUpdates() <-chan ServiceConfig {
return w.updates
}
func (w *ConfigWatcher) Close() error {
return w.client.Close()
}
This approach allows multiple instances of an application to share configuration and receive updates in real-time.
Runtime Reconfiguration
Supporting configuration changes without restarts improves availability in cloud environments.
package main
import (
"encoding/json"
"log"
"net/http"
"sync"
"time"
)
type RuntimeConfig struct {
mu sync.RWMutex
LogLevel string `json:"logLevel"`
CacheSize int `json:"cacheSize"`
DatabasePool int `json:"databasePool"`
Timeouts map[string]int `json:"timeouts"`
AllowedIPs []string `json:"allowedIPs"`
// Callbacks for components that need to react to config changes
logLevelChanged func(string)
cacheSizeChanged func(int)
dbPoolChanged func(int)
}
func NewRuntimeConfig() *RuntimeConfig {
return &RuntimeConfig{
LogLevel: "info",
CacheSize: 1000,
DatabasePool: 10,
Timeouts: map[string]int{
"http": 30,
"database": 10,
"cache": 5,
},
AllowedIPs: []string{"127.0.0.1"},
}
}
func (c *RuntimeConfig) RegisterLogLevelCallback(fn func(string)) {
c.mu.Lock()
defer c.mu.Unlock()
c.logLevelChanged = fn
}
func (c *RuntimeConfig) RegisterCacheSizeCallback(fn func(int)) {
c.mu.Lock()
defer c.mu.Unlock()
c.cacheSizeChanged = fn
}
func (c *RuntimeConfig) RegisterDBPoolCallback(fn func(int)) {
c.mu.Lock()
defer c.mu.Unlock()
c.dbPoolChanged = fn
}
func (c *RuntimeConfig) GetLogLevel() string {
c.mu.RLock()
defer c.mu.RUnlock()
return c.LogLevel
}
func (c *RuntimeConfig) GetCacheSize() int {
c.mu.RLock()
defer c.mu.RUnlock()
return c.CacheSize
}
func (c *RuntimeConfig) GetDatabasePool() int {
c.mu.RLock()
defer c.mu.RUnlock()
return c.DatabasePool
}
func (c *RuntimeConfig) GetTimeout(name string) time.Duration {
c.mu.RLock()
defer c.mu.RUnlock()
seconds, exists := c.Timeouts[name]
if !exists {
return 30 * time.Second // Default
}
return time.Duration(seconds) * time.Second
}
func (c *RuntimeConfig) IsIPAllowed(ip string) bool {
c.mu.RLock()
defer c.mu.RUnlock()
for _, allowedIP := range c.AllowedIPs {
if allowedIP == ip || allowedIP == "*" {
return true
}
}
return false
}
func (c *RuntimeConfig) Update(updated *RuntimeConfig) {
c.mu.Lock()
defer c.mu.Unlock()
// Track what changed
logLevelChanged := c.LogLevel != updated.LogLevel
cacheSizeChanged := c.CacheSize != updated.CacheSize
dbPoolChanged := c.DatabasePool != updated.DatabasePool
// Update values
c.LogLevel = updated.LogLevel
c.CacheSize = updated.CacheSize
c.DatabasePool = updated.DatabasePool
c.Timeouts = updated.Timeouts
c.AllowedIPs = updated.AllowedIPs
// Call callbacks outside of the lock
go func() {
if logLevelChanged && c.logLevelChanged != nil {
c.logLevelChanged(updated.LogLevel)
}
if cacheSizeChanged && c.cacheSizeChanged != nil {
c.cacheSizeChanged(updated.CacheSize)
}
if dbPoolChanged && c.dbPoolChanged != nil {
c.dbPoolChanged(updated.DatabasePool)
}
}()
}
// HTTP handler for updating configuration
func (c *RuntimeConfig) UpdateHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
var updated RuntimeConfig
decoder := json.NewDecoder(r.Body)
if err := decoder.Decode(&updated); err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Invalid configuration format"))
return
}
c.Update(&updated)
log.Printf("Configuration updated: %+v", updated)
w.WriteHeader(http.StatusOK)
w.Write([]byte("Configuration updated successfully"))
}
func setupConfigEndpoints(config *RuntimeConfig) {
http.HandleFunc("/config", config.UpdateHandler)
http.HandleFunc("/config/status", func(w http.ResponseWriter, r *http.Request) {
config.mu.RLock()
defer config.mu.RUnlock()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(config)
})
}
This implementation provides thread-safe access to configuration, callback notifications for changes, and HTTP endpoints for runtime updates.
Configuration Templating
Templates allow customizing configuration based on deployment environments.
package main
import (
"bytes"
"fmt"
"os"
"text/template"
"gopkg.in/yaml.v2"
)
type ServerConfig struct {
Environment string `yaml:"environment"`
Host string `yaml:"host"`
Port int `yaml:"port"`
LogDir string `yaml:"logDir"`
DataDir string `yaml:"dataDir"`
AdminEmail string `yaml:"adminEmail"`
Features struct {
Metrics bool `yaml:"metrics"`
Tracing bool `yaml:"tracing"`
Profiling bool `yaml:"profiling"`
} `yaml:"features"`
}
func loadTemplatedConfig(templatePath string) (*ServerConfig, error) {
// Read the template file
templateData, err := os.ReadFile(templatePath)
if err != nil {
return nil, fmt.Errorf("failed to read template: %w", err)
}
// Create template
tmpl, err := template.New("config").Parse(string(templateData))
if err != nil {
return nil, fmt.Errorf("failed to parse template: %w", err)
}
// Define variables to inject into template
vars := map[string]string{
"Environment": getEnvWithDefault("APP_ENV", "development"),
"HostName": getEnvWithDefault("HOSTNAME", "localhost"),
"BaseDir": getEnvWithDefault("APP_BASE_DIR", "/app"),
"AdminEmail": getEnvWithDefault("ADMIN_EMAIL", "[email protected]"),
}
// Execute template
var rendered bytes.Buffer
if err := tmpl.Execute(&rendered, vars); err != nil {
return nil, fmt.Errorf("failed to render template: %w", err)
}
// Parse the rendered YAML
var config ServerConfig
if err := yaml.Unmarshal(rendered.Bytes(), &config); err != nil {
return nil, fmt.Errorf("failed to parse rendered config: %w", err)
}
return &config, nil
}
func getEnvWithDefault(key, defaultValue string) string {
value, exists := os.LookupEnv(key)
if !exists {
return defaultValue
}
return value
}
This approach lets you create a single configuration template with placeholders that adapt to each environment.
Configuration Validation
Validating configuration early prevents runtime failures and improves reliability.
package main
import (
"errors"
"fmt"
"net"
"os"
"regexp"
"strings"
"github.com/go-playground/validator/v10"
)
type APIConfig struct {
Server struct {
Host string `yaml:"host" validate:"hostname|ip"`
Port int `yaml:"port" validate:"required,gt=0,lt=65536"`
ReadTimeout int `yaml:"readTimeout" validate:"required,gt=0"`
WriteTimeout int `yaml:"writeTimeout" validate:"required,gt=0"`
MaxHeaderBytes int `yaml:"maxHeaderBytes" validate:"required,gt=0"`
} `yaml:"server"`
Database struct {
DSN string `yaml:"dsn" validate:"required"`
MaxOpenConns int `yaml:"maxOpenConns" validate:"required,gt=0"`
MaxIdleConns int `yaml:"maxIdleConns" validate:"required,gte=0"`
ConnMaxLifetime int `yaml:"connMaxLifetime" validate:"required,gt=0"`
} `yaml:"database"`
Auth struct {
JWTSecret string `yaml:"jwtSecret" validate:"required,min=16"`
TokenExpiry int `yaml:"tokenExpiry" validate:"required,gt=0"`
RefreshEnabled bool `yaml:"refreshEnabled"`
} `yaml:"auth"`
Logging struct {
Level string `yaml:"level" validate:"required,oneof=debug info warn error fatal"`
Format string `yaml:"format" validate:"required,oneof=json text"`
Output string `yaml:"output" validate:"required"`
} `yaml:"logging"`
CORS struct {
Enabled bool `yaml:"enabled"`
AllowedOrigins []string `yaml:"allowedOrigins" validate:"required_if=Enabled true,dive,url"`
AllowedMethods []string `yaml:"allowedMethods" validate:"required_if=Enabled true,dive,oneof=GET POST PUT PATCH DELETE OPTIONS HEAD"`
} `yaml:"cors"`
}
func ValidateConfig(config *APIConfig) error {
validate := validator.New()
// Register custom validators
validate.RegisterValidation("hostname", validateHostname)
// Schema validation
if err := validate.Struct(config); err != nil {
return fmt.Errorf("config validation failed: %w", err)
}
// Semantic validation
if err := validateDatabaseDSN(config.Database.DSN); err != nil {
return fmt.Errorf("invalid database DSN: %w", err)
}
if config.Database.MaxIdleConns > config.Database.MaxOpenConns {
return errors.New("maxIdleConns cannot be greater than maxOpenConns")
}
if config.Logging.Output != "stdout" && config.Logging.Output != "stderr" {
// Check if output file is writable
file, err := os.OpenFile(config.Logging.Output, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return fmt.Errorf("log file %s is not writable: %w", config.Logging.Output, err)
}
file.Close()
}
return nil
}
func validateHostname(fl validator.FieldLevel) bool {
hostname := fl.Field().String()
// Check if it's an IP
if net.ParseIP(hostname) != nil {
return true
}
// Check if it's a valid hostname
hostnameRegex := regexp.MustCompile(`^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$`)
return hostnameRegex.MatchString(hostname)
}
func validateDatabaseDSN(dsn string) error {
if !strings.Contains(dsn, ":") {
return errors.New("DSN format incorrect")
}
// Additional DSN validation logic would go here
// This is simplified; real validation would be DB-specific
return nil
}
Comprehensive validation catches issues early, making applications more resilient in cloud environments.
Configuration Logging and Auditing
Proper logging of configuration enables troubleshooting and auditing.
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
type Config struct {
AppName string `json:"appName"`
Environment string `json:"environment"`
Server struct {
Port int `json:"port"`
Host string `json:"host"`
TLSEnable bool `json:"tlsEnable"`
} `json:"server"`
Database struct {
Host string `json:"host"`
Port int `json:"port"`
Name string `json:"name"`
User string `json:"user"`
Password string `json:"-"` // Sensitive field, don't log
} `json:"database"`
Tracing struct {
Enabled bool `json:"enabled"`
Ratio float64 `json:"ratio"`
} `json:"tracing"`
}
type ConfigLogger struct {
logger zerolog.Logger
config Config
}
func NewConfigLogger(config Config) *ConfigLogger {
// Create log directory if it doesn't exist
logDir := "logs"
if err := os.MkdirAll(logDir, 0755); err != nil {
log.Fatal().Err(err).Msg("Failed to create log directory")
}
// Create log file with timestamp
logFile := filepath.Join(logDir, fmt.Sprintf("config-%s.log",
time.Now().Format("2006-01-02")))
file, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
if err != nil {
log.Fatal().Err(err).Msg("Failed to open log file")
}
// Configure logger
logger := zerolog.New(file).With().Timestamp().Logger()
return &ConfigLogger{
logger: logger,
config: config,
}
}
func (cl *ConfigLogger) LogInitialConfig() {
cl.logger.Info().
Str("event", "config_loaded").
Str("app", cl.config.AppName).
Str("env", cl.config.Environment).
Interface("config", cl.config).
Msg("Application configuration loaded")
