Add worker reading extra_records_path from file (#2271)

* consolidate scheduled tasks into one goroutine

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

* rename Tailcfg dns struct

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

* add dns.extra_records_path option

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

* prettier lint

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

* go-fmt

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

---------

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
This commit is contained in:
Kristoffer Dalby 2024-12-13 07:52:40 +00:00 committed by GitHub
parent 89a648c7dd
commit 380fcdba17
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 388 additions and 81 deletions

View file

@ -27,6 +27,7 @@ import (
"github.com/juanfont/headscale/hscontrol/db"
"github.com/juanfont/headscale/hscontrol/derp"
derpServer "github.com/juanfont/headscale/hscontrol/derp/server"
"github.com/juanfont/headscale/hscontrol/dns"
"github.com/juanfont/headscale/hscontrol/mapper"
"github.com/juanfont/headscale/hscontrol/notifier"
"github.com/juanfont/headscale/hscontrol/policy"
@ -88,8 +89,9 @@ type Headscale struct {
DERPMap *tailcfg.DERPMap
DERPServer *derpServer.DERPServer
polManOnce sync.Once
polMan policy.PolicyManager
polManOnce sync.Once
polMan policy.PolicyManager
extraRecordMan *dns.ExtraRecordsMan
mapper *mapper.Mapper
nodeNotifier *notifier.Notifier
@ -184,7 +186,7 @@ func NewHeadscale(cfg *types.Config) (*Headscale, error) {
}
app.authProvider = authProvider
if app.cfg.DNSConfig != nil && app.cfg.DNSConfig.Proxied { // if MagicDNS
if app.cfg.TailcfgDNSConfig != nil && app.cfg.TailcfgDNSConfig.Proxied { // if MagicDNS
// TODO(kradalby): revisit why this takes a list.
var magicDNSDomains []dnsname.FQDN
@ -196,11 +198,11 @@ func NewHeadscale(cfg *types.Config) (*Headscale, error) {
}
// we might have routes already from Split DNS
if app.cfg.DNSConfig.Routes == nil {
app.cfg.DNSConfig.Routes = make(map[string][]*dnstype.Resolver)
if app.cfg.TailcfgDNSConfig.Routes == nil {
app.cfg.TailcfgDNSConfig.Routes = make(map[string][]*dnstype.Resolver)
}
for _, d := range magicDNSDomains {
app.cfg.DNSConfig.Routes[d.WithoutTrailingDot()] = nil
app.cfg.TailcfgDNSConfig.Routes[d.WithoutTrailingDot()] = nil
}
}
@ -237,23 +239,38 @@ func (h *Headscale) redirect(w http.ResponseWriter, req *http.Request) {
http.Redirect(w, req, target, http.StatusFound)
}
// expireExpiredNodes expires nodes that have an explicit expiry set
// after that expiry time has passed.
func (h *Headscale) expireExpiredNodes(ctx context.Context, every time.Duration) {
ticker := time.NewTicker(every)
func (h *Headscale) scheduledTasks(ctx context.Context) {
expireTicker := time.NewTicker(updateInterval)
defer expireTicker.Stop()
lastCheck := time.Unix(0, 0)
var update types.StateUpdate
var changed bool
lastExpiryCheck := time.Unix(0, 0)
derpTicker := time.NewTicker(h.cfg.DERP.UpdateFrequency)
defer derpTicker.Stop()
// If we dont want auto update, just stop the ticker
if !h.cfg.DERP.AutoUpdate {
derpTicker.Stop()
}
var extraRecordsUpdate <-chan []tailcfg.DNSRecord
if h.extraRecordMan != nil {
extraRecordsUpdate = h.extraRecordMan.UpdateCh()
} else {
extraRecordsUpdate = make(chan []tailcfg.DNSRecord)
}
for {
select {
case <-ctx.Done():
ticker.Stop()
log.Info().Caller().Msg("scheduled task worker is shutting down.")
return
case <-ticker.C:
case <-expireTicker.C:
var update types.StateUpdate
var changed bool
if err := h.db.Write(func(tx *gorm.DB) error {
lastCheck, update, changed = db.ExpireExpiredNodes(tx, lastCheck)
lastExpiryCheck, update, changed = db.ExpireExpiredNodes(tx, lastExpiryCheck)
return nil
}); err != nil {
@ -267,24 +284,8 @@ func (h *Headscale) expireExpiredNodes(ctx context.Context, every time.Duration)
ctx := types.NotifyCtx(context.Background(), "expire-expired", "na")
h.nodeNotifier.NotifyAll(ctx, update)
}
}
}
}
// scheduledDERPMapUpdateWorker refreshes the DERPMap stored on the global object
// at a set interval.
func (h *Headscale) scheduledDERPMapUpdateWorker(cancelChan <-chan struct{}) {
log.Info().
Dur("frequency", h.cfg.DERP.UpdateFrequency).
Msg("Setting up a DERPMap update worker")
ticker := time.NewTicker(h.cfg.DERP.UpdateFrequency)
for {
select {
case <-cancelChan:
return
case <-ticker.C:
case <-derpTicker.C:
log.Info().Msg("Fetching DERPMap updates")
h.DERPMap = derp.GetDERPMap(h.cfg.DERP)
if h.cfg.DERP.ServerEnabled && h.cfg.DERP.AutomaticallyAddEmbeddedDerpRegion {
@ -297,6 +298,19 @@ func (h *Headscale) scheduledDERPMapUpdateWorker(cancelChan <-chan struct{}) {
Type: types.StateDERPUpdated,
DERPMap: h.DERPMap,
})
case records, ok := <-extraRecordsUpdate:
if !ok {
continue
}
h.cfg.TailcfgDNSConfig.ExtraRecords = records
ctx := types.NotifyCtx(context.Background(), "dns-extrarecord", "all")
h.nodeNotifier.NotifyAll(ctx, types.StateUpdate{
// TODO(kradalby): We can probably do better than sending a full update here,
// but for now this will ensure that all of the nodes get the new records.
Type: types.StateFullUpdate,
})
}
}
}
@ -568,12 +582,6 @@ func (h *Headscale) Serve() error {
go h.DERPServer.ServeSTUN()
}
if h.cfg.DERP.AutoUpdate {
derpMapCancelChannel := make(chan struct{})
defer func() { derpMapCancelChannel <- struct{}{} }()
go h.scheduledDERPMapUpdateWorker(derpMapCancelChannel)
}
if len(h.DERPMap.Regions) == 0 {
return errEmptyInitialDERPMap
}
@ -591,9 +599,21 @@ func (h *Headscale) Serve() error {
h.ephemeralGC.Schedule(node.ID, h.cfg.EphemeralNodeInactivityTimeout)
}
expireNodeCtx, expireNodeCancel := context.WithCancel(context.Background())
defer expireNodeCancel()
go h.expireExpiredNodes(expireNodeCtx, updateInterval)
if h.cfg.DNSConfig.ExtraRecordsPath != "" {
h.extraRecordMan, err = dns.NewExtraRecordsManager(h.cfg.DNSConfig.ExtraRecordsPath)
if err != nil {
return fmt.Errorf("setting up extrarecord manager: %w", err)
}
h.cfg.TailcfgDNSConfig.ExtraRecords = h.extraRecordMan.Records()
go h.extraRecordMan.Run()
defer h.extraRecordMan.Close()
}
// Start all scheduled tasks, e.g. expiring nodes, derp updates and
// records updates
scheduleCtx, scheduleCancel := context.WithCancel(context.Background())
defer scheduleCancel()
go h.scheduledTasks(scheduleCtx)
if zl.GlobalLevel() == zl.TraceLevel {
zerolog.RespLog = true
@ -847,7 +867,7 @@ func (h *Headscale) Serve() error {
Str("signal", sig.String()).
Msg("Received signal to stop, shutting down gracefully")
expireNodeCancel()
scheduleCancel()
h.ephemeralGC.Close()
// Gracefully shut down servers

View file

@ -390,7 +390,6 @@ func (h *Headscale) handleAuthKey(
http.Error(writer, "Internal server error", http.StatusInternalServerError)
return
}
}
err = h.db.Write(func(tx *gorm.DB) error {

View file

@ -373,6 +373,5 @@ func TestConstraints(t *testing.T) {
tt.run(t, db.DB.Debug())
})
}
}

View file

@ -0,0 +1,155 @@
package dns
import (
"crypto/sha256"
"encoding/json"
"fmt"
"os"
"sync"
"github.com/fsnotify/fsnotify"
"github.com/rs/zerolog/log"
"tailscale.com/tailcfg"
"tailscale.com/util/set"
)
type ExtraRecordsMan struct {
mu sync.RWMutex
records set.Set[tailcfg.DNSRecord]
watcher *fsnotify.Watcher
path string
updateCh chan []tailcfg.DNSRecord
closeCh chan struct{}
hashes map[string][32]byte
}
// NewExtraRecordsManager creates a new ExtraRecordsMan and starts watching the file at the given path.
func NewExtraRecordsManager(path string) (*ExtraRecordsMan, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, fmt.Errorf("creating watcher: %w", err)
}
fi, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("getting file info: %w", err)
}
if fi.IsDir() {
return nil, fmt.Errorf("path is a directory, only file is supported: %s", path)
}
records, hash, err := readExtraRecordsFromPath(path)
if err != nil {
return nil, fmt.Errorf("reading extra records from path: %w", err)
}
er := &ExtraRecordsMan{
watcher: watcher,
path: path,
records: set.SetOf(records),
hashes: map[string][32]byte{
path: hash,
},
closeCh: make(chan struct{}),
updateCh: make(chan []tailcfg.DNSRecord),
}
err = watcher.Add(path)
if err != nil {
return nil, fmt.Errorf("adding path to watcher: %w", err)
}
log.Trace().Caller().Strs("watching", watcher.WatchList()).Msg("started filewatcher")
return er, nil
}
func (e *ExtraRecordsMan) Records() []tailcfg.DNSRecord {
e.mu.RLock()
defer e.mu.RUnlock()
return e.records.Slice()
}
func (e *ExtraRecordsMan) Run() {
for {
select {
case <-e.closeCh:
return
case event, ok := <-e.watcher.Events:
if !ok {
log.Error().Caller().Msgf("file watcher event channel closing")
return
}
log.Trace().Caller().Str("path", event.Name).Str("op", event.Op.String()).Msg("extra records received filewatch event")
if event.Name != e.path {
continue
}
e.updateRecords()
case err, ok := <-e.watcher.Errors:
if !ok {
log.Error().Caller().Msgf("file watcher error channel closing")
return
}
log.Error().Caller().Err(err).Msgf("extra records filewatcher returned error: %q", err)
}
}
}
func (e *ExtraRecordsMan) Close() {
e.watcher.Close()
close(e.closeCh)
}
func (e *ExtraRecordsMan) UpdateCh() <-chan []tailcfg.DNSRecord {
return e.updateCh
}
func (e *ExtraRecordsMan) updateRecords() {
records, newHash, err := readExtraRecordsFromPath(e.path)
if err != nil {
log.Error().Caller().Err(err).Msgf("reading extra records from path: %s", e.path)
return
}
e.mu.Lock()
defer e.mu.Unlock()
// If there has not been any change, ignore the update.
if oldHash, ok := e.hashes[e.path]; ok {
if newHash == oldHash {
return
}
}
oldCount := e.records.Len()
e.records = set.SetOf(records)
e.hashes[e.path] = newHash
log.Trace().Caller().Interface("records", e.records).Msgf("extra records updated from path, count old: %d, new: %d", oldCount, e.records.Len())
e.updateCh <- e.records.Slice()
}
// readExtraRecordsFromPath reads a JSON file of tailcfg.DNSRecord
// and returns the records and the hash of the file.
func readExtraRecordsFromPath(path string) ([]tailcfg.DNSRecord, [32]byte, error) {
b, err := os.ReadFile(path)
if err != nil {
return nil, [32]byte{}, fmt.Errorf("reading path: %s, err: %w", path, err)
}
var records []tailcfg.DNSRecord
err = json.Unmarshal(b, &records)
if err != nil {
return nil, [32]byte{}, fmt.Errorf("unmarshalling records, content: %q: %w", string(b), err)
}
hash := sha256.Sum256(b)
return records, hash, nil
}

View file

@ -116,11 +116,11 @@ func generateDNSConfig(
cfg *types.Config,
node *types.Node,
) *tailcfg.DNSConfig {
if cfg.DNSConfig == nil {
if cfg.TailcfgDNSConfig == nil {
return nil
}
dnsConfig := cfg.DNSConfig.Clone()
dnsConfig := cfg.TailcfgDNSConfig.Clone()
addNextDNSMetadata(dnsConfig.Resolvers, node)

View file

@ -117,7 +117,7 @@ func TestDNSConfigMapResponse(t *testing.T) {
got := generateDNSConfig(
&types.Config{
DNSConfig: &dnsConfigOrig,
TailcfgDNSConfig: &dnsConfigOrig,
},
nodeInShared1,
)
@ -349,7 +349,7 @@ func Test_fullMapResponse(t *testing.T) {
derpMap: &tailcfg.DERPMap{},
cfg: &types.Config{
BaseDomain: "",
DNSConfig: &tailcfg.DNSConfig{},
TailcfgDNSConfig: &tailcfg.DNSConfig{},
LogTail: types.LogTailConfig{Enabled: false},
RandomizeClientPort: false,
},
@ -381,7 +381,7 @@ func Test_fullMapResponse(t *testing.T) {
derpMap: &tailcfg.DERPMap{},
cfg: &types.Config{
BaseDomain: "",
DNSConfig: &tailcfg.DNSConfig{},
TailcfgDNSConfig: &tailcfg.DNSConfig{},
LogTail: types.LogTailConfig{Enabled: false},
RandomizeClientPort: false,
},
@ -424,7 +424,7 @@ func Test_fullMapResponse(t *testing.T) {
derpMap: &tailcfg.DERPMap{},
cfg: &types.Config{
BaseDomain: "",
DNSConfig: &tailcfg.DNSConfig{},
TailcfgDNSConfig: &tailcfg.DNSConfig{},
LogTail: types.LogTailConfig{Enabled: false},
RandomizeClientPort: false,
},

View file

@ -187,7 +187,7 @@ func TestTailNode(t *testing.T) {
polMan, _ := policy.NewPolicyManagerForTest(tt.pol, []types.User{}, types.Nodes{tt.node})
cfg := &types.Config{
BaseDomain: tt.baseDomain,
DNSConfig: tt.dnsConfig,
TailcfgDNSConfig: tt.dnsConfig,
RandomizeClientPort: false,
}
got, err := tailNode(

View file

@ -447,7 +447,7 @@ func (a *AuthProviderOIDC) createOrUpdateUserFromClaim(
// This check is for legacy, if the user cannot be found by the OIDC identifier
// look it up by username. This should only be needed once.
// This branch will presist for a number of versions after the OIDC migration and
// This branch will persist for a number of versions after the OIDC migration and
// then be removed following a deprecation.
// TODO(kradalby): Remove when strip_email_domain and migration is removed
// after #2170 is cleaned up.
@ -536,7 +536,7 @@ func renderOIDCCallbackTemplate(
// TODO(kradalby): Reintroduce when strip_email_domain is removed
// after #2170 is cleaned up
// DEPRECATED: DO NOT USE
// DEPRECATED: DO NOT USE.
func getUserName(
claims *types.OIDCClaims,
stripEmaildomain bool,

View file

@ -72,7 +72,14 @@ type Config struct {
ACMEURL string
ACMEEmail string
DNSConfig *tailcfg.DNSConfig
// DNSConfig is the headscale representation of the DNS configuration.
// It is kept in the config update for some settings that are
// not directly converted into a tailcfg.DNSConfig.
DNSConfig DNSConfig
// TailcfgDNSConfig is the tailcfg representation of the DNS configuration,
// it can be used directly when sending Netmaps to clients.
TailcfgDNSConfig *tailcfg.DNSConfig
UnixSocket string
UnixSocketPermission fs.FileMode
@ -90,11 +97,12 @@ type Config struct {
}
type DNSConfig struct {
MagicDNS bool `mapstructure:"magic_dns"`
BaseDomain string `mapstructure:"base_domain"`
Nameservers Nameservers
SearchDomains []string `mapstructure:"search_domains"`
ExtraRecords []tailcfg.DNSRecord `mapstructure:"extra_records"`
MagicDNS bool `mapstructure:"magic_dns"`
BaseDomain string `mapstructure:"base_domain"`
Nameservers Nameservers
SearchDomains []string `mapstructure:"search_domains"`
ExtraRecords []tailcfg.DNSRecord `mapstructure:"extra_records"`
ExtraRecordsPath string `mapstructure:"extra_records_path"`
}
type Nameservers struct {
@ -253,7 +261,6 @@ func LoadConfig(path string, isFile bool) error {
viper.SetDefault("dns.nameservers.global", []string{})
viper.SetDefault("dns.nameservers.split", map[string]string{})
viper.SetDefault("dns.search_domains", []string{})
viper.SetDefault("dns.extra_records", []tailcfg.DNSRecord{})
viper.SetDefault("derp.server.enabled", false)
viper.SetDefault("derp.server.stun.enabled", true)
@ -344,6 +351,10 @@ func validateServerConfig() error {
}
}
if viper.IsSet("dns.extra_records") && viper.IsSet("dns.extra_records_path") {
log.Fatal().Msg("Fatal config error: dns.extra_records and dns.extra_records_path are mutually exclusive. Please remove one of them from your config file")
}
// Collect any validation errors and return them all at once
var errorText string
if (viper.GetString("tls_letsencrypt_hostname") != "") &&
@ -586,6 +597,7 @@ func dns() (DNSConfig, error) {
dns.Nameservers.Global = viper.GetStringSlice("dns.nameservers.global")
dns.Nameservers.Split = viper.GetStringMapStringSlice("dns.nameservers.split")
dns.SearchDomains = viper.GetStringSlice("dns.search_domains")
dns.ExtraRecordsPath = viper.GetString("dns.extra_records_path")
if viper.IsSet("dns.extra_records") {
var extraRecords []tailcfg.DNSRecord
@ -871,7 +883,8 @@ func LoadServerConfig() (*Config, error) {
TLS: tlsConfig(),
DNSConfig: dnsToTailcfgDNS(dnsConfig),
DNSConfig: dnsConfig,
TailcfgDNSConfig: dnsToTailcfgDNS(dnsConfig),
ACMEEmail: viper.GetString("acme_email"),
ACMEURL: viper.GetString("acme_url"),

View file

@ -280,9 +280,9 @@ func TestReadConfigFromEnv(t *testing.T) {
// "foo.bar.com": {"1.1.1.1"},
},
},
ExtraRecords: []tailcfg.DNSRecord{
// {Name: "prometheus.myvpn.example.com", Type: "A", Value: "100.64.0.4"},
},
// ExtraRecords: []tailcfg.DNSRecord{
// {Name: "prometheus.myvpn.example.com", Type: "A", Value: "100.64.0.4"},
// },
SearchDomains: []string{"test.com", "bar.com"},
},
},

View file

@ -189,7 +189,6 @@ func GenerateIPv6DNSRootDomain(ipPrefix netip.Prefix) []dnsname.FQDN {
// NormalizeToFQDNRules will replace forbidden chars in user
// it can also return an error if the user doesn't respect RFC 952 and 1123.
func NormalizeToFQDNRules(name string, stripEmailDomain bool) (string, error) {
name = strings.ToLower(name)
name = strings.ReplaceAll(name, "'", "")
atIdx := strings.Index(name, "@")