Restore the cache update code from version 2.1.4 for now

dnscrypt-proxy/config.go

@@ -892,8 +892,8 @@ func (config *Config) loadSource(proxy *Proxy, cfgSourceName string, cfgSource *
 	if cfgSource.FormatStr == "" {
 		cfgSource.FormatStr = "v2"
 	}
-	if cfgSource.RefreshDelay < 24 {
+	if cfgSource.RefreshDelay <= 0 {
-		cfgSource.RefreshDelay = 24
+		cfgSource.RefreshDelay = 72
 	} else if cfgSource.RefreshDelay > 168 {
 		cfgSource.RefreshDelay = 168
 	}

dnscrypt-proxy/sources.go

@@ -3,10 +3,10 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"math"
 	"math/rand"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -23,18 +23,21 @@ const (
 	SourceFormatV2 = iota
 )
 
-const MinimumPrefetchInterval time.Duration = 10 * time.Minute
+const (
+	DefaultPrefetchDelay    time.Duration = 24 * time.Hour
+	MinimumPrefetchInterval time.Duration = 10 * time.Minute
+)
 
 type Source struct {
-	name        string
+	name                    string
-	urls        []*url.URL
+	urls                    []*url.URL
-	bin         []byte // copy of the file content - there's something wrong in our logic, we shouldn't need to keep that in memory
+	format                  SourceFormat
-	minisignKey *minisign.PublicKey
+	bin                     []byte
-	cacheFile   string
+	minisignKey             *minisign.PublicKey
-	prefix      string
+	cacheFile               string
-	cacheTTL    time.Duration
+	cacheTTL, prefetchDelay time.Duration
-	refresh     time.Time
+	refresh                 time.Time
-	format      SourceFormat
+	prefix                  string
 }
 
 func (source *Source) checkSignature(bin, sig []byte) (err error) {
@@ -48,7 +51,7 @@ func (source *Source) checkSignature(bin, sig []byte) (err error) {
 // timeNow() can be replaced by tests to provide a static value
 var timeNow = time.Now
 
-func (source *Source) fetchFromCache(now time.Time) (remaining time.Duration, err error) {
+func (source *Source) fetchFromCache(now time.Time) (ttl time.Duration, err error) {
 	var bin, sig []byte
 	if bin, err = os.ReadFile(source.cacheFile); err != nil {
 		return
@@ -65,8 +68,8 @@ func (source *Source) fetchFromCache(now time.Time) (remaining time.Duration, er
 		return
 	}
 	if elapsed := now.Sub(fi.ModTime()); elapsed < source.cacheTTL {
-		remaining = source.cacheTTL - elapsed
+		ttl = source.prefetchDelay - elapsed
-		dlog.Debugf("Source [%s] cache file [%s] is still fresh, next update in %v min", source.name, source.cacheFile, math.Round(remaining.Minutes()))
+		dlog.Debugf("Source [%s] cache file [%s] is still fresh, next update: %v", source.name, source.cacheFile, ttl)
 	} else {
 		dlog.Debugf("Source [%s] cache file [%s] needs to be refreshed", source.name, source.cacheFile)
 	}
@@ -95,24 +98,25 @@ func writeSource(f string, bin, sig []byte) (err error) {
 	return fSig.Commit()
 }
 
-func (source *Source) updateCache(bin, sig []byte, now time.Time) error {
+func (source *Source) updateCache(bin, sig []byte, now time.Time) {
 	f := source.cacheFile
-	// If the data is unchanged, update the files timestamps only
+	var writeErr error // an error writing cache isn't fatal
-	if bytes.Equal(source.bin, bin) {
+	defer func() {
-		_ = os.Chtimes(f, now, now)
+		source.bin = bin
-		_ = os.Chtimes(f+".minisig", now, now)
+		if writeErr == nil {
-		return nil
+			return
+		}
+		if absPath, absErr := filepath.Abs(f); absErr == nil {
+			f = absPath
+		}
+		dlog.Warnf("%s: %s", f, writeErr)
+	}()
+	if !bytes.Equal(source.bin, bin) {
+		if writeErr = writeSource(f, bin, sig); writeErr != nil {
+			return
+		}
 	}
-	// Otherwise, write the new data and signature
+	writeErr = os.Chtimes(f, now, now)
-	if err := writeSource(f, bin, sig); err != nil {
-		dlog.Warnf("Source [%s] failed to update cache file [%s]: %v", source.name, f, err)
-		return err
-	}
-	source.bin = bin // In-memory copy of the cache file content
-	// The tests require the timestamps to be updated, no idea why
-	_ = os.Chtimes(f, now, now)
-	_ = os.Chtimes(f+".minisig", now, now)
-	return nil
 }
 
 func (source *Source) parseURLs(urls []string) {
@@ -130,23 +134,23 @@ func fetchFromURL(xTransport *XTransport, u *url.URL) (bin []byte, err error) {
 	return bin, err
 }
 
-func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (time.Duration, error) {
+func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (ttl time.Duration, err error) {
-	remaining, err := source.fetchFromCache(now)
+	if ttl, err = source.fetchFromCache(now); err != nil {
-	if err != nil {
 		if len(source.urls) == 0 {
-			dlog.Fatalf("Source [%s] cache file [%s] not present and no valid URL", source.name, source.cacheFile)
+			dlog.Errorf("Source [%s] cache file [%s] not present and no valid URL", source.name, source.cacheFile)
-			return 0, err
+			return
 		}
 		dlog.Debugf("Source [%s] cache file [%s] not present", source.name, source.cacheFile)
 	}
-	if len(source.urls) == 0 {
+	if len(source.urls) > 0 {
-		dlog.Debugf("No URL to update [%s]", source.name)
+		defer func() {
-		return 24 * time.Hour, nil
+			source.refresh = now.Add(ttl)
+		}()
 	}
-	if remaining > 0 {
+	if len(source.urls) == 0 || ttl > 0 {
-		source.refresh = now.Add(remaining)
+		return
-		return remaining, nil
 	}
+	ttl = MinimumPrefetchInterval
 	var bin, sig []byte
 	for _, srcURL := range source.urls {
 		dlog.Infof("Source [%s] loading from URL [%s]", source.name, srcURL)
@@ -167,13 +171,11 @@ func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (tim
 		dlog.Debugf("Source [%s] failed signature check using URL [%s]", source.name, srcURL)
 	}
 	if err != nil {
-		source.refresh = now.Add(MinimumPrefetchInterval)
+		return
-		return MinimumPrefetchInterval, err
 	}
 	source.updateCache(bin, sig, now)
-	remaining = source.cacheTTL
+	ttl = source.prefetchDelay
-	source.refresh = now.Add(remaining)
+	return
-	return remaining, nil
 }
 
 // NewSource loads a new source using the given cacheFile and urls, ensuring it has a valid signature
@@ -187,12 +189,16 @@ func NewSource(
 	refreshDelay time.Duration,
 	prefix string,
 ) (source *Source, err error) {
+	if refreshDelay < DefaultPrefetchDelay {
+		refreshDelay = DefaultPrefetchDelay
+	}
 	source = &Source{
-		name:      name,
+		name:          name,
-		urls:      []*url.URL{},
+		urls:          []*url.URL{},
-		cacheFile: cacheFile,
+		cacheFile:     cacheFile,
-		cacheTTL:  refreshDelay,
+		cacheTTL:      refreshDelay,
-		prefix:    prefix,
+		prefetchDelay: DefaultPrefetchDelay,
+		prefix:        prefix,
 	}
 	if formatStr == "v2" {
 		source.format = SourceFormatV2
@@ -223,7 +229,7 @@ func PrefetchSources(xTransport *XTransport, sources []*Source) time.Duration {
 		if delay, err := source.fetchWithCache(xTransport, now); err != nil {
 			dlog.Infof("Prefetching [%s] failed: %v, will retry in %v", source.name, err, interval)
 		} else {
-			dlog.Debugf("Prefetching [%s] succeeded, next update: %v min", source.name, math.Round(delay.Minutes()))
+			dlog.Debugf("Prefetching [%s] succeeded, next update in %v min", source.name, delay)
 			if delay >= MinimumPrefetchInterval && (interval == MinimumPrefetchInterval || interval > delay) {
 				interval = delay
 			}
@@ -248,8 +254,8 @@ func (source *Source) parseV2() ([]RegisteredServer, error) {
 		stampErrs = append(stampErrs, stampErr)
 		dlog.Warn(stampErr)
 	}
-	bin := string(source.bin)
+	in := string(source.bin)
-	parts := strings.Split(bin, "## ")
+	parts := strings.Split(in, "## ")
 	if len(parts) < 2 {
 		return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
 	}

dnscrypt-proxy/sources_test.go

@@ -42,8 +42,6 @@ const (
 	TestStatePathErr                           // unparseable path to files (download only)
 )
 
-const DefaultPrefetchDelay time.Duration = 24 * time.Hour
-
 type SourceTestData struct {
 	n                         int // subtest counter
 	xTransport                *XTransport
@@ -352,7 +350,7 @@ func prepSourceTestDownload(
 	}
 	if e.success {
 		e.err = ""
-		e.delay = e.Source.cacheTTL
+		e.delay = DefaultPrefetchDelay
 	} else {
 		e.delay = MinimumPrefetchInterval
 	}
@@ -373,7 +371,7 @@ func setupSourceTestCase(t *testing.T, d *SourceTestData, i int,
 	}
 	e.Source = &Source{
 		name: id, urls: []*url.URL{}, format: SourceFormatV2, minisignKey: d.key,
-		cacheFile: e.cachePath, cacheTTL: DefaultPrefetchDelay * 3,
+		cacheFile: e.cachePath, cacheTTL: DefaultPrefetchDelay * 3, prefetchDelay: DefaultPrefetchDelay,
 	}
 	if cacheTest != nil {
 		prepSourceTestCache(t, d, e, d.sources[i], *cacheTest)
@@ -407,9 +405,9 @@ func TestNewSource(t *testing.T) {
 		refreshDelay time.Duration
 		e            *SourceTestExpect
 	}{
-		{"", "", 0, &SourceTestExpect{err: " ", Source: &Source{name: "short refresh delay", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay, prefix: ""}}},
+		{"", "", 0, &SourceTestExpect{err: " ", Source: &Source{name: "short refresh delay", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay, prefetchDelay: DefaultPrefetchDelay, prefix: ""}}},
-		{"v1", d.keyStr, DefaultPrefetchDelay * 2, &SourceTestExpect{err: "Unsupported source format", Source: &Source{name: "old format", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 2}}},
+		{"v1", d.keyStr, DefaultPrefetchDelay * 2, &SourceTestExpect{err: "Unsupported source format", Source: &Source{name: "old format", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 2, prefetchDelay: DefaultPrefetchDelay}}},
-		{"v2", "", DefaultPrefetchDelay * 3, &SourceTestExpect{err: "Invalid encoded public key", Source: &Source{name: "invalid public key", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 3}}},
+		{"v2", "", DefaultPrefetchDelay * 3, &SourceTestExpect{err: "Invalid encoded public key", Source: &Source{name: "invalid public key", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 3, prefetchDelay: DefaultPrefetchDelay}}},
 	} {
 		t.Run(tt.e.Source.name, func(t *testing.T) {
 			got, err := NewSource(
@@ -480,7 +478,6 @@ func TestPrefetchSources(t *testing.T) {
 			s := &Source{}
 			*s = *e.Source
 			s.bin = nil
-			s.refresh = d.timeNow
 			sources = append(sources, s)
 			expects = append(expects, e)
 		}