Preliminary support for remote sources

vendor/github.com/dchest/safefile/.travis.yml

@@ -0,0 +1,8 @@
+language: go
+
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - 1.4
+  - tip

vendor/github.com/dchest/safefile/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2013 Dmitry Chestnykh <dmitry@codingrobots.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+
+   * Redistributions in binary form must reproduce the above
+     copyright notice, this list of conditions and the following
+     disclaimer in the documentation and/or other materials
+     provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/dchest/safefile/README.md

@@ -0,0 +1,44 @@
+# safefile
+
+[![Build Status](https://travis-ci.org/dchest/safefile.svg)](https://travis-ci.org/dchest/safefile) [![Windows Build status](https://ci.appveyor.com/api/projects/status/owlifxeekg75t2ho?svg=true)](https://ci.appveyor.com/project/dchest/safefile)
+
+Go package safefile implements safe "atomic" saving of files.
+
+Instead of truncating and overwriting the destination file, it creates a
+temporary file in the same directory, writes to it, and then renames the
+temporary file to the original name when calling Commit.
+
+
+### Installation
+
+```
+$ go get github.com/dchest/safefile
+```
+
+### Documentation
+
+ <https://godoc.org/github.com/dchest/safefile>
+
+### Example
+
+```go
+f, err := safefile.Create("/home/ken/report.txt", 0644)
+if err != nil {
+	// ...
+}
+// Created temporary file /home/ken/sf-ppcyksu5hyw2mfec.tmp
+
+defer f.Close()
+
+_, err = io.WriteString(f, "Hello world")
+if err != nil {
+	// ...
+}
+// Wrote "Hello world" to /home/ken/sf-ppcyksu5hyw2mfec.tmp
+
+err = f.Commit()
+if err != nil {
+    // ...
+}
+// Renamed /home/ken/sf-ppcyksu5hyw2mfec.tmp to /home/ken/report.txt
+```

vendor/github.com/dchest/safefile/appveyor.yml

@@ -0,0 +1,24 @@
+version: "{build}"
+
+os: Windows Server 2012 R2
+
+clone_folder: c:\projects\src\github.com\dchest\safefile
+
+environment:
+ PATH: c:\projects\bin;%PATH%
+ GOPATH: c:\projects
+ NOTIFY_TIMEOUT: 5s
+
+install:
+ - go version
+ - go get golang.org/x/tools/cmd/vet
+ - go get -v -t ./...
+
+build_script:
+ - go tool vet -all .
+ - go build ./...
+ - go test -v -race ./...
+
+test: off
+
+deploy: off

vendor/github.com/dchest/safefile/rename.go

@@ -0,0 +1,9 @@
+// +build !plan9,!windows windows,go1.5
+
+package safefile
+
+import "os"
+
+func rename(oldname, newname string) error {
+	return os.Rename(oldname, newname)
+}

vendor/github.com/dchest/safefile/rename_nonatomic.go

@@ -0,0 +1,51 @@
+// +build plan9 windows,!go1.5
+
+// os.Rename on Windows before Go 1.5 and Plan 9 will not overwrite existing
+// files, thus we cannot guarantee atomic saving of file by doing rename.
+// We will have to do some voodoo to minimize data loss on those systems.
+
+package safefile
+
+import (
+	"os"
+	"path/filepath"
+)
+
+func rename(oldname, newname string) error {
+	err := os.Rename(oldname, newname)
+	if err != nil {
+		// If newname exists ("original"), we will try renaming it to a
+		// new temporary name, then renaming oldname to the newname,
+		// and deleting the renamed original. If system crashes between
+		// renaming and deleting, the original file will still be available
+		// under the temporary name, so users can manually recover data.
+		// (No automatic recovery is possible because after crash the
+		// temporary name is not known.)
+		var origtmp string
+		for {
+			origtmp, err = makeTempName(newname, filepath.Base(newname))
+			if err != nil {
+				return err
+			}
+			_, err = os.Stat(origtmp)
+			if err == nil {
+				continue // most likely will never happen
+			}
+			break
+		}
+		err = os.Rename(newname, origtmp)
+		if err != nil {
+			return err
+		}
+		err = os.Rename(oldname, newname)
+		if err != nil {
+			// Rename still fails, try to revert original rename,
+			// ignoring errors.
+			os.Rename(origtmp, newname)
+			return err
+		}
+		// Rename succeeded, now delete original file.
+		os.Remove(origtmp)
+	}
+	return nil
+}

vendor/github.com/dchest/safefile/safefile.go

@@ -0,0 +1,197 @@
+// Copyright 2013 Dmitry Chestnykh. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package safefile implements safe "atomic" saving of files.
+//
+// Instead of truncating and overwriting the destination file, it creates a
+// temporary file in the same directory, writes to it, and then renames the
+// temporary file to the original name when calling Commit.
+//
+// Example:
+//
+//  f, err := safefile.Create("/home/ken/report.txt", 0644)
+//  if err != nil {
+//  	// ...
+//  }
+//  // Created temporary file /home/ken/sf-ppcyksu5hyw2mfec.tmp
+//
+//  defer f.Close()
+//
+//  _, err = io.WriteString(f, "Hello world")
+//  if err != nil {
+//  	// ...
+//  }
+//  // Wrote "Hello world" to /home/ken/sf-ppcyksu5hyw2mfec.tmp
+//
+//  err = f.Commit()
+//  if err != nil {
+//      // ...
+//  }
+//  // Renamed /home/ken/sf-ppcyksu5hyw2mfec.tmp to /home/ken/report.txt
+//
+package safefile
+
+import (
+	"crypto/rand"
+	"encoding/base32"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ErrAlreadyCommitted error is returned when calling Commit on a file that
+// has been already successfully committed.
+var ErrAlreadyCommitted = errors.New("file already committed")
+
+type File struct {
+	*os.File
+	origName    string
+	closeFunc   func(*File) error
+	isClosed    bool // if true, temporary file has been closed, but not renamed
+	isCommitted bool // if true, the file has been successfully committed
+}
+
+func makeTempName(origname, prefix string) (tempname string, err error) {
+	origname = filepath.Clean(origname)
+	if len(origname) == 0 || origname[len(origname)-1] == filepath.Separator {
+		return "", os.ErrInvalid
+	}
+	// Generate 10 random bytes.
+	// This gives 80 bits of entropy, good enough
+	// for making temporary file name unpredictable.
+	var rnd [10]byte
+	if _, err := rand.Read(rnd[:]); err != nil {
+		return "", err
+	}
+	name := prefix + "-" + strings.ToLower(base32.StdEncoding.EncodeToString(rnd[:])) + ".tmp"
+	return filepath.Join(filepath.Dir(origname), name), nil
+}
+
+// Create creates a temporary file in the same directory as filename,
+// which will be renamed to the given filename when calling Commit.
+func Create(filename string, perm os.FileMode) (*File, error) {
+	for {
+		tempname, err := makeTempName(filename, "sf")
+		if err != nil {
+			return nil, err
+		}
+		f, err := os.OpenFile(tempname, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
+		if err != nil {
+			if os.IsExist(err) {
+				continue
+			}
+			return nil, err
+		}
+		return &File{
+			File:      f,
+			origName:  filename,
+			closeFunc: closeUncommitted,
+		}, nil
+	}
+}
+
+// OrigName returns the original filename given to Create.
+func (f *File) OrigName() string {
+	return f.origName
+}
+
+// Close closes temporary file and removes it.
+// If the file has been committed, Close is no-op.
+func (f *File) Close() error {
+	return f.closeFunc(f)
+}
+
+func closeUncommitted(f *File) error {
+	err0 := f.File.Close()
+	err1 := os.Remove(f.Name())
+	f.closeFunc = closeAgainError
+	if err0 != nil {
+		return err0
+	}
+	return err1
+}
+
+func closeAfterFailedRename(f *File) error {
+	// Remove temporary file.
+	//
+	// The note from Commit function applies here too, as we may be
+	// removing a different file. However, since we rely on our temporary
+	// names being unpredictable, this should not be a concern.
+	f.closeFunc = closeAgainError
+	return os.Remove(f.Name())
+}
+
+func closeCommitted(f *File) error {
+	// noop
+	return nil
+}
+
+func closeAgainError(f *File) error {
+	return os.ErrInvalid
+}
+
+// Commit safely commits data into the original file by syncing temporary
+// file to disk, closing it and renaming to the original file name.
+//
+// In case of success, the temporary file is closed and no longer exists
+// on disk. It is safe to call Close after Commit: the operation will do
+// nothing.
+//
+// In case of error, the temporary file is still opened and exists on disk;
+// it must be closed by callers by calling Close or by trying to commit again.
+
+// Note that when trying to Commit again after a failed Commit when the file
+// has been closed, but not renamed to its original name (the new commit will
+// try again to rename it), safefile cannot guarantee that the temporary file
+// has not been changed, or that it is the same temporary file we were dealing
+// with.  However, since the temporary name is unpredictable, it is unlikely
+// that this happened accidentally. If complete atomicity is needed, do not
+// Commit again after error, write the file again.
+func (f *File) Commit() error {
+	if f.isCommitted {
+		return ErrAlreadyCommitted
+	}
+	if !f.isClosed {
+		// Sync to disk.
+		err := f.Sync()
+		if err != nil {
+			return err
+		}
+		// Close underlying os.File.
+		err = f.File.Close()
+		if err != nil {
+			return err
+		}
+		f.isClosed = true
+	}
+	// Rename.
+	err := rename(f.Name(), f.origName)
+	if err != nil {
+		f.closeFunc = closeAfterFailedRename
+		return err
+	}
+	f.closeFunc = closeCommitted
+	f.isCommitted = true
+	return nil
+}
+
+// WriteFile is a safe analog of ioutil.WriteFile.
+func WriteFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := Create(filename, perm)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	n, err := f.Write(data)
+	if err != nil {
+		return err
+	}
+	if err == nil && n < len(data) {
+		err = io.ErrShortWrite
+		return err
+	}
+	return f.Commit()
+}

vendor/github.com/dchest/safefile/safefile_test.go

@@ -0,0 +1,155 @@
+package safefile
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func ensureFileContains(name, data string) error {
+	b, err := ioutil.ReadFile(name)
+	if err != nil {
+		return err
+	}
+	if string(b) != data {
+		return fmt.Errorf("wrong data in file: expected %s, got %s", data, string(b))
+	}
+	return nil
+}
+
+func tempFileName(count int) string {
+	return filepath.Join(os.TempDir(), fmt.Sprintf("safefile-test-%d-%x", count, time.Now().UnixNano()))
+}
+
+var testData = "Hello, this is a test data"
+
+func testInTempDir() error {
+	name := tempFileName(0)
+	defer os.Remove(name)
+	f, err := Create(name, 0666)
+	if err != nil {
+		return err
+	}
+	if name != f.OrigName() {
+		f.Close()
+		return fmt.Errorf("name %q differs from OrigName: %q", name, f.OrigName())
+	}
+	_, err = io.WriteString(f, testData)
+	if err != nil {
+		f.Close()
+		return err
+	}
+	err = f.Commit()
+	if err != nil {
+		f.Close()
+		return err
+	}
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+	return ensureFileContains(name, testData)
+}
+
+func TestMakeTempName(t *testing.T) {
+	// Make sure temp name is random.
+	m := make(map[string]bool)
+	for i := 0; i < 100; i++ {
+		name, err := makeTempName("/tmp", "sf")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if m[name] {
+			t.Fatal("repeated file name")
+		}
+		m[name] = true
+	}
+}
+
+func TestFile(t *testing.T) {
+	err := testInTempDir()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestWriteFile(t *testing.T) {
+	name := tempFileName(1)
+	err := WriteFile(name, []byte(testData), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ensureFileContains(name, testData)
+	if err != nil {
+		os.Remove(name)
+		t.Fatal(err)
+	}
+	os.Remove(name)
+}
+
+func TestAbandon(t *testing.T) {
+	name := tempFileName(2)
+	f, err := Create(name, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = f.Close()
+	if err != nil {
+		t.Fatalf("Abandon failed: %s", err)
+	}
+	// Make sure temporary file doesn't exist.
+	_, err = os.Stat(f.Name())
+	if err != nil && !os.IsNotExist(err) {
+		t.Fatal(err)
+	}
+}
+
+func TestDoubleCommit(t *testing.T) {
+	name := tempFileName(3)
+	f, err := Create(name, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = f.Commit()
+	if err != nil {
+		os.Remove(name)
+		t.Fatalf("First commit failed: %s", err)
+	}
+	err = f.Commit()
+	if err != ErrAlreadyCommitted {
+		os.Remove(name)
+		t.Fatalf("Second commit didn't fail: %s", err)
+	}
+	err = f.Close()
+	if err != nil {
+		os.Remove(name)
+		t.Fatalf("Close failed: %s", err)
+	}
+	os.Remove(name)
+}
+
+func TestOverwriting(t *testing.T) {
+	name := tempFileName(4)
+	defer os.Remove(name)
+
+	olddata := "This is old data"
+	err := ioutil.WriteFile(name, []byte(olddata), 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newdata := "This is new data"
+	err = WriteFile(name, []byte(newdata), 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = ensureFileContains(name, newdata)
+	if err != nil {
+		t.Fatal(err)
+	}
+}