add -retrygitcommands cli parameter or retry_git_commands g10k config…

… setting for corrupted Git caches #76

.gitignore

@@ -1,2 +1,9 @@
 *.swp
 g10k
+.modules/
+Puppetfile*
+cache/
+example/
+*.zip
+modules/
+tmp/

README.md

@@ -105,10 +105,12 @@ Usage of ./g10k:
         which Puppetfile to use in -puppetfile mode (default "./Puppetfile")
   -quiet
         no output, defaults to false
+  -retrygitcommands
+        if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing
   -usecachefallback
         if g10k should try to use its cache for sources and modules instead of failing
   -usemove
-        do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! Var(&Useful for g10k runs inside a Docker container)
+        do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)
   -verbose
         log verbose output, defaults to false
   -version
@@ -246,7 +248,7 @@ sources:
     warn_if_branch_is_missing: true
 ```
 
-If you then call g10k with that config file and the following parameter `-branch nonExistingBranch`. You should get:
+If you then call g10k with this config file and the following parameter `-branch nonExistingBranch`. You should get:
 
 ```
 WARNING: Couldn't find specified branch 'nonExistingBranch' anywhere in source 'example' (https://github.com/xorpaul/g10k-environment.git)
@@ -267,7 +269,7 @@ sources:
     basedir: '/tmp/failing/'
 ```
 
-If you then call g10k with that config file and at least the `info` verbosity level, you should get: 
+If you then call g10k with this config file and at least the `info` verbosity level, you should get: 
 
 ```
 Failed to populate module /tmp/failing/master/modules//sensu/ but ignore-unreachable is set. Continuing...
@@ -288,7 +290,7 @@ sources:
     exit_if_unreachable: true
 ```
 
-If you then call g10k with that config file. You should get:
+If you then call g10k with this config file. You should get:
 
 ```
 WARN: git repository git://github.com/xorpaul/g10k-environment-unavailable.git does not exist or is unreachable at this moment!
@@ -309,7 +311,7 @@ sources:
     basedir: '/tmp/example/'
 ```
 
-If you then call g10k with that config file and your github.com repository is unavailable your g10k run tries to find a suitable cached version of your modules:
+If you then call g10k with this config file and your github.com repository is unavailable your g10k run tries to find a suitable cached version of your modules:
 
 ```
 WARN: git repository https://github.com/puppetlabs/puppetlabs-firewall.git does not exist or is unreachable at this moment!
@@ -319,6 +321,26 @@ if your g10k did manage to at least once cache this git repository.
 
 If there is no useable cache available your g10k run still fails.
 
+- You can let g10k retry to git clone or update the local repository if it failed before and was left in a corrupted state:
+
+```
+---
+:cachedir: '/tmp/g10k'
+retry_git_commands: true
+
+sources:
+  example:
+    remote: 'https://github.com/xorpaul/g10k-environment.git'
+    basedir: '/tmp/example/'
+```
+
+If you then call g10k with this config file and have a corrupted local Git repository, g10k deletes the local cache and retries the Git clone command once:
+
+```
+WARN: git command failed: git --git-dir /tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git remote update --prune deleting local cached repository and retrying...
+```
+
+See [#76](https://github.com/xorpaul/g10k/issues/76) for details.
 # building
 ```
 # only initially needed to resolve all dependencies

config.go

@@ -67,6 +67,10 @@ func readConfigfile(configFile string) ConfigSettings {
 		config.UseCacheFallback = true
 	}
 
+	if retryGitCommands {
+		config.RetryGitCommands = true
+	}
+
 	// set default max Go routines for Forge and Git module resolution if none is given
 	if !(config.Maxworker > 0) {
 		config.Maxworker = maxworker

g10k.go

@@ -18,6 +18,7 @@ var (
 	force                  bool
 	usemove                bool
 	usecacheFallback       bool
+	retryGitCommands       bool
 	pfMode                 bool
 	pfLocation             string
 	dryRun                 bool
@@ -68,6 +69,7 @@ type ConfigSettings struct {
 	Maxworker                int  `yaml:"maxworker"`
 	MaxExtractworker         int  `yaml:"maxextractworker"`
 	UseCacheFallback         bool `yaml:"use_cache_fallback"`
+	RetryGitCommands         bool `yaml:"retry_git_commands"`
 }
 
 type Forge struct {
@@ -158,14 +160,15 @@ func main() {
 	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
 	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
 	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
-	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! Var(&Useful for g10k runs inside a Docker container)")
+	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
 	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
 	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
 	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
 	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
 	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
 	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
 	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
+	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
 	flag.Parse()
 
 	configFile = *configFileFlag

g10k_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -419,7 +420,7 @@ func TestResolvConfigExitIfUnreachableFalse(t *testing.T) {
 	}
 
 	if 0 != exitCode {
-		t.Errorf("resolvePuppetEnvironment() terminated with %v, but we expected exit status %v", exitCode, 1)
+		t.Errorf("resolvePuppetEnvironment() terminated with %v, but we expected exit status %v", exitCode, 0)
 	}
 	if !strings.Contains(string(out), "WARN: git repository git://github.com/xorpaul/g10k-environment-unavailable.git does not exist or is unreachable at this moment!\nWARNING: Could not resolve git repository in source 'example' (git://github.com/xorpaul/g10k-environment-unavailable.git)") {
 		t.Errorf("resolvePuppetEnvironment() terminated with the correct exit code, but the expected output was missing. out: %s", string(out))
@@ -436,7 +437,7 @@ func TestConfigUseCacheFallback(t *testing.T) {
 	} else {
 
 		// get the module to cache it
-		doMirrorOrUpdate("https://github.com/puppetlabs/puppetlabs-firewall.git", "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git/", "false", false)
+		doMirrorOrUpdate("https://github.com/puppetlabs/puppetlabs-firewall.git", "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git/", "false", false, 0)
 
 		// rename the cached module dir to match the otherwise failing single_fail env
 		unresolvableGitDir := "/tmp/g10k/modules/https-__.com_puppetlabs_puppetlabs-firewall.git/"
@@ -485,7 +486,7 @@ func TestConfigUseCacheFallbackFalse(t *testing.T) {
 	} else {
 
 		// get the module to cache it
-		doMirrorOrUpdate("https://github.com/puppetlabs/puppetlabs-firewall.git", "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git/", "false", false)
+		doMirrorOrUpdate("https://github.com/puppetlabs/puppetlabs-firewall.git", "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git/", "false", false, 0)
 
 		// rename the cached module dir to match the otherwise failing single_fail env
 		unresolvableGitDir := "/tmp/g10k/modules/https-__.com_puppetlabs_puppetlabs-firewall.git/"
@@ -889,3 +890,53 @@ func TestResolvePuppetfileControlBranch(t *testing.T) {
 	debug = false
 
 }
+
+func TestConfigRetryGitCommands(t *testing.T) {
+	quiet = true
+	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
+	config = readConfigfile("tests/" + funcName + ".yaml")
+	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
+		resolvePuppetEnvironment("single_git")
+		return
+	} else {
+
+		localGitRepoDir := "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git/"
+		purgeDir(localGitRepoDir, funcName)
+
+		// get the module to cache it
+		doMirrorOrUpdate("https://github.com/puppetlabs/puppetlabs-firewall.git", localGitRepoDir, "false", false, 0)
+
+		// corrupt the local git module repository
+
+		matches, _ := filepath.Glob(localGitRepoDir + "objects/pack/*.idx")
+		for _, m := range matches {
+			if err := os.RemoveAll(m); err != nil {
+				t.Error("Error: deleting Git *.idx file to corrupt the local Git repository")
+			}
+			f, _ := os.Create(m)
+			defer f.Close()
+			f.WriteString("foobar")
+			f.Sync()
+		}
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$")
+	cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1")
+	out, err := cmd.CombinedOutput()
+
+	exitCode := 0
+	if msg, ok := err.(*exec.ExitError); ok { // there is error code
+		exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus()
+	}
+
+	if 0 != exitCode {
+		t.Errorf("resolvePuppetEnvironment() terminated with %v, but we expected exit status %v", exitCode, 0)
+	}
+	//fmt.Println(string(out))
+	if !strings.Contains(string(out), "WARN: git command failed: git --git-dir /tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git remote update --prune deleting local cached repository and retrying...") {
+		t.Errorf("resolvePuppetEnvironment() terminated with the correct exit code, but the expected output was missing. out: %s", string(out))
+	}
+	//if !fileExists("/tmp/example/single_fail/modules/firewall/metadata.json") {
+	//	t.Errorf("resolvePuppetEnvironment() terminated with the correct exit code and the correct output, but the resulting module was missing")
+	//}
+}

git.go

@@ -80,7 +80,7 @@ func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
 			repoDir := strings.Replace(strings.Replace(url, "/", "_", -1), ":", "-", -1)
 			workDir := config.ModulesCacheDir + repoDir
 
-			doMirrorOrUpdate(url, workDir, privateKey, gm.ignoreUnreachable)
+			doMirrorOrUpdate(url, workDir, privateKey, gm.ignoreUnreachable, 1)
 			//	doCloneOrPull(source, workDir, targetDir, sa.Remote, branch, sa.PrivateKey)
 		}(url, privateKey, gm, bar)
 		done <- true
@@ -91,7 +91,7 @@ func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
 	wg.Wait()
 }
 
-func doMirrorOrUpdate(url string, workDir string, sshPrivateKey string, allowFail bool) bool {
+func doMirrorOrUpdate(url string, workDir string, sshPrivateKey string, allowFail bool, retryCount int) bool {
 	needSSHKey := true
 	if strings.Contains(url, "github.com") || len(sshPrivateKey) == 0 {
 		needSSHKey = false
@@ -113,6 +113,10 @@ func doMirrorOrUpdate(url string, workDir string, sshPrivateKey string, allowFai
 		Warnf("WARN: git repository " + url + " does not exist or is unreachable at this moment!")
 		if config.UseCacheFallback {
 			Warnf("WARN: Trying to use cache for " + url + " git repository")
+		} else if config.RetryGitCommands && retryCount > 0 {
+			Warnf("WARN: git command failed: " + gitCmd + " deleting local cached repository and retrying...")
+			purgeDir(workDir, "doMirrorOrUpdate, because git command failed, retrying")
+			return doMirrorOrUpdate(url, workDir, sshPrivateKey, false, retryCount-1)
 		}
 		return false
 	}

helper.go

@@ -162,7 +162,7 @@ func executeCommand(command string, timeout int, allowFail bool) ExecResult {
 		Verbosef("Executing " + command + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
 	}
 	if err != nil {
-		if !allowFail && !config.UseCacheFallback {
+		if !allowFail && !config.UseCacheFallback && !config.RetryGitCommands {
 			Fatalf("executeCommand(): git command failed: " + command + " " + err.Error() + "\nOutput: " + string(out) +
 				"\nIf you are using GitLab be sure that you added your deploy key to your repository")
 		} else {

puppetfile.go

@@ -40,17 +40,14 @@ func resolvePuppetEnvironment(envBranch string) {
 			sa.Basedir = checkDirAndCreate(sa.Basedir, "basedir for source "+source)
 			Debugf("Puppet environment: " + source + " (" + fmt.Sprintf("%+v", sa) + ")")
 
-			// check for a valid source that has all neccessary attributes (basedir, remote, SSH key exists if given)
+			// check for a valid source that has all neccessary attributes (basedir, remote, SSH key exist if given)
 			sourceSanityCheck(source, sa)
 
 			workDir := config.EnvCacheDir + source + ".git"
 			// check if sa.Basedir exists
 			checkDirAndCreate(sa.Basedir, "basedir")
 
-			//if !strings.Contains(source, "hiera") && !strings.Contains(source, "files") {
-			//	gitKey = sa.PrivateKey
-			//}
-			if success := doMirrorOrUpdate(sa.Remote, workDir, sa.PrivateKey, true); success {
+			if success := doMirrorOrUpdate(sa.Remote, workDir, sa.PrivateKey, true, 1); success {
 
 				// get all branches
 				er := executeCommand("git --git-dir "+workDir+" branch", config.Timeout, false)

tests/TestConfigRetryGitCommands.yaml

@@ -0,0 +1,8 @@
+---
+:cachedir: '/tmp/g10k'
+retry_git_commands: true
+
+sources:
+  example:
+    remote: 'git://github.com/xorpaul/g10k-environment.git'
+    basedir: '/tmp/example/'