add retrys

go.mod

@@ -5,4 +5,10 @@ go 1.20
 require (
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	golang.org/x/net v0.8.0
+	gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
+)
+
+require (
+	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
+	github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect
 )

go.sum

@@ -1,4 +1,10 @@
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 h1:JAEbJn3j/FrhdWA9jW8B5ajsLIjeuEHLi8xE4fk997o=
+github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
 golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e h1:bJHzu9Qwc9wQRWJ/WVkJGAfs+riucl/tKAFNxf9pzqk=
+gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e/go.mod h1:tve0rTLdGlwnXF7iBO9rbAEyeXvuuPx0n4DvXS/Nw7o=

main.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/logrusorgru/aurora"
 	"golang.org/x/net/proxy"
+	"gopkg.in/matryer/try.v1"
 )
 
 type Post struct {
@@ -200,15 +201,30 @@ func work(wn int, state *workState, wc chan *Post, transport http.Transport) {
 		progress := aurora.Sprintf(aurora.Green("[%d/%d]"), state.Completed, state.Total)
 		workerText := aurora.Sprintf(aurora.Cyan("[w%d]"), wn)
 
-		fmt.Println(aurora.Sprintf(
+		err := try.Do(func(attempt int) (retry bool, err error) {
-			"%s %s Downloading post %d -> %s",
+			if attempt == 1 {
-			progress,
+				fmt.Println(aurora.Sprintf(
-			workerText,
+					"%s %s Downloading post %d -> %s",
-			post.ID,
+					progress,
-			getSavePath(post, &state.SaveDirectory),
+					workerText,
-		))
+					post.ID,
+					getSavePath(post, &state.SaveDirectory),
+				))
+			} else {
+				fmt.Println(aurora.Sprintf(
+					"%s %s (%d) Retry Download post %d -> %s",
+					progress,
+					workerText,
+					attempt,
+					post.ID,
+					getSavePath(post, &state.SaveDirectory),
+				))
+			}
+
+			err = downloadPost(post, state.SaveDirectory, transport)
+			return attempt < 5, err
+		})
 
-		err := downloadPost(post, state.SaveDirectory, transport)
 		if err != nil {
 			fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err)
 			state.Failures++
@@ -268,7 +284,7 @@ func main() {
 	tags := flag.String("tags", "", "Tags to search for")
 	maxConcurrents := flag.Int("concurrents", 30, "Maximum amount of concurrent downloads")
 	saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to")
-	postLimit := flag.Int("limit", 9999999999, "Maximum amount of posts to grab from e621")
+	postLimit := flag.Int("limit", 9999999999, "Maximum amount of posts to grab from rule34")
 	proxyAddr := flag.String("proxy", "", "Proxy address to parsing")
 	timeout := flag.Int("timeout", 10, "Timeout proxy to parsing")