feat: multi-page functionality

e621/e621.go

@@ -54,7 +54,7 @@ func (date *SerializedDate) Time() time.Time {
 }
 
 // GetPostsForTags gets a list of e621 Posts
-func GetPostsForTags(tags string, limit int, sfw bool) ([]Post, error) {
+func GetPostsForTags(tags string, limit int, sfw bool, page int) ([]Post, error) {
 	client := &http.Client{}
 
 	var domain string
@@ -71,6 +71,7 @@ func GetPostsForTags(tags string, limit int, sfw bool) ([]Post, error) {
 	qs := req.URL.Query()
 	qs.Add("tags", tags)
 	qs.Add("limit", strconv.Itoa(limit))
+	qs.Add("page", strconv.Itoa(page))
 
 	req.URL.RawQuery = qs.Encode()
 

main.go

@@ -17,29 +17,40 @@ func main() {
 	postLimit := flag.Int("limit", 10, "Maximum amount of posts to grab from e621")
 	saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to")
 	sfw := flag.Bool("sfw", false, "Download posts from e926 instead of e621")
+	pages := flag.Int("pages", 1, "Number of search result pages to download")
 
 	flag.Parse()
 
-	fmt.Printf("Fetching posts for \"%v\" (limit %v)\n", *tags, *postLimit)
+	fmt.Printf("Fetching posts for \"%s\" (limit=%d, pages=%d)\n", *tags, *postLimit, *pages)
 
-	posts, err := e621.GetPostsForTags(*tags, *postLimit, *sfw)
+	var allPosts []e621.Post
+
+	for i := 1; i <= *pages; i++ {
+		fmt.Printf("Fetching page %d/%d...", i, *pages)
+
+		posts, err := e621.GetPostsForTags(*tags, *postLimit, *sfw, i)
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)
 		}
 
-	fmt.Printf("Found %d posts. Starting download with %d workers...\n\n", len(posts), *maxConcurrents)
+		fmt.Printf(" fetched %d posts\n", len(posts))
+
+		allPosts = append(allPosts, posts...)
+	}
+
+	fmt.Printf("Found %d posts. Starting download with %d workers...\n\n", len(allPosts), *maxConcurrents)
 
 	cwd, _ := os.Getwd()
 	absSaveDir := path.Join(cwd, *saveDirectory)
 
-	err = os.MkdirAll(absSaveDir, 0755)
+	err := os.MkdirAll(absSaveDir, 0755)
 	if err != nil {
 		fmt.Printf("Cannot create output directory (%s). Do you have the right permissions?\n", absSaveDir)
 		os.Exit(1)
 	}
 
-	successes, failures, _ := concurrent.BeginDownload(&posts, saveDirectory, maxConcurrents)
+	successes, failures, _ := concurrent.BeginDownload(&allPosts, saveDirectory, maxConcurrents)
 
 	fmt.Printf("\nAll done! %d posts downloaded and saved. (%d failed to download)\n", *successes, *failures)
 }