feat: multi-page functionality
This commit is contained in:
parent
d6d942a7d5
commit
d6ed8c94df
|
@ -54,7 +54,7 @@ func (date *SerializedDate) Time() time.Time {
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPostsForTags gets a list of e621 Posts
|
// GetPostsForTags gets a list of e621 Posts
|
||||||
func GetPostsForTags(tags string, limit int, sfw bool) ([]Post, error) {
|
func GetPostsForTags(tags string, limit int, sfw bool, page int) ([]Post, error) {
|
||||||
client := &http.Client{}
|
client := &http.Client{}
|
||||||
|
|
||||||
var domain string
|
var domain string
|
||||||
|
@ -71,6 +71,7 @@ func GetPostsForTags(tags string, limit int, sfw bool) ([]Post, error) {
|
||||||
qs := req.URL.Query()
|
qs := req.URL.Query()
|
||||||
qs.Add("tags", tags)
|
qs.Add("tags", tags)
|
||||||
qs.Add("limit", strconv.Itoa(limit))
|
qs.Add("limit", strconv.Itoa(limit))
|
||||||
|
qs.Add("page", strconv.Itoa(page))
|
||||||
|
|
||||||
req.URL.RawQuery = qs.Encode()
|
req.URL.RawQuery = qs.Encode()
|
||||||
|
|
||||||
|
|
27
main.go
27
main.go
|
@ -17,29 +17,40 @@ func main() {
|
||||||
postLimit := flag.Int("limit", 10, "Maximum amount of posts to grab from e621")
|
postLimit := flag.Int("limit", 10, "Maximum amount of posts to grab from e621")
|
||||||
saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to")
|
saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to")
|
||||||
sfw := flag.Bool("sfw", false, "Download posts from e926 instead of e621")
|
sfw := flag.Bool("sfw", false, "Download posts from e926 instead of e621")
|
||||||
|
pages := flag.Int("pages", 1, "Number of search result pages to download")
|
||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
fmt.Printf("Fetching posts for \"%v\" (limit %v)\n", *tags, *postLimit)
|
fmt.Printf("Fetching posts for \"%s\" (limit=%d, pages=%d)\n", *tags, *postLimit, *pages)
|
||||||
|
|
||||||
posts, err := e621.GetPostsForTags(*tags, *postLimit, *sfw)
|
var allPosts []e621.Post
|
||||||
if err != nil {
|
|
||||||
fmt.Println(err)
|
for i := 1; i <= *pages; i++ {
|
||||||
os.Exit(1)
|
fmt.Printf("Fetching page %d/%d...", i, *pages)
|
||||||
|
|
||||||
|
posts, err := e621.GetPostsForTags(*tags, *postLimit, *sfw, i)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf(" fetched %d posts\n", len(posts))
|
||||||
|
|
||||||
|
allPosts = append(allPosts, posts...)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Found %d posts. Starting download with %d workers...\n\n", len(posts), *maxConcurrents)
|
fmt.Printf("Found %d posts. Starting download with %d workers...\n\n", len(allPosts), *maxConcurrents)
|
||||||
|
|
||||||
cwd, _ := os.Getwd()
|
cwd, _ := os.Getwd()
|
||||||
absSaveDir := path.Join(cwd, *saveDirectory)
|
absSaveDir := path.Join(cwd, *saveDirectory)
|
||||||
|
|
||||||
err = os.MkdirAll(absSaveDir, 0755)
|
err := os.MkdirAll(absSaveDir, 0755)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("Cannot create output directory (%s). Do you have the right permissions?\n", absSaveDir)
|
fmt.Printf("Cannot create output directory (%s). Do you have the right permissions?\n", absSaveDir)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
successes, failures, _ := concurrent.BeginDownload(&posts, saveDirectory, maxConcurrents)
|
successes, failures, _ := concurrent.BeginDownload(&allPosts, saveDirectory, maxConcurrents)
|
||||||
|
|
||||||
fmt.Printf("\nAll done! %d posts downloaded and saved. (%d failed to download)\n", *successes, *failures)
|
fmt.Printf("\nAll done! %d posts downloaded and saved. (%d failed to download)\n", *successes, *failures)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue