BitHeaven
/
e6dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: multi-page functionality

This commit is contained in:
TJ Horner 2019-09-05 22:05:44 -04:00
parent d6d942a7d5
commit d6ed8c94df
2 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
e621/e621.go
View File

@ -54,7 +54,7 @@ func (date *SerializedDate) Time() time.Time {
} }
// GetPostsForTags gets a list of e621 Posts // GetPostsForTags gets a list of e621 Posts
func GetPostsForTags(tags string, limit int, sfw bool) ([]Post, error) { func GetPostsForTags(tags string, limit int, sfw bool, page int) ([]Post, error) {
client := &http.Client{} client := &http.Client{}
var domain string var domain string
@ -71,6 +71,7 @@ func GetPostsForTags(tags string, limit int, sfw bool) ([]Post, error) {
qs := req.URL.Query() qs := req.URL.Query()
qs.Add("tags", tags) qs.Add("tags", tags)
qs.Add("limit", strconv.Itoa(limit)) qs.Add("limit", strconv.Itoa(limit))
qs.Add("page", strconv.Itoa(page))
req.URL.RawQuery = qs.Encode() req.URL.RawQuery = qs.Encode()

27
main.go
View File

@ -17,29 +17,40 @@ func main() {
postLimit := flag.Int("limit", 10, "Maximum amount of posts to grab from e621") postLimit := flag.Int("limit", 10, "Maximum amount of posts to grab from e621")
saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to") saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to")
sfw := flag.Bool("sfw", false, "Download posts from e926 instead of e621") sfw := flag.Bool("sfw", false, "Download posts from e926 instead of e621")
pages := flag.Int("pages", 1, "Number of search result pages to download")
flag.Parse() flag.Parse()
fmt.Printf("Fetching posts for \"%v\" (limit %v)\n", *tags, *postLimit) fmt.Printf("Fetching posts for \"%s\" (limit=%d, pages=%d)\n", *tags, *postLimit, *pages)
posts, err := e621.GetPostsForTags(*tags, *postLimit, *sfw) var allPosts []e621.Post
if err != nil {
fmt.Println(err) for i := 1; i <= *pages; i++ {
os.Exit(1) fmt.Printf("Fetching page %d/%d...", i, *pages)
posts, err := e621.GetPostsForTags(*tags, *postLimit, *sfw, i)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Printf(" fetched %d posts\n", len(posts))
allPosts = append(allPosts, posts...)
} }
fmt.Printf("Found %d posts. Starting download with %d workers...\n\n", len(posts), *maxConcurrents) fmt.Printf("Found %d posts. Starting download with %d workers...\n\n", len(allPosts), *maxConcurrents)
cwd, _ := os.Getwd() cwd, _ := os.Getwd()
absSaveDir := path.Join(cwd, *saveDirectory) absSaveDir := path.Join(cwd, *saveDirectory)
err = os.MkdirAll(absSaveDir, 0755) err := os.MkdirAll(absSaveDir, 0755)
if err != nil { if err != nil {
fmt.Printf("Cannot create output directory (%s). Do you have the right permissions?\n", absSaveDir) fmt.Printf("Cannot create output directory (%s). Do you have the right permissions?\n", absSaveDir)
os.Exit(1) os.Exit(1)
} }
successes, failures, _ := concurrent.BeginDownload(&posts, saveDirectory, maxConcurrents) successes, failures, _ := concurrent.BeginDownload(&allPosts, saveDirectory, maxConcurrents)
fmt.Printf("\nAll done! %d posts downloaded and saved. (%d failed to download)\n", *successes, *failures) fmt.Printf("\nAll done! %d posts downloaded and saved. (%d failed to download)\n", *successes, *failures)
} }