I think this is buggy

This commit is contained in:
TJ Horner 2019-03-29 11:42:11 -07:00
parent 17a0252c69
commit dbcddc7b9e
1 changed files with 20 additions and 23 deletions

View File

@ -19,6 +19,7 @@ type workState struct {
Completed int Completed int
Successes int Successes int
Failures int Failures int
SaveDirectory string
} }
// BeginDownload takes a slice of posts, a directory to save them in, and a // BeginDownload takes a slice of posts, a directory to save them in, and a
@ -26,13 +27,8 @@ type workState struct {
// been processed. It returns the number of successes, failures, and the total // been processed. It returns the number of successes, failures, and the total
// amount of posts. // amount of posts.
func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *int) (*int, *int, *int) { func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *int) (*int, *int, *int) {
// Channel for worker goroutines to notify the main goroutine that it is done // Channel for main goroutine to give workers a post when they are done downloading one
// downloading a post wc := make(chan *e621.Post)
done := make(chan interface{})
// Channel for main goroutine to give workers a post when they are done downloading
// one
pc := make(chan *e621.Post)
var current int var current int
@ -40,6 +36,7 @@ func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *in
state := workState{ state := workState{
Total: total, Total: total,
SaveDirectory: *saveDirectory,
} }
// If we have more workers than posts, then we don't need all of them // If we have more workers than posts, then we don't need all of them
@ -49,18 +46,18 @@ func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *in
for i := 0; i < *maxConcurrents; i++ { for i := 0; i < *maxConcurrents; i++ {
// Create our workers // Create our workers
go work(i+1, *saveDirectory, &state, done, pc) go work(i+1, &state, wc)
// Give them their initial posts // Give them their initial posts
pc <- &(*posts)[current] wc <- &(*posts)[current]
current++ current++
time.Sleep(time.Millisecond * 50) time.Sleep(time.Millisecond * 50)
} }
for { for {
// Wait for a worker to be done // Wait for a worker to be done (they send nil to wc)
<-done <-wc
// If we finished downloading all posts, break out of the loop // If we finished downloading all posts, break out of the loop
if state.Successes+state.Failures == total { if state.Successes+state.Failures == total {
@ -69,24 +66,24 @@ func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *in
// If there's no more posts to give, stop the worker // If there's no more posts to give, stop the worker
if current >= total { if current >= total {
pc <- nil wc <- nil
continue continue
} }
// Give the worker the next post in the array // Give the worker the next post in the array
pc <- &(*posts)[current] wc <- &(*posts)[current]
current++ current++
} }
return &state.Successes, &state.Failures, &total return &state.Successes, &state.Failures, &total
} }
func work(wn int, directory string, state *workState, done chan interface{}, pc chan *e621.Post) { func work(wn int, state *workState, wc chan *e621.Post) {
for { for {
state.Completed++ state.Completed++
// Wait for a post from main // Wait for a post from main
post := <-pc post := <-wc
if post == nil { // nil means there aren't any more posts, so we're OK to break if post == nil { // nil means there aren't any more posts, so we're OK to break
return return
} }
@ -100,10 +97,10 @@ func work(wn int, directory string, state *workState, done chan interface{}, pc
workerText, workerText,
post.ID, post.ID,
humanize.Bytes(uint64(post.FileSize)), humanize.Bytes(uint64(post.FileSize)),
getSavePath(post, &directory), getSavePath(post, &state.SaveDirectory),
)) ))
err := downloadPost(post, directory) err := downloadPost(post, state.SaveDirectory)
if err != nil { if err != nil {
fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err) fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err)
state.Failures++ state.Failures++
@ -111,7 +108,7 @@ func work(wn int, directory string, state *workState, done chan interface{}, pc
state.Successes++ state.Successes++
} }
done <- nil wc <- nil
} }
} }