I think this is buggy
This commit is contained in:
parent
17a0252c69
commit
dbcddc7b9e
|
@ -15,10 +15,11 @@ import (
|
||||||
// workState stores the state of all the jobs and
|
// workState stores the state of all the jobs and
|
||||||
// is shared across workers
|
// is shared across workers
|
||||||
type workState struct {
|
type workState struct {
|
||||||
Total int
|
Total int
|
||||||
Completed int
|
Completed int
|
||||||
Successes int
|
Successes int
|
||||||
Failures int
|
Failures int
|
||||||
|
SaveDirectory string
|
||||||
}
|
}
|
||||||
|
|
||||||
// BeginDownload takes a slice of posts, a directory to save them in, and a
|
// BeginDownload takes a slice of posts, a directory to save them in, and a
|
||||||
|
@ -26,20 +27,16 @@ type workState struct {
|
||||||
// been processed. It returns the number of successes, failures, and the total
|
// been processed. It returns the number of successes, failures, and the total
|
||||||
// amount of posts.
|
// amount of posts.
|
||||||
func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *int) (*int, *int, *int) {
|
func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *int) (*int, *int, *int) {
|
||||||
// Channel for worker goroutines to notify the main goroutine that it is done
|
// Channel for main goroutine to give workers a post when they are done downloading one
|
||||||
// downloading a post
|
wc := make(chan *e621.Post)
|
||||||
done := make(chan interface{})
|
|
||||||
|
|
||||||
// Channel for main goroutine to give workers a post when they are done downloading
|
|
||||||
// one
|
|
||||||
pc := make(chan *e621.Post)
|
|
||||||
|
|
||||||
var current int
|
var current int
|
||||||
|
|
||||||
total := len(*posts)
|
total := len(*posts)
|
||||||
|
|
||||||
state := workState{
|
state := workState{
|
||||||
Total: total,
|
Total: total,
|
||||||
|
SaveDirectory: *saveDirectory,
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we have more workers than posts, then we don't need all of them
|
// If we have more workers than posts, then we don't need all of them
|
||||||
|
@ -49,18 +46,18 @@ func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *in
|
||||||
|
|
||||||
for i := 0; i < *maxConcurrents; i++ {
|
for i := 0; i < *maxConcurrents; i++ {
|
||||||
// Create our workers
|
// Create our workers
|
||||||
go work(i+1, *saveDirectory, &state, done, pc)
|
go work(i+1, &state, wc)
|
||||||
|
|
||||||
// Give them their initial posts
|
// Give them their initial posts
|
||||||
pc <- &(*posts)[current]
|
wc <- &(*posts)[current]
|
||||||
current++
|
current++
|
||||||
|
|
||||||
time.Sleep(time.Millisecond * 50)
|
time.Sleep(time.Millisecond * 50)
|
||||||
}
|
}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
// Wait for a worker to be done
|
// Wait for a worker to be done (they send nil to wc)
|
||||||
<-done
|
<-wc
|
||||||
|
|
||||||
// If we finished downloading all posts, break out of the loop
|
// If we finished downloading all posts, break out of the loop
|
||||||
if state.Successes+state.Failures == total {
|
if state.Successes+state.Failures == total {
|
||||||
|
@ -69,24 +66,24 @@ func BeginDownload(posts *[]e621.Post, saveDirectory *string, maxConcurrents *in
|
||||||
|
|
||||||
// If there's no more posts to give, stop the worker
|
// If there's no more posts to give, stop the worker
|
||||||
if current >= total {
|
if current >= total {
|
||||||
pc <- nil
|
wc <- nil
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Give the worker the next post in the array
|
// Give the worker the next post in the array
|
||||||
pc <- &(*posts)[current]
|
wc <- &(*posts)[current]
|
||||||
current++
|
current++
|
||||||
}
|
}
|
||||||
|
|
||||||
return &state.Successes, &state.Failures, &total
|
return &state.Successes, &state.Failures, &total
|
||||||
}
|
}
|
||||||
|
|
||||||
func work(wn int, directory string, state *workState, done chan interface{}, pc chan *e621.Post) {
|
func work(wn int, state *workState, wc chan *e621.Post) {
|
||||||
for {
|
for {
|
||||||
state.Completed++
|
state.Completed++
|
||||||
|
|
||||||
// Wait for a post from main
|
// Wait for a post from main
|
||||||
post := <-pc
|
post := <-wc
|
||||||
if post == nil { // nil means there aren't any more posts, so we're OK to break
|
if post == nil { // nil means there aren't any more posts, so we're OK to break
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -100,10 +97,10 @@ func work(wn int, directory string, state *workState, done chan interface{}, pc
|
||||||
workerText,
|
workerText,
|
||||||
post.ID,
|
post.ID,
|
||||||
humanize.Bytes(uint64(post.FileSize)),
|
humanize.Bytes(uint64(post.FileSize)),
|
||||||
getSavePath(post, &directory),
|
getSavePath(post, &state.SaveDirectory),
|
||||||
))
|
))
|
||||||
|
|
||||||
err := downloadPost(post, directory)
|
err := downloadPost(post, state.SaveDirectory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err)
|
fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err)
|
||||||
state.Failures++
|
state.Failures++
|
||||||
|
@ -111,7 +108,7 @@ func work(wn int, directory string, state *workState, done chan interface{}, pc
|
||||||
state.Successes++
|
state.Successes++
|
||||||
}
|
}
|
||||||
|
|
||||||
done <- nil
|
wc <- nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue