s3store: Parallelize part uploads and information retrieval (#478)

* Add first draft of parallel upload queue

* s3store: Use queue for parallel uploads

* Revert "Add first draft of parallel upload queue"

This reverts commit 86a329c.

* Revert "s3store: Use queue for parallel uploads"

This reverts commit 29b59a2.

* s3store: Cache results from listing parts and checking incomplete object

* s3store: Remove debugging output`

* s3store: Make requests for fetching info concurrently

* s3store: Make parallel uploads work and tests pass

* s3store: Add semaphore package

* s3store: Add comments to semaphore package

* s3store: Encapsulate more logic into s3PartProducer

* s3store: Refactor WriteChunk

* s3store: Remove TODO

* s3store: Acquire lock before uploading

* cli: Add flag for setting concurrency limit

* s3store: One more comment

cmd/tusd/cli/flags.go

@@ -27,6 +27,7 @@ var Flags struct {
 	S3PartSize              int64
 	S3DisableContentHashes  bool
 	S3DisableSSL            bool
+	S3ConcurrentPartUploads int
 	GCSBucket               string
 	GCSObjectPrefix         string
 	EnabledHooksString      string
@@ -68,6 +69,7 @@ func ParseFlags() {
 	flag.Int64Var(&Flags.S3PartSize, "s3-part-size", 50*1024*1024, "Size in bytes of the individual upload requests made to the S3 API. Defaults to 50MiB (experimental and may be removed in the future)")
 	flag.BoolVar(&Flags.S3DisableContentHashes, "s3-disable-content-hashes", false, "Disable the calculation of MD5 and SHA256 hashes for the content that gets uploaded to S3 for minimized CPU usage (experimental and may be removed in the future)")
 	flag.BoolVar(&Flags.S3DisableSSL, "s3-disable-ssl", false, "Disable SSL and only use HTTP for communication with S3 (experimental and may be removed in the future)")
+	flag.IntVar(&Flags.S3ConcurrentPartUploads, "s3-concurrent-part-uploads", 10, "Number of concurrent part uploads to S3 (experimental and may be removed in the future)")
 	flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)")
 	flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names (can't contain underscore character)")
 	flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "pre-create,post-create,post-receive,post-terminate,post-finish", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable default events")

internal/semaphore/semaphore.go

@@ -0,0 +1,20 @@
+// Package semaphore implements a basic semaphore for coordinating and limiting
+// non-exclusive, concurrent access.
+package semaphore
+
+type Semaphore chan struct{}
+
+// New creates a semaphore with the given concurrency limit.
+func New(concurrency int) Semaphore {
+	return make(chan struct{}, concurrency)
+}
+
+// Acquire will block until the semaphore can be acquired.
+func (s Semaphore) Acquire() {
+	s <- struct{}{}
+}
+
+// Release frees the acquired slot in the semaphore.
+func (s Semaphore) Release() {
+	<-s
+}