diff --git a/Makefile b/Makefile index e772c5ba2..fe14edecb 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,6 @@ gen-mock: $(MOCKGEN) mockgen -typed -destination=./internal/storage/fake/mock_client.go -package=fake github.com/artefactual-sdps/enduro/internal/storage Client mockgen -typed -destination=./internal/storage/fake/mock_storage.go -package=fake github.com/artefactual-sdps/enduro/internal/storage Service mockgen -typed -destination=./internal/storage/persistence/fake/mock_persistence.go -package=fake github.com/artefactual-sdps/enduro/internal/storage/persistence Storage - mockgen -typed -destination=./internal/upload/fake/mock_upload.go -package=fake github.com/artefactual-sdps/enduro/internal/upload Service mockgen -typed -destination=./internal/watcher/fake/mock_service.go -package=fake github.com/artefactual-sdps/enduro/internal/watcher Service mockgen -typed -destination=./internal/watcher/fake/mock_watcher.go -package=fake github.com/artefactual-sdps/enduro/internal/watcher Watcher @@ -249,11 +248,11 @@ tparse: $(TPARSE) go test -count=1 -json -cover $(TEST_PACKAGES) | tparse -follow -all -notests upload-sample-transfer: # @HELP Upload sample transfer (small.zip). -upload-sample-transfer: ADDRESS ?= localhost:9000 +upload-sample-transfer: ADDRESS ?= localhost:9002 upload-sample-transfer: curl \ -F "file=@$(CURDIR)/internal/testdata/zipped_transfer/small.zip" \ - http://$(ADDRESS)/upload/upload + http://$(ADDRESS)/package/upload workflowcheck: # @HELP Detect non-determinism in workflow functions. workflowcheck: $(WORKFLOWCHECK) diff --git a/cmd/enduro-a3m-worker/main.go b/cmd/enduro-a3m-worker/main.go index a25f54e67..927c5a1f9 100644 --- a/cmd/enduro-a3m-worker/main.go +++ b/cmd/enduro-a3m-worker/main.go @@ -163,6 +163,8 @@ func main() { &auth.NoopTokenVerifier{}, nil, cfg.Temporal.TaskQueue, + nil, + 0, ) } diff --git a/cmd/enduro-am-worker/main.go b/cmd/enduro-am-worker/main.go index 43e6f9856..adc0a340c 100644 --- a/cmd/enduro-am-worker/main.go +++ b/cmd/enduro-am-worker/main.go @@ -180,6 +180,8 @@ func main() { &auth.NoopTokenVerifier{}, nil, cfg.Temporal.TaskQueue, + nil, + 0, ) } diff --git a/cmd/enduro/main.go b/cmd/enduro/main.go index a99d7d6e4..b364b006e 100644 --- a/cmd/enduro/main.go +++ b/cmd/enduro/main.go @@ -19,6 +19,7 @@ import ( "github.com/oklog/run" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/pflag" + "go.artefactual.dev/tools/bucket" "go.artefactual.dev/tools/log" temporal_tools "go.artefactual.dev/tools/temporal" "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" @@ -50,7 +51,6 @@ import ( storage_entdb "github.com/artefactual-sdps/enduro/internal/storage/persistence/ent/db" storage_workflows "github.com/artefactual-sdps/enduro/internal/storage/workflows" "github.com/artefactual-sdps/enduro/internal/telemetry" - "github.com/artefactual-sdps/enduro/internal/upload" "github.com/artefactual-sdps/enduro/internal/version" "github.com/artefactual-sdps/enduro/internal/watcher" "github.com/artefactual-sdps/enduro/internal/workflow" @@ -215,6 +215,14 @@ func main() { ) } + // Set up upload bucket. + uploadBucket, err := bucket.NewWithConfig(ctx, &cfg.Upload.Bucket) + if err != nil { + logger.Error(err, "Error setting up upload bucket.") + os.Exit(1) + } + defer uploadBucket.Close() + // Set up the package service. var pkgsvc package_.Service { @@ -227,6 +235,8 @@ func main() { tokenVerifier, ticketProvider, cfg.Temporal.TaskQueue, + uploadBucket, + cfg.Upload.MaxSize, ) } @@ -264,17 +274,6 @@ func main() { } } - // Set up the upload service. - var uploadsvc upload.Service - { - uploadsvc, err = upload.NewService(logger.WithName("upload"), cfg.Upload, upload.UPLOAD_MAX_SIZE, tokenVerifier) - if err != nil { - logger.Error(err, "Error setting up upload service.") - os.Exit(1) - } - defer uploadsvc.Close() - } - // Set up the watcher service. var wsvc watcher.Service { @@ -293,7 +292,7 @@ func main() { g.Add( func() error { - srv = api.HTTPServer(logger, tp, &cfg.API, pkgsvc, storagesvc, uploadsvc) + srv = api.HTTPServer(logger, tp, &cfg.API, pkgsvc, storagesvc) return srv.ListenAndServe() }, func(err error) { @@ -317,6 +316,8 @@ func main() { &auth.NoopTokenVerifier{}, ticketProvider, cfg.Temporal.TaskQueue, + uploadBucket, + cfg.Upload.MaxSize, ) storagesvc, err = storage.NewService( @@ -332,23 +333,11 @@ func main() { os.Exit(1) } - uploadsvc, err = upload.NewService( - logger.WithName("internal-upload"), - cfg.Upload, - upload.UPLOAD_MAX_SIZE, - &auth.NoopTokenVerifier{}, - ) - if err != nil { - logger.Error(err, "Error setting up internal upload service.") - os.Exit(1) - } - defer uploadsvc.Close() - var srv *http.Server g.Add( func() error { - srv = api.HTTPServer(logger, tp, &cfg.InternalAPI, pkgsvc, storagesvc, uploadsvc) + srv = api.HTTPServer(logger, tp, &cfg.InternalAPI, pkgsvc, storagesvc) return srv.ListenAndServe() }, func(err error) { diff --git a/docs/src/admin-manual/iac.md b/docs/src/admin-manual/iac.md index 79f4318ca..7ef9fd0f3 100644 --- a/docs/src/admin-manual/iac.md +++ b/docs/src/admin-manual/iac.md @@ -150,7 +150,7 @@ The `*` attribute will provide full access to the API. | POST | /package/{id}/move | `package:move` | | GET | /package/{id}/preservation-actions | `package:listActions` | | POST | /package/{id}/reject | `package:review` | -| POST | /upload/upload | `package:upload` | +| POST | /package/upload | `package:upload` | | GET | /storage/location | `storage:location:list` | | POST | /storage/location | `storage:location:create` | | GET | /storage/location/{uuid} | `storage:location:read` | diff --git a/enduro.toml b/enduro.toml index b2a435a00..cab34f2b4 100644 --- a/enduro.toml +++ b/enduro.toml @@ -152,7 +152,7 @@ pollInterval = "10s" # no time limit. transferDeadline = "1h" -# TransferSourcePath is the path to an Archivematica transfer source directory. +# transferSourcePath is the path to an Archivematica transfer source directory. # It is used in the POST /api/v2beta/package "path" parameter to start a # transfer via the API. TransferSourcePath must be prefixed with the UUID of an # AMSS transfer source directory, optionally followed by a relative path from @@ -180,10 +180,18 @@ path = "" passphrase = "" # Secret: set (if required) with env var ENDURO_AM_SFTP_PRIVATEKEY_PASSPHRASE. [upload] +# maxSize is the maximum upload size allowed by the server in bytes. +# Default: 102400000. +maxSize = 102400000 + +# upload.bucket section configures a bucket where the files will be placed. +# Make sure it matches the configuration from one of the watchers to trigger +# the processing workflow after upload. +[upload.bucket] endpoint = "http://minio.enduro-sdps:9000" pathStyle = true -key = "minio" -secret = "minio123" +accessKey = "minio" +secretKey = "minio123" region = "us-west-1" bucket = "sips" diff --git a/internal/api/api.go b/internal/api/api.go index 9e66e0f27..eb7a43258 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -27,13 +27,10 @@ import ( packagesvr "github.com/artefactual-sdps/enduro/internal/api/gen/http/package_/server" storagesvr "github.com/artefactual-sdps/enduro/internal/api/gen/http/storage/server" swaggersvr "github.com/artefactual-sdps/enduro/internal/api/gen/http/swagger/server" - uploadsvr "github.com/artefactual-sdps/enduro/internal/api/gen/http/upload/server" "github.com/artefactual-sdps/enduro/internal/api/gen/package_" "github.com/artefactual-sdps/enduro/internal/api/gen/storage" - "github.com/artefactual-sdps/enduro/internal/api/gen/upload" intpkg "github.com/artefactual-sdps/enduro/internal/package_" intstorage "github.com/artefactual-sdps/enduro/internal/storage" - intupload "github.com/artefactual-sdps/enduro/internal/upload" "github.com/artefactual-sdps/enduro/internal/version" ) @@ -46,7 +43,6 @@ func HTTPServer( config *Config, pkgsvc intpkg.Service, storagesvc intstorage.Service, - uploadsvc intupload.Service, ) *http.Server { dec := goahttp.RequestDecoder enc := goahttp.ResponseEncoder @@ -70,12 +66,6 @@ func HTTPServer( storageServer.Download = writeTimeout(intstorage.Download(storagesvc, mux, dec), 0) storagesvr.Mount(mux, storageServer) - // Upload service. - uploadEndpoints := upload.NewEndpoints(uploadsvc) - uploadErrorHandler := errorHandler(logger, "Upload error.") - uploadServer := uploadsvr.New(uploadEndpoints, mux, dec, enc, uploadErrorHandler, nil) - uploadsvr.Mount(mux, uploadServer) - // Swagger service. swaggerService := swaggersvr.New(nil, nil, nil, nil, nil, nil, http.FS(openAPIJSON)) swaggersvr.Mount(mux, swaggerService) diff --git a/internal/api/design/design.go b/internal/api/design/design.go index efe3fc185..c0751ff91 100644 --- a/internal/api/design/design.go +++ b/internal/api/design/design.go @@ -39,7 +39,7 @@ var _ = API("enduro", func() { Title("Enduro API") Randomizer(expr.NewDeterministicRandomizer()) Server("enduro", func() { - Services("package", "storage", "swagger", "upload") + Services("package", "storage", "swagger") Host("localhost", func() { URI("http://localhost:9000") }) diff --git a/internal/api/design/package_.go b/internal/api/design/package_.go index 3aa99aba5..7f589c977 100644 --- a/internal/api/design/package_.go +++ b/internal/api/design/package_.go @@ -219,6 +219,47 @@ var _ = Service("package", func() { Response("failed_dependency", StatusFailedDependency) }) }) + Method("upload", func() { + Description("Upload a package to trigger an ingest workflow") + Security(JWTAuth, func() { + Scope("package:upload") + }) + Payload(func() { + Attribute("content_type", String, "Content-Type header, must define value for multipart boundary.", func() { + Default("multipart/form-data; boundary=goa") + Pattern("multipart/[^;]+; boundary=.+") + Example("multipart/form-data; boundary=goa") + }) + Token("token", String) + }) + + Error( + "invalid_media_type", + ErrorResult, + "Error returned when the Content-Type header does not define a multipart request.", + ) + Error( + "invalid_multipart_request", + ErrorResult, + "Error returned when the request body is not a valid multipart content.", + ) + Error("internal_error", ErrorResult, "Fault while processing upload.") + + HTTP(func() { + POST("/upload") + Header("content_type:Content-Type") + + // Bypass request body decoder code generation to alleviate need for + // loading the entire request body in memory. The service gets + // direct access to the HTTP request body reader. + SkipRequestBodyEncodeDecode() + + // Define error HTTP statuses. + Response("invalid_media_type", StatusBadRequest) + Response("invalid_multipart_request", StatusBadRequest) + Response("internal_error", StatusInternalServerError) + }) + }) }) var EnumPackageStatus = func() { diff --git a/internal/api/design/upload.go b/internal/api/design/upload.go deleted file mode 100644 index b97d089fa..000000000 --- a/internal/api/design/upload.go +++ /dev/null @@ -1,57 +0,0 @@ -package design - -import ( - . "goa.design/goa/v3/dsl" -) - -var _ = Service("upload", func() { - Description("The upload service handles file submissions to the SIPs bucket.") - Error("unauthorized", String, "Unauthorized") - Error("forbidden", String, "Forbidden") - HTTP(func() { - Path("/upload") - Response("unauthorized", StatusUnauthorized) - Response("forbidden", StatusForbidden) - }) - Method("upload", func() { - Description("Upload a package to trigger an ingest workflow") - Security(JWTAuth, func() { - Scope("package:upload") - }) - Payload(func() { - Attribute("content_type", String, "Content-Type header, must define value for multipart boundary.", func() { - Default("multipart/form-data; boundary=goa") - Pattern("multipart/[^;]+; boundary=.+") - Example("multipart/form-data; boundary=goa") - }) - Token("token", String) - }) - - Error( - "invalid_media_type", - ErrorResult, - "Error returned when the Content-Type header does not define a multipart request.", - ) - Error( - "invalid_multipart_request", - ErrorResult, - "Error returned when the request body is not a valid multipart content.", - ) - Error("internal_error", ErrorResult, "Fault while processing upload.") - - HTTP(func() { - POST("/upload") - Header("content_type:Content-Type") - - // Bypass request body decoder code generation to alleviate need for - // loading the entire request body in memory. The service gets - // direct access to the HTTP request body reader. - SkipRequestBodyEncodeDecode() - - // Define error HTTP statuses. - Response("invalid_media_type", StatusBadRequest) - Response("invalid_multipart_request", StatusBadRequest) - Response("internal_error", StatusInternalServerError) - }) - }) -}) diff --git a/internal/api/gen/http/cli/enduro/cli.go b/internal/api/gen/http/cli/enduro/cli.go index 91e87361a..cb15f7497 100644 --- a/internal/api/gen/http/cli/enduro/cli.go +++ b/internal/api/gen/http/cli/enduro/cli.go @@ -16,7 +16,6 @@ import ( package_c "github.com/artefactual-sdps/enduro/internal/api/gen/http/package_/client" storagec "github.com/artefactual-sdps/enduro/internal/api/gen/http/storage/client" - uploadc "github.com/artefactual-sdps/enduro/internal/api/gen/http/upload/client" goahttp "goa.design/goa/v3/http" goa "goa.design/goa/v3/pkg" ) @@ -25,9 +24,8 @@ import ( // // command (subcommand1|subcommand2|...) func UsageCommands() string { - return `package (monitor-request|monitor|list|show|preservation-actions|confirm|reject|move|move-status) + return `package (monitor-request|monitor|list|show|preservation-actions|confirm|reject|move|move-status|upload) storage (create|submit|update|download|move|move-status|reject|show|locations|add-location|show-location|location-packages) -upload upload ` } @@ -41,7 +39,6 @@ func UsageExamples() string { "object_key": "d1845cb6-a5ea-474a-9ab8-26f9bcd919f5", "status": "in_review" }' --token "abc123"` + "\n" + - os.Args[0] + ` upload upload --content-type "multipart/form-data; boundary=goa" --token "abc123" --stream "goa.png"` + "\n" + "" } @@ -101,6 +98,11 @@ func ParseEndpoint( package_MoveStatusIDFlag = package_MoveStatusFlags.String("id", "REQUIRED", "Identifier of package to move") package_MoveStatusTokenFlag = package_MoveStatusFlags.String("token", "", "") + package_UploadFlags = flag.NewFlagSet("upload", flag.ExitOnError) + package_UploadContentTypeFlag = package_UploadFlags.String("content-type", "multipart/form-data; boundary=goa", "") + package_UploadTokenFlag = package_UploadFlags.String("token", "", "") + package_UploadStreamFlag = package_UploadFlags.String("stream", "REQUIRED", "path to file containing the streamed request body") + storageFlags = flag.NewFlagSet("storage", flag.ContinueOnError) storageCreateFlags = flag.NewFlagSet("create", flag.ExitOnError) @@ -151,13 +153,6 @@ func ParseEndpoint( storageLocationPackagesFlags = flag.NewFlagSet("location-packages", flag.ExitOnError) storageLocationPackagesUUIDFlag = storageLocationPackagesFlags.String("uuid", "REQUIRED", "Identifier of location") storageLocationPackagesTokenFlag = storageLocationPackagesFlags.String("token", "", "") - - uploadFlags = flag.NewFlagSet("upload", flag.ContinueOnError) - - uploadUploadFlags = flag.NewFlagSet("upload", flag.ExitOnError) - uploadUploadContentTypeFlag = uploadUploadFlags.String("content-type", "multipart/form-data; boundary=goa", "") - uploadUploadTokenFlag = uploadUploadFlags.String("token", "", "") - uploadUploadStreamFlag = uploadUploadFlags.String("stream", "REQUIRED", "path to file containing the streamed request body") ) package_Flags.Usage = package_Usage package_MonitorRequestFlags.Usage = package_MonitorRequestUsage @@ -169,6 +164,7 @@ func ParseEndpoint( package_RejectFlags.Usage = package_RejectUsage package_MoveFlags.Usage = package_MoveUsage package_MoveStatusFlags.Usage = package_MoveStatusUsage + package_UploadFlags.Usage = package_UploadUsage storageFlags.Usage = storageUsage storageCreateFlags.Usage = storageCreateUsage @@ -184,9 +180,6 @@ func ParseEndpoint( storageShowLocationFlags.Usage = storageShowLocationUsage storageLocationPackagesFlags.Usage = storageLocationPackagesUsage - uploadFlags.Usage = uploadUsage - uploadUploadFlags.Usage = uploadUploadUsage - if err := flag.CommandLine.Parse(os.Args[1:]); err != nil { return nil, nil, err } @@ -206,8 +199,6 @@ func ParseEndpoint( svcf = package_Flags case "storage": svcf = storageFlags - case "upload": - svcf = uploadFlags default: return nil, nil, fmt.Errorf("unknown service %q", svcn) } @@ -252,6 +243,9 @@ func ParseEndpoint( case "move-status": epf = package_MoveStatusFlags + case "upload": + epf = package_UploadFlags + } case "storage": @@ -294,13 +288,6 @@ func ParseEndpoint( } - case "upload": - switch epn { - case "upload": - epf = uploadUploadFlags - - } - } } if epf == nil { @@ -351,6 +338,12 @@ func ParseEndpoint( case "move-status": endpoint = c.MoveStatus() data, err = package_c.BuildMoveStatusPayload(*package_MoveStatusIDFlag, *package_MoveStatusTokenFlag) + case "upload": + endpoint = c.Upload() + data, err = package_c.BuildUploadPayload(*package_UploadContentTypeFlag, *package_UploadTokenFlag) + if err == nil { + data, err = package_c.BuildUploadStreamPayload(data, *package_UploadStreamFlag) + } } case "storage": c := storagec.NewClient(scheme, host, doer, enc, dec, restore) @@ -392,16 +385,6 @@ func ParseEndpoint( endpoint = c.LocationPackages() data, err = storagec.BuildLocationPackagesPayload(*storageLocationPackagesUUIDFlag, *storageLocationPackagesTokenFlag) } - case "upload": - c := uploadc.NewClient(scheme, host, doer, enc, dec, restore) - switch epn { - case "upload": - endpoint = c.Upload() - data, err = uploadc.BuildUploadPayload(*uploadUploadContentTypeFlag, *uploadUploadTokenFlag) - if err == nil { - data, err = uploadc.BuildUploadStreamPayload(data, *uploadUploadStreamFlag) - } - } } } if err != nil { @@ -427,6 +410,7 @@ COMMAND: reject: Signal the package has been reviewed and rejected move: Move a package to a permanent storage location move-status: Retrieve the status of a permanent storage location move of the package + upload: Upload a package to trigger an ingest workflow Additional help: %[1]s package COMMAND --help @@ -550,6 +534,19 @@ Example: `, os.Args[0]) } +func package_UploadUsage() { + fmt.Fprintf(os.Stderr, `%[1]s [flags] package upload -content-type STRING -token STRING -stream STRING + +Upload a package to trigger an ingest workflow + -content-type STRING: + -token STRING: + -stream STRING: path to file containing the streamed request body + +Example: + %[1]s package upload --content-type "multipart/form-data; boundary=goa" --token "abc123" --stream "goa.png" +`, os.Args[0]) +} + // storageUsage displays the usage of the storage command and its subcommands. func storageUsage() { fmt.Fprintf(os.Stderr, `The storage service manages the storage of packages. @@ -737,29 +734,3 @@ Example: %[1]s storage location-packages --uuid "d1845cb6-a5ea-474a-9ab8-26f9bcd919f5" --token "abc123" `, os.Args[0]) } - -// uploadUsage displays the usage of the upload command and its subcommands. -func uploadUsage() { - fmt.Fprintf(os.Stderr, `The upload service handles file submissions to the SIPs bucket. -Usage: - %[1]s [globalflags] upload COMMAND [flags] - -COMMAND: - upload: Upload a package to trigger an ingest workflow - -Additional help: - %[1]s upload COMMAND --help -`, os.Args[0]) -} -func uploadUploadUsage() { - fmt.Fprintf(os.Stderr, `%[1]s [flags] upload upload -content-type STRING -token STRING -stream STRING - -Upload a package to trigger an ingest workflow - -content-type STRING: - -token STRING: - -stream STRING: path to file containing the streamed request body - -Example: - %[1]s upload upload --content-type "multipart/form-data; boundary=goa" --token "abc123" --stream "goa.png" -`, os.Args[0]) -} diff --git a/internal/api/gen/http/openapi.json b/internal/api/gen/http/openapi.json index 550c6be4d..9145a267d 100644 --- a/internal/api/gen/http/openapi.json +++ b/internal/api/gen/http/openapi.json @@ -1384,6 +1384,165 @@ "title": "Mediatype identifier: application/vnd.enduro.stored-package; view=default", "type": "object" }, + "PackageUploadInternalErrorResponseBody": { + "description": "Fault while processing upload. (default view)", + "example": { + "fault": false, + "id": "123abc", + "message": "parameter 'p' must be an integer", + "name": "bad_request", + "temporary": false, + "timeout": false + }, + "properties": { + "fault": { + "description": "Is the error a server-side fault?", + "example": false, + "type": "boolean" + }, + "id": { + "description": "ID is a unique identifier for this particular occurrence of the problem.", + "example": "123abc", + "type": "string" + }, + "message": { + "description": "Message is a human-readable explanation specific to this occurrence of the problem.", + "example": "parameter 'p' must be an integer", + "type": "string" + }, + "name": { + "description": "Name is the name of this class of errors.", + "example": "bad_request", + "type": "string" + }, + "temporary": { + "description": "Is the error temporary?", + "example": false, + "type": "boolean" + }, + "timeout": { + "description": "Is the error a timeout?", + "example": false, + "type": "boolean" + } + }, + "required": [ + "name", + "id", + "message", + "temporary", + "timeout", + "fault" + ], + "title": "Mediatype identifier: application/vnd.goa.error; view=default", + "type": "object" + }, + "PackageUploadInvalidMediaTypeResponseBody": { + "description": "Error returned when the Content-Type header does not define a multipart request. (default view)", + "example": { + "fault": false, + "id": "123abc", + "message": "parameter 'p' must be an integer", + "name": "bad_request", + "temporary": false, + "timeout": false + }, + "properties": { + "fault": { + "description": "Is the error a server-side fault?", + "example": false, + "type": "boolean" + }, + "id": { + "description": "ID is a unique identifier for this particular occurrence of the problem.", + "example": "123abc", + "type": "string" + }, + "message": { + "description": "Message is a human-readable explanation specific to this occurrence of the problem.", + "example": "parameter 'p' must be an integer", + "type": "string" + }, + "name": { + "description": "Name is the name of this class of errors.", + "example": "bad_request", + "type": "string" + }, + "temporary": { + "description": "Is the error temporary?", + "example": false, + "type": "boolean" + }, + "timeout": { + "description": "Is the error a timeout?", + "example": false, + "type": "boolean" + } + }, + "required": [ + "name", + "id", + "message", + "temporary", + "timeout", + "fault" + ], + "title": "Mediatype identifier: application/vnd.goa.error; view=default", + "type": "object" + }, + "PackageUploadInvalidMultipartRequestResponseBody": { + "description": "Error returned when the request body is not a valid multipart content. (default view)", + "example": { + "fault": false, + "id": "123abc", + "message": "parameter 'p' must be an integer", + "name": "bad_request", + "temporary": false, + "timeout": false + }, + "properties": { + "fault": { + "description": "Is the error a server-side fault?", + "example": false, + "type": "boolean" + }, + "id": { + "description": "ID is a unique identifier for this particular occurrence of the problem.", + "example": "123abc", + "type": "string" + }, + "message": { + "description": "Message is a human-readable explanation specific to this occurrence of the problem.", + "example": "parameter 'p' must be an integer", + "type": "string" + }, + "name": { + "description": "Name is the name of this class of errors.", + "example": "bad_request", + "type": "string" + }, + "temporary": { + "description": "Is the error temporary?", + "example": false, + "type": "boolean" + }, + "timeout": { + "description": "Is the error a timeout?", + "example": false, + "type": "boolean" + } + }, + "required": [ + "name", + "id", + "message", + "temporary", + "timeout", + "fault" + ], + "title": "Mediatype identifier: application/vnd.goa.error; view=default", + "type": "object" + }, "StorageAddLocationNotValidResponseBody": { "description": "add_location_not_valid_response_body result type (default view)", "example": { @@ -2618,165 +2777,6 @@ ], "title": "Mediatype identifier: application/vnd.goa.error; view=default", "type": "object" - }, - "UploadUploadInternalErrorResponseBody": { - "description": "Fault while processing upload. (default view)", - "example": { - "fault": false, - "id": "123abc", - "message": "parameter 'p' must be an integer", - "name": "bad_request", - "temporary": false, - "timeout": false - }, - "properties": { - "fault": { - "description": "Is the error a server-side fault?", - "example": false, - "type": "boolean" - }, - "id": { - "description": "ID is a unique identifier for this particular occurrence of the problem.", - "example": "123abc", - "type": "string" - }, - "message": { - "description": "Message is a human-readable explanation specific to this occurrence of the problem.", - "example": "parameter 'p' must be an integer", - "type": "string" - }, - "name": { - "description": "Name is the name of this class of errors.", - "example": "bad_request", - "type": "string" - }, - "temporary": { - "description": "Is the error temporary?", - "example": false, - "type": "boolean" - }, - "timeout": { - "description": "Is the error a timeout?", - "example": false, - "type": "boolean" - } - }, - "required": [ - "name", - "id", - "message", - "temporary", - "timeout", - "fault" - ], - "title": "Mediatype identifier: application/vnd.goa.error; view=default", - "type": "object" - }, - "UploadUploadInvalidMediaTypeResponseBody": { - "description": "Error returned when the Content-Type header does not define a multipart request. (default view)", - "example": { - "fault": false, - "id": "123abc", - "message": "parameter 'p' must be an integer", - "name": "bad_request", - "temporary": false, - "timeout": false - }, - "properties": { - "fault": { - "description": "Is the error a server-side fault?", - "example": false, - "type": "boolean" - }, - "id": { - "description": "ID is a unique identifier for this particular occurrence of the problem.", - "example": "123abc", - "type": "string" - }, - "message": { - "description": "Message is a human-readable explanation specific to this occurrence of the problem.", - "example": "parameter 'p' must be an integer", - "type": "string" - }, - "name": { - "description": "Name is the name of this class of errors.", - "example": "bad_request", - "type": "string" - }, - "temporary": { - "description": "Is the error temporary?", - "example": false, - "type": "boolean" - }, - "timeout": { - "description": "Is the error a timeout?", - "example": false, - "type": "boolean" - } - }, - "required": [ - "name", - "id", - "message", - "temporary", - "timeout", - "fault" - ], - "title": "Mediatype identifier: application/vnd.goa.error; view=default", - "type": "object" - }, - "UploadUploadInvalidMultipartRequestResponseBody": { - "description": "Error returned when the request body is not a valid multipart content. (default view)", - "example": { - "fault": false, - "id": "123abc", - "message": "parameter 'p' must be an integer", - "name": "bad_request", - "temporary": false, - "timeout": false - }, - "properties": { - "fault": { - "description": "Is the error a server-side fault?", - "example": false, - "type": "boolean" - }, - "id": { - "description": "ID is a unique identifier for this particular occurrence of the problem.", - "example": "123abc", - "type": "string" - }, - "message": { - "description": "Message is a human-readable explanation specific to this occurrence of the problem.", - "example": "parameter 'p' must be an integer", - "type": "string" - }, - "name": { - "description": "Name is the name of this class of errors.", - "example": "bad_request", - "type": "string" - }, - "temporary": { - "description": "Is the error temporary?", - "example": false, - "type": "boolean" - }, - "timeout": { - "description": "Is the error a timeout?", - "example": false, - "type": "boolean" - } - }, - "required": [ - "name", - "id", - "message", - "temporary", - "timeout", - "fault" - ], - "title": "Mediatype identifier: application/vnd.goa.error; view=default", - "type": "object" } }, "host": "localhost:9000", @@ -2979,6 +2979,70 @@ ] } }, + "/package/upload": { + "post": { + "description": "Upload a package to trigger an ingest workflow\n\n**Required security scopes for jwt**:\n * `package:upload`", + "operationId": "package#upload", + "parameters": [ + { + "default": "multipart/form-data; boundary=goa", + "description": "Content-Type header, must define value for multipart boundary.", + "in": "header", + "name": "Content-Type", + "pattern": "multipart/[^;]+; boundary=.+", + "required": false, + "type": "string" + }, + { + "in": "header", + "name": "Authorization", + "required": false, + "type": "string" + } + ], + "responses": { + "204": { + "description": "No Content response." + }, + "400": { + "description": "Bad Request response.", + "schema": { + "$ref": "#/definitions/PackageUploadInvalidMultipartRequestResponseBody" + } + }, + "401": { + "description": "Unauthorized response.", + "schema": { + "type": "string" + } + }, + "403": { + "description": "Forbidden response.", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error response.", + "schema": { + "$ref": "#/definitions/PackageUploadInternalErrorResponseBody" + } + } + }, + "schemes": [ + "http" + ], + "security": [ + { + "jwt_header_Authorization": [] + } + ], + "summary": "upload package", + "tags": [ + "package" + ] + } + }, "/package/{id}": { "get": { "description": "Show package by ID\n\n**Required security scopes for jwt**:\n * `package:read`", @@ -4250,70 +4314,6 @@ "swagger" ] } - }, - "/upload/upload": { - "post": { - "description": "Upload a package to trigger an ingest workflow\n\n**Required security scopes for jwt**:\n * `package:upload`", - "operationId": "upload#upload", - "parameters": [ - { - "default": "multipart/form-data; boundary=goa", - "description": "Content-Type header, must define value for multipart boundary.", - "in": "header", - "name": "Content-Type", - "pattern": "multipart/[^;]+; boundary=.+", - "required": false, - "type": "string" - }, - { - "in": "header", - "name": "Authorization", - "required": false, - "type": "string" - } - ], - "responses": { - "204": { - "description": "No Content response." - }, - "400": { - "description": "Bad Request response.", - "schema": { - "$ref": "#/definitions/UploadUploadInvalidMultipartRequestResponseBody" - } - }, - "401": { - "description": "Unauthorized response.", - "schema": { - "type": "string" - } - }, - "403": { - "description": "Forbidden response.", - "schema": { - "type": "string" - } - }, - "500": { - "description": "Internal Server Error response.", - "schema": { - "$ref": "#/definitions/UploadUploadInternalErrorResponseBody" - } - } - }, - "schemes": [ - "http" - ], - "security": [ - { - "jwt_header_Authorization": [] - } - ], - "summary": "upload upload", - "tags": [ - "upload" - ] - } } }, "produces": [ diff --git a/internal/api/gen/http/openapi.yaml b/internal/api/gen/http/openapi.yaml index b5090d9b8..8786cb912 100644 --- a/internal/api/gen/http/openapi.yaml +++ b/internal/api/gen/http/openapi.yaml @@ -465,6 +465,52 @@ paths: - http security: - jwt_header_Authorization: [] + /package/upload: + post: + tags: + - package + summary: upload package + description: |- + Upload a package to trigger an ingest workflow + + **Required security scopes for jwt**: + * `package:upload` + operationId: package#upload + parameters: + - name: Content-Type + in: header + description: Content-Type header, must define value for multipart boundary. + required: false + type: string + default: multipart/form-data; boundary=goa + pattern: multipart/[^;]+; boundary=.+ + - name: Authorization + in: header + required: false + type: string + responses: + "204": + description: No Content response. + "400": + description: Bad Request response. + schema: + $ref: '#/definitions/PackageUploadInvalidMultipartRequestResponseBody' + "401": + description: Unauthorized response. + schema: + type: string + "403": + description: Forbidden response. + schema: + type: string + "500": + description: Internal Server Error response. + schema: + $ref: '#/definitions/PackageUploadInternalErrorResponseBody' + schemes: + - http + security: + - jwt_header_Authorization: [] /storage/location: get: tags: @@ -1057,52 +1103,6 @@ paths: type: file schemes: - http - /upload/upload: - post: - tags: - - upload - summary: upload upload - description: |- - Upload a package to trigger an ingest workflow - - **Required security scopes for jwt**: - * `package:upload` - operationId: upload#upload - parameters: - - name: Content-Type - in: header - description: Content-Type header, must define value for multipart boundary. - required: false - type: string - default: multipart/form-data; boundary=goa - pattern: multipart/[^;]+; boundary=.+ - - name: Authorization - in: header - required: false - type: string - responses: - "204": - description: No Content response. - "400": - description: Bad Request response. - schema: - $ref: '#/definitions/UploadUploadInvalidMultipartRequestResponseBody' - "401": - description: Unauthorized response. - schema: - type: string - "403": - description: Forbidden response. - schema: - type: string - "500": - description: Internal Server Error response. - schema: - $ref: '#/definitions/UploadUploadInternalErrorResponseBody' - schemes: - - http - security: - - jwt_header_Authorization: [] definitions: EnduroPackagePreservationActionResponseBody: title: 'Mediatype identifier: application/vnd.enduro.package-preservation-action; view=default' @@ -2213,6 +2213,135 @@ definitions: - id - status - created_at + PackageUploadInternalErrorResponseBody: + title: 'Mediatype identifier: application/vnd.goa.error; view=default' + type: object + properties: + fault: + type: boolean + description: Is the error a server-side fault? + example: false + id: + type: string + description: ID is a unique identifier for this particular occurrence of the problem. + example: 123abc + message: + type: string + description: Message is a human-readable explanation specific to this occurrence of the problem. + example: parameter 'p' must be an integer + name: + type: string + description: Name is the name of this class of errors. + example: bad_request + temporary: + type: boolean + description: Is the error temporary? + example: false + timeout: + type: boolean + description: Is the error a timeout? + example: false + description: Fault while processing upload. (default view) + example: + fault: false + id: 123abc + message: parameter 'p' must be an integer + name: bad_request + temporary: false + timeout: false + required: + - name + - id + - message + - temporary + - timeout + - fault + PackageUploadInvalidMediaTypeResponseBody: + title: 'Mediatype identifier: application/vnd.goa.error; view=default' + type: object + properties: + fault: + type: boolean + description: Is the error a server-side fault? + example: false + id: + type: string + description: ID is a unique identifier for this particular occurrence of the problem. + example: 123abc + message: + type: string + description: Message is a human-readable explanation specific to this occurrence of the problem. + example: parameter 'p' must be an integer + name: + type: string + description: Name is the name of this class of errors. + example: bad_request + temporary: + type: boolean + description: Is the error temporary? + example: false + timeout: + type: boolean + description: Is the error a timeout? + example: false + description: Error returned when the Content-Type header does not define a multipart request. (default view) + example: + fault: false + id: 123abc + message: parameter 'p' must be an integer + name: bad_request + temporary: false + timeout: false + required: + - name + - id + - message + - temporary + - timeout + - fault + PackageUploadInvalidMultipartRequestResponseBody: + title: 'Mediatype identifier: application/vnd.goa.error; view=default' + type: object + properties: + fault: + type: boolean + description: Is the error a server-side fault? + example: false + id: + type: string + description: ID is a unique identifier for this particular occurrence of the problem. + example: 123abc + message: + type: string + description: Message is a human-readable explanation specific to this occurrence of the problem. + example: parameter 'p' must be an integer + name: + type: string + description: Name is the name of this class of errors. + example: bad_request + temporary: + type: boolean + description: Is the error temporary? + example: false + timeout: + type: boolean + description: Is the error a timeout? + example: false + description: Error returned when the request body is not a valid multipart content. (default view) + example: + fault: false + id: 123abc + message: parameter 'p' must be an integer + name: bad_request + temporary: false + timeout: false + required: + - name + - id + - message + - temporary + - timeout + - fault StorageAddLocationNotValidResponseBody: title: 'Mediatype identifier: application/vnd.goa.error; view=default' type: object @@ -3194,135 +3323,6 @@ definitions: - temporary - timeout - fault - UploadUploadInternalErrorResponseBody: - title: 'Mediatype identifier: application/vnd.goa.error; view=default' - type: object - properties: - fault: - type: boolean - description: Is the error a server-side fault? - example: false - id: - type: string - description: ID is a unique identifier for this particular occurrence of the problem. - example: 123abc - message: - type: string - description: Message is a human-readable explanation specific to this occurrence of the problem. - example: parameter 'p' must be an integer - name: - type: string - description: Name is the name of this class of errors. - example: bad_request - temporary: - type: boolean - description: Is the error temporary? - example: false - timeout: - type: boolean - description: Is the error a timeout? - example: false - description: Fault while processing upload. (default view) - example: - fault: false - id: 123abc - message: parameter 'p' must be an integer - name: bad_request - temporary: false - timeout: false - required: - - name - - id - - message - - temporary - - timeout - - fault - UploadUploadInvalidMediaTypeResponseBody: - title: 'Mediatype identifier: application/vnd.goa.error; view=default' - type: object - properties: - fault: - type: boolean - description: Is the error a server-side fault? - example: false - id: - type: string - description: ID is a unique identifier for this particular occurrence of the problem. - example: 123abc - message: - type: string - description: Message is a human-readable explanation specific to this occurrence of the problem. - example: parameter 'p' must be an integer - name: - type: string - description: Name is the name of this class of errors. - example: bad_request - temporary: - type: boolean - description: Is the error temporary? - example: false - timeout: - type: boolean - description: Is the error a timeout? - example: false - description: Error returned when the Content-Type header does not define a multipart request. (default view) - example: - fault: false - id: 123abc - message: parameter 'p' must be an integer - name: bad_request - temporary: false - timeout: false - required: - - name - - id - - message - - temporary - - timeout - - fault - UploadUploadInvalidMultipartRequestResponseBody: - title: 'Mediatype identifier: application/vnd.goa.error; view=default' - type: object - properties: - fault: - type: boolean - description: Is the error a server-side fault? - example: false - id: - type: string - description: ID is a unique identifier for this particular occurrence of the problem. - example: 123abc - message: - type: string - description: Message is a human-readable explanation specific to this occurrence of the problem. - example: parameter 'p' must be an integer - name: - type: string - description: Name is the name of this class of errors. - example: bad_request - temporary: - type: boolean - description: Is the error temporary? - example: false - timeout: - type: boolean - description: Is the error a timeout? - example: false - description: Error returned when the request body is not a valid multipart content. (default view) - example: - fault: false - id: 123abc - message: parameter 'p' must be an integer - name: bad_request - temporary: false - timeout: false - required: - - name - - id - - message - - temporary - - timeout - - fault securityDefinitions: jwt_header_Authorization: type: apiKey diff --git a/internal/api/gen/http/openapi3.json b/internal/api/gen/http/openapi3.json index 454cf6c18..921e6ae30 100644 --- a/internal/api/gen/http/openapi3.json +++ b/internal/api/gen/http/openapi3.json @@ -1650,6 +1650,88 @@ ] } }, + "/package/upload": { + "post": { + "description": "Upload a package to trigger an ingest workflow", + "operationId": "package#upload", + "parameters": [ + { + "allowEmptyValue": true, + "description": "Content-Type header, must define value for multipart boundary.", + "example": "multipart/form-data; boundary=goa", + "in": "header", + "name": "Content-Type", + "schema": { + "default": "multipart/form-data; boundary=goa", + "description": "Content-Type header, must define value for multipart boundary.", + "example": "multipart/form-data; boundary=goa", + "pattern": "multipart/[^;]+; boundary=.+", + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "No Content response." + }, + "400": { + "content": { + "application/vnd.goa.error": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "invalid_multipart_request: Error returned when the request body is not a valid multipart content." + }, + "401": { + "content": { + "application/json": { + "example": "abc123", + "schema": { + "example": "abc123", + "type": "string" + } + } + }, + "description": "unauthorized: Unauthorized response." + }, + "403": { + "content": { + "application/json": { + "example": "abc123", + "schema": { + "example": "abc123", + "type": "string" + } + } + }, + "description": "forbidden: Forbidden response." + }, + "500": { + "content": { + "application/vnd.goa.error": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "internal_error: Fault while processing upload." + } + }, + "security": [ + { + "jwt_header_Authorization": [ + "package:upload" + ] + } + ], + "summary": "upload package", + "tags": [ + "package" + ] + } + }, "/package/{id}": { "get": { "description": "Show package by ID", @@ -3371,88 +3453,6 @@ "swagger" ] } - }, - "/upload/upload": { - "post": { - "description": "Upload a package to trigger an ingest workflow", - "operationId": "upload#upload", - "parameters": [ - { - "allowEmptyValue": true, - "description": "Content-Type header, must define value for multipart boundary.", - "example": "multipart/form-data; boundary=goa", - "in": "header", - "name": "Content-Type", - "schema": { - "default": "multipart/form-data; boundary=goa", - "description": "Content-Type header, must define value for multipart boundary.", - "example": "multipart/form-data; boundary=goa", - "pattern": "multipart/[^;]+; boundary=.+", - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "No Content response." - }, - "400": { - "content": { - "application/vnd.goa.error": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - }, - "description": "invalid_multipart_request: Error returned when the request body is not a valid multipart content." - }, - "401": { - "content": { - "application/json": { - "example": "abc123", - "schema": { - "example": "abc123", - "type": "string" - } - } - }, - "description": "unauthorized: Unauthorized response." - }, - "403": { - "content": { - "application/json": { - "example": "abc123", - "schema": { - "example": "abc123", - "type": "string" - } - } - }, - "description": "forbidden: Forbidden response." - }, - "500": { - "content": { - "application/vnd.goa.error": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - }, - "description": "internal_error: Fault while processing upload." - } - }, - "security": [ - { - "jwt_header_Authorization": [ - "package:upload" - ] - } - ], - "summary": "upload upload", - "tags": [ - "upload" - ] - } } }, "security": [ @@ -3477,10 +3477,6 @@ { "description": "The swagger service serves the API swagger definition.", "name": "swagger" - }, - { - "description": "The upload service handles file submissions to the SIPs bucket.", - "name": "upload" } ] } diff --git a/internal/api/gen/http/openapi3.yaml b/internal/api/gen/http/openapi3.yaml index 2794eeb36..4b512a3d5 100644 --- a/internal/api/gen/http/openapi3.yaml +++ b/internal/api/gen/http/openapi3.yaml @@ -613,6 +613,59 @@ paths: - package:read - package:review - package:upload + /package/upload: + post: + tags: + - package + summary: upload package + description: Upload a package to trigger an ingest workflow + operationId: package#upload + parameters: + - name: Content-Type + in: header + description: Content-Type header, must define value for multipart boundary. + allowEmptyValue: true + schema: + type: string + description: Content-Type header, must define value for multipart boundary. + default: multipart/form-data; boundary=goa + example: multipart/form-data; boundary=goa + pattern: multipart/[^;]+; boundary=.+ + example: multipart/form-data; boundary=goa + responses: + "204": + description: No Content response. + "400": + description: 'invalid_multipart_request: Error returned when the request body is not a valid multipart content.' + content: + application/vnd.goa.error: + schema: + $ref: '#/components/schemas/Error' + "401": + description: 'unauthorized: Unauthorized response.' + content: + application/json: + schema: + type: string + example: abc123 + example: abc123 + "403": + description: 'forbidden: Forbidden response.' + content: + application/json: + schema: + type: string + example: abc123 + example: abc123 + "500": + description: 'internal_error: Fault while processing upload.' + content: + application/vnd.goa.error: + schema: + $ref: '#/components/schemas/Error' + security: + - jwt_header_Authorization: + - package:upload /storage/location: get: tags: @@ -1342,59 +1395,6 @@ paths: description: File downloaded security: - jwt_header_: [] - /upload/upload: - post: - tags: - - upload - summary: upload upload - description: Upload a package to trigger an ingest workflow - operationId: upload#upload - parameters: - - name: Content-Type - in: header - description: Content-Type header, must define value for multipart boundary. - allowEmptyValue: true - schema: - type: string - description: Content-Type header, must define value for multipart boundary. - default: multipart/form-data; boundary=goa - example: multipart/form-data; boundary=goa - pattern: multipart/[^;]+; boundary=.+ - example: multipart/form-data; boundary=goa - responses: - "204": - description: No Content response. - "400": - description: 'invalid_multipart_request: Error returned when the request body is not a valid multipart content.' - content: - application/vnd.goa.error: - schema: - $ref: '#/components/schemas/Error' - "401": - description: 'unauthorized: Unauthorized response.' - content: - application/json: - schema: - type: string - example: abc123 - example: abc123 - "403": - description: 'forbidden: Forbidden response.' - content: - application/json: - schema: - type: string - example: abc123 - example: abc123 - "500": - description: 'internal_error: Fault while processing upload.' - content: - application/vnd.goa.error: - schema: - $ref: '#/components/schemas/Error' - security: - - jwt_header_Authorization: - - package:upload components: schemas: AddLocationRequestBody: @@ -2456,7 +2456,5 @@ tags: description: The storage service manages the storage of packages. - name: swagger description: The swagger service serves the API swagger definition. - - name: upload - description: The upload service handles file submissions to the SIPs bucket. security: - jwt_header_: [] diff --git a/internal/api/gen/http/package_/client/cli.go b/internal/api/gen/http/package_/client/cli.go index ff5b49547..880b3420e 100644 --- a/internal/api/gen/http/package_/client/cli.go +++ b/internal/api/gen/http/package_/client/cli.go @@ -307,3 +307,30 @@ func BuildMoveStatusPayload(package_MoveStatusID string, package_MoveStatusToken return v, nil } + +// BuildUploadPayload builds the payload for the package upload endpoint from +// CLI flags. +func BuildUploadPayload(package_UploadContentType string, package_UploadToken string) (*package_.UploadPayload, error) { + var err error + var contentType string + { + if package_UploadContentType != "" { + contentType = package_UploadContentType + err = goa.MergeErrors(err, goa.ValidatePattern("content_type", contentType, "multipart/[^;]+; boundary=.+")) + if err != nil { + return nil, err + } + } + } + var token *string + { + if package_UploadToken != "" { + token = &package_UploadToken + } + } + v := &package_.UploadPayload{} + v.ContentType = contentType + v.Token = token + + return v, nil +} diff --git a/internal/api/gen/http/package_/client/client.go b/internal/api/gen/http/package_/client/client.go index 8aa18ddc1..553cf63a9 100644 --- a/internal/api/gen/http/package_/client/client.go +++ b/internal/api/gen/http/package_/client/client.go @@ -52,6 +52,9 @@ type Client struct { // endpoint. MoveStatusDoer goahttp.Doer + // Upload Doer is the HTTP client used to make requests to the upload endpoint. + UploadDoer goahttp.Doer + // CORS Doer is the HTTP client used to make requests to the endpoint. CORSDoer goahttp.Doer @@ -91,6 +94,7 @@ func NewClient( RejectDoer: doer, MoveDoer: doer, MoveStatusDoer: doer, + UploadDoer: doer, CORSDoer: doer, RestoreResponseBody: restoreBody, scheme: scheme, @@ -335,3 +339,27 @@ func (c *Client) MoveStatus() goa.Endpoint { return decodeResponse(resp) } } + +// Upload returns an endpoint that makes HTTP requests to the package service +// upload server. +func (c *Client) Upload() goa.Endpoint { + var ( + encodeRequest = EncodeUploadRequest(c.encoder) + decodeResponse = DecodeUploadResponse(c.decoder, c.RestoreResponseBody) + ) + return func(ctx context.Context, v any) (any, error) { + req, err := c.BuildUploadRequest(ctx, v) + if err != nil { + return nil, err + } + err = encodeRequest(req, v) + if err != nil { + return nil, err + } + resp, err := c.UploadDoer.Do(req) + if err != nil { + return nil, goahttp.ErrRequestError("package", "upload", err) + } + return decodeResponse(resp) + } +} diff --git a/internal/api/gen/http/package_/client/encode_decode.go b/internal/api/gen/http/package_/client/encode_decode.go index f5065a8d1..a0701218c 100644 --- a/internal/api/gen/http/package_/client/encode_decode.go +++ b/internal/api/gen/http/package_/client/encode_decode.go @@ -14,6 +14,7 @@ import ( "io" "net/http" "net/url" + "os" "strings" package_ "github.com/artefactual-sdps/enduro/internal/api/gen/package_" @@ -1218,6 +1219,170 @@ func DecodeMoveStatusResponse(decoder func(*http.Response) goahttp.Decoder, rest } } +// BuildUploadRequest instantiates a HTTP request object with method and path +// set to call the "package" service "upload" endpoint +func (c *Client) BuildUploadRequest(ctx context.Context, v any) (*http.Request, error) { + var ( + body io.Reader + ) + rd, ok := v.(*package_.UploadRequestData) + if !ok { + return nil, goahttp.ErrInvalidType("package", "upload", "package_.UploadRequestData", v) + } + body = rd.Body + u := &url.URL{Scheme: c.scheme, Host: c.host, Path: UploadPackagePath()} + req, err := http.NewRequest("POST", u.String(), body) + if err != nil { + return nil, goahttp.ErrInvalidURL("package", "upload", u.String(), err) + } + if ctx != nil { + req = req.WithContext(ctx) + } + + return req, nil +} + +// EncodeUploadRequest returns an encoder for requests sent to the package +// upload server. +func EncodeUploadRequest(encoder func(*http.Request) goahttp.Encoder) func(*http.Request, any) error { + return func(req *http.Request, v any) error { + data, ok := v.(*package_.UploadRequestData) + if !ok { + return goahttp.ErrInvalidType("package", "upload", "*package_.UploadRequestData", v) + } + p := data.Payload + { + head := p.ContentType + req.Header.Set("Content-Type", head) + } + if p.Token != nil { + head := *p.Token + if !strings.Contains(head, " ") { + req.Header.Set("Authorization", "Bearer "+head) + } else { + req.Header.Set("Authorization", head) + } + } + return nil + } +} + +// DecodeUploadResponse returns a decoder for responses returned by the package +// upload endpoint. restoreBody controls whether the response body should be +// restored after having been read. +// DecodeUploadResponse may return the following errors: +// - "invalid_media_type" (type *goa.ServiceError): http.StatusBadRequest +// - "invalid_multipart_request" (type *goa.ServiceError): http.StatusBadRequest +// - "internal_error" (type *goa.ServiceError): http.StatusInternalServerError +// - "forbidden" (type package_.Forbidden): http.StatusForbidden +// - "unauthorized" (type package_.Unauthorized): http.StatusUnauthorized +// - error: internal error +func DecodeUploadResponse(decoder func(*http.Response) goahttp.Decoder, restoreBody bool) func(*http.Response) (any, error) { + return func(resp *http.Response) (any, error) { + if restoreBody { + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + resp.Body = io.NopCloser(bytes.NewBuffer(b)) + defer func() { + resp.Body = io.NopCloser(bytes.NewBuffer(b)) + }() + } else { + defer resp.Body.Close() + } + switch resp.StatusCode { + case http.StatusNoContent: + return nil, nil + case http.StatusBadRequest: + en := resp.Header.Get("goa-error") + switch en { + case "invalid_media_type": + var ( + body UploadInvalidMediaTypeResponseBody + err error + ) + err = decoder(resp).Decode(&body) + if err != nil { + return nil, goahttp.ErrDecodingError("package", "upload", err) + } + err = ValidateUploadInvalidMediaTypeResponseBody(&body) + if err != nil { + return nil, goahttp.ErrValidationError("package", "upload", err) + } + return nil, NewUploadInvalidMediaType(&body) + case "invalid_multipart_request": + var ( + body UploadInvalidMultipartRequestResponseBody + err error + ) + err = decoder(resp).Decode(&body) + if err != nil { + return nil, goahttp.ErrDecodingError("package", "upload", err) + } + err = ValidateUploadInvalidMultipartRequestResponseBody(&body) + if err != nil { + return nil, goahttp.ErrValidationError("package", "upload", err) + } + return nil, NewUploadInvalidMultipartRequest(&body) + default: + body, _ := io.ReadAll(resp.Body) + return nil, goahttp.ErrInvalidResponse("package", "upload", resp.StatusCode, string(body)) + } + case http.StatusInternalServerError: + var ( + body UploadInternalErrorResponseBody + err error + ) + err = decoder(resp).Decode(&body) + if err != nil { + return nil, goahttp.ErrDecodingError("package", "upload", err) + } + err = ValidateUploadInternalErrorResponseBody(&body) + if err != nil { + return nil, goahttp.ErrValidationError("package", "upload", err) + } + return nil, NewUploadInternalError(&body) + case http.StatusForbidden: + var ( + body string + err error + ) + err = decoder(resp).Decode(&body) + if err != nil { + return nil, goahttp.ErrDecodingError("package", "upload", err) + } + return nil, NewUploadForbidden(body) + case http.StatusUnauthorized: + var ( + body string + err error + ) + err = decoder(resp).Decode(&body) + if err != nil { + return nil, goahttp.ErrDecodingError("package", "upload", err) + } + return nil, NewUploadUnauthorized(body) + default: + body, _ := io.ReadAll(resp.Body) + return nil, goahttp.ErrInvalidResponse("package", "upload", resp.StatusCode, string(body)) + } + } +} + +// // BuildUploadStreamPayload creates a streaming endpoint request payload from +// the method payload and the path to the file to be streamed +func BuildUploadStreamPayload(payload any, fpath string) (*package_.UploadRequestData, error) { + f, err := os.Open(fpath) + if err != nil { + return nil, err + } + return &package_.UploadRequestData{ + Payload: payload.(*package_.UploadPayload), + Body: f, + }, nil +} + // unmarshalEnduroStoredPackageResponseBodyToPackageEnduroStoredPackage builds // a value of type *package_.EnduroStoredPackage from a value of type // *EnduroStoredPackageResponseBody. diff --git a/internal/api/gen/http/package_/client/paths.go b/internal/api/gen/http/package_/client/paths.go index 896dd209a..77d9d5698 100644 --- a/internal/api/gen/http/package_/client/paths.go +++ b/internal/api/gen/http/package_/client/paths.go @@ -56,3 +56,8 @@ func MovePackagePath(id uint) string { func MoveStatusPackagePath(id uint) string { return fmt.Sprintf("/package/%v/move", id) } + +// UploadPackagePath returns the URL path to the package service upload HTTP endpoint. +func UploadPackagePath() string { + return "/package/upload" +} diff --git a/internal/api/gen/http/package_/client/types.go b/internal/api/gen/http/package_/client/types.go index e80754ae0..5c7f88dce 100644 --- a/internal/api/gen/http/package_/client/types.go +++ b/internal/api/gen/http/package_/client/types.go @@ -329,6 +329,61 @@ type MoveStatusNotFoundResponseBody struct { ID *uint `form:"id,omitempty" json:"id,omitempty" xml:"id,omitempty"` } +// UploadInvalidMediaTypeResponseBody is the type of the "package" service +// "upload" endpoint HTTP response body for the "invalid_media_type" error. +type UploadInvalidMediaTypeResponseBody struct { + // Name is the name of this class of errors. + Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` + // ID is a unique identifier for this particular occurrence of the problem. + ID *string `form:"id,omitempty" json:"id,omitempty" xml:"id,omitempty"` + // Message is a human-readable explanation specific to this occurrence of the + // problem. + Message *string `form:"message,omitempty" json:"message,omitempty" xml:"message,omitempty"` + // Is the error temporary? + Temporary *bool `form:"temporary,omitempty" json:"temporary,omitempty" xml:"temporary,omitempty"` + // Is the error a timeout? + Timeout *bool `form:"timeout,omitempty" json:"timeout,omitempty" xml:"timeout,omitempty"` + // Is the error a server-side fault? + Fault *bool `form:"fault,omitempty" json:"fault,omitempty" xml:"fault,omitempty"` +} + +// UploadInvalidMultipartRequestResponseBody is the type of the "package" +// service "upload" endpoint HTTP response body for the +// "invalid_multipart_request" error. +type UploadInvalidMultipartRequestResponseBody struct { + // Name is the name of this class of errors. + Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` + // ID is a unique identifier for this particular occurrence of the problem. + ID *string `form:"id,omitempty" json:"id,omitempty" xml:"id,omitempty"` + // Message is a human-readable explanation specific to this occurrence of the + // problem. + Message *string `form:"message,omitempty" json:"message,omitempty" xml:"message,omitempty"` + // Is the error temporary? + Temporary *bool `form:"temporary,omitempty" json:"temporary,omitempty" xml:"temporary,omitempty"` + // Is the error a timeout? + Timeout *bool `form:"timeout,omitempty" json:"timeout,omitempty" xml:"timeout,omitempty"` + // Is the error a server-side fault? + Fault *bool `form:"fault,omitempty" json:"fault,omitempty" xml:"fault,omitempty"` +} + +// UploadInternalErrorResponseBody is the type of the "package" service +// "upload" endpoint HTTP response body for the "internal_error" error. +type UploadInternalErrorResponseBody struct { + // Name is the name of this class of errors. + Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` + // ID is a unique identifier for this particular occurrence of the problem. + ID *string `form:"id,omitempty" json:"id,omitempty" xml:"id,omitempty"` + // Message is a human-readable explanation specific to this occurrence of the + // problem. + Message *string `form:"message,omitempty" json:"message,omitempty" xml:"message,omitempty"` + // Is the error temporary? + Temporary *bool `form:"temporary,omitempty" json:"temporary,omitempty" xml:"temporary,omitempty"` + // Is the error a timeout? + Timeout *bool `form:"timeout,omitempty" json:"timeout,omitempty" xml:"timeout,omitempty"` + // Is the error a server-side fault? + Fault *bool `form:"fault,omitempty" json:"fault,omitempty" xml:"fault,omitempty"` +} + // EnduroStoredPackageCollectionResponseBody is used to define fields on // response body types. type EnduroStoredPackageCollectionResponseBody []*EnduroStoredPackageResponseBody @@ -873,6 +928,66 @@ func NewMoveStatusUnauthorized(body string) package_.Unauthorized { return v } +// NewUploadInvalidMediaType builds a package service upload endpoint +// invalid_media_type error. +func NewUploadInvalidMediaType(body *UploadInvalidMediaTypeResponseBody) *goa.ServiceError { + v := &goa.ServiceError{ + Name: *body.Name, + ID: *body.ID, + Message: *body.Message, + Temporary: *body.Temporary, + Timeout: *body.Timeout, + Fault: *body.Fault, + } + + return v +} + +// NewUploadInvalidMultipartRequest builds a package service upload endpoint +// invalid_multipart_request error. +func NewUploadInvalidMultipartRequest(body *UploadInvalidMultipartRequestResponseBody) *goa.ServiceError { + v := &goa.ServiceError{ + Name: *body.Name, + ID: *body.ID, + Message: *body.Message, + Temporary: *body.Temporary, + Timeout: *body.Timeout, + Fault: *body.Fault, + } + + return v +} + +// NewUploadInternalError builds a package service upload endpoint +// internal_error error. +func NewUploadInternalError(body *UploadInternalErrorResponseBody) *goa.ServiceError { + v := &goa.ServiceError{ + Name: *body.Name, + ID: *body.ID, + Message: *body.Message, + Temporary: *body.Temporary, + Timeout: *body.Timeout, + Fault: *body.Fault, + } + + return v +} + +// NewUploadForbidden builds a package service upload endpoint forbidden error. +func NewUploadForbidden(body string) package_.Forbidden { + v := package_.Forbidden(body) + + return v +} + +// NewUploadUnauthorized builds a package service upload endpoint unauthorized +// error. +func NewUploadUnauthorized(body string) package_.Unauthorized { + v := package_.Unauthorized(body) + + return v +} + // ValidateMonitorResponseBody runs the validations defined on // MonitorResponseBody func ValidateMonitorResponseBody(body *MonitorResponseBody) (err error) { @@ -1226,6 +1341,78 @@ func ValidateMoveStatusNotFoundResponseBody(body *MoveStatusNotFoundResponseBody return } +// ValidateUploadInvalidMediaTypeResponseBody runs the validations defined on +// upload_invalid_media_type_response_body +func ValidateUploadInvalidMediaTypeResponseBody(body *UploadInvalidMediaTypeResponseBody) (err error) { + if body.Name == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("name", "body")) + } + if body.ID == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("id", "body")) + } + if body.Message == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("message", "body")) + } + if body.Temporary == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("temporary", "body")) + } + if body.Timeout == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("timeout", "body")) + } + if body.Fault == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("fault", "body")) + } + return +} + +// ValidateUploadInvalidMultipartRequestResponseBody runs the validations +// defined on upload_invalid_multipart_request_response_body +func ValidateUploadInvalidMultipartRequestResponseBody(body *UploadInvalidMultipartRequestResponseBody) (err error) { + if body.Name == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("name", "body")) + } + if body.ID == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("id", "body")) + } + if body.Message == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("message", "body")) + } + if body.Temporary == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("temporary", "body")) + } + if body.Timeout == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("timeout", "body")) + } + if body.Fault == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("fault", "body")) + } + return +} + +// ValidateUploadInternalErrorResponseBody runs the validations defined on +// upload_internal_error_response_body +func ValidateUploadInternalErrorResponseBody(body *UploadInternalErrorResponseBody) (err error) { + if body.Name == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("name", "body")) + } + if body.ID == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("id", "body")) + } + if body.Message == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("message", "body")) + } + if body.Temporary == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("temporary", "body")) + } + if body.Timeout == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("timeout", "body")) + } + if body.Fault == nil { + err = goa.MergeErrors(err, goa.MissingFieldError("fault", "body")) + } + return +} + // ValidateEnduroStoredPackageCollectionResponseBody runs the validations // defined on EnduroStored-PackageCollectionResponseBody func ValidateEnduroStoredPackageCollectionResponseBody(body EnduroStoredPackageCollectionResponseBody) (err error) { diff --git a/internal/api/gen/http/package_/server/encode_decode.go b/internal/api/gen/http/package_/server/encode_decode.go index bed5ec7c7..7d1cce477 100644 --- a/internal/api/gen/http/package_/server/encode_decode.go +++ b/internal/api/gen/http/package_/server/encode_decode.go @@ -1003,6 +1003,122 @@ func EncodeMoveStatusError(encoder func(context.Context, http.ResponseWriter) go } } +// EncodeUploadResponse returns an encoder for responses returned by the +// package upload endpoint. +func EncodeUploadResponse(encoder func(context.Context, http.ResponseWriter) goahttp.Encoder) func(context.Context, http.ResponseWriter, any) error { + return func(ctx context.Context, w http.ResponseWriter, v any) error { + w.WriteHeader(http.StatusNoContent) + return nil + } +} + +// DecodeUploadRequest returns a decoder for requests sent to the package +// upload endpoint. +func DecodeUploadRequest(mux goahttp.Muxer, decoder func(*http.Request) goahttp.Decoder) func(*http.Request) (any, error) { + return func(r *http.Request) (any, error) { + var ( + contentType string + token *string + err error + ) + contentTypeRaw := r.Header.Get("Content-Type") + if contentTypeRaw != "" { + contentType = contentTypeRaw + } else { + contentType = "multipart/form-data; boundary=goa" + } + err = goa.MergeErrors(err, goa.ValidatePattern("content_type", contentType, "multipart/[^;]+; boundary=.+")) + tokenRaw := r.Header.Get("Authorization") + if tokenRaw != "" { + token = &tokenRaw + } + if err != nil { + return nil, err + } + payload := NewUploadPayload(contentType, token) + if payload.Token != nil { + if strings.Contains(*payload.Token, " ") { + // Remove authorization scheme prefix (e.g. "Bearer") + cred := strings.SplitN(*payload.Token, " ", 2)[1] + payload.Token = &cred + } + } + + return payload, nil + } +} + +// EncodeUploadError returns an encoder for errors returned by the upload +// package endpoint. +func EncodeUploadError(encoder func(context.Context, http.ResponseWriter) goahttp.Encoder, formatter func(ctx context.Context, err error) goahttp.Statuser) func(context.Context, http.ResponseWriter, error) error { + encodeError := goahttp.ErrorEncoder(encoder, formatter) + return func(ctx context.Context, w http.ResponseWriter, v error) error { + var en goa.GoaErrorNamer + if !errors.As(v, &en) { + return encodeError(ctx, w, v) + } + switch en.GoaErrorName() { + case "invalid_media_type": + var res *goa.ServiceError + errors.As(v, &res) + enc := encoder(ctx, w) + var body any + if formatter != nil { + body = formatter(ctx, res) + } else { + body = NewUploadInvalidMediaTypeResponseBody(res) + } + w.Header().Set("goa-error", res.GoaErrorName()) + w.WriteHeader(http.StatusBadRequest) + return enc.Encode(body) + case "invalid_multipart_request": + var res *goa.ServiceError + errors.As(v, &res) + enc := encoder(ctx, w) + var body any + if formatter != nil { + body = formatter(ctx, res) + } else { + body = NewUploadInvalidMultipartRequestResponseBody(res) + } + w.Header().Set("goa-error", res.GoaErrorName()) + w.WriteHeader(http.StatusBadRequest) + return enc.Encode(body) + case "internal_error": + var res *goa.ServiceError + errors.As(v, &res) + enc := encoder(ctx, w) + var body any + if formatter != nil { + body = formatter(ctx, res) + } else { + body = NewUploadInternalErrorResponseBody(res) + } + w.Header().Set("goa-error", res.GoaErrorName()) + w.WriteHeader(http.StatusInternalServerError) + return enc.Encode(body) + case "forbidden": + var res package_.Forbidden + errors.As(v, &res) + enc := encoder(ctx, w) + body := res + w.Header().Set("goa-error", res.GoaErrorName()) + w.WriteHeader(http.StatusForbidden) + return enc.Encode(body) + case "unauthorized": + var res package_.Unauthorized + errors.As(v, &res) + enc := encoder(ctx, w) + body := res + w.Header().Set("goa-error", res.GoaErrorName()) + w.WriteHeader(http.StatusUnauthorized) + return enc.Encode(body) + default: + return encodeError(ctx, w, v) + } + } +} + // marshalPackageEnduroStoredPackageToEnduroStoredPackageResponseBody builds a // value of type *EnduroStoredPackageResponseBody from a value of type // *package_.EnduroStoredPackage. diff --git a/internal/api/gen/http/package_/server/paths.go b/internal/api/gen/http/package_/server/paths.go index f190ee8bb..361545e8f 100644 --- a/internal/api/gen/http/package_/server/paths.go +++ b/internal/api/gen/http/package_/server/paths.go @@ -56,3 +56,8 @@ func MovePackagePath(id uint) string { func MoveStatusPackagePath(id uint) string { return fmt.Sprintf("/package/%v/move", id) } + +// UploadPackagePath returns the URL path to the package service upload HTTP endpoint. +func UploadPackagePath() string { + return "/package/upload" +} diff --git a/internal/api/gen/http/package_/server/server.go b/internal/api/gen/http/package_/server/server.go index 5840de3fa..36ecca4a6 100644 --- a/internal/api/gen/http/package_/server/server.go +++ b/internal/api/gen/http/package_/server/server.go @@ -32,6 +32,7 @@ type Server struct { Reject http.Handler Move http.Handler MoveStatus http.Handler + Upload http.Handler CORS http.Handler } @@ -76,6 +77,7 @@ func New( {"Reject", "POST", "/package/{id}/reject"}, {"Move", "POST", "/package/{id}/move"}, {"MoveStatus", "GET", "/package/{id}/move"}, + {"Upload", "POST", "/package/upload"}, {"CORS", "OPTIONS", "/package/monitor"}, {"CORS", "OPTIONS", "/package"}, {"CORS", "OPTIONS", "/package/{id}"}, @@ -83,6 +85,7 @@ func New( {"CORS", "OPTIONS", "/package/{id}/confirm"}, {"CORS", "OPTIONS", "/package/{id}/reject"}, {"CORS", "OPTIONS", "/package/{id}/move"}, + {"CORS", "OPTIONS", "/package/upload"}, }, MonitorRequest: NewMonitorRequestHandler(e.MonitorRequest, mux, decoder, encoder, errhandler, formatter), Monitor: NewMonitorHandler(e.Monitor, mux, decoder, encoder, errhandler, formatter, upgrader, configurer.MonitorFn), @@ -93,6 +96,7 @@ func New( Reject: NewRejectHandler(e.Reject, mux, decoder, encoder, errhandler, formatter), Move: NewMoveHandler(e.Move, mux, decoder, encoder, errhandler, formatter), MoveStatus: NewMoveStatusHandler(e.MoveStatus, mux, decoder, encoder, errhandler, formatter), + Upload: NewUploadHandler(e.Upload, mux, decoder, encoder, errhandler, formatter), CORS: NewCORSHandler(), } } @@ -111,6 +115,7 @@ func (s *Server) Use(m func(http.Handler) http.Handler) { s.Reject = m(s.Reject) s.Move = m(s.Move) s.MoveStatus = m(s.MoveStatus) + s.Upload = m(s.Upload) s.CORS = m(s.CORS) } @@ -128,6 +133,7 @@ func Mount(mux goahttp.Muxer, h *Server) { MountRejectHandler(mux, h.Reject) MountMoveHandler(mux, h.Move) MountMoveStatusHandler(mux, h.MoveStatus) + MountUploadHandler(mux, h.Upload) MountCORSHandler(mux, h.CORS) } @@ -610,6 +616,58 @@ func NewMoveStatusHandler( }) } +// MountUploadHandler configures the mux to serve the "package" service +// "upload" endpoint. +func MountUploadHandler(mux goahttp.Muxer, h http.Handler) { + f, ok := HandlePackageOrigin(h).(http.HandlerFunc) + if !ok { + f = func(w http.ResponseWriter, r *http.Request) { + h.ServeHTTP(w, r) + } + } + mux.Handle("POST", "/package/upload", otelhttp.WithRouteTag("/package/upload", f).ServeHTTP) +} + +// NewUploadHandler creates a HTTP handler which loads the HTTP request and +// calls the "package" service "upload" endpoint. +func NewUploadHandler( + endpoint goa.Endpoint, + mux goahttp.Muxer, + decoder func(*http.Request) goahttp.Decoder, + encoder func(context.Context, http.ResponseWriter) goahttp.Encoder, + errhandler func(context.Context, http.ResponseWriter, error), + formatter func(ctx context.Context, err error) goahttp.Statuser, +) http.Handler { + var ( + decodeRequest = DecodeUploadRequest(mux, decoder) + encodeResponse = EncodeUploadResponse(encoder) + encodeError = EncodeUploadError(encoder, formatter) + ) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := context.WithValue(r.Context(), goahttp.AcceptTypeKey, r.Header.Get("Accept")) + ctx = context.WithValue(ctx, goa.MethodKey, "upload") + ctx = context.WithValue(ctx, goa.ServiceKey, "package") + payload, err := decodeRequest(r) + if err != nil { + if err := encodeError(ctx, w, err); err != nil { + errhandler(ctx, w, err) + } + return + } + data := &package_.UploadRequestData{Payload: payload.(*package_.UploadPayload), Body: r.Body} + res, err := endpoint(ctx, data) + if err != nil { + if err := encodeError(ctx, w, err); err != nil { + errhandler(ctx, w, err) + } + return + } + if err := encodeResponse(ctx, w, res); err != nil { + errhandler(ctx, w, err) + } + }) +} + // MountCORSHandler configures the mux to serve the CORS endpoints for the // service package. func MountCORSHandler(mux goahttp.Muxer, h http.Handler) { @@ -621,6 +679,7 @@ func MountCORSHandler(mux goahttp.Muxer, h http.Handler) { mux.Handle("OPTIONS", "/package/{id}/confirm", h.ServeHTTP) mux.Handle("OPTIONS", "/package/{id}/reject", h.ServeHTTP) mux.Handle("OPTIONS", "/package/{id}/move", h.ServeHTTP) + mux.Handle("OPTIONS", "/package/upload", h.ServeHTTP) } // NewCORSHandler creates a HTTP handler which returns a simple 204 response. diff --git a/internal/api/gen/http/package_/server/types.go b/internal/api/gen/http/package_/server/types.go index 060b4f557..d27575f3e 100644 --- a/internal/api/gen/http/package_/server/types.go +++ b/internal/api/gen/http/package_/server/types.go @@ -329,6 +329,61 @@ type MoveStatusNotFoundResponseBody struct { ID uint `form:"id" json:"id" xml:"id"` } +// UploadInvalidMediaTypeResponseBody is the type of the "package" service +// "upload" endpoint HTTP response body for the "invalid_media_type" error. +type UploadInvalidMediaTypeResponseBody struct { + // Name is the name of this class of errors. + Name string `form:"name" json:"name" xml:"name"` + // ID is a unique identifier for this particular occurrence of the problem. + ID string `form:"id" json:"id" xml:"id"` + // Message is a human-readable explanation specific to this occurrence of the + // problem. + Message string `form:"message" json:"message" xml:"message"` + // Is the error temporary? + Temporary bool `form:"temporary" json:"temporary" xml:"temporary"` + // Is the error a timeout? + Timeout bool `form:"timeout" json:"timeout" xml:"timeout"` + // Is the error a server-side fault? + Fault bool `form:"fault" json:"fault" xml:"fault"` +} + +// UploadInvalidMultipartRequestResponseBody is the type of the "package" +// service "upload" endpoint HTTP response body for the +// "invalid_multipart_request" error. +type UploadInvalidMultipartRequestResponseBody struct { + // Name is the name of this class of errors. + Name string `form:"name" json:"name" xml:"name"` + // ID is a unique identifier for this particular occurrence of the problem. + ID string `form:"id" json:"id" xml:"id"` + // Message is a human-readable explanation specific to this occurrence of the + // problem. + Message string `form:"message" json:"message" xml:"message"` + // Is the error temporary? + Temporary bool `form:"temporary" json:"temporary" xml:"temporary"` + // Is the error a timeout? + Timeout bool `form:"timeout" json:"timeout" xml:"timeout"` + // Is the error a server-side fault? + Fault bool `form:"fault" json:"fault" xml:"fault"` +} + +// UploadInternalErrorResponseBody is the type of the "package" service +// "upload" endpoint HTTP response body for the "internal_error" error. +type UploadInternalErrorResponseBody struct { + // Name is the name of this class of errors. + Name string `form:"name" json:"name" xml:"name"` + // ID is a unique identifier for this particular occurrence of the problem. + ID string `form:"id" json:"id" xml:"id"` + // Message is a human-readable explanation specific to this occurrence of the + // problem. + Message string `form:"message" json:"message" xml:"message"` + // Is the error temporary? + Temporary bool `form:"temporary" json:"temporary" xml:"temporary"` + // Is the error a timeout? + Timeout bool `form:"timeout" json:"timeout" xml:"timeout"` + // Is the error a server-side fault? + Fault bool `form:"fault" json:"fault" xml:"fault"` +} + // EnduroStoredPackageCollectionResponseBody is used to define fields on // response body types. type EnduroStoredPackageCollectionResponseBody []*EnduroStoredPackageResponseBody @@ -699,6 +754,48 @@ func NewMoveStatusNotFoundResponseBody(res *package_.PackageNotFound) *MoveStatu return body } +// NewUploadInvalidMediaTypeResponseBody builds the HTTP response body from the +// result of the "upload" endpoint of the "package" service. +func NewUploadInvalidMediaTypeResponseBody(res *goa.ServiceError) *UploadInvalidMediaTypeResponseBody { + body := &UploadInvalidMediaTypeResponseBody{ + Name: res.Name, + ID: res.ID, + Message: res.Message, + Temporary: res.Temporary, + Timeout: res.Timeout, + Fault: res.Fault, + } + return body +} + +// NewUploadInvalidMultipartRequestResponseBody builds the HTTP response body +// from the result of the "upload" endpoint of the "package" service. +func NewUploadInvalidMultipartRequestResponseBody(res *goa.ServiceError) *UploadInvalidMultipartRequestResponseBody { + body := &UploadInvalidMultipartRequestResponseBody{ + Name: res.Name, + ID: res.ID, + Message: res.Message, + Temporary: res.Temporary, + Timeout: res.Timeout, + Fault: res.Fault, + } + return body +} + +// NewUploadInternalErrorResponseBody builds the HTTP response body from the +// result of the "upload" endpoint of the "package" service. +func NewUploadInternalErrorResponseBody(res *goa.ServiceError) *UploadInternalErrorResponseBody { + body := &UploadInternalErrorResponseBody{ + Name: res.Name, + ID: res.ID, + Message: res.Message, + Temporary: res.Temporary, + Timeout: res.Timeout, + Fault: res.Fault, + } + return body +} + // NewMonitorRequestPayload builds a package service monitor_request endpoint // payload. func NewMonitorRequestPayload(token *string) *package_.MonitorRequestPayload { @@ -790,6 +887,15 @@ func NewMoveStatusPayload(id uint, token *string) *package_.MoveStatusPayload { return v } +// NewUploadPayload builds a package service upload endpoint payload. +func NewUploadPayload(contentType string, token *string) *package_.UploadPayload { + v := &package_.UploadPayload{} + v.ContentType = contentType + v.Token = token + + return v +} + // ValidateConfirmRequestBody runs the validations defined on ConfirmRequestBody func ValidateConfirmRequestBody(body *ConfirmRequestBody) (err error) { if body.LocationID == nil { diff --git a/internal/api/gen/http/upload/client/cli.go b/internal/api/gen/http/upload/client/cli.go deleted file mode 100644 index 68f2f8a99..000000000 --- a/internal/api/gen/http/upload/client/cli.go +++ /dev/null @@ -1,41 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload HTTP client CLI support package -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package client - -import ( - upload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - goa "goa.design/goa/v3/pkg" -) - -// BuildUploadPayload builds the payload for the upload upload endpoint from -// CLI flags. -func BuildUploadPayload(uploadUploadContentType string, uploadUploadToken string) (*upload.UploadPayload, error) { - var err error - var contentType string - { - if uploadUploadContentType != "" { - contentType = uploadUploadContentType - err = goa.MergeErrors(err, goa.ValidatePattern("content_type", contentType, "multipart/[^;]+; boundary=.+")) - if err != nil { - return nil, err - } - } - } - var token *string - { - if uploadUploadToken != "" { - token = &uploadUploadToken - } - } - v := &upload.UploadPayload{} - v.ContentType = contentType - v.Token = token - - return v, nil -} diff --git a/internal/api/gen/http/upload/client/client.go b/internal/api/gen/http/upload/client/client.go deleted file mode 100644 index ccf9045d0..000000000 --- a/internal/api/gen/http/upload/client/client.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload client HTTP transport -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package client - -import ( - "context" - "net/http" - - goahttp "goa.design/goa/v3/http" - goa "goa.design/goa/v3/pkg" -) - -// Client lists the upload service endpoint HTTP clients. -type Client struct { - // Upload Doer is the HTTP client used to make requests to the upload endpoint. - UploadDoer goahttp.Doer - - // CORS Doer is the HTTP client used to make requests to the endpoint. - CORSDoer goahttp.Doer - - // RestoreResponseBody controls whether the response bodies are reset after - // decoding so they can be read again. - RestoreResponseBody bool - - scheme string - host string - encoder func(*http.Request) goahttp.Encoder - decoder func(*http.Response) goahttp.Decoder -} - -// NewClient instantiates HTTP clients for all the upload service servers. -func NewClient( - scheme string, - host string, - doer goahttp.Doer, - enc func(*http.Request) goahttp.Encoder, - dec func(*http.Response) goahttp.Decoder, - restoreBody bool, -) *Client { - return &Client{ - UploadDoer: doer, - CORSDoer: doer, - RestoreResponseBody: restoreBody, - scheme: scheme, - host: host, - decoder: dec, - encoder: enc, - } -} - -// Upload returns an endpoint that makes HTTP requests to the upload service -// upload server. -func (c *Client) Upload() goa.Endpoint { - var ( - encodeRequest = EncodeUploadRequest(c.encoder) - decodeResponse = DecodeUploadResponse(c.decoder, c.RestoreResponseBody) - ) - return func(ctx context.Context, v any) (any, error) { - req, err := c.BuildUploadRequest(ctx, v) - if err != nil { - return nil, err - } - err = encodeRequest(req, v) - if err != nil { - return nil, err - } - resp, err := c.UploadDoer.Do(req) - if err != nil { - return nil, goahttp.ErrRequestError("upload", "upload", err) - } - return decodeResponse(resp) - } -} diff --git a/internal/api/gen/http/upload/client/encode_decode.go b/internal/api/gen/http/upload/client/encode_decode.go deleted file mode 100644 index 3c28b163a..000000000 --- a/internal/api/gen/http/upload/client/encode_decode.go +++ /dev/null @@ -1,186 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload HTTP client encoders and decoders -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package client - -import ( - "bytes" - "context" - "io" - "net/http" - "net/url" - "os" - "strings" - - upload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - goahttp "goa.design/goa/v3/http" -) - -// BuildUploadRequest instantiates a HTTP request object with method and path -// set to call the "upload" service "upload" endpoint -func (c *Client) BuildUploadRequest(ctx context.Context, v any) (*http.Request, error) { - var ( - body io.Reader - ) - rd, ok := v.(*upload.UploadRequestData) - if !ok { - return nil, goahttp.ErrInvalidType("upload", "upload", "upload.UploadRequestData", v) - } - body = rd.Body - u := &url.URL{Scheme: c.scheme, Host: c.host, Path: UploadUploadPath()} - req, err := http.NewRequest("POST", u.String(), body) - if err != nil { - return nil, goahttp.ErrInvalidURL("upload", "upload", u.String(), err) - } - if ctx != nil { - req = req.WithContext(ctx) - } - - return req, nil -} - -// EncodeUploadRequest returns an encoder for requests sent to the upload -// upload server. -func EncodeUploadRequest(encoder func(*http.Request) goahttp.Encoder) func(*http.Request, any) error { - return func(req *http.Request, v any) error { - data, ok := v.(*upload.UploadRequestData) - if !ok { - return goahttp.ErrInvalidType("upload", "upload", "*upload.UploadRequestData", v) - } - p := data.Payload - { - head := p.ContentType - req.Header.Set("Content-Type", head) - } - if p.Token != nil { - head := *p.Token - if !strings.Contains(head, " ") { - req.Header.Set("Authorization", "Bearer "+head) - } else { - req.Header.Set("Authorization", head) - } - } - return nil - } -} - -// DecodeUploadResponse returns a decoder for responses returned by the upload -// upload endpoint. restoreBody controls whether the response body should be -// restored after having been read. -// DecodeUploadResponse may return the following errors: -// - "invalid_media_type" (type *goa.ServiceError): http.StatusBadRequest -// - "invalid_multipart_request" (type *goa.ServiceError): http.StatusBadRequest -// - "internal_error" (type *goa.ServiceError): http.StatusInternalServerError -// - "forbidden" (type upload.Forbidden): http.StatusForbidden -// - "unauthorized" (type upload.Unauthorized): http.StatusUnauthorized -// - error: internal error -func DecodeUploadResponse(decoder func(*http.Response) goahttp.Decoder, restoreBody bool) func(*http.Response) (any, error) { - return func(resp *http.Response) (any, error) { - if restoreBody { - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - resp.Body = io.NopCloser(bytes.NewBuffer(b)) - defer func() { - resp.Body = io.NopCloser(bytes.NewBuffer(b)) - }() - } else { - defer resp.Body.Close() - } - switch resp.StatusCode { - case http.StatusNoContent: - return nil, nil - case http.StatusBadRequest: - en := resp.Header.Get("goa-error") - switch en { - case "invalid_media_type": - var ( - body UploadInvalidMediaTypeResponseBody - err error - ) - err = decoder(resp).Decode(&body) - if err != nil { - return nil, goahttp.ErrDecodingError("upload", "upload", err) - } - err = ValidateUploadInvalidMediaTypeResponseBody(&body) - if err != nil { - return nil, goahttp.ErrValidationError("upload", "upload", err) - } - return nil, NewUploadInvalidMediaType(&body) - case "invalid_multipart_request": - var ( - body UploadInvalidMultipartRequestResponseBody - err error - ) - err = decoder(resp).Decode(&body) - if err != nil { - return nil, goahttp.ErrDecodingError("upload", "upload", err) - } - err = ValidateUploadInvalidMultipartRequestResponseBody(&body) - if err != nil { - return nil, goahttp.ErrValidationError("upload", "upload", err) - } - return nil, NewUploadInvalidMultipartRequest(&body) - default: - body, _ := io.ReadAll(resp.Body) - return nil, goahttp.ErrInvalidResponse("upload", "upload", resp.StatusCode, string(body)) - } - case http.StatusInternalServerError: - var ( - body UploadInternalErrorResponseBody - err error - ) - err = decoder(resp).Decode(&body) - if err != nil { - return nil, goahttp.ErrDecodingError("upload", "upload", err) - } - err = ValidateUploadInternalErrorResponseBody(&body) - if err != nil { - return nil, goahttp.ErrValidationError("upload", "upload", err) - } - return nil, NewUploadInternalError(&body) - case http.StatusForbidden: - var ( - body string - err error - ) - err = decoder(resp).Decode(&body) - if err != nil { - return nil, goahttp.ErrDecodingError("upload", "upload", err) - } - return nil, NewUploadForbidden(body) - case http.StatusUnauthorized: - var ( - body string - err error - ) - err = decoder(resp).Decode(&body) - if err != nil { - return nil, goahttp.ErrDecodingError("upload", "upload", err) - } - return nil, NewUploadUnauthorized(body) - default: - body, _ := io.ReadAll(resp.Body) - return nil, goahttp.ErrInvalidResponse("upload", "upload", resp.StatusCode, string(body)) - } - } -} - -// // BuildUploadStreamPayload creates a streaming endpoint request payload from -// the method payload and the path to the file to be streamed -func BuildUploadStreamPayload(payload any, fpath string) (*upload.UploadRequestData, error) { - f, err := os.Open(fpath) - if err != nil { - return nil, err - } - return &upload.UploadRequestData{ - Payload: payload.(*upload.UploadPayload), - Body: f, - }, nil -} diff --git a/internal/api/gen/http/upload/client/paths.go b/internal/api/gen/http/upload/client/paths.go deleted file mode 100644 index da2d89154..000000000 --- a/internal/api/gen/http/upload/client/paths.go +++ /dev/null @@ -1,14 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// HTTP request path constructors for the upload service. -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package client - -// UploadUploadPath returns the URL path to the upload service upload HTTP endpoint. -func UploadUploadPath() string { - return "/upload/upload" -} diff --git a/internal/api/gen/http/upload/client/types.go b/internal/api/gen/http/upload/client/types.go deleted file mode 100644 index c85f3ce2a..000000000 --- a/internal/api/gen/http/upload/client/types.go +++ /dev/null @@ -1,201 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload HTTP client types -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package client - -import ( - upload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - goa "goa.design/goa/v3/pkg" -) - -// UploadInvalidMediaTypeResponseBody is the type of the "upload" service -// "upload" endpoint HTTP response body for the "invalid_media_type" error. -type UploadInvalidMediaTypeResponseBody struct { - // Name is the name of this class of errors. - Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` - // ID is a unique identifier for this particular occurrence of the problem. - ID *string `form:"id,omitempty" json:"id,omitempty" xml:"id,omitempty"` - // Message is a human-readable explanation specific to this occurrence of the - // problem. - Message *string `form:"message,omitempty" json:"message,omitempty" xml:"message,omitempty"` - // Is the error temporary? - Temporary *bool `form:"temporary,omitempty" json:"temporary,omitempty" xml:"temporary,omitempty"` - // Is the error a timeout? - Timeout *bool `form:"timeout,omitempty" json:"timeout,omitempty" xml:"timeout,omitempty"` - // Is the error a server-side fault? - Fault *bool `form:"fault,omitempty" json:"fault,omitempty" xml:"fault,omitempty"` -} - -// UploadInvalidMultipartRequestResponseBody is the type of the "upload" -// service "upload" endpoint HTTP response body for the -// "invalid_multipart_request" error. -type UploadInvalidMultipartRequestResponseBody struct { - // Name is the name of this class of errors. - Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` - // ID is a unique identifier for this particular occurrence of the problem. - ID *string `form:"id,omitempty" json:"id,omitempty" xml:"id,omitempty"` - // Message is a human-readable explanation specific to this occurrence of the - // problem. - Message *string `form:"message,omitempty" json:"message,omitempty" xml:"message,omitempty"` - // Is the error temporary? - Temporary *bool `form:"temporary,omitempty" json:"temporary,omitempty" xml:"temporary,omitempty"` - // Is the error a timeout? - Timeout *bool `form:"timeout,omitempty" json:"timeout,omitempty" xml:"timeout,omitempty"` - // Is the error a server-side fault? - Fault *bool `form:"fault,omitempty" json:"fault,omitempty" xml:"fault,omitempty"` -} - -// UploadInternalErrorResponseBody is the type of the "upload" service "upload" -// endpoint HTTP response body for the "internal_error" error. -type UploadInternalErrorResponseBody struct { - // Name is the name of this class of errors. - Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` - // ID is a unique identifier for this particular occurrence of the problem. - ID *string `form:"id,omitempty" json:"id,omitempty" xml:"id,omitempty"` - // Message is a human-readable explanation specific to this occurrence of the - // problem. - Message *string `form:"message,omitempty" json:"message,omitempty" xml:"message,omitempty"` - // Is the error temporary? - Temporary *bool `form:"temporary,omitempty" json:"temporary,omitempty" xml:"temporary,omitempty"` - // Is the error a timeout? - Timeout *bool `form:"timeout,omitempty" json:"timeout,omitempty" xml:"timeout,omitempty"` - // Is the error a server-side fault? - Fault *bool `form:"fault,omitempty" json:"fault,omitempty" xml:"fault,omitempty"` -} - -// NewUploadInvalidMediaType builds a upload service upload endpoint -// invalid_media_type error. -func NewUploadInvalidMediaType(body *UploadInvalidMediaTypeResponseBody) *goa.ServiceError { - v := &goa.ServiceError{ - Name: *body.Name, - ID: *body.ID, - Message: *body.Message, - Temporary: *body.Temporary, - Timeout: *body.Timeout, - Fault: *body.Fault, - } - - return v -} - -// NewUploadInvalidMultipartRequest builds a upload service upload endpoint -// invalid_multipart_request error. -func NewUploadInvalidMultipartRequest(body *UploadInvalidMultipartRequestResponseBody) *goa.ServiceError { - v := &goa.ServiceError{ - Name: *body.Name, - ID: *body.ID, - Message: *body.Message, - Temporary: *body.Temporary, - Timeout: *body.Timeout, - Fault: *body.Fault, - } - - return v -} - -// NewUploadInternalError builds a upload service upload endpoint -// internal_error error. -func NewUploadInternalError(body *UploadInternalErrorResponseBody) *goa.ServiceError { - v := &goa.ServiceError{ - Name: *body.Name, - ID: *body.ID, - Message: *body.Message, - Temporary: *body.Temporary, - Timeout: *body.Timeout, - Fault: *body.Fault, - } - - return v -} - -// NewUploadForbidden builds a upload service upload endpoint forbidden error. -func NewUploadForbidden(body string) upload.Forbidden { - v := upload.Forbidden(body) - - return v -} - -// NewUploadUnauthorized builds a upload service upload endpoint unauthorized -// error. -func NewUploadUnauthorized(body string) upload.Unauthorized { - v := upload.Unauthorized(body) - - return v -} - -// ValidateUploadInvalidMediaTypeResponseBody runs the validations defined on -// upload_invalid_media_type_response_body -func ValidateUploadInvalidMediaTypeResponseBody(body *UploadInvalidMediaTypeResponseBody) (err error) { - if body.Name == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("name", "body")) - } - if body.ID == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("id", "body")) - } - if body.Message == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("message", "body")) - } - if body.Temporary == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("temporary", "body")) - } - if body.Timeout == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("timeout", "body")) - } - if body.Fault == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("fault", "body")) - } - return -} - -// ValidateUploadInvalidMultipartRequestResponseBody runs the validations -// defined on upload_invalid_multipart_request_response_body -func ValidateUploadInvalidMultipartRequestResponseBody(body *UploadInvalidMultipartRequestResponseBody) (err error) { - if body.Name == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("name", "body")) - } - if body.ID == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("id", "body")) - } - if body.Message == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("message", "body")) - } - if body.Temporary == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("temporary", "body")) - } - if body.Timeout == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("timeout", "body")) - } - if body.Fault == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("fault", "body")) - } - return -} - -// ValidateUploadInternalErrorResponseBody runs the validations defined on -// upload_internal_error_response_body -func ValidateUploadInternalErrorResponseBody(body *UploadInternalErrorResponseBody) (err error) { - if body.Name == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("name", "body")) - } - if body.ID == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("id", "body")) - } - if body.Message == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("message", "body")) - } - if body.Temporary == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("temporary", "body")) - } - if body.Timeout == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("timeout", "body")) - } - if body.Fault == nil { - err = goa.MergeErrors(err, goa.MissingFieldError("fault", "body")) - } - return -} diff --git a/internal/api/gen/http/upload/server/encode_decode.go b/internal/api/gen/http/upload/server/encode_decode.go deleted file mode 100644 index f1d490a31..000000000 --- a/internal/api/gen/http/upload/server/encode_decode.go +++ /dev/null @@ -1,136 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload HTTP server encoders and decoders -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package server - -import ( - "context" - "errors" - "net/http" - "strings" - - upload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - goahttp "goa.design/goa/v3/http" - goa "goa.design/goa/v3/pkg" -) - -// EncodeUploadResponse returns an encoder for responses returned by the upload -// upload endpoint. -func EncodeUploadResponse(encoder func(context.Context, http.ResponseWriter) goahttp.Encoder) func(context.Context, http.ResponseWriter, any) error { - return func(ctx context.Context, w http.ResponseWriter, v any) error { - w.WriteHeader(http.StatusNoContent) - return nil - } -} - -// DecodeUploadRequest returns a decoder for requests sent to the upload upload -// endpoint. -func DecodeUploadRequest(mux goahttp.Muxer, decoder func(*http.Request) goahttp.Decoder) func(*http.Request) (any, error) { - return func(r *http.Request) (any, error) { - var ( - contentType string - token *string - err error - ) - contentTypeRaw := r.Header.Get("Content-Type") - if contentTypeRaw != "" { - contentType = contentTypeRaw - } else { - contentType = "multipart/form-data; boundary=goa" - } - err = goa.MergeErrors(err, goa.ValidatePattern("content_type", contentType, "multipart/[^;]+; boundary=.+")) - tokenRaw := r.Header.Get("Authorization") - if tokenRaw != "" { - token = &tokenRaw - } - if err != nil { - return nil, err - } - payload := NewUploadPayload(contentType, token) - if payload.Token != nil { - if strings.Contains(*payload.Token, " ") { - // Remove authorization scheme prefix (e.g. "Bearer") - cred := strings.SplitN(*payload.Token, " ", 2)[1] - payload.Token = &cred - } - } - - return payload, nil - } -} - -// EncodeUploadError returns an encoder for errors returned by the upload -// upload endpoint. -func EncodeUploadError(encoder func(context.Context, http.ResponseWriter) goahttp.Encoder, formatter func(ctx context.Context, err error) goahttp.Statuser) func(context.Context, http.ResponseWriter, error) error { - encodeError := goahttp.ErrorEncoder(encoder, formatter) - return func(ctx context.Context, w http.ResponseWriter, v error) error { - var en goa.GoaErrorNamer - if !errors.As(v, &en) { - return encodeError(ctx, w, v) - } - switch en.GoaErrorName() { - case "invalid_media_type": - var res *goa.ServiceError - errors.As(v, &res) - enc := encoder(ctx, w) - var body any - if formatter != nil { - body = formatter(ctx, res) - } else { - body = NewUploadInvalidMediaTypeResponseBody(res) - } - w.Header().Set("goa-error", res.GoaErrorName()) - w.WriteHeader(http.StatusBadRequest) - return enc.Encode(body) - case "invalid_multipart_request": - var res *goa.ServiceError - errors.As(v, &res) - enc := encoder(ctx, w) - var body any - if formatter != nil { - body = formatter(ctx, res) - } else { - body = NewUploadInvalidMultipartRequestResponseBody(res) - } - w.Header().Set("goa-error", res.GoaErrorName()) - w.WriteHeader(http.StatusBadRequest) - return enc.Encode(body) - case "internal_error": - var res *goa.ServiceError - errors.As(v, &res) - enc := encoder(ctx, w) - var body any - if formatter != nil { - body = formatter(ctx, res) - } else { - body = NewUploadInternalErrorResponseBody(res) - } - w.Header().Set("goa-error", res.GoaErrorName()) - w.WriteHeader(http.StatusInternalServerError) - return enc.Encode(body) - case "forbidden": - var res upload.Forbidden - errors.As(v, &res) - enc := encoder(ctx, w) - body := res - w.Header().Set("goa-error", res.GoaErrorName()) - w.WriteHeader(http.StatusForbidden) - return enc.Encode(body) - case "unauthorized": - var res upload.Unauthorized - errors.As(v, &res) - enc := encoder(ctx, w) - body := res - w.Header().Set("goa-error", res.GoaErrorName()) - w.WriteHeader(http.StatusUnauthorized) - return enc.Encode(body) - default: - return encodeError(ctx, w, v) - } - } -} diff --git a/internal/api/gen/http/upload/server/paths.go b/internal/api/gen/http/upload/server/paths.go deleted file mode 100644 index 4ce645a90..000000000 --- a/internal/api/gen/http/upload/server/paths.go +++ /dev/null @@ -1,14 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// HTTP request path constructors for the upload service. -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package server - -// UploadUploadPath returns the URL path to the upload service upload HTTP endpoint. -func UploadUploadPath() string { - return "/upload/upload" -} diff --git a/internal/api/gen/http/upload/server/server.go b/internal/api/gen/http/upload/server/server.go deleted file mode 100644 index 5de828921..000000000 --- a/internal/api/gen/http/upload/server/server.go +++ /dev/null @@ -1,184 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload HTTP server -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package server - -import ( - "context" - "net/http" - "os" - - upload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - otelhttp "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - goahttp "goa.design/goa/v3/http" - goa "goa.design/goa/v3/pkg" - "goa.design/plugins/v3/cors" -) - -// Server lists the upload service endpoint HTTP handlers. -type Server struct { - Mounts []*MountPoint - Upload http.Handler - CORS http.Handler -} - -// MountPoint holds information about the mounted endpoints. -type MountPoint struct { - // Method is the name of the service method served by the mounted HTTP handler. - Method string - // Verb is the HTTP method used to match requests to the mounted handler. - Verb string - // Pattern is the HTTP request path pattern used to match requests to the - // mounted handler. - Pattern string -} - -// New instantiates HTTP handlers for all the upload service endpoints using -// the provided encoder and decoder. The handlers are mounted on the given mux -// using the HTTP verb and path defined in the design. errhandler is called -// whenever a response fails to be encoded. formatter is used to format errors -// returned by the service methods prior to encoding. Both errhandler and -// formatter are optional and can be nil. -func New( - e *upload.Endpoints, - mux goahttp.Muxer, - decoder func(*http.Request) goahttp.Decoder, - encoder func(context.Context, http.ResponseWriter) goahttp.Encoder, - errhandler func(context.Context, http.ResponseWriter, error), - formatter func(ctx context.Context, err error) goahttp.Statuser, -) *Server { - return &Server{ - Mounts: []*MountPoint{ - {"Upload", "POST", "/upload/upload"}, - {"CORS", "OPTIONS", "/upload/upload"}, - }, - Upload: NewUploadHandler(e.Upload, mux, decoder, encoder, errhandler, formatter), - CORS: NewCORSHandler(), - } -} - -// Service returns the name of the service served. -func (s *Server) Service() string { return "upload" } - -// Use wraps the server handlers with the given middleware. -func (s *Server) Use(m func(http.Handler) http.Handler) { - s.Upload = m(s.Upload) - s.CORS = m(s.CORS) -} - -// MethodNames returns the methods served. -func (s *Server) MethodNames() []string { return upload.MethodNames[:] } - -// Mount configures the mux to serve the upload endpoints. -func Mount(mux goahttp.Muxer, h *Server) { - MountUploadHandler(mux, h.Upload) - MountCORSHandler(mux, h.CORS) -} - -// Mount configures the mux to serve the upload endpoints. -func (s *Server) Mount(mux goahttp.Muxer) { - Mount(mux, s) -} - -// MountUploadHandler configures the mux to serve the "upload" service "upload" -// endpoint. -func MountUploadHandler(mux goahttp.Muxer, h http.Handler) { - f, ok := HandleUploadOrigin(h).(http.HandlerFunc) - if !ok { - f = func(w http.ResponseWriter, r *http.Request) { - h.ServeHTTP(w, r) - } - } - mux.Handle("POST", "/upload/upload", otelhttp.WithRouteTag("/upload/upload", f).ServeHTTP) -} - -// NewUploadHandler creates a HTTP handler which loads the HTTP request and -// calls the "upload" service "upload" endpoint. -func NewUploadHandler( - endpoint goa.Endpoint, - mux goahttp.Muxer, - decoder func(*http.Request) goahttp.Decoder, - encoder func(context.Context, http.ResponseWriter) goahttp.Encoder, - errhandler func(context.Context, http.ResponseWriter, error), - formatter func(ctx context.Context, err error) goahttp.Statuser, -) http.Handler { - var ( - decodeRequest = DecodeUploadRequest(mux, decoder) - encodeResponse = EncodeUploadResponse(encoder) - encodeError = EncodeUploadError(encoder, formatter) - ) - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := context.WithValue(r.Context(), goahttp.AcceptTypeKey, r.Header.Get("Accept")) - ctx = context.WithValue(ctx, goa.MethodKey, "upload") - ctx = context.WithValue(ctx, goa.ServiceKey, "upload") - payload, err := decodeRequest(r) - if err != nil { - if err := encodeError(ctx, w, err); err != nil { - errhandler(ctx, w, err) - } - return - } - data := &upload.UploadRequestData{Payload: payload.(*upload.UploadPayload), Body: r.Body} - res, err := endpoint(ctx, data) - if err != nil { - if err := encodeError(ctx, w, err); err != nil { - errhandler(ctx, w, err) - } - return - } - if err := encodeResponse(ctx, w, res); err != nil { - errhandler(ctx, w, err) - } - }) -} - -// MountCORSHandler configures the mux to serve the CORS endpoints for the -// service upload. -func MountCORSHandler(mux goahttp.Muxer, h http.Handler) { - h = HandleUploadOrigin(h) - mux.Handle("OPTIONS", "/upload/upload", h.ServeHTTP) -} - -// NewCORSHandler creates a HTTP handler which returns a simple 204 response. -func NewCORSHandler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(204) - }) -} - -// HandleUploadOrigin applies the CORS response headers corresponding to the -// origin for the service upload. -func HandleUploadOrigin(h http.Handler) http.Handler { - originStr0, present := os.LookupEnv("ENDURO_API_CORS_ORIGIN") - if !present { - panic("CORS origin environment variable \"ENDURO_API_CORS_ORIGIN\" not set!") - } - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - origin := r.Header.Get("Origin") - if origin == "" { - // Not a CORS request - h.ServeHTTP(w, r) - return - } - if cors.MatchOrigin(origin, originStr0) { - w.Header().Set("Access-Control-Allow-Origin", origin) - w.Header().Set("Vary", "Origin") - if acrm := r.Header.Get("Access-Control-Request-Method"); acrm != "" { - // We are handling a preflight request - w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, DELETE, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type") - w.WriteHeader(204) - return - } - h.ServeHTTP(w, r) - return - } - h.ServeHTTP(w, r) - return - }) -} diff --git a/internal/api/gen/http/upload/server/types.go b/internal/api/gen/http/upload/server/types.go deleted file mode 100644 index 498d6dd7a..000000000 --- a/internal/api/gen/http/upload/server/types.go +++ /dev/null @@ -1,120 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload HTTP server types -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package server - -import ( - upload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - goa "goa.design/goa/v3/pkg" -) - -// UploadInvalidMediaTypeResponseBody is the type of the "upload" service -// "upload" endpoint HTTP response body for the "invalid_media_type" error. -type UploadInvalidMediaTypeResponseBody struct { - // Name is the name of this class of errors. - Name string `form:"name" json:"name" xml:"name"` - // ID is a unique identifier for this particular occurrence of the problem. - ID string `form:"id" json:"id" xml:"id"` - // Message is a human-readable explanation specific to this occurrence of the - // problem. - Message string `form:"message" json:"message" xml:"message"` - // Is the error temporary? - Temporary bool `form:"temporary" json:"temporary" xml:"temporary"` - // Is the error a timeout? - Timeout bool `form:"timeout" json:"timeout" xml:"timeout"` - // Is the error a server-side fault? - Fault bool `form:"fault" json:"fault" xml:"fault"` -} - -// UploadInvalidMultipartRequestResponseBody is the type of the "upload" -// service "upload" endpoint HTTP response body for the -// "invalid_multipart_request" error. -type UploadInvalidMultipartRequestResponseBody struct { - // Name is the name of this class of errors. - Name string `form:"name" json:"name" xml:"name"` - // ID is a unique identifier for this particular occurrence of the problem. - ID string `form:"id" json:"id" xml:"id"` - // Message is a human-readable explanation specific to this occurrence of the - // problem. - Message string `form:"message" json:"message" xml:"message"` - // Is the error temporary? - Temporary bool `form:"temporary" json:"temporary" xml:"temporary"` - // Is the error a timeout? - Timeout bool `form:"timeout" json:"timeout" xml:"timeout"` - // Is the error a server-side fault? - Fault bool `form:"fault" json:"fault" xml:"fault"` -} - -// UploadInternalErrorResponseBody is the type of the "upload" service "upload" -// endpoint HTTP response body for the "internal_error" error. -type UploadInternalErrorResponseBody struct { - // Name is the name of this class of errors. - Name string `form:"name" json:"name" xml:"name"` - // ID is a unique identifier for this particular occurrence of the problem. - ID string `form:"id" json:"id" xml:"id"` - // Message is a human-readable explanation specific to this occurrence of the - // problem. - Message string `form:"message" json:"message" xml:"message"` - // Is the error temporary? - Temporary bool `form:"temporary" json:"temporary" xml:"temporary"` - // Is the error a timeout? - Timeout bool `form:"timeout" json:"timeout" xml:"timeout"` - // Is the error a server-side fault? - Fault bool `form:"fault" json:"fault" xml:"fault"` -} - -// NewUploadInvalidMediaTypeResponseBody builds the HTTP response body from the -// result of the "upload" endpoint of the "upload" service. -func NewUploadInvalidMediaTypeResponseBody(res *goa.ServiceError) *UploadInvalidMediaTypeResponseBody { - body := &UploadInvalidMediaTypeResponseBody{ - Name: res.Name, - ID: res.ID, - Message: res.Message, - Temporary: res.Temporary, - Timeout: res.Timeout, - Fault: res.Fault, - } - return body -} - -// NewUploadInvalidMultipartRequestResponseBody builds the HTTP response body -// from the result of the "upload" endpoint of the "upload" service. -func NewUploadInvalidMultipartRequestResponseBody(res *goa.ServiceError) *UploadInvalidMultipartRequestResponseBody { - body := &UploadInvalidMultipartRequestResponseBody{ - Name: res.Name, - ID: res.ID, - Message: res.Message, - Temporary: res.Temporary, - Timeout: res.Timeout, - Fault: res.Fault, - } - return body -} - -// NewUploadInternalErrorResponseBody builds the HTTP response body from the -// result of the "upload" endpoint of the "upload" service. -func NewUploadInternalErrorResponseBody(res *goa.ServiceError) *UploadInternalErrorResponseBody { - body := &UploadInternalErrorResponseBody{ - Name: res.Name, - ID: res.ID, - Message: res.Message, - Temporary: res.Temporary, - Timeout: res.Timeout, - Fault: res.Fault, - } - return body -} - -// NewUploadPayload builds a upload service upload endpoint payload. -func NewUploadPayload(contentType string, token *string) *upload.UploadPayload { - v := &upload.UploadPayload{} - v.ContentType = contentType - v.Token = token - - return v -} diff --git a/internal/api/gen/package_/client.go b/internal/api/gen/package_/client.go index f3a0ca7e4..604cefa7e 100644 --- a/internal/api/gen/package_/client.go +++ b/internal/api/gen/package_/client.go @@ -10,6 +10,7 @@ package package_ import ( "context" + "io" goa "goa.design/goa/v3/pkg" ) @@ -25,10 +26,11 @@ type Client struct { RejectEndpoint goa.Endpoint MoveEndpoint goa.Endpoint MoveStatusEndpoint goa.Endpoint + UploadEndpoint goa.Endpoint } // NewClient initializes a "package" service client given the endpoints. -func NewClient(monitorRequest, monitor, list, show, preservationActions, confirm, reject, move, moveStatus goa.Endpoint) *Client { +func NewClient(monitorRequest, monitor, list, show, preservationActions, confirm, reject, move, moveStatus, upload goa.Endpoint) *Client { return &Client{ MonitorRequestEndpoint: monitorRequest, MonitorEndpoint: monitor, @@ -39,6 +41,7 @@ func NewClient(monitorRequest, monitor, list, show, preservationActions, confirm RejectEndpoint: reject, MoveEndpoint: move, MoveStatusEndpoint: moveStatus, + UploadEndpoint: upload, } } @@ -172,3 +175,16 @@ func (c *Client) MoveStatus(ctx context.Context, p *MoveStatusPayload) (res *Mov } return ires.(*MoveStatusResult), nil } + +// Upload calls the "upload" endpoint of the "package" service. +// Upload may return the following errors: +// - "invalid_media_type" (type *goa.ServiceError): Error returned when the Content-Type header does not define a multipart request. +// - "invalid_multipart_request" (type *goa.ServiceError): Error returned when the request body is not a valid multipart content. +// - "internal_error" (type *goa.ServiceError): Fault while processing upload. +// - "unauthorized" (type Unauthorized) +// - "forbidden" (type Forbidden) +// - error: internal error +func (c *Client) Upload(ctx context.Context, p *UploadPayload, req io.ReadCloser) (err error) { + _, err = c.UploadEndpoint(ctx, &UploadRequestData{Payload: p, Body: req}) + return +} diff --git a/internal/api/gen/package_/endpoints.go b/internal/api/gen/package_/endpoints.go index 95e213b5d..0f6670a33 100644 --- a/internal/api/gen/package_/endpoints.go +++ b/internal/api/gen/package_/endpoints.go @@ -10,6 +10,7 @@ package package_ import ( "context" + "io" goa "goa.design/goa/v3/pkg" "goa.design/goa/v3/security" @@ -26,6 +27,7 @@ type Endpoints struct { Reject goa.Endpoint Move goa.Endpoint MoveStatus goa.Endpoint + Upload goa.Endpoint } // MonitorEndpointInput holds both the payload and the server stream of the @@ -37,6 +39,15 @@ type MonitorEndpointInput struct { Stream MonitorServerStream } +// UploadRequestData holds both the payload and the HTTP request body reader of +// the "upload" method. +type UploadRequestData struct { + // Payload is the method payload. + Payload *UploadPayload + // Body streams the HTTP request body. + Body io.ReadCloser +} + // NewEndpoints wraps the methods of the "package" service with endpoints. func NewEndpoints(s Service) *Endpoints { // Casting service to Auther interface @@ -51,6 +62,7 @@ func NewEndpoints(s Service) *Endpoints { Reject: NewRejectEndpoint(s, a.JWTAuth), Move: NewMoveEndpoint(s, a.JWTAuth), MoveStatus: NewMoveStatusEndpoint(s, a.JWTAuth), + Upload: NewUploadEndpoint(s, a.JWTAuth), } } @@ -65,6 +77,7 @@ func (e *Endpoints) Use(m func(goa.Endpoint) goa.Endpoint) { e.Reject = m(e.Reject) e.Move = m(e.Move) e.MoveStatus = m(e.MoveStatus) + e.Upload = m(e.Upload) } // NewMonitorRequestEndpoint returns an endpoint function that calls the method @@ -269,3 +282,26 @@ func NewMoveStatusEndpoint(s Service, authJWTFn security.AuthJWTFunc) goa.Endpoi return s.MoveStatus(ctx, p) } } + +// NewUploadEndpoint returns an endpoint function that calls the method +// "upload" of service "package". +func NewUploadEndpoint(s Service, authJWTFn security.AuthJWTFunc) goa.Endpoint { + return func(ctx context.Context, req any) (any, error) { + ep := req.(*UploadRequestData) + var err error + sc := security.JWTScheme{ + Name: "jwt", + Scopes: []string{"package:list", "package:listActions", "package:move", "package:read", "package:review", "package:upload", "storage:location:create", "storage:location:list", "storage:location:listPackages", "storage:location:read", "storage:package:create", "storage:package:download", "storage:package:move", "storage:package:read", "storage:package:review", "storage:package:submit"}, + RequiredScopes: []string{"package:upload"}, + } + var token string + if ep.Payload.Token != nil { + token = *ep.Payload.Token + } + ctx, err = authJWTFn(ctx, token, &sc) + if err != nil { + return nil, err + } + return nil, s.Upload(ctx, ep.Payload, ep.Body) + } +} diff --git a/internal/api/gen/package_/service.go b/internal/api/gen/package_/service.go index e724d4d07..20868449a 100644 --- a/internal/api/gen/package_/service.go +++ b/internal/api/gen/package_/service.go @@ -10,6 +10,7 @@ package package_ import ( "context" + "io" package_views "github.com/artefactual-sdps/enduro/internal/api/gen/package_/views" "github.com/google/uuid" @@ -37,6 +38,8 @@ type Service interface { Move(context.Context, *MovePayload) (err error) // Retrieve the status of a permanent storage location move of the package MoveStatus(context.Context, *MoveStatusPayload) (res *MoveStatusResult, err error) + // Upload a package to trigger an ingest workflow + Upload(context.Context, *UploadPayload, io.ReadCloser) (err error) } // Auther defines the authorization functions to be implemented by the service. @@ -59,7 +62,7 @@ const ServiceName = "package" // MethodNames lists the service method names as defined in the design. These // are the same values that are set in the endpoint request contexts under the // MethodKey key. -var MethodNames = [9]string{"monitor_request", "monitor", "list", "show", "preservation_actions", "confirm", "reject", "move", "move_status"} +var MethodNames = [10]string{"monitor_request", "monitor", "list", "show", "preservation_actions", "confirm", "reject", "move", "move_status", "upload"} // MonitorServerStream is the interface a "monitor" endpoint server stream must // satisfy. @@ -297,6 +300,13 @@ type ShowPayload struct { Token *string } +// UploadPayload is the payload type of the package service upload method. +type UploadPayload struct { + // Content-Type header, must define value for multipart boundary. + ContentType string + Token *string +} + // Forbidden type Forbidden string @@ -378,6 +388,21 @@ func MakeFailedDependency(err error) *goa.ServiceError { return goa.NewServiceError(err, "failed_dependency", false, false, false) } +// MakeInvalidMediaType builds a goa.ServiceError from an error. +func MakeInvalidMediaType(err error) *goa.ServiceError { + return goa.NewServiceError(err, "invalid_media_type", false, false, false) +} + +// MakeInvalidMultipartRequest builds a goa.ServiceError from an error. +func MakeInvalidMultipartRequest(err error) *goa.ServiceError { + return goa.NewServiceError(err, "invalid_multipart_request", false, false, false) +} + +// MakeInternalError builds a goa.ServiceError from an error. +func MakeInternalError(err error) *goa.ServiceError { + return goa.NewServiceError(err, "internal_error", false, false, false) +} + // NewEnduroStoredPackage initializes result type EnduroStoredPackage from // viewed result type EnduroStoredPackage. func NewEnduroStoredPackage(vres *package_views.EnduroStoredPackage) *EnduroStoredPackage { diff --git a/internal/api/gen/upload/client.go b/internal/api/gen/upload/client.go deleted file mode 100644 index 8bc25a5d0..000000000 --- a/internal/api/gen/upload/client.go +++ /dev/null @@ -1,41 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload client -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package upload - -import ( - "context" - "io" - - goa "goa.design/goa/v3/pkg" -) - -// Client is the "upload" service client. -type Client struct { - UploadEndpoint goa.Endpoint -} - -// NewClient initializes a "upload" service client given the endpoints. -func NewClient(upload goa.Endpoint) *Client { - return &Client{ - UploadEndpoint: upload, - } -} - -// Upload calls the "upload" endpoint of the "upload" service. -// Upload may return the following errors: -// - "invalid_media_type" (type *goa.ServiceError): Error returned when the Content-Type header does not define a multipart request. -// - "invalid_multipart_request" (type *goa.ServiceError): Error returned when the request body is not a valid multipart content. -// - "internal_error" (type *goa.ServiceError): Fault while processing upload. -// - "unauthorized" (type Unauthorized) -// - "forbidden" (type Forbidden) -// - error: internal error -func (c *Client) Upload(ctx context.Context, p *UploadPayload, req io.ReadCloser) (err error) { - _, err = c.UploadEndpoint(ctx, &UploadRequestData{Payload: p, Body: req}) - return -} diff --git a/internal/api/gen/upload/endpoints.go b/internal/api/gen/upload/endpoints.go deleted file mode 100644 index 561e58ce8..000000000 --- a/internal/api/gen/upload/endpoints.go +++ /dev/null @@ -1,68 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload endpoints -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package upload - -import ( - "context" - "io" - - goa "goa.design/goa/v3/pkg" - "goa.design/goa/v3/security" -) - -// Endpoints wraps the "upload" service endpoints. -type Endpoints struct { - Upload goa.Endpoint -} - -// UploadRequestData holds both the payload and the HTTP request body reader of -// the "upload" method. -type UploadRequestData struct { - // Payload is the method payload. - Payload *UploadPayload - // Body streams the HTTP request body. - Body io.ReadCloser -} - -// NewEndpoints wraps the methods of the "upload" service with endpoints. -func NewEndpoints(s Service) *Endpoints { - // Casting service to Auther interface - a := s.(Auther) - return &Endpoints{ - Upload: NewUploadEndpoint(s, a.JWTAuth), - } -} - -// Use applies the given middleware to all the "upload" service endpoints. -func (e *Endpoints) Use(m func(goa.Endpoint) goa.Endpoint) { - e.Upload = m(e.Upload) -} - -// NewUploadEndpoint returns an endpoint function that calls the method -// "upload" of service "upload". -func NewUploadEndpoint(s Service, authJWTFn security.AuthJWTFunc) goa.Endpoint { - return func(ctx context.Context, req any) (any, error) { - ep := req.(*UploadRequestData) - var err error - sc := security.JWTScheme{ - Name: "jwt", - Scopes: []string{"package:list", "package:listActions", "package:move", "package:read", "package:review", "package:upload", "storage:location:create", "storage:location:list", "storage:location:listPackages", "storage:location:read", "storage:package:create", "storage:package:download", "storage:package:move", "storage:package:read", "storage:package:review", "storage:package:submit"}, - RequiredScopes: []string{"package:upload"}, - } - var token string - if ep.Payload.Token != nil { - token = *ep.Payload.Token - } - ctx, err = authJWTFn(ctx, token, &sc) - if err != nil { - return nil, err - } - return nil, s.Upload(ctx, ep.Payload, ep.Body) - } -} diff --git a/internal/api/gen/upload/service.go b/internal/api/gen/upload/service.go deleted file mode 100644 index 3077a54f8..000000000 --- a/internal/api/gen/upload/service.go +++ /dev/null @@ -1,211 +0,0 @@ -// Code generated by goa v3.15.2, DO NOT EDIT. -// -// upload service -// -// Command: -// $ goa gen github.com/artefactual-sdps/enduro/internal/api/design -o -// internal/api - -package upload - -import ( - "context" - "io" - - "github.com/google/uuid" - goa "goa.design/goa/v3/pkg" - "goa.design/goa/v3/security" -) - -// The upload service handles file submissions to the SIPs bucket. -type Service interface { - // Upload a package to trigger an ingest workflow - Upload(context.Context, *UploadPayload, io.ReadCloser) (err error) -} - -// Auther defines the authorization functions to be implemented by the service. -type Auther interface { - // JWTAuth implements the authorization logic for the JWT security scheme. - JWTAuth(ctx context.Context, token string, schema *security.JWTScheme) (context.Context, error) -} - -// APIName is the name of the API as defined in the design. -const APIName = "enduro" - -// APIVersion is the version of the API as defined in the design. -const APIVersion = "0.0.1" - -// ServiceName is the name of the service as defined in the design. This is the -// same value that is set in the endpoint request contexts under the ServiceKey -// key. -const ServiceName = "upload" - -// MethodNames lists the service method names as defined in the design. These -// are the same values that are set in the endpoint request contexts under the -// MethodKey key. -var MethodNames = [1]string{"upload"} - -// PreservationAction describes a preservation action. -type EnduroPackagePreservationAction struct { - ID uint - WorkflowID string - Type string - Status string - StartedAt string - CompletedAt *string - Tasks EnduroPackagePreservationTaskCollection - PackageID *uint -} - -// PreservationTask describes a preservation action task. -type EnduroPackagePreservationTask struct { - ID uint - TaskID string - Name string - Status string - StartedAt string - CompletedAt *string - Note *string - PreservationActionID *uint -} - -type EnduroPackagePreservationTaskCollection []*EnduroPackagePreservationTask - -// StoredPackage describes a package retrieved by the service. -type EnduroStoredPackage struct { - // Identifier of package - ID uint - // Name of the package - Name *string - // Identifier of storage location - LocationID *uuid.UUID - // Status of the package - Status string - // Identifier of processing workflow - WorkflowID *string - // Identifier of latest processing workflow run - RunID *string - // Identifier of AIP - AipID *string - // Creation datetime - CreatedAt string - // Start datetime - StartedAt *string - // Completion datetime - CompletedAt *string -} - -type MonitorPingEvent struct { - Message *string -} - -type PackageCreatedEvent struct { - // Identifier of package - ID uint - Item *EnduroStoredPackage -} - -type PackageLocationUpdatedEvent struct { - // Identifier of package - ID uint - // Identifier of storage location - LocationID uuid.UUID -} - -type PackageStatusUpdatedEvent struct { - // Identifier of package - ID uint - Status string -} - -type PackageUpdatedEvent struct { - // Identifier of package - ID uint - Item *EnduroStoredPackage -} - -type PreservationActionCreatedEvent struct { - // Identifier of preservation action - ID uint - Item *EnduroPackagePreservationAction -} - -type PreservationActionUpdatedEvent struct { - // Identifier of preservation action - ID uint - Item *EnduroPackagePreservationAction -} - -type PreservationTaskCreatedEvent struct { - // Identifier of preservation task - ID uint - Item *EnduroPackagePreservationTask -} - -type PreservationTaskUpdatedEvent struct { - // Identifier of preservation task - ID uint - Item *EnduroPackagePreservationTask -} - -// UploadPayload is the payload type of the upload service upload method. -type UploadPayload struct { - // Content-Type header, must define value for multipart boundary. - ContentType string - Token *string -} - -// Forbidden -type Forbidden string - -// Unauthorized -type Unauthorized string - -// Error returns an error description. -func (e Forbidden) Error() string { - return "Forbidden" -} - -// ErrorName returns "forbidden". -// -// Deprecated: Use GoaErrorName - https://github.com/goadesign/goa/issues/3105 -func (e Forbidden) ErrorName() string { - return e.GoaErrorName() -} - -// GoaErrorName returns "forbidden". -func (e Forbidden) GoaErrorName() string { - return "forbidden" -} - -// Error returns an error description. -func (e Unauthorized) Error() string { - return "Unauthorized" -} - -// ErrorName returns "unauthorized". -// -// Deprecated: Use GoaErrorName - https://github.com/goadesign/goa/issues/3105 -func (e Unauthorized) ErrorName() string { - return e.GoaErrorName() -} - -// GoaErrorName returns "unauthorized". -func (e Unauthorized) GoaErrorName() string { - return "unauthorized" -} - -// MakeInvalidMediaType builds a goa.ServiceError from an error. -func MakeInvalidMediaType(err error) *goa.ServiceError { - return goa.NewServiceError(err, "invalid_media_type", false, false, false) -} - -// MakeInvalidMultipartRequest builds a goa.ServiceError from an error. -func MakeInvalidMultipartRequest(err error) *goa.ServiceError { - return goa.NewServiceError(err, "invalid_multipart_request", false, false, false) -} - -// MakeInternalError builds a goa.ServiceError from an error. -func MakeInternalError(err error) *goa.ServiceError { - return goa.NewServiceError(err, "internal_error", false, false, false) -} diff --git a/internal/config/config.go b/internal/config/config.go index 9243c38a7..719456b58 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,12 +19,12 @@ import ( "github.com/artefactual-sdps/enduro/internal/api" "github.com/artefactual-sdps/enduro/internal/db" "github.com/artefactual-sdps/enduro/internal/event" + "github.com/artefactual-sdps/enduro/internal/package_" "github.com/artefactual-sdps/enduro/internal/preprocessing" "github.com/artefactual-sdps/enduro/internal/pres" "github.com/artefactual-sdps/enduro/internal/storage" "github.com/artefactual-sdps/enduro/internal/telemetry" "github.com/artefactual-sdps/enduro/internal/temporal" - "github.com/artefactual-sdps/enduro/internal/upload" "github.com/artefactual-sdps/enduro/internal/watcher" ) @@ -49,7 +49,7 @@ type Configuration struct { Preservation pres.Config Storage storage.Config Temporal temporal.Config - Upload upload.Config + Upload package_.UploadConfig Watcher watcher.Config Telemetry telemetry.Config } @@ -80,6 +80,7 @@ func Read(config *Configuration, configFile string) (found bool, configFileUsed v.SetDefault("preservation.taskqueue", temporal.A3mWorkerTaskQueue) v.SetDefault("storage.taskqueue", temporal.GlobalTaskQueue) v.SetDefault("temporal.taskqueue", temporal.GlobalTaskQueue) + v.SetDefault("upload.maxSize", 102400000) v.SetEnvPrefix("enduro") v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.AutomaticEnv() diff --git a/internal/package_/package_.go b/internal/package_/package_.go index 9b6643383..727c4e119 100644 --- a/internal/package_/package_.go +++ b/internal/package_/package_.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/jmoiron/sqlx" temporalsdk_client "go.temporal.io/sdk/client" + "gocloud.dev/blob" "github.com/artefactual-sdps/enduro/internal/api/auth" goapackage "github.com/artefactual-sdps/enduro/internal/api/gen/package_" @@ -61,6 +62,8 @@ type packageImpl struct { tokenVerifier auth.TokenVerifier ticketProvider *auth.TicketProvider taskQueue string + uploadBucket *blob.Bucket + uploadMaxSize int64 } var _ Service = (*packageImpl)(nil) @@ -74,6 +77,8 @@ func NewService( tokenVerifier auth.TokenVerifier, ticketProvider *auth.TicketProvider, taskQueue string, + uploadBucket *blob.Bucket, + uploadMaxSize int64, ) *packageImpl { return &packageImpl{ logger: logger, @@ -84,6 +89,8 @@ func NewService( tokenVerifier: tokenVerifier, ticketProvider: ticketProvider, taskQueue: taskQueue, + uploadBucket: uploadBucket, + uploadMaxSize: uploadMaxSize, } } diff --git a/internal/package_/package__test.go b/internal/package_/package__test.go index b9d7237e2..81d346a4c 100644 --- a/internal/package_/package__test.go +++ b/internal/package_/package__test.go @@ -11,6 +11,7 @@ import ( "go.artefactual.dev/tools/mockutil" temporalsdk_mocks "go.temporal.io/sdk/mocks" "go.uber.org/mock/gomock" + "gocloud.dev/blob" "gotest.tools/v3/assert" "github.com/artefactual-sdps/enduro/internal/api/auth" @@ -21,7 +22,7 @@ import ( persistence_fake "github.com/artefactual-sdps/enduro/internal/persistence/fake" ) -func testSvc(t *testing.T) (package_.Service, *persistence_fake.MockService) { +func testSvc(t *testing.T, b *blob.Bucket, s int64) (package_.Service, *persistence_fake.MockService) { t.Helper() psvc := persistence_fake.NewMockService(gomock.NewController(t)) @@ -34,6 +35,8 @@ func testSvc(t *testing.T) (package_.Service, *persistence_fake.MockService) { &auth.NoopTokenVerifier{}, &auth.TicketProvider{}, "test", + b, + s, ) return pkgSvc, psvc @@ -93,7 +96,7 @@ func TestCreatePackage(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - pkgSvc, perSvc := testSvc(t) + pkgSvc, perSvc := testSvc(t, nil, 0) if tt.mock != nil { tt.mock(perSvc, tt.pkg) } diff --git a/internal/package_/preservation_action_test.go b/internal/package_/preservation_action_test.go index d6dccdcac..b37495439 100644 --- a/internal/package_/preservation_action_test.go +++ b/internal/package_/preservation_action_test.go @@ -121,7 +121,7 @@ func TestCreatePreservationAction(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - pkgSvc, perSvc := testSvc(t) + pkgSvc, perSvc := testSvc(t, nil, 0) if tt.mock != nil { tt.mock(perSvc, tt.pa) } diff --git a/internal/package_/preservation_task_test.go b/internal/package_/preservation_task_test.go index 7162b9217..6233112ae 100644 --- a/internal/package_/preservation_task_test.go +++ b/internal/package_/preservation_task_test.go @@ -120,7 +120,7 @@ func TestCreatePreservationTask(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - pkgSvc, perSvc := testSvc(t) + pkgSvc, perSvc := testSvc(t, nil, 0) if tt.mock != nil { tt.mock(perSvc, tt.pt) } @@ -282,7 +282,7 @@ func TestCompletePreservationTask(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - pkgSvc, perSvc := testSvc(t) + pkgSvc, perSvc := testSvc(t, nil, 0) pt := datatypes.PreservationTask{ ID: 1, } diff --git a/internal/package_/upload.go b/internal/package_/upload.go new file mode 100644 index 000000000..87d2bc9aa --- /dev/null +++ b/internal/package_/upload.go @@ -0,0 +1,64 @@ +package package_ + +import ( + "context" + "errors" + "io" + "mime" + "mime/multipart" + + "go.artefactual.dev/tools/bucket" + "gocloud.dev/blob" + + goapackage "github.com/artefactual-sdps/enduro/internal/api/gen/package_" +) + +type UploadConfig struct { + MaxSize int64 + Bucket bucket.Config +} + +// Validate implements config.ConfigurationValidator. +func (c UploadConfig) Validate() error { + if c.Bucket.URL != "" && (c.Bucket.Bucket != "" || c.Bucket.Region != "") { + return errors.New("URL and rest of the [upload.bucket] configuration options are mutually exclusive") + } + return nil +} + +func (w *goaWrapper) Upload(ctx context.Context, payload *goapackage.UploadPayload, req io.ReadCloser) error { + defer req.Close() + + lr := io.LimitReader(req, int64(w.uploadMaxSize)) + + _, params, err := mime.ParseMediaType(payload.ContentType) + if err != nil { + return goapackage.MakeInvalidMediaType(errors.New("invalid media type")) + } + mr := multipart.NewReader(lr, params["boundary"]) + + part, err := mr.NextPart() + if err == io.EOF { + return nil + } + if err != nil { + return goapackage.MakeInvalidMultipartRequest(errors.New("invalid multipart request")) + } + + wr, err := w.uploadBucket.NewWriter(ctx, part.FileName(), &blob.WriterOptions{}) + if err != nil { + return err + } + + _, copyErr := io.Copy(wr, part) + closeErr := wr.Close() + + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + + return nil +} diff --git a/internal/package_/upload_test.go b/internal/package_/upload_test.go new file mode 100644 index 000000000..a063efe1c --- /dev/null +++ b/internal/package_/upload_test.go @@ -0,0 +1,123 @@ +package package__test + +import ( + "context" + "io" + "strings" + "testing" + + "go.artefactual.dev/tools/bucket" + goa "goa.design/goa/v3/pkg" + "gocloud.dev/blob/memblob" + "gotest.tools/v3/assert" + + goapackage "github.com/artefactual-sdps/enduro/internal/api/gen/package_" + "github.com/artefactual-sdps/enduro/internal/package_" +) + +func TestConfig(t *testing.T) { + t.Parallel() + + t.Run("Returns error if config is invalid", func(t *testing.T) { + t.Parallel() + + c := package_.UploadConfig{ + Bucket: bucket.Config{ + URL: "s3blob://my-bucket", + Bucket: "my-bucket", + Region: "planet-earth", + }, + } + err := c.Validate() + assert.ErrorContains(t, err, "URL and rest of the [upload.bucket] configuration options are mutually exclusive") + }) + + t.Run("Validates if only URL is provided", func(t *testing.T) { + t.Parallel() + + c := package_.UploadConfig{ + Bucket: bucket.Config{ + URL: "s3blob://my-bucket", + }, + } + err := c.Validate() + assert.NilError(t, err) + }) + + t.Run("Validates if only bucket options are provided", func(t *testing.T) { + t.Parallel() + + c := package_.UploadConfig{ + Bucket: bucket.Config{ + Bucket: "my-bucket", + Region: "planet-earth", + }, + } + err := c.Validate() + assert.NilError(t, err) + }) +} + +const multipartBody = `Content-Type: multipart/form-data; boundary="foobar" + +--foobar +Content-Disposition: form-data; name="field1"; filename="first.txt" +Content-Type: text/plain + +first +--foobar +Content-Disposition: form-data; name="field2"; filename="second.txt" +Content-Type: text/plain + +second +--foobar-- +` + +func TestUpload(t *testing.T) { + t.Parallel() + + t.Run("Writes only the first multipart of the request to the bucket", func(t *testing.T) { + t.Parallel() + + b := memblob.OpenBucket(nil) + svc, _ := testSvc(t, b, 102400000) + ctx := context.Background() + r := io.NopCloser(strings.NewReader(multipartBody)) + + err := svc.Goa().Upload(ctx, &goapackage.UploadPayload{ContentType: "multipart/form-data; boundary=foobar"}, r) + assert.NilError(t, err) + + data, err := b.ReadAll(ctx, "first.txt") + assert.NilError(t, err) + assert.Equal(t, string(data), "first") + + _, err = b.ReadAll(ctx, "second.txt") + assert.ErrorContains(t, err, `blob (key "second.txt") (code=NotFound): blob not found`) + }) + + t.Run("Returns invalid_media_type if media type cannot be parsed", func(t *testing.T) { + t.Parallel() + + b := memblob.OpenBucket(nil) + svc, _ := testSvc(t, b, 102400000) + ctx := context.Background() + r := io.NopCloser(strings.NewReader(multipartBody)) + + err := svc.Goa().Upload(ctx, &goapackage.UploadPayload{ContentType: "invalid type"}, r) + assert.Equal(t, err.(*goa.ServiceError).Name, "invalid_media_type") + assert.ErrorContains(t, err, "invalid media type") + }) + + t.Run("Returns invalid_multipart_request if request size is bigger than maximum size", func(t *testing.T) { + t.Parallel() + + b := memblob.OpenBucket(nil) + svc, _ := testSvc(t, b, 1) + ctx := context.Background() + r := io.NopCloser(strings.NewReader(multipartBody)) + + err := svc.Goa().Upload(ctx, &goapackage.UploadPayload{ContentType: "multipart/form-data; boundary=foobar"}, r) + assert.Equal(t, err.(*goa.ServiceError).Name, "invalid_multipart_request") + assert.ErrorContains(t, err, "invalid multipart request") + }) +} diff --git a/internal/upload/config.go b/internal/upload/config.go deleted file mode 100644 index 263ac5a4e..000000000 --- a/internal/upload/config.go +++ /dev/null @@ -1,25 +0,0 @@ -package upload - -import ( - "errors" -) - -type Config struct { - URL string - Region string - Endpoint string - PathStyle bool - Profile string - Key string - Secret string - Token string - Bucket string -} - -// Validate implements config.ConfigurationValidator. -func (c Config) Validate() error { - if c.URL != "" && (c.Bucket != "" || c.Region != "") { - return errors.New("URL and rest of the [upload] configuration options are mutually exclusive") - } - return nil -} diff --git a/internal/upload/config_test.go b/internal/upload/config_test.go deleted file mode 100644 index 0f0881673..000000000 --- a/internal/upload/config_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package upload_test - -import ( - "testing" - - "gotest.tools/v3/assert" - - "github.com/artefactual-sdps/enduro/internal/upload" -) - -func TestConfig(t *testing.T) { - t.Parallel() - - t.Run("Returns error if config is invalid", func(t *testing.T) { - t.Parallel() - - c := upload.Config{ - URL: "s3blob://my-bucket", - Bucket: "my-bucket", - Region: "planet-earth", - } - err := c.Validate() - assert.ErrorContains(t, err, "URL and rest of the [upload] configuration options are mutually exclusive") - }) - - t.Run("Validates if only URL is provided", func(t *testing.T) { - t.Parallel() - - c := upload.Config{ - URL: "s3blob://my-bucket", - } - err := c.Validate() - assert.NilError(t, err) - }) - - t.Run("Validates if only bucket options are provided", func(t *testing.T) { - t.Parallel() - - c := upload.Config{ - Bucket: "my-bucket", - Region: "planet-earth", - } - err := c.Validate() - assert.NilError(t, err) - }) -} diff --git a/internal/upload/fake/mock_upload.go b/internal/upload/fake/mock_upload.go deleted file mode 100644 index 7d9c52971..000000000 --- a/internal/upload/fake/mock_upload.go +++ /dev/null @@ -1,157 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/artefactual-sdps/enduro/internal/upload (interfaces: Service) -// -// Generated by this command: -// -// mockgen -typed -destination=./internal/upload/fake/mock_upload.go -package=fake github.com/artefactual-sdps/enduro/internal/upload Service -// - -// Package fake is a generated GoMock package. -package fake - -import ( - context "context" - io "io" - reflect "reflect" - - upload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - gomock "go.uber.org/mock/gomock" - blob "gocloud.dev/blob" -) - -// MockService is a mock of Service interface. -type MockService struct { - ctrl *gomock.Controller - recorder *MockServiceMockRecorder -} - -// MockServiceMockRecorder is the mock recorder for MockService. -type MockServiceMockRecorder struct { - mock *MockService -} - -// NewMockService creates a new mock instance. -func NewMockService(ctrl *gomock.Controller) *MockService { - mock := &MockService{ctrl: ctrl} - mock.recorder = &MockServiceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockService) EXPECT() *MockServiceMockRecorder { - return m.recorder -} - -// Bucket mocks base method. -func (m *MockService) Bucket() *blob.Bucket { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Bucket") - ret0, _ := ret[0].(*blob.Bucket) - return ret0 -} - -// Bucket indicates an expected call of Bucket. -func (mr *MockServiceMockRecorder) Bucket() *MockServiceBucketCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockService)(nil).Bucket)) - return &MockServiceBucketCall{Call: call} -} - -// MockServiceBucketCall wrap *gomock.Call -type MockServiceBucketCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockServiceBucketCall) Return(arg0 *blob.Bucket) *MockServiceBucketCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockServiceBucketCall) Do(f func() *blob.Bucket) *MockServiceBucketCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockServiceBucketCall) DoAndReturn(f func() *blob.Bucket) *MockServiceBucketCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Close mocks base method. -func (m *MockService) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockServiceMockRecorder) Close() *MockServiceCloseCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockService)(nil).Close)) - return &MockServiceCloseCall{Call: call} -} - -// MockServiceCloseCall wrap *gomock.Call -type MockServiceCloseCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockServiceCloseCall) Return(arg0 error) *MockServiceCloseCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockServiceCloseCall) Do(f func() error) *MockServiceCloseCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockServiceCloseCall) DoAndReturn(f func() error) *MockServiceCloseCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Upload mocks base method. -func (m *MockService) Upload(arg0 context.Context, arg1 *upload.UploadPayload, arg2 io.ReadCloser) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Upload", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Upload indicates an expected call of Upload. -func (mr *MockServiceMockRecorder) Upload(arg0, arg1, arg2 any) *MockServiceUploadCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upload", reflect.TypeOf((*MockService)(nil).Upload), arg0, arg1, arg2) - return &MockServiceUploadCall{Call: call} -} - -// MockServiceUploadCall wrap *gomock.Call -type MockServiceUploadCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockServiceUploadCall) Return(arg0 error) *MockServiceUploadCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockServiceUploadCall) Do(f func(context.Context, *upload.UploadPayload, io.ReadCloser) error) *MockServiceUploadCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockServiceUploadCall) DoAndReturn(f func(context.Context, *upload.UploadPayload, io.ReadCloser) error) *MockServiceUploadCall { - c.Call = c.Call.DoAndReturn(f) - return c -} diff --git a/internal/upload/goa_test.go b/internal/upload/goa_test.go deleted file mode 100644 index d73596cc8..000000000 --- a/internal/upload/goa_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package upload - -import ( - "context" - "fmt" - "testing" - - "github.com/go-logr/logr/funcr" - "go.uber.org/mock/gomock" - "goa.design/goa/v3/security" - "gotest.tools/v3/assert" - - "github.com/artefactual-sdps/enduro/internal/api/auth" - authfake "github.com/artefactual-sdps/enduro/internal/api/auth/fake" -) - -func TestJWTAuth(t *testing.T) { - t.Parallel() - - type test struct { - name string - mock func(tv *authfake.MockTokenVerifier, claims *auth.Claims) - claims *auth.Claims - scopes []string - logged string - wantErr error - } - for _, tt := range []test{ - { - name: "Verifies and adds claims to context", - mock: func(tv *authfake.MockTokenVerifier, claims *auth.Claims) { - tv.EXPECT(). - Verify(context.Background(), "abc"). - Return(claims, nil) - }, - claims: &auth.Claims{ - Email: "info@artefactual.com", - EmailVerified: true, - Attributes: []string{"*"}, - }, - scopes: []string{"package:read"}, - }, - { - name: "Fails with unauthorized error", - mock: func(tv *authfake.MockTokenVerifier, claims *auth.Claims) { - tv.EXPECT(). - Verify(context.Background(), "abc"). - Return(nil, auth.ErrUnauthorized) - }, - wantErr: ErrUnauthorized, - }, - { - name: "Fails with unauthorized error (logging)", - mock: func(tv *authfake.MockTokenVerifier, claims *auth.Claims) { - tv.EXPECT(). - Verify(context.Background(), "abc"). - Return(nil, fmt.Errorf("fail")) - }, - logged: `"level"=1 "msg"="failed to verify token" "err"="fail"`, - wantErr: ErrUnauthorized, - }, - { - name: "Fails with forbidden error", - mock: func(tv *authfake.MockTokenVerifier, claims *auth.Claims) { - tv.EXPECT(). - Verify(context.Background(), "abc"). - Return(claims, nil) - }, - claims: &auth.Claims{ - Email: "info@artefactual.com", - EmailVerified: true, - Attributes: []string{"package:list"}, - }, - scopes: []string{"package:read"}, - wantErr: ErrForbidden, - }, - } { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - var logged string - logger := funcr.New( - func(prefix, args string) { logged = args }, - funcr.Options{Verbosity: 1}, - ) - - tvMock := authfake.NewMockTokenVerifier(gomock.NewController(t)) - tt.mock(tvMock, tt.claims) - svc := &serviceImpl{ - logger: logger, - tokenVerifier: tvMock, - } - - ctx, err := svc.JWTAuth(context.Background(), "abc", &security.JWTScheme{RequiredScopes: tt.scopes}) - assert.Equal(t, logged, tt.logged) - if tt.wantErr != nil { - assert.ErrorIs(t, err, tt.wantErr) - return - } - assert.NilError(t, err) - assert.DeepEqual(t, auth.UserClaimsFromContext(ctx), tt.claims) - }) - } -} diff --git a/internal/upload/service.go b/internal/upload/service.go deleted file mode 100644 index 1900b21b3..000000000 --- a/internal/upload/service.go +++ /dev/null @@ -1,153 +0,0 @@ -package upload - -import ( - "context" - "errors" - "io" - "mime" - "mime/multipart" - "time" - - "github.com/go-logr/logr" - "go.artefactual.dev/tools/bucket" - "goa.design/goa/v3/security" - "gocloud.dev/blob" - - "github.com/artefactual-sdps/enduro/internal/api/auth" - goaupload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" -) - -const UPLOAD_MAX_SIZE = 102400000 // 100 MB - -type Service interface { - Upload(ctx context.Context, payload *goaupload.UploadPayload, req io.ReadCloser) error - - Bucket() *blob.Bucket - Close() error -} - -type serviceImpl struct { - logger logr.Logger - config Config - bucket *blob.Bucket - uploadMaxSize int - tokenVerifier auth.TokenVerifier -} - -var _ Service = (*serviceImpl)(nil) - -var ( - ErrUnauthorized error = goaupload.Unauthorized("Unauthorized") - ErrForbidden error = goaupload.Forbidden("Forbidden") -) - -func NewService( - logger logr.Logger, - config Config, - uploadMaxSize int, - tokenVerifier auth.TokenVerifier, -) (s *serviceImpl, err error) { - s = &serviceImpl{ - logger: logger, - config: config, - uploadMaxSize: uploadMaxSize, - tokenVerifier: tokenVerifier, - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) - defer cancel() - - err = s.openBucket(ctx, config) - if err != nil { - return nil, err - } - - return s, nil -} - -func (s *serviceImpl) openBucket(ctx context.Context, config Config) error { - if b, err := bucket.NewWithConfig(ctx, &bucket.Config{ - URL: config.URL, - Endpoint: config.Endpoint, - Bucket: config.Bucket, - AccessKey: config.Key, - SecretKey: config.Secret, - Token: config.Token, - Profile: config.Profile, - Region: config.Region, - PathStyle: config.PathStyle, - }); err != nil { - return err - } else { - s.bucket = b - } - - return nil -} - -func (s *serviceImpl) JWTAuth( - ctx context.Context, - token string, - scheme *security.JWTScheme, -) (context.Context, error) { - claims, err := s.tokenVerifier.Verify(ctx, token) - if err != nil { - if !errors.Is(err, auth.ErrUnauthorized) { - s.logger.V(1).Info("failed to verify token", "err", err) - } - return ctx, ErrUnauthorized - } - - if !claims.CheckAttributes(scheme.RequiredScopes) { - return ctx, ErrForbidden - } - - ctx = auth.WithUserClaims(ctx, claims) - - return ctx, nil -} - -func (s *serviceImpl) Bucket() *blob.Bucket { - return s.bucket -} - -func (s *serviceImpl) Close() error { - return s.bucket.Close() -} - -func (s *serviceImpl) Upload(ctx context.Context, payload *goaupload.UploadPayload, req io.ReadCloser) error { - defer req.Close() - - lr := io.LimitReader(req, int64(s.uploadMaxSize)) - - _, params, err := mime.ParseMediaType(payload.ContentType) - if err != nil { - return goaupload.MakeInvalidMediaType(errors.New("invalid media type")) - } - mr := multipart.NewReader(lr, params["boundary"]) - - part, err := mr.NextPart() - if err == io.EOF { - return nil - } - if err != nil { - return goaupload.MakeInvalidMultipartRequest(errors.New("invalid multipart request")) - } - - w, err := s.bucket.NewWriter(ctx, part.FileName(), &blob.WriterOptions{}) - if err != nil { - return err - } - - _, copyErr := io.Copy(w, part) - closeErr := w.Close() - - if copyErr != nil { - return copyErr - } - if closeErr != nil { - return closeErr - } - - return nil -} diff --git a/internal/upload/service_test.go b/internal/upload/service_test.go deleted file mode 100644 index 66e04092f..000000000 --- a/internal/upload/service_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package upload_test - -import ( - "context" - "io" - "strings" - "testing" - - "github.com/go-logr/logr" - "go.artefactual.dev/tools/ref" - goa "goa.design/goa/v3/pkg" - _ "gocloud.dev/blob/memblob" - "gotest.tools/v3/assert" - - "github.com/artefactual-sdps/enduro/internal/api/auth" - goaupload "github.com/artefactual-sdps/enduro/internal/api/gen/upload" - "github.com/artefactual-sdps/enduro/internal/upload" -) - -type setUpAttrs struct { - logger *logr.Logger - config *upload.Config - uploadMaxSize *int - tokenVerifier auth.TokenVerifier -} - -func setUpService(t *testing.T, attrs *setUpAttrs) upload.Service { - t.Helper() - - params := setUpAttrs{ - logger: ref.New(logr.Discard()), - config: ref.New(upload.Config{URL: "mem://my-bucket"}), - uploadMaxSize: ref.New(upload.UPLOAD_MAX_SIZE), - tokenVerifier: &auth.OIDCTokenVerifier{}, - } - if attrs.logger != nil { - params.logger = attrs.logger - } - if attrs.config != nil { - params.config = attrs.config - } - if attrs.uploadMaxSize != nil { - params.uploadMaxSize = attrs.uploadMaxSize - } - if attrs.tokenVerifier != nil { - params.tokenVerifier = attrs.tokenVerifier - } - - s, err := upload.NewService( - *params.logger, - *params.config, - *params.uploadMaxSize, - params.tokenVerifier, - ) - assert.NilError(t, err) - - return s -} - -const multipartBody = `Content-Type: multipart/form-data; boundary="foobar" - ---foobar -Content-Disposition: form-data; name="field1"; filename="first.txt" -Content-Type: text/plain - -first ---foobar -Content-Disposition: form-data; name="field2"; filename="second.txt" -Content-Type: text/plain - -second ---foobar-- -` - -func TestNewService(t *testing.T) { - t.Parallel() - - _, err := upload.NewService( - logr.Discard(), - upload.Config{URL: "mem://my-bucket"}, - upload.UPLOAD_MAX_SIZE, - &auth.OIDCTokenVerifier{}, - ) - assert.NilError(t, err) -} - -func TestServiceUpload(t *testing.T) { - t.Parallel() - - t.Run("Writes only the first multipart of the request to the bucket", func(t *testing.T) { - t.Parallel() - - attrs := &setUpAttrs{} - svc := setUpService(t, attrs) - ctx := context.Background() - r := io.NopCloser(strings.NewReader(multipartBody)) - - err := svc.Upload(ctx, &goaupload.UploadPayload{ContentType: "multipart/form-data; boundary=foobar"}, r) - assert.NilError(t, err) - - b := svc.Bucket() - data, err := b.ReadAll(ctx, "first.txt") - assert.NilError(t, err) - assert.Equal(t, string(data), "first") - - _, err = b.ReadAll(ctx, "second.txt") - assert.ErrorContains(t, err, `blob (key "second.txt") (code=NotFound): blob not found`) - }) - - t.Run("Returns invalid_media_type if media type cannot be parsed", func(t *testing.T) { - t.Parallel() - - attrs := &setUpAttrs{ - uploadMaxSize: ref.New(1), - } - svc := setUpService(t, attrs) - ctx := context.Background() - r := io.NopCloser(strings.NewReader(multipartBody)) - - err := svc.Upload(ctx, &goaupload.UploadPayload{ContentType: "invalid type"}, r) - assert.Equal(t, err.(*goa.ServiceError).Name, "invalid_media_type") - assert.ErrorContains(t, err, "invalid media type") - }) - - t.Run("Returns invalid_multipart_request if request size is bigger than maximum size", func(t *testing.T) { - t.Parallel() - - attrs := &setUpAttrs{ - uploadMaxSize: ref.New(1), - } - svc := setUpService(t, attrs) - ctx := context.Background() - r := io.NopCloser(strings.NewReader(multipartBody)) - - err := svc.Upload(ctx, &goaupload.UploadPayload{ContentType: "multipart/form-data; boundary=foobar"}, r) - assert.Equal(t, err.(*goa.ServiceError).Name, "invalid_multipart_request") - assert.ErrorContains(t, err, "invalid multipart request") - }) -}