diff --git a/changelog/unreleased/http-tpc.md b/changelog/unreleased/http-tpc.md new file mode 100644 index 0000000000..dd1f80c4dd --- /dev/null +++ b/changelog/unreleased/http-tpc.md @@ -0,0 +1,15 @@ +Enhancement: Add support for HTTP TPC + +We have added support for HTTP Third Party Copy. +This allows remote data transfers between storages managed by either two different reva servers, +or a reva server and a Grid (WLCG/ESCAPE) site server. + +Such remote transfers are expected to be driven by [GFAL](https://cern.ch/dmc-docs/gfal2/gfal2.html), +the underlying library used by [FTS](https://cern.ch/fts), and [Rucio](https://rucio.cern.ch). + +In addition, the oidcmapping package has been refactored to +support the standard OIDC use cases as well when no mapping +is defined. + +https://github.com/cs3org/reva/issues/1787 +https://github.com/cs3org/reva/pull/2007 diff --git a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md index a7309eb3e1..e05ce5cc7c 100644 --- a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md +++ b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L57) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L59) {{< highlight toml >}} [auth.manager.oidcmapping] insecure = false @@ -17,7 +17,7 @@ insecure = false {{% /dir %}} {{% dir name="issuer" type="string" default="" %}} -The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L58) +The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L60) {{< highlight toml >}} [auth.manager.oidcmapping] issuer = "" @@ -25,7 +25,7 @@ issuer = "" {{% /dir %}} {{% dir name="id_claim" type="string" default="sub" %}} -The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L59) +The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L61) {{< highlight toml >}} [auth.manager.oidcmapping] id_claim = "sub" @@ -33,7 +33,7 @@ id_claim = "sub" {{% /dir %}} {{% dir name="uid_claim" type="string" default="" %}} -The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L60) +The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L62) {{< highlight toml >}} [auth.manager.oidcmapping] uid_claim = "" @@ -41,26 +41,42 @@ uid_claim = "" {{% /dir %}} {{% dir name="gid_claim" type="string" default="" %}} -The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L61) +The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L63) {{< highlight toml >}} [auth.manager.oidcmapping] gid_claim = "" {{< /highlight >}} {{% /dir %}} +{{% dir name="gatewaysvc" type="string" default="" %}} +The endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L64) +{{< highlight toml >}} +[auth.manager.oidcmapping] +gatewaysvc = "" +{{< /highlight >}} +{{% /dir %}} + {{% dir name="userprovidersvc" type="string" default="" %}} -The endpoint at which the GRPC userprovider is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L62) +The endpoint at which the GRPC userprovider is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L65) {{< highlight toml >}} [auth.manager.oidcmapping] userprovidersvc = "" {{< /highlight >}} {{% /dir %}} -{{% dir name="usersmapping" type="string" default="" %}} - The OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L63) +{{% dir name="users_mapping" type="string" default="" %}} + The optional OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L66) +{{< highlight toml >}} +[auth.manager.oidcmapping] +users_mapping = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="group_claim" type="string" default="" %}} + The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L67) {{< highlight toml >}} [auth.manager.oidcmapping] -usersmapping = "" +group_claim = "" {{< /highlight >}} {{% /dir %}} diff --git a/examples/oidc-mapping/users-oidcmapping.toml b/examples/oidc-mapping-tpc/oidcmapping-1.toml similarity index 57% rename from examples/oidc-mapping/users-oidcmapping.toml rename to examples/oidc-mapping-tpc/oidcmapping-1.toml index 54bba68ac3..166c96880f 100644 --- a/examples/oidc-mapping/users-oidcmapping.toml +++ b/examples/oidc-mapping-tpc/oidcmapping-1.toml @@ -12,12 +12,18 @@ auth_manager = "oidcmapping" [grpc.services.authprovider.auth_managers.json] users = "users.json" [grpc.services.authprovider.auth_managers.oidcmapping] -issuer = "http://iam-login-service:8080/" -userprovidersvc = "0.0.0.0:13000" +gatewaysvc = "localhost:19000" +issuer = "https://iam-escape.cloud.cnaf.infn.it/" +# ESCAPE adopted the WLCG groups as group claims +group_claim = "wlcg.groups" # The OIDC users mapping file path -usersmapping = "/go/src/github/cs3org/reva/examples/oidc-mapping/users-oidcmapping.json" +users_mapping = "users-oidcmapping-1.demo.json" +# If your local identity provider service configuration includes further claims, +# please configure them also here +#uid_claim = "" +#gid_claim = "" [grpc.services.userprovider] driver = "json" [grpc.services.userprovider.drivers.json] -users = "users.json" +users = "users.demo.json" diff --git a/examples/oidc-mapping-tpc/oidcmapping-2.toml b/examples/oidc-mapping-tpc/oidcmapping-2.toml new file mode 100644 index 0000000000..51eb894eed --- /dev/null +++ b/examples/oidc-mapping-tpc/oidcmapping-2.toml @@ -0,0 +1,29 @@ +[shared] +jwt_secret = "Pive-Fumkiu4" + +# This toml config file will start a reva service that: +# - handles user metadata and user preferences +# - serves the grpc services on port 14000 +[grpc] +address = "0.0.0.0:14000" + +[grpc.services.authprovider] +auth_manager = "oidcmapping" +[grpc.services.authprovider.auth_managers.json] +users = "users.json" +[grpc.services.authprovider.auth_managers.oidcmapping] +gatewaysvc = "localhost:17000" +issuer = "https://iam-escape.cloud.cnaf.infn.it/" +# ESCAPE adopted the WLCG groups as group claims +group_claim = "wlcg.groups" +# The OIDC users mapping file path +users_mapping = "users-oidcmapping-2.demo.json" +# If your local identity provider service configuration includes further claims, +# please configure them also here +#uid_claim = "" +#gid_claim = "" + +[grpc.services.userprovider] +driver = "json" +[grpc.services.userprovider.drivers.json] +users = "users.demo.json" diff --git a/examples/oidc-mapping-tpc/providers.demo.json b/examples/oidc-mapping-tpc/providers.demo.json new file mode 100644 index 0000000000..05aa6c78d3 --- /dev/null +++ b/examples/oidc-mapping-tpc/providers.demo.json @@ -0,0 +1,198 @@ +[ + { + "name": "cernbox", + "full_name": "CERNBox", + "organization": "CERN", + "domain": "cernbox.cern.ch", + "homepage": "https://cernbox.web.cern.ch", + "description": "CERNBox provides cloud data storage to all CERN users.", + "services": [ + { + "endpoint": { + "type": { + "name": "OCM", + "description": "CERNBox Open Cloud Mesh API" + }, + "name": "CERNBox - OCM API", + "path": "http://127.0.0.1:19001/ocm/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:19001/" + }, + { + "endpoint": { + "type": { + "name": "Webdav", + "description": "CERNBox Webdav API" + }, + "name": "CERNBox - Webdav API", + "path": "http://127.0.0.1:19001/remote.php/webdav/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:19001/" + }, + { + "endpoint": { + "type": { + "name": "Gateway", + "description": "CERNBox GRPC Gateway" + }, + "name": "CERNBox - GRPC Gateway", + "path": "127.0.0.1:19000", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:19000" + } + ] + }, + { + "name": "oc-cesnet", + "full_name": "ownCloud@CESNET", + "organization": "CESNET", + "domain": "cesnet.cz", + "homepage": "https://owncloud.cesnet.cz", + "description": "OwnCloud has been designed for individual users.", + "services": [ + { + "endpoint": { + "type": { + "name": "OCM", + "description": "CESNET Open Cloud Mesh API" + }, + "name": "CESNET - OCM API", + "path": "http://127.0.0.1:17001/ocm/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:17001/" + }, + { + "endpoint": { + "type": { + "name": "Webdav", + "description": "CESNET Webdav API" + }, + "name": "CESNET - Webdav API", + "path": "http://127.0.0.1:17001/remote.php/webdav/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:17001/" + }, + { + "endpoint": { + "type": { + "name": "Gateway", + "description": "CESNET GRPC Gateway" + }, + "name": "CESNET - GRPC Gateway", + "path": "127.0.0.1:17000", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:17000" + } + ] + }, + { + "name": "example", + "full_name": "ownCloud@Example", + "organization": "Example", + "domain": "example.org", + "homepage": "http://example.org", + "description": "Example cloud storage.", + "services": [ + { + "endpoint": { + "type": { + "name": "OCM", + "description": "Example Open Cloud Mesh API" + }, + "name": "Example - OCM API", + "path": "http://127.0.0.1:19001/ocm/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:19001/" + }, + { + "endpoint": { + "type": { + "name": "Webdav", + "description": "Example Webdav API" + }, + "name": "Example - Webdav API", + "path": "http://127.0.0.1:19001/remote.php/webdav/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:19001/" + }, + { + "endpoint": { + "type": { + "name": "Gateway", + "description": "Example GRPC Gateway" + }, + "name": "Example - GRPC Gateway", + "path": "127.0.0.1:19000", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:19000" + } + ] + }, + { + "name": "test", + "full_name": "ownCloud@Test", + "organization": "Test", + "domain": "test.org", + "homepage": "http://test.org", + "description": "Test cloud storage.", + "services": [ + { + "endpoint": { + "type": { + "name": "OCM", + "description": "Test Open Cloud Mesh API" + }, + "name": "Test - OCM API", + "path": "http://127.0.0.1:19001/ocm/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:19001/" + }, + { + "endpoint": { + "type": { + "name": "Webdav", + "description": "Test Webdav API" + }, + "name": "Test - Webdav API", + "path": "http://127.0.0.1:19001/remote.php/webdav/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "http://127.0.0.1:19001/" + }, + { + "endpoint": { + "type": { + "name": "Gateway", + "description": "Test GRPC Gateway" + }, + "name": "Test - GRPC Gateway", + "path": "127.0.0.1:19000", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:19000" + } + ] + } +] diff --git a/examples/oidc-mapping-tpc/server-1.toml b/examples/oidc-mapping-tpc/server-1.toml new file mode 100644 index 0000000000..c212892e59 --- /dev/null +++ b/examples/oidc-mapping-tpc/server-1.toml @@ -0,0 +1,67 @@ +[grpc] +address = "0.0.0.0:19000" + +[shared] +jwt_secret = "jwt_secret" +gatewaysvc = "localhost:19000" + +# services to enable +[grpc.services.gateway] +commit_share_to_storage_grant = true +commit_share_to_storage_ref = true + +[grpc.services.authprovider] +[grpc.services.authprovider.auth_managers.json] +users = "users.demo.json" + +[grpc.services.userprovider.drivers.json] +users = "users.demo.json" + +[grpc.services.authregistry] +[grpc.services.authregistry.drivers.static.rules] +bearer = "localhost:13000" + +[grpc.services.storageregistry] +[grpc.services.storageregistry.drivers.static] +home_provider = "/home" + +[grpc.services.storageregistry.drivers.static.rules] +"/home" = {"address" = "localhost:19000"} +"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:19000"} + +[grpc.services.storageprovider] +driver = "localhome" +mount_path = "/home" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://localhost:19001/data" +enable_home_creation = true + +[grpc.services.usershareprovider] +[grpc.services.groupprovider] +[grpc.services.publicshareprovider] +[grpc.services.ocmcore] + +[grpc.services.ocmshareprovider] +gateway_addr = "0.0.0.0:19000" + +[grpc.services.ocminvitemanager] +[grpc.services.ocmproviderauthorizer] +[grpc.services.ocmproviderauthorizer.drivers.json] +providers = "providers.demo.json" + +[http.middlewares.providerauthorizer.drivers.json] +providers = "providers.demo.json" + +[http] +address = "0.0.0.0:19001" + +[http.services.dataprovider] +driver = "localhome" + +[http.services.datagateway] +[http.services.prometheus] +[http.services.ocmd] +[http.services.ocs] +[http.services.ocdav] +enable_http_tpc = true diff --git a/examples/oidc-mapping-tpc/server-2.toml b/examples/oidc-mapping-tpc/server-2.toml new file mode 100644 index 0000000000..259c4b77d8 --- /dev/null +++ b/examples/oidc-mapping-tpc/server-2.toml @@ -0,0 +1,67 @@ +[grpc] +address = "0.0.0.0:17000" + +[shared] +jwt_secret = "jwt_secret" +gatewaysvc = "localhost:17000" + +# services to enable +[grpc.services.gateway] +commit_share_to_storage_grant = true +commit_share_to_storage_ref = true + +[grpc.services.authprovider] +[grpc.services.authprovider.auth_managers.json] +users = "users.demo.json" + +[grpc.services.userprovider.drivers.json] +users = "users.demo.json" + +[grpc.services.authregistry] +[grpc.services.authregistry.drivers.static.rules] +bearer = "localhost:14000" + +[grpc.services.storageregistry] +[grpc.services.storageregistry.drivers.static] +home_provider = "/home" + +[grpc.services.storageregistry.drivers.static.rules] +"/home" = {"address" = "localhost:17000"} +"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:17000"} + +[grpc.services.storageprovider] +driver = "localhome" +mount_path = "/home" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://localhost:17001/data" +enable_home_creation = true + +[grpc.services.usershareprovider] +[grpc.services.groupprovider] +[grpc.services.publicshareprovider] +[grpc.services.ocmcore] + +[grpc.services.ocmshareprovider] +gateway_addr = "0.0.0.0:17000" + +[grpc.services.ocminvitemanager] +[grpc.services.ocmproviderauthorizer] +[grpc.services.ocmproviderauthorizer.drivers.json] +providers = "providers.demo.json" + +[http.middlewares.providerauthorizer.drivers.json] +providers = "providers.demo.json" + +[http] +address = "0.0.0.0:17001" + +[http.services.dataprovider] +driver = "localhome" + +[http.services.datagateway] +[http.services.prometheus] +[http.services.ocmd] +[http.services.ocs] +[http.services.ocdav] +enable_http_tpc = true diff --git a/examples/oidc-mapping/users-oidcmapping.json b/examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json similarity index 62% rename from examples/oidc-mapping/users-oidcmapping.json rename to examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json index 8678ed60b0..2eed64a504 100644 --- a/examples/oidc-mapping/users-oidcmapping.json +++ b/examples/oidc-mapping-tpc/users-oidcmapping-1.demo.json @@ -5,8 +5,8 @@ "username": "einstein" }, { - "oidc_issuer": "http://iam-login-service:8080/", - "oidc_group": "Sciencemesh", + "oidc_issuer": "https://iam-escape.cloud.cnaf.infn.it/", + "oidc_group": "/escape", "username": "marie" } ] diff --git a/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json b/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json new file mode 100644 index 0000000000..0742a2b1cd --- /dev/null +++ b/examples/oidc-mapping-tpc/users-oidcmapping-2.demo.json @@ -0,0 +1,7 @@ +[ + { + "oidc_issuer": "https://iam-escape.cloud.cnaf.infn.it/", + "oidc_group": "/escape", + "username": "einstein" + } +] diff --git a/examples/oidc-mapping/users.json b/examples/oidc-mapping-tpc/users.demo.json similarity index 64% rename from examples/oidc-mapping/users.json rename to examples/oidc-mapping-tpc/users.demo.json index 342e54b900..38932b65a0 100644 --- a/examples/oidc-mapping/users.json +++ b/examples/oidc-mapping-tpc/users.demo.json @@ -2,7 +2,8 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "reva-oidc-escape:20080" + "idp": "http://localhost:20080", + "type": 1 }, "username": "einstein", "secret": "relativity", @@ -13,7 +14,8 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "reva-oidc-escape:20080" + "idp": "http://localhost:20080", + "type": 1 }, "username": "marie", "secret": "radioactivity", @@ -24,7 +26,8 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "reva-oidc-escape:20080" + "idp": "http://localhost:20080", + "type": 1 }, "username": "richard", "secret": "superfluidity", @@ -34,13 +37,14 @@ }, { "id": { - "opaque_id": "4029579c-6ad5-4cec-a9ce-e843f77de452", - "idp": "reva-oidc-escape:20080" + "opaque_id": "0e4d9dc1-8349-49fe-8afc-6b844aec1cf6", + "idp": "http://localhost:20080", + "type": 7 }, - "username": "jimmie", - "secret": "spokenword", - "mail": "jimmie@surfsara.nl", - "display_name": "Jimmie Rigg", - "groups": ["sailing-lovers", "violin-haters", "physics-lovers"] + "username": "lwaccount", + "secret": "lightweight", + "mail": "lwaccount@example.org", + "display_name": "Lightweight Test Account", + "groups": ["guest-users", "weight-loss-club", "physics-lovers"] } ] diff --git a/examples/oidc-mapping/gateway.toml b/examples/oidc-mapping/gateway.toml deleted file mode 100644 index 7e43c757de..0000000000 --- a/examples/oidc-mapping/gateway.toml +++ /dev/null @@ -1,39 +0,0 @@ -[shared] -jwt_secret = "jwt_secret" - -# services to enable -[grpc.services.gateway] -commit_share_to_storage_grant = true -commit_share_to_storage_ref = true - -[grpc.services.storageregistry] -[grpc.services.storageregistry.drivers.static] -home_provider = "/home" - -[grpc.services.authregistry.drivers.static.rules] -oidcmapping = "localhost:13000" - -[grpc.services.storageregistry.drivers.static.rules."/home"] -address = "localhost:17000" -[grpc.services.storageregistry.drivers.static.rules."/reva"] -address = "localhost:18000" -[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"] -address = "localhost:18000" - -[grpc.services.authregistry] -[grpc.services.usershareprovider] -[grpc.services.groupprovider] -[grpc.services.publicshareprovider] -[grpc.services.ocmcore] - -[grpc.services.ocmshareprovider] -gateway_addr = "0.0.0.0:19000" - -[grpc.services.ocminvitemanager] -[grpc.services.ocmproviderauthorizer] - -[http.services.datagateway] -[http.services.prometheus] -[http.services.ocmd] -[http.services.ocdav] -[http.services.ocs] diff --git a/examples/oidc-mapping/storage-home.toml b/examples/oidc-mapping/storage-home.toml deleted file mode 100644 index 0323c24455..0000000000 --- a/examples/oidc-mapping/storage-home.toml +++ /dev/null @@ -1,17 +0,0 @@ -[grpc] -address = "0.0.0.0:17000" - -[grpc.services.storageprovider] -driver = "localhome" -mount_path = "/home" -mount_id = "123e4567-e89b-12d3-a456-426655440000" -data_server_url = "http://localhost:17001/data" - -[grpc.services.storageprovider.drivers.localhome] -shadow = "shadowfolder" - -[http] -address = "0.0.0.0:17001" - -[http.services.dataprovider] -driver = "localhome" diff --git a/internal/grpc/interceptors/recovery/recovery.go b/internal/grpc/interceptors/recovery/recovery.go index 7abda41ad6..7ebe0983ae 100644 --- a/internal/grpc/interceptors/recovery/recovery.go +++ b/internal/grpc/interceptors/recovery/recovery.go @@ -47,6 +47,6 @@ func NewStream() grpc.StreamServerInterceptor { func recoveryFunc(ctx context.Context, p interface{}) (err error) { debug.PrintStack() log := appctx.GetLogger(ctx) - log.Error().Msgf("%+v", p) + log.Error().Msgf("%+v; stack: %s", p, debug.Stack()) return status.Errorf(codes.Internal, "%s", p) } diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index eefb8d74e1..55a6ea6fc0 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -52,18 +52,35 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string) ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "copy") defer span.End() + if s.c.EnableHTTPTpc { + if r.Header.Get("Source") != "" { + // HTTP Third-Party Copy Pull mode + s.handleTPCPull(ctx, w, r, ns) + return + } else if r.Header.Get("Destination") != "" { + // HTTP Third-Party Copy Push mode + s.handleTPCPush(ctx, w, r, ns) + return + } + } + + // Local copy: in this case Destination is mandatory src := path.Join(ns, r.URL.Path) dst, err := extractDestination(r) if err != nil { + appctx.GetLogger(ctx).Warn().Msg("HTTP COPY: failed to extract destination") w.WriteHeader(http.StatusBadRequest) return } + for _, r := range nameRules { if !r.Test(dst) { + appctx.GetLogger(ctx).Warn().Msgf("HTTP COPY: destination %s failed validation", dst) w.WriteHeader(http.StatusBadRequest) return } } + dst = path.Join(ns, dst) sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger() diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 034bc8c5fa..0f51ef2a82 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -96,10 +96,12 @@ type Config struct { // Example: if WebdavNamespace is /users/{{substr 0 1 .Username}}/{{.Username}} // and received path is /docs the internal path will be: // /users///docs - WebdavNamespace string `mapstructure:"webdav_namespace"` - GatewaySvc string `mapstructure:"gatewaysvc"` - Timeout int64 `mapstructure:"timeout"` - Insecure bool `mapstructure:"insecure"` + WebdavNamespace string `mapstructure:"webdav_namespace"` + GatewaySvc string `mapstructure:"gatewaysvc"` + Timeout int64 `mapstructure:"timeout"` + Insecure bool `mapstructure:"insecure"` + // If true, HTTP COPY will expect the HTTP-TPC (third-party copy) headers + EnableHTTPTpc bool `mapstructure:"enable_http_tpc"` PublicURL string `mapstructure:"public_url"` FavoriteStorageDriver string `mapstructure:"favorite_storage_driver"` FavoriteStorageDrivers map[string]map[string]interface{} `mapstructure:"favorite_storage_drivers"` diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go new file mode 100644 index 0000000000..c31b6066a4 --- /dev/null +++ b/internal/http/services/owncloud/ocdav/tpc.go @@ -0,0 +1,415 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocdav + +import ( + "context" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + "time" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/internal/http/services/datagateway" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/rhttp" +) + +const ( + // PerfMarkerResponseTime corresponds to the interval at which a performance marker is sent back to the TPC client + PerfMarkerResponseTime float64 = 5 +) + +// PerfResponse provides a single chunk of permormance marker response +type PerfResponse struct { + Timestamp time.Time + Bytes uint64 + Index int + Count int +} + +func (p *PerfResponse) getPerfResponseString() string { + var sb strings.Builder + sb.WriteString("Perf Marker\n") + sb.WriteString("Timestamp: " + strconv.FormatInt(p.Timestamp.Unix(), 10) + "\n") + sb.WriteString("Stripe Bytes Transferred: " + strconv.FormatUint(p.Bytes, 10) + "\n") + sb.WriteString("Strip Index: " + strconv.Itoa(p.Index) + "\n") + sb.WriteString("Total Stripe Count: " + strconv.Itoa(p.Count) + "\n") + sb.WriteString("End\n") + return sb.String() +} + +// WriteCounter counts the number of bytes transferred and reports +// back to the TPC client about the progress of the transfer +// through the performance marker response stream. +type WriteCounter struct { + Total uint64 + PrevTime time.Time + w http.ResponseWriter +} + +// SendPerfMarker flushes a single chunk (performance marker) as +// part of the chunked transfer encoding scheme. +func (wc *WriteCounter) SendPerfMarker(size uint64) { + flusher, ok := wc.w.(http.Flusher) + if !ok { + panic("expected http.ResponseWriter to be an http.Flusher") + } + perfResp := PerfResponse{time.Now(), size, 0, 1} + pString := perfResp.getPerfResponseString() + fmt.Fprintln(wc.w, pString) + flusher.Flush() +} + +func (wc *WriteCounter) Write(p []byte) (int, error) { + + n := len(p) + wc.Total += uint64(n) + NowTime := time.Now() + + diff := NowTime.Sub(wc.PrevTime).Seconds() + if diff >= PerfMarkerResponseTime { + wc.SendPerfMarker(wc.Total) + wc.PrevTime = NowTime + } + return n, nil +} + +// +// An example of an HTTP TPC Pull +// +// +-----------------+ GET +----------------+ +// | Src server | <---------------- | Dest server | +// | (Remote) | ----------------> | (Reva) | +// +-----------------+ Data +----------------+ +// ^ +// | +// | COPY +// | +// +----------+ +// | Client | +// +----------+ + +// handleTPCPull performs a GET request on the remote site and upload it +// the requested reva endpoint. +func (s *svc) handleTPCPull(ctx context.Context, w http.ResponseWriter, r *http.Request, ns string) { + src := r.Header.Get("Source") + dst := path.Join(ns, r.URL.Path) + sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger() + + overwrite, err := extractOverwrite(w, r) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + m := fmt.Sprintf("Overwrite header is set to incorrect value %v", overwrite) + sublog.Warn().Msgf("HTTP TPC Pull: %s", m) + b, err := Marshal(exception{ + code: SabredavBadRequest, + message: m, + }) + HandleWebdavError(&sublog, w, b, err) + return + } + sublog.Debug().Str("overwrite", overwrite).Msg("TPC Pull") + + // get Gateway client + client, err := s.getClient() + if err != nil { + sublog.Error().Err(err).Msg("error getting grpc client") + w.WriteHeader(http.StatusInternalServerError) + return + } + + // check if destination exists + ref := &provider.Reference{Path: dst} + dstStatReq := &provider.StatRequest{Ref: ref} + dstStatRes, err := client.Stat(ctx, dstStatReq) + if err != nil { + sublog.Error().Err(err).Msg("error sending grpc stat request") + w.WriteHeader(http.StatusInternalServerError) + return + } + if dstStatRes.Status.Code != rpc.Code_CODE_OK && dstStatRes.Status.Code != rpc.Code_CODE_NOT_FOUND { + HandleErrorStatus(&sublog, w, dstStatRes.Status) + return + } + if dstStatRes.Status.Code == rpc.Code_CODE_OK && overwrite == "F" { + sublog.Warn().Str("overwrite", overwrite).Msg("Destination already exists") + w.WriteHeader(http.StatusPreconditionFailed) // 412, see https://tools.ietf.org/html/rfc4918#section-9.8.5 + return + } + + err = s.performHTTPPull(ctx, client, r, w, ns) + if err != nil { + sublog.Error().Err(err).Msg("error performing TPC Pull") + return + } + fmt.Fprintf(w, "success: Created") +} + +func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, ns string) error { + src := r.Header.Get("Source") + dst := path.Join(ns, r.URL.Path) + sublog := appctx.GetLogger(ctx) + sublog.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Pull") + + // get http client for remote + httpClient := &http.Client{} + + req, err := http.NewRequest("GET", src, nil) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + + // add authentication header + bearerHeader := r.Header.Get(HeaderTransferAuth) + req.Header.Add("Authorization", bearerHeader) + + // do download + httpDownloadRes, err := httpClient.Do(req) // lgtm[go/request-forgery] + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + defer httpDownloadRes.Body.Close() + + if httpDownloadRes.StatusCode == http.StatusNotImplemented { + w.WriteHeader(http.StatusBadRequest) + return errtypes.NotSupported("Third-Party copy not supported, source might be a folder") + } + if httpDownloadRes.StatusCode != http.StatusOK { + w.WriteHeader(httpDownloadRes.StatusCode) + return errtypes.InternalError(fmt.Sprintf("Remote GET returned status code %d", httpDownloadRes.StatusCode)) + } + + // get upload url + uReq := &provider.InitiateFileUploadRequest{ + Ref: &provider.Reference{Path: dst}, + Opaque: &typespb.Opaque{ + Map: map[string]*typespb.OpaqueEntry{ + "sizedeferred": { + Value: []byte("true"), + }, + }, + }, + } + uRes, err := client.InitiateFileUpload(ctx, uReq) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + + if uRes.Status.Code != rpc.Code_CODE_OK { + w.WriteHeader(http.StatusInternalServerError) + return fmt.Errorf("status code %d", uRes.Status.Code) + } + + var uploadEP, uploadToken string + for _, p := range uRes.Protocols { + if p.Protocol == "simple" { + uploadEP, uploadToken = p.UploadEndpoint, p.Token + } + } + + // send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured) + w.WriteHeader(http.StatusAccepted) + wc := WriteCounter{0, time.Now(), w} + tempReader := io.TeeReader(httpDownloadRes.Body, &wc) + + // do Upload + httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, tempReader) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken) + httpUploadRes, err := s.client.Do(httpUploadReq) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + + defer httpUploadRes.Body.Close() + if httpUploadRes.StatusCode != http.StatusOK { + w.WriteHeader(httpUploadRes.StatusCode) + return err + } + return nil +} + +// +// An example of an HTTP TPC Push +// +// +-----------------+ PUT +----------------+ +// | Dest server | <---------------- | Src server | +// | (Remote) | ----------------> | (Reva) | +// +-----------------+ Done +----------------+ +// ^ +// | +// | COPY +// | +// +----------+ +// | Client | +// +----------+ + +// handleTPCPush performs a PUT request on the remote site and while downloading +// data from the requested reva endpoint. +func (s *svc) handleTPCPush(ctx context.Context, w http.ResponseWriter, r *http.Request, ns string) { + src := path.Join(ns, r.URL.Path) + dst := r.Header.Get("Destination") + sublog := appctx.GetLogger(ctx).With().Str("src", src).Str("dst", dst).Logger() + + overwrite, err := extractOverwrite(w, r) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + m := fmt.Sprintf("Overwrite header is set to incorrect value %v", overwrite) + sublog.Warn().Msgf("HTTP TPC Push: %s", m) + b, err := Marshal(exception{ + code: SabredavBadRequest, + message: m, + }) + HandleWebdavError(&sublog, w, b, err) + return + } + + sublog.Debug().Str("overwrite", overwrite).Msg("TPC Push") + + // get Gateway client + client, err := s.getClient() + if err != nil { + sublog.Error().Err(err).Msg("error getting grpc client") + w.WriteHeader(http.StatusInternalServerError) + return + } + + ref := &provider.Reference{Path: src} + srcStatReq := &provider.StatRequest{Ref: ref} + srcStatRes, err := client.Stat(ctx, srcStatReq) + if err != nil { + sublog.Error().Err(err).Msg("error sending grpc stat request") + w.WriteHeader(http.StatusInternalServerError) + return + } + if srcStatRes.Status.Code != rpc.Code_CODE_OK && srcStatRes.Status.Code != rpc.Code_CODE_NOT_FOUND { + HandleErrorStatus(&sublog, w, srcStatRes.Status) + return + } + if srcStatRes.Info.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + sublog.Error().Msg("Third-Party copy of a folder is not supported") + w.WriteHeader(http.StatusBadRequest) + return + } + + err = s.performHTTPPush(ctx, client, r, w, srcStatRes.Info, ns) + if err != nil { + sublog.Error().Err(err).Msg("error performing TPC Push") + return + } + fmt.Fprintf(w, "success: Created") +} + +func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClient, r *http.Request, w http.ResponseWriter, srcInfo *provider.ResourceInfo, ns string) error { + src := path.Join(ns, r.URL.Path) + dst := r.Header.Get("Destination") + + sublog := appctx.GetLogger(ctx) + sublog.Debug().Str("src", src).Str("dst", dst).Msg("Performing HTTP Push") + + // get download url + dReq := &provider.InitiateFileDownloadRequest{ + Ref: &provider.Reference{Path: src}, + } + + dRes, err := client.InitiateFileDownload(ctx, dReq) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + + if dRes.Status.Code != rpc.Code_CODE_OK { + w.WriteHeader(http.StatusInternalServerError) + return fmt.Errorf("status code %d", dRes.Status.Code) + } + + var downloadEP, downloadToken string + for _, p := range dRes.Protocols { + if p.Protocol == "simple" { + downloadEP, downloadToken = p.DownloadEndpoint, p.Token + } + } + + // do download + httpDownloadReq, err := rhttp.NewRequest(ctx, "GET", downloadEP, nil) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + httpDownloadReq.Header.Set(datagateway.TokenTransportHeader, downloadToken) + + httpDownloadRes, err := s.client.Do(httpDownloadReq) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + defer httpDownloadRes.Body.Close() + if httpDownloadRes.StatusCode != http.StatusOK { + w.WriteHeader(httpDownloadRes.StatusCode) + return fmt.Errorf("Remote PUT returned status code %d", httpDownloadRes.StatusCode) + } + + // send performance markers periodically every PerfMarkerResponseTime (5 seconds unless configured) + w.WriteHeader(http.StatusAccepted) + wc := WriteCounter{0, time.Now(), w} + tempReader := io.TeeReader(httpDownloadRes.Body, &wc) + + // get http client for a remote call + httpClient := &http.Client{} + req, err := http.NewRequest("PUT", dst, tempReader) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + + // add authentication header and content length + bearerHeader := r.Header.Get(HeaderTransferAuth) + req.Header.Add("Authorization", bearerHeader) + req.ContentLength = int64(srcInfo.GetSize()) + + // do Upload + httpUploadRes, err := httpClient.Do(req) // lgtm[go/request-forgery] + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return err + } + defer httpUploadRes.Body.Close() + + if httpUploadRes.StatusCode != http.StatusOK { + w.WriteHeader(httpUploadRes.StatusCode) + return err + } + + return nil +} diff --git a/internal/http/services/owncloud/ocdav/webdav.go b/internal/http/services/owncloud/ocdav/webdav.go index 246a7a8e01..b33ff86898 100644 --- a/internal/http/services/owncloud/ocdav/webdav.go +++ b/internal/http/services/owncloud/ocdav/webdav.go @@ -74,6 +74,7 @@ const ( HeaderUploadOffset = "Upload-Offset" HeaderOCMtime = "X-OC-Mtime" HeaderExpectedEntityLength = "X-Expected-Entity-Length" + HeaderTransferAuth = "TransferHeaderAuthorization" ) // WebDavHandler implements a dav endpoint diff --git a/pkg/auth/manager/oidcmapping/oidcmapping.go b/pkg/auth/manager/oidcmapping/oidcmapping.go index 8dbef88079..8f502f704f 100644 --- a/pkg/auth/manager/oidcmapping/oidcmapping.go +++ b/pkg/auth/manager/oidcmapping/oidcmapping.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "strings" "time" oidc "github.com/coreos/go-oidc" @@ -37,6 +38,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/sharedconf" "github.com/juliangruber/go-intersect" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -54,13 +56,14 @@ type mgr struct { } type config struct { - Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` - Issuer string `mapstructure:"issuer" docs:";The issuer of the OIDC token."` - IDClaim string `mapstructure:"id_claim" docs:"sub;The claim containing the ID of the user."` - UIDClaim string `mapstructure:"uid_claim" docs:";The claim containing the UID of the user."` - GIDClaim string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."` - UserProviderSvc string `mapstructure:"userprovidersvc" docs:";The endpoint at which the GRPC userprovider is exposed."` - UsersMapping string `mapstructure:"usersmapping" docs:"; The OIDC users mapping file path"` + Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` + Issuer string `mapstructure:"issuer" docs:";The issuer of the OIDC token."` + IDClaim string `mapstructure:"id_claim" docs:"sub;The claim containing the ID of the user."` + UIDClaim string `mapstructure:"uid_claim" docs:";The claim containing the UID of the user."` + GIDClaim string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."` + GatewaySvc string `mapstructure:"gatewaysvc" docs:";The endpoint at which the GRPC gateway is exposed."` + UsersMapping string `mapstructure:"users_mapping" docs:"; The optional OIDC users mapping file path"` + GroupClaim string `mapstructure:"group_claim" docs:"; The group claim to be looked up to map the user (default to 'groups')."` } type oidcUserMapping struct { @@ -71,8 +74,14 @@ type oidcUserMapping struct { func (c *config) init() { if c.IDClaim == "" { + // sub is stable and defined as unique. the user manager needs to take care of the sub to user metadata lookup c.IDClaim = "sub" } + if c.GroupClaim == "" { + c.GroupClaim = "groups" + } + + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) } func parseConfig(m map[string]interface{}) (*config, error) { @@ -103,21 +112,23 @@ func (am *mgr) Configure(m map[string]interface{}) error { am.c = c am.oidcUsersMapping = map[string]*oidcUserMapping{} + if c.UsersMapping == "" { + // no mapping defined, leave the map empty and move on + return nil + } + f, err := ioutil.ReadFile(c.UsersMapping) if err != nil { - return fmt.Errorf("oidcmapping: error reading oidc users mapping file: +%v", err) + return fmt.Errorf("oidc: error reading the users mapping file: +%v", err) } - oidcUsers := []*oidcUserMapping{} - err = json.Unmarshal(f, &oidcUsers) if err != nil { - return fmt.Errorf("oidcmapping: error unmarshalling oidc users mapping file: +%v", err) + return fmt.Errorf("oidc: error unmarshalling the users mapping file: +%v", err) } - for _, u := range oidcUsers { if _, found := am.oidcUsersMapping[u.OIDCGroup]; found { - return errors.New("oidcmapping: mapping error, multiple users mapped to a single group") + return fmt.Errorf("oidc: mapping error, group \"%s\" is mapped to multiple users", u.OIDCGroup) } am.oidcUsersMapping[u.OIDCGroup] = u } @@ -134,7 +145,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) oidcProvider, err := am.getOIDCProvider(ctx) if err != nil { - return nil, nil, fmt.Errorf("oidcmapping: error creating oidc provider: +%v", err) + return nil, nil, fmt.Errorf("oidc: error creating oidc provider: +%v", err) } oauth2Token := &oauth2.Token{ @@ -144,119 +155,96 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) // query the oidc provider for user info userInfo, err := oidcProvider.UserInfo(ctx, oauth2.StaticTokenSource(oauth2Token)) if err != nil { - return nil, nil, fmt.Errorf("oidcmapping: error getting userinfo: +%v", err) + return nil, nil, fmt.Errorf("oidc: error getting userinfo: +%v", err) } - // claims contains the standard OIDC claims like issuer, iat, aud, ... and any other non-standard one. + // claims contains the standard OIDC claims like iss, iat, aud, ... and any other non-standard one. // TODO(labkode): make claims configuration dynamic from the config file so we can add arbitrary mappings from claims to user struct. + // For now, only the group claim is dynamic. var claims map[string]interface{} if err := userInfo.Claims(&claims); err != nil { - return nil, nil, fmt.Errorf("oidcmapping: error unmarshaling userinfo claims: %v", err) + return nil, nil, fmt.Errorf("oidc: error unmarshaling userinfo claims: %v", err) } log.Debug().Interface("claims", claims).Interface("userInfo", userInfo).Msg("unmarshalled userinfo") - if claims["issuer"] == nil { // This is not set in simplesamlphp - claims["issuer"] = am.c.Issuer + if claims["iss"] == nil { // This is not set in simplesamlphp + claims["iss"] = am.c.Issuer } if claims["email_verified"] == nil { // This is not set in simplesamlphp claims["email_verified"] = false } - if claims["email"] == nil { - return nil, nil, fmt.Errorf("oidcmapping: no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope") + if claims["preferred_username"] == nil { + claims["preferred_username"] = claims[am.c.IDClaim] } - if claims["preferred_username"] == nil || claims["name"] == nil { - return nil, nil, fmt.Errorf("oidcmapping: no \"preferred_username\" or \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope") + if claims["preferred_username"] == nil { + claims["preferred_username"] = claims["email"] } - if claims["groups"] == nil { - return nil, nil, fmt.Errorf("oidcmapping: no \"groups\" attribute found in userinfo") + if claims["name"] == nil { + claims["name"] = claims[am.c.IDClaim] } - - // discover the user username - var username string - mappings := make([]string, 0, len(am.oidcUsersMapping)) - for _, m := range am.oidcUsersMapping { - if m.OIDCIssuer == claims["issuer"] { - mappings = append(mappings, m.OIDCGroup) - } + if claims["name"] == nil { + return nil, nil, fmt.Errorf("no \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope") } - intersection := intersect.Simple(claims["groups"], mappings) - if len(intersection) > 1 { - // multiple mappings is not implemented, we don't know which one to choose - return nil, nil, errtypes.PermissionDenied("oidcmapping: mapping failed, more than one mapping found") - } - if len(intersection) == 1 { - for _, m := range intersection { - username = am.oidcUsersMapping[m.(string)].Username - } - } - - var uid, gid float64 - if am.c.UIDClaim != "" { - uid, _ = claims[am.c.UIDClaim].(float64) - } - if am.c.GIDClaim != "" { - gid, _ = claims[am.c.GIDClaim].(float64) + if claims["email"] == nil { + return nil, nil, fmt.Errorf("no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope") } - gwc, err := pool.GetUserProviderServiceClient(am.c.UserProviderSvc) + err = am.resolveUser(ctx, claims) if err != nil { - return nil, nil, errors.Wrap(err, "oidcmapping: error getting gateway grpc client") + return nil, nil, errors.Wrapf(err, "oidc: error resolving username for external user '%v'", claims["email"]) } userID := &user.UserId{ - OpaqueId: "", - Idp: "", - Type: user.UserType_USER_TYPE_PRIMARY, + OpaqueId: claims[am.c.IDClaim].(string), // a stable non reassignable id + Idp: claims["iss"].(string), // in the scope of this issuer + Type: getUserType(claims[am.c.IDClaim].(string)), } - if username != "" { - getUserByClaimResp, err := gwc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{ - Claim: "username", - Value: username, - }) - if err != nil { - return nil, nil, errors.Wrapf(err, "oidcmapping: error getting user by claim username (\"%v\")", username) - } - if getUserByClaimResp.Status.Code != rpc.Code_CODE_OK { - return nil, nil, status.NewErrorFromCode(getUserByClaimResp.Status.Code, "oidcmapping") - } - - userID.Idp = getUserByClaimResp.GetUser().GetId().Idp - userID.Type = getUserByClaimResp.GetUser().GetId().Type - userID.OpaqueId = getUserByClaimResp.GetUser().GetId().OpaqueId - } else { - username = claims["preferred_username"].(string) - userID.OpaqueId = claims[am.c.IDClaim].(string) - userID.Idp = claims["issuer"].(string) + gwc, err := pool.GetGatewayServiceClient(am.c.GatewaySvc) + if err != nil { + return nil, nil, errors.Wrap(err, "oidc: error getting gateway grpc client") } - getGroupsResp, err := gwc.GetUserGroups(ctx, &user.GetUserGroupsRequest{ UserId: userID, }) if err != nil { - return nil, nil, errors.Wrap(err, "oidcmapping: error getting user groups") + return nil, nil, errors.Wrapf(err, "oidc: error getting user groups for '%+v'", userID) } if getGroupsResp.Status.Code != rpc.Code_CODE_OK { - return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidcmapping") + return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidc") + } + + var uid, gid int64 + if am.c.UIDClaim != "" { + uid, _ = claims[am.c.UIDClaim].(int64) + } + if am.c.GIDClaim != "" { + gid, _ = claims[am.c.GIDClaim].(int64) } u := &user.User{ Id: userID, - Username: username, + Username: claims["preferred_username"].(string), Groups: getGroupsResp.Groups, Mail: claims["email"].(string), MailVerified: claims["email_verified"].(bool), DisplayName: claims["name"].(string), - UidNumber: int64(uid), - GidNumber: int64(gid), + UidNumber: uid, + GidNumber: gid, } - log.Debug().Msgf("returning user: %v", u) var scopes map[string]*authpb.Scope - scopes, err = scope.AddOwnerScope(nil) - if err != nil { - return nil, nil, err + if userID != nil && userID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT { + scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil) + if err != nil { + return nil, nil, err + } + } else { + scopes, err = scope.AddOwnerScope(nil) + if err != nil { + return nil, nil, err + } } return u, scopes, nil @@ -286,16 +274,87 @@ func (am *mgr) getOIDCProvider(ctx context.Context) (*oidc.Provider, error) { } // Initialize a provider by specifying the issuer URL. - // Once initialized is a singleton that is reused if further requests. + // Once initialized this is a singleton that is reused for further requests. // The provider is responsible to verify the token sent by the client // against the security keys oftentimes available in the .well-known endpoint. provider, err := oidc.NewProvider(ctx, am.c.Issuer) - if err != nil { - log.Error().Err(err).Msg("oidcmapping: error creating a new oidc provider") - return nil, fmt.Errorf("oidcmapping: error creating a new oidc provider: %+v", err) + log.Error().Err(err).Msg("oidc: error creating a new oidc provider") + return nil, fmt.Errorf("oidc: error creating a new oidc provider: %+v", err) } am.provider = provider return am.provider, nil } + +func (am *mgr) resolveUser(ctx context.Context, claims map[string]interface{}) error { + if len(am.oidcUsersMapping) > 0 { + var username string + + // map and discover the user's username when a mapping is defined + if claims[am.c.GroupClaim] == nil { + // we are required to perform a user mapping but the group claim is not available + return fmt.Errorf("no \"%s\" claim found in userinfo to map user", am.c.GroupClaim) + } + mappings := make([]string, 0, len(am.oidcUsersMapping)) + for _, m := range am.oidcUsersMapping { + if m.OIDCIssuer == claims["iss"] { + mappings = append(mappings, m.OIDCGroup) + } + } + + intersection := intersect.Simple(claims[am.c.GroupClaim], mappings) + if len(intersection) > 1 { + // multiple mappings are not implemented as we cannot decide which one to choose + return errtypes.PermissionDenied("more than one user mapping entry exists for the given group claims") + } + if len(intersection) == 0 { + return errtypes.PermissionDenied("no user mapping found for the given group claim(s)") + } + for _, m := range intersection { + username = am.oidcUsersMapping[m.(string)].Username + } + + upsc, err := pool.GetUserProviderServiceClient(am.c.GatewaySvc) + if err != nil { + return errors.Wrap(err, "error getting user provider grpc client") + } + getUserByClaimResp, err := upsc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{ + Claim: "username", + Value: username, + }) + if err != nil { + return errors.Wrapf(err, "error getting user by username '%v'", username) + } + if getUserByClaimResp.Status.Code != rpc.Code_CODE_OK { + return status.NewErrorFromCode(getUserByClaimResp.Status.Code, "oidc") + } + + // take the properties of the mapped target user to override the claims + claims["preferred_username"] = username + claims[am.c.IDClaim] = getUserByClaimResp.GetUser().GetId().OpaqueId + claims["iss"] = getUserByClaimResp.GetUser().GetId().Idp + if am.c.UIDClaim != "" { + claims[am.c.UIDClaim] = getUserByClaimResp.GetUser().UidNumber + } + if am.c.GIDClaim != "" { + claims[am.c.GIDClaim] = getUserByClaimResp.GetUser().GidNumber + } + appctx.GetLogger(ctx).Debug().Str("username", username).Interface("claims", claims).Msg("resolveUser: claims overridden from mapped user") + } + return nil +} + +func getUserType(upn string) user.UserType { + var t user.UserType + switch { + case strings.HasPrefix(upn, "guest"): + t = user.UserType_USER_TYPE_LIGHTWEIGHT + case strings.Contains(upn, "@"): + t = user.UserType_USER_TYPE_FEDERATED + default: + t = user.UserType_USER_TYPE_PRIMARY + } + return t + +} diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go index c291f7f1cf..7fc1c3c427 100644 --- a/pkg/cbox/user/rest/rest.go +++ b/pkg/cbox/user/rest/rest.go @@ -20,7 +20,6 @@ package rest import ( "context" - "errors" "fmt" "net/url" "regexp" @@ -34,6 +33,7 @@ import ( "github.com/cs3org/reva/pkg/user/manager/registry" "github.com/gomodule/redigo/redis" "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" ) func init() { @@ -198,8 +198,15 @@ func (m *manager) getInternalUserID(ctx context.Context, uid *userpb.UserId) (st return internalID, nil } -func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]interface{}) *userpb.User { - upn, _ := userData["upn"].(string) +func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]interface{}) (*userpb.User, error) { + id, ok := userData["id"].(string) + if !ok { + return nil, errors.New("parseAndCacheUser: Missing id in userData") + } + upn, ok := userData["upn"].(string) + if !ok { + return nil, errors.New("parseAndCacheUser: Missing upn in userData") + } mail, _ := userData["primaryAccountEmail"].(string) name, _ := userData["displayName"].(string) uidNumber, _ := userData["uid"].(float64) @@ -225,12 +232,11 @@ func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]int log := appctx.GetLogger(ctx) log.Error().Err(err).Msg("rest: error caching user details") } - if err := m.cacheInternalID(userID, userData["id"].(string)); err != nil { + if err := m.cacheInternalID(userID, id); err != nil { log := appctx.GetLogger(ctx) - log.Error().Err(err).Msg("rest: error caching user details") + log.Error().Err(err).Msg("rest: error caching internal ID") } - return u - + return u, nil } func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingGroups bool) (*userpb.User, error) { @@ -249,7 +255,10 @@ func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingG if err != nil { return nil, err } - u = m.parseAndCacheUser(ctx, userData) + u, err = m.parseAndCacheUser(ctx, userData) + if err != nil { + return nil, err + } } if !skipFetchingGroups { @@ -280,16 +289,21 @@ func (m *manager) GetUserByClaim(ctx context.Context, claim, value string, skipF return nil, errors.New("rest: invalid field: " + claim) } - userData, err := m.getUserByParam(ctx, claim, value) - if err != nil { - // Lightweight accounts need to be fetched by email - if strings.HasPrefix(value, "guest:") { - if userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(value, "guest:")); err != nil { - return nil, err - } + var userData map[string]interface{} + if strings.HasPrefix(value, "guest:") { + // Lightweight accounts need to be fetched by email, regardless of the demanded claim + if userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(value, "guest:")); err != nil { + return nil, err + } + } else { + if userData, err = m.getUserByParam(ctx, claim, value); err != nil { + return nil, errors.Wrap(err, "rest: failed getUserByParam, claim="+claim+", value="+value) } } - u := m.parseAndCacheUser(ctx, userData) + u, err := m.parseAndCacheUser(ctx, userData) + if err != nil { + return nil, err + } if !skipFetchingGroups { userGroups, err := m.GetUserGroups(ctx, u.Id) @@ -300,7 +314,6 @@ func (m *manager) GetUserByClaim(ctx context.Context, claim, value string, skipF } return u, nil - } func (m *manager) findUsersByFilter(ctx context.Context, url string, users map[string]*userpb.User, skipFetchingGroups bool) error {