From 8930b9ba54390f2d142dbe33285e5c523ce76a84 Mon Sep 17 00:00:00 2001 From: Jason Patton Date: Wed, 4 Oct 2023 16:44:41 -0500 Subject: [PATCH] HTCONDOR-1731 Try replacing dots in token names with underscores While adding support for underscores in URL schemes was laudible, there is still a problem parsing the (illegal) URLs upstream in HTCondor. Until we come up with a permanent solution, our workaround is to have users submit with a "." in place of the "_" (similar to how the client handles underscores internally) and have any plugins do the conversion from "." to "_" when doing token discovery. Here I have refactored the HTCondor-specific token discovery code into its own function, which will search for both tokens with and without "." replaced with "_" in their filenames. The TestGetToken test was expanded to test URL schemes using both "_" and "." and token files named with both "_" and ".". --- main.go | 50 ++++++++++++++++++++++++++++++++++--------------- main_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/main.go b/main.go index 82f1e56..fab6233 100644 --- a/main.go +++ b/main.go @@ -154,22 +154,10 @@ func getToken(destination *url.URL, namespace namespaces.Namespace, isWrite bool } // Finally, look in the HTCondor runtime - token_filename := "scitokens.use" - if len(token_name) > 0 { - token_filename = token_name + ".use" - } - log.Debugln("Looking for token file:", token_filename) - if credsDir, isCondorCredsSet := os.LookupEnv("_CONDOR_CREDS"); token_location == "" && isCondorCredsSet { - // Token wasn't specified on the command line or environment, try the default scitoken - if _, err := os.Stat(filepath.Join(credsDir, token_filename)); err != nil { - log.Warningln("Environment variable _CONDOR_CREDS is set, but file being point to does not exist:", err) - } else { - token_location = filepath.Join(credsDir, token_filename) - } - } - if _, err := os.Stat(".condor_creds/" + token_filename); err == nil && token_location == "" { - token_location, _ = filepath.Abs(".condor_creds/" + token_filename) + if token_location == "" { + token_location = discoverHTCondorToken(token_name) } + if token_location == "" { value, err := AcquireToken(destination, namespace, isWrite) if err == nil { @@ -268,6 +256,38 @@ func correctURLWithUnderscore(sourceFile string) (string, string) { return sourceFile, originalScheme } +func discoverHTCondorToken(token_name string) (string) { + token_location := "" + + // Tokens with dots in their name may need to have dots converted to underscores. + if strings.Contains(token_name, ".") { + underscore_token_name := strings.ReplaceAll(token_name, ".", "_") + // If we find a token after replacing dots, then we're already done. + token_location = discoverHTCondorToken(underscore_token_name) + if token_location != "" { + return token_location + } + } + + token_filename := "scitokens.use" + if len(token_name) > 0 { + token_filename = token_name + ".use" + } + log.Debugln("Looking for token file:", token_filename) + if credsDir, isCondorCredsSet := os.LookupEnv("_CONDOR_CREDS"); token_location == "" && isCondorCredsSet { + // Token wasn't specified on the command line or environment, try the default scitoken + if _, err := os.Stat(filepath.Join(credsDir, token_filename)); err != nil { + log.Warningln("Environment variable _CONDOR_CREDS is set, but file being point to does not exist:", err) + } else { + token_location = filepath.Join(credsDir, token_filename) + } + } + if _, err := os.Stat(".condor_creds/" + token_filename); err == nil && token_location == "" { + token_location, _ = filepath.Abs(".condor_creds/" + token_filename) + } + return token_location +} + // Start the transfer, whether read or write back func DoStashCPSingle(sourceFile string, destination string, methods []string, recursive bool) (bytesTransferred int64, err error) { diff --git a/main_test.go b/main_test.go index 0eaf849..b11be57 100644 --- a/main_test.go +++ b/main_test.go @@ -119,6 +119,59 @@ func TestGetToken(t *testing.T) { assert.Equal(t, token_contents, token) os.Unsetenv("_CONDOR_CREDS") + // _CONDOR_CREDS/renamed_handle1.use via renamed_handle1+osdf:///user/ligo/frames + token_contents = "bearer_token_file_contents renamed_handle1.use" + tmpFile = []byte(token_contents) + tmpDir = t.TempDir() + bearer_token_file = filepath.Join(tmpDir, "renamed_handle1.use") + err = os.WriteFile(bearer_token_file, tmpFile, 0644) + assert.NoError(t, err) + os.Setenv("_CONDOR_CREDS", tmpDir) + // Use a valid URL, then replace the scheme + renamedUrl, err = url.Parse("renamed.handle1+osdf:///user/ligo/frames") + renamedUrl.Scheme = "renamed_handle1+osdf" + assert.NoError(t, err) + renamedNamespace, err = namespaces.MatchNamespace("/user/ligo/frames") + assert.NoError(t, err) + token, err = getToken(renamedUrl, renamedNamespace, false, "") + assert.NoError(t, err) + assert.Equal(t, token_contents, token) + os.Unsetenv("_CONDOR_CREDS") + + // _CONDOR_CREDS/renamed_handle2.use via renamed.handle2+osdf:///user/ligo/frames + token_contents = "bearer_token_file_contents renamed.handle2.use" + tmpFile = []byte(token_contents) + tmpDir = t.TempDir() + bearer_token_file = filepath.Join(tmpDir, "renamed_handle2.use") + err = os.WriteFile(bearer_token_file, tmpFile, 0644) + assert.NoError(t, err) + os.Setenv("_CONDOR_CREDS", tmpDir) + renamedUrl, err = url.Parse("renamed.handle2+osdf:///user/ligo/frames") + assert.NoError(t, err) + renamedNamespace, err = namespaces.MatchNamespace("/user/ligo/frames") + assert.NoError(t, err) + token, err = getToken(renamedUrl, renamedNamespace, false, "") + assert.NoError(t, err) + assert.Equal(t, token_contents, token) + os.Unsetenv("_CONDOR_CREDS") + + // _CONDOR_CREDS/renamed.handle3.use via renamed.handle3+osdf:///user/ligo/frames + token_contents = "bearer_token_file_contents renamed.handle3.use" + tmpFile = []byte(token_contents) + tmpDir = t.TempDir() + bearer_token_file = filepath.Join(tmpDir, "renamed.handle3.use") + err = os.WriteFile(bearer_token_file, tmpFile, 0644) + assert.NoError(t, err) + os.Setenv("_CONDOR_CREDS", tmpDir) + renamedUrl, err = url.Parse("renamed.handle3+osdf:///user/ligo/frames") + assert.NoError(t, err) + renamedNamespace, err = namespaces.MatchNamespace("/user/ligo/frames") + assert.NoError(t, err) + token, err = getToken(renamedUrl, renamedNamespace, false, "") + assert.NoError(t, err) + assert.Equal(t, token_contents, token) + os.Unsetenv("_CONDOR_CREDS") + // _CONDOR_CREDS/renamed.use token_contents = "bearer_token_file_contents renamed.use" tmpFile = []byte(token_contents)