Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for S3 session keys #108

Merged
merged 2 commits into from
Dec 6, 2015
Merged

Support for S3 session keys #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f75d529
TWebFile: error handling for Status Code 301 without Location
holzman Nov 20, 2015
5ccdddc
TS3WebFile - add support for session keys
holzman Dec 1, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions net/net/inc/TS3HTTPRequest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ class TS3HTTPRequest : public TObject {
TString MakeAuthPrefix() const;
TString MakeHostHeader() const;
TString MakeDateHeader() const;
TString MakeTokenHeader() const;
TS3HTTPRequest& SetTimeStamp();

public:
Expand Down Expand Up @@ -102,6 +103,8 @@ class TS3HTTPRequest : public TObject {
TS3HTTPRequest& SetAuthKeys(const TString& accessKey, const TString& secretKey);
TS3HTTPRequest& SetAuthType(TS3HTTPRequest::EAuthType authType);

TString fToken;

ClassDef(TS3HTTPRequest, 0) // Create generic HTTP request for Amazon S3 and Google Storage services
};

Expand Down
6 changes: 3 additions & 3 deletions net/net/inc/TS3WebFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@ class TS3WebFile: public TWebFile {

private:
TS3WebFile();
Bool_t ParseOptions(Option_t* options, TString& accessKey, TString& secretKey);
Bool_t GetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv,
TString& outAccessKey, TString& outSecretKey);
Bool_t ParseOptions(Option_t* options, TString& accessKey, TString& secretKey, TString& token);
Bool_t GetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv, const char* tokenEnv,
TString& outAccessKey, TString& outSecretKey, TString &outToken);

protected:
// Super-class methods extended by this class
Expand Down
21 changes: 21 additions & 0 deletions net/net/src/TS3HTTPRequest.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,10 @@ TString TS3HTTPRequest::ComputeSignature(TS3HTTPRequest::EHTTPVerb httpVerb) con
toSign += "x-goog-api-version:1\n"; // Lowercase, no spaces around ':'
}

if (!fToken.IsNull()) {
toSign += "x-amz-security-token:" + fToken + "\n";
}

toSign += "/" + fBucket + fObjectKey;

unsigned char digest[SHA_DIGEST_LENGTH] = {0};
Expand Down Expand Up @@ -211,6 +215,19 @@ TString TS3HTTPRequest::MakeAuthPrefix() const
}
}

////////////////////////////////////////////////////////////////////////////////
/// Returns the token header for this HTTP request

TString TS3HTTPRequest::MakeTokenHeader() const
{

if (fToken.IsNull())
return "";

return TString::Format("x-amz-security-token: %s",
(const char*) fToken.Data());
}

////////////////////////////////////////////////////////////////////////////////
/// Returns the authentication header for this HTTP request

Expand Down Expand Up @@ -238,6 +255,10 @@ TString TS3HTTPRequest::GetRequest(TS3HTTPRequest::EHTTPVerb httpVerb, Bool_t ap
(const char*)MakeRequestLine(httpVerb),
(const char*)MakeHostHeader(),
(const char*)MakeDateHeader());
TString tokenHeader = MakeTokenHeader();
if (!tokenHeader.IsNull())
request += tokenHeader + "\r\n";

TString authHeader = MakeAuthHeader(httpVerb);
if (!authHeader.IsNull())
request += authHeader + "\r\n";
Expand Down
48 changes: 36 additions & 12 deletions net/net/src/TS3WebFile.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
// S3_SECRET_KEY, or //
// b) by specifying them when opening each file. //
// //
// (You can also access session keys via the S3_SESSION_KEY //
// environment variable, or by specifying it on open.) //
// //
// The first method is convenient if all the S3 files you want to //
// access are hosted by a single provider. The second one is more //
// flexible as it allows you to specify which credentials to use //
Expand Down Expand Up @@ -105,7 +108,10 @@ ClassImp(TS3WebFile)
/// open several files hosted by different providers in the same program/macro,
/// where the environemntal variables solution is not convenient (see below).
///
/// If you need to specify both NOPROXY and AUTH separate them by ' '
/// You can also specify the session key (if needed) by adding a string of the form
/// TOKEN=mySessionToken.
///
/// If you need to specify more than one option, separate them by ' '
/// (blank), for instance:
/// "NOPROXY AUTH=F38XYZABCDeFgH4D0E1F:V+frt4re7J1euSNFnmaf8wwmI4AAAE7kzxZ/TTM+"
///
Expand All @@ -114,6 +120,8 @@ ClassImp(TS3WebFile)
/// "NOPROXY AUTH=F38XYZABCDeFgH4D0E1F:V+frt4re7J1euSNFnmaf8wwmI4AAAE7kzxZ/TTM+");
/// TFile* f2 = TFile::Open("s3://host.example.com/bucket/path/to/my/file",
/// "AUTH=F38XYZABCDeFgH4D0E1F:V+frt4re7J1euSNFnmaf8wwmI4AAAE7kzxZ/TTM+");
/// TFile* f3 = TFile::Open("s3://host.example.com/bucket/path/to/my/file",
/// "TOKEN=AQoDYXdzEM///////////wEa8AHEYmCinjD+TsGEjtgKSMAT6wnY");
///
/// If there is no authentication information in the 'options' argument
/// (i.e. not AUTH="....") the values of the environmental variables
Expand All @@ -136,12 +144,14 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
TString errorMsg;
TString accessKey;
TString secretKey;
TString token;

TPMERegexp rex("^([a]?s3|s3http[s]?|gs|gshttp[s]?){1}://([^/]+)/([^/]+)/([^/].*)", "i");
if (rex.Match(TString(path)) != 5) {
errorMsg = TString::Format("invalid S3 path '%s'", path);
doMakeZombie = kTRUE;
}
else if (!ParseOptions(options, accessKey, secretKey)) {
else if (!ParseOptions(options, accessKey, secretKey, token)) {
errorMsg = TString::Format("could not parse options '%s'", options);
doMakeZombie = kTRUE;
}
Expand Down Expand Up @@ -173,8 +183,10 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
// variables.
const char* kAccessKeyEnv = "S3_ACCESS_KEY";
const char* kSecretKeyEnv = "S3_SECRET_KEY";
const char* kSessionToken = "S3_SESSION_TOKEN";
if (accessKey.IsNull())
GetCredentialsFromEnv(kAccessKeyEnv, kSecretKeyEnv, accessKey, secretKey);
GetCredentialsFromEnv(kAccessKeyEnv, kSecretKeyEnv, kSessionToken,
accessKey, secretKey, token);

// Initialize the S3 HTTP request
fS3Request.SetHost(fUrl.GetHost());
Expand All @@ -188,6 +200,8 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
// Set the authentication information we need to use
// for this file
fS3Request.SetAuthKeys(accessKey, secretKey);
if (!token.IsNull()) { fS3Request.fToken = token; }

if (rex[1].BeginsWith("gs"))
fS3Request.SetAuthType(TS3HTTPRequest::kGoogle);
else
Expand All @@ -214,7 +228,7 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)


////////////////////////////////////////////////////////////////////////////////
/// Extracts the S3 authentication key pair (access key and secret key)
/// Extracts the S3 authentication key pair (access key and secret key) and session token (if needed)
/// from the options. The authentication credentials can be specified in
/// the options provided to the constructor of this class as a string
/// containing: "AUTH=<access key>:<secret key>" and can include other
Expand All @@ -223,7 +237,7 @@ TS3WebFile::TS3WebFile(const char* path, Option_t* options)
/// For instance:
/// "NOPROXY AUTH=F38XYZABCDeFgHiJkLm:V+frt4re7J1euSNFnmaf8wwmI401234E7kzxZ/TTM+"

Bool_t TS3WebFile::ParseOptions(Option_t* options, TString& accessKey, TString& secretKey)
Bool_t TS3WebFile::ParseOptions(Option_t* options, TString& accessKey, TString& secretKey, TString& token)
{
TString optStr = (const char*)options;
if (optStr.IsNull())
Expand All @@ -235,13 +249,17 @@ Bool_t TS3WebFile::ParseOptions(Option_t* options, TString& accessKey, TString&
CheckProxy();

// Look in the options string for the authentication information.
TPMERegexp rex_token("(^TOKEN=|^.* TOKEN=)([\\S]+)[\\s]*.*$", "i");
if (rex_token.Match(optStr) == 3) {
token = rex_token[2];
}

TPMERegexp rex("(^AUTH=|^.* AUTH=)([a-z0-9]+):([a-z0-9+/]+)[\\s]*.*$", "i");
if (rex.Match(optStr) < 4) {
Error("ParseOptions", "expecting options of the form \"AUTH=myAccessKey:mySecretKey\"");
return kFALSE;
if (rex.Match(optStr) == 4) {
accessKey = rex[2];
secretKey = rex[3];
}
accessKey = rex[2];
secretKey = rex[3];

if (gDebug > 0)
Info("ParseOptions", "using authentication information from 'options' argument");
return kTRUE;
Expand Down Expand Up @@ -327,18 +345,24 @@ void TS3WebFile::ProcessHttpHeader(const TString& headerLine)
fUseMultiRange = multirangeServers.Contains(serverId, TString::kIgnoreCase) ? kTRUE : kFALSE;
}


////////////////////////////////////////////////////////////////////////////////
/// Sets the access and secret keys from the environmental variables, if
/// they are both set.

Bool_t TS3WebFile::GetCredentialsFromEnv(const char* accessKeyEnv, const char* secretKeyEnv,
TString& outAccessKey, TString& outSecretKey)
const char* tokenEnv, TString& outAccessKey,
TString& outSecretKey, TString& outToken)
{
// Look first in the recommended environmental variables. Both variables
// must be set.
TString accKey = gSystem->Getenv(accessKeyEnv);
TString secKey = gSystem->Getenv(secretKeyEnv);
TString token = gSystem->Getenv(tokenEnv);

if (!token.IsNull()) {
outToken = token;
}

if (!accKey.IsNull() && !secKey.IsNull()) {
outAccessKey = accKey;
outSecretKey = secKey;
Expand Down
27 changes: 23 additions & 4 deletions net/net/src/TWebFile.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -660,12 +660,19 @@ Int_t TWebFile::GetFromWeb10(char *buf, Int_t len, const TString &msg)
Int_t n, ret = 0, nranges = 0, ltot = 0, redirect = 0;
TString boundary, boundaryEnd;
Long64_t first = -1, last = -1, tot;
TString redir;

while ((n = GetLine(fSocket, line, sizeof(line))) >= 0) {
if (n == 0) {
if (ret < 0)
return ret;
if (redirect) {
if (redir.IsNull()) {
// Some sites (s3.amazonaws.com) do not return a Location field on 301
Error("GetFromWeb10", "error - permanent redirect (301) without location from host %s", fUrl.GetHost());
return -1;
}

ws.ReOpen();
// set message to reflect the redirectLocation and add bytes field
TString msg_1 = fMsgReadBuffer10;
Expand Down Expand Up @@ -775,14 +782,19 @@ Int_t TWebFile::GetFromWeb10(char *buf, Int_t len, const TString &msg)
#endif
if (fSize == -1) fSize = tot;
} else if (res.BeginsWith("Location:") && redirect) {
TString redir = res(10, 1000);
redir = res(10, 1000);
if (redirect == 2) // temp redirect
SetMsgReadBuffer10(redir, kTRUE);
else // permanent redirect
SetMsgReadBuffer10(redir, kFALSE);
}
}

if (redirect && redir.IsNull()) {
ret = -1;
Error("GetFromWeb10", "error - permanent redirect (301) without location from host %s", fUrl.GetHost());
}

if (n == -1 && fHTTP11) {
if (gDebug > 0)
Info("GetFromWeb10", "HTTP/1.1 socket closed, reopen");
Expand Down Expand Up @@ -945,6 +957,7 @@ Int_t TWebFile::GetHead()

char line[8192];
Int_t n, ret = 0, redirect = 0;
TString redir;

while ((n = GetLine(s, line, sizeof(line))) >= 0) {
if (n == 0) {
Expand All @@ -958,8 +971,14 @@ Int_t TWebFile::GetHead()
}
if (ret < 0)
return ret;
if (redirect)
return GetHead();
if (redirect) {
if (redir.IsNull()) {
// Some sites (s3.amazonaws.com) do not return a Location field on 301
Error("GetHead", "error - permanent redirect (301) without location from host %s", fUrl.GetHost());
return -1;
}
return GetHead();
}
return 0;
}

Expand Down Expand Up @@ -1015,7 +1034,7 @@ Int_t TWebFile::GetHead()
TString slen = res(16, 1000);
fSize = slen.Atoll();
} else if (res.BeginsWith("Location:") && redirect) {
TString redir = res(10, 1000);
redir = res(10, 1000);
if (redirect == 2) // temp redirect
SetMsgReadBuffer10(redir, kTRUE);
else // permanent redirect
Expand Down