Skip to content

Commit

Permalink
Merge branch 'master' into batch-expire
Browse files Browse the repository at this point in the history
  • Loading branch information
shtripat authored Nov 13, 2023
2 parents 81b004f + 8c8e625 commit 4ffa93f
Show file tree
Hide file tree
Showing 8 changed files with 340 additions and 41 deletions.
10 changes: 8 additions & 2 deletions cmd/admin-trace.go
Original file line number Diff line number Diff line change
Expand Up @@ -900,6 +900,7 @@ type statItem struct {
CallStatsCount int `json:"callStatsCount,omitempty"`
CallStats callStats `json:"callStats,omitempty"`
TTFB time.Duration `json:"ttfb,omitempty"`
MaxTTFB time.Duration `json:"maxTTFB,omitempty"`
MaxDur time.Duration `json:"maxDuration"`
MinDur time.Duration `json:"minDuration"`
}
Expand Down Expand Up @@ -939,6 +940,9 @@ func (s *statTrace) add(t madmin.ServiceTraceInfo) {
if got.MaxDur < t.Trace.Duration {
got.MaxDur = t.Trace.Duration
}
if got.MaxTTFB < t.Trace.HTTP.CallStats.TimeToFirstByte {
got.MaxTTFB = t.Trace.HTTP.CallStats.TimeToFirstByte
}
if got.MinDur <= 0 {
got.MinDur = t.Trace.Duration
}
Expand Down Expand Up @@ -1078,9 +1082,10 @@ func (m *traceStatsUI) View() string {
console.Colorize("metrics-top-title", "Count"),
console.Colorize("metrics-top-title", "RPM"),
console.Colorize("metrics-top-title", "Avg Time"),
console.Colorize("metrics-top-title", "TTFB Time"),
console.Colorize("metrics-top-title", "Min Time"),
console.Colorize("metrics-top-title", "Max Time"),
console.Colorize("metrics-top-title", "Avg TTFB"),
console.Colorize("metrics-top-title", "Max TTFB"),
console.Colorize("metrics-top-title", "Errors"),
console.Colorize("metrics-top-title", "RX Avg"),
console.Colorize("metrics-top-title", "TX Avg"),
Expand Down Expand Up @@ -1128,9 +1133,10 @@ func (m *traceStatsUI) View() string {
console.Colorize("metrics-number-secondary", fmt.Sprintf("(%0.1f%%)", float64(v.Count)/float64(totalCnt)*100)),
console.Colorize("metrics-number", fmt.Sprintf("%0.1f", float64(v.Count)/dur.Minutes())),
console.Colorize(avgColor, fmt.Sprintf("%v", avg.Round(time.Microsecond))),
console.Colorize(avgColor, fmt.Sprintf("%v", avgTTFB.Round(time.Microsecond))),
console.Colorize(minColor, v.MinDur),
console.Colorize(maxColor, v.MaxDur),
console.Colorize(avgColor, fmt.Sprintf("%v", avgTTFB.Round(time.Microsecond))),
console.Colorize(maxColor, v.MaxTTFB),
errs,
rx,
tx,
Expand Down
124 changes: 95 additions & 29 deletions cmd/client-admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,18 @@ import (
"net"
"net/http"
"net/url"
"os"
"sync"
"time"

"github.com/klauspost/compress/gzhttp"

"github.com/minio/pkg/v2/env"

"github.com/mattn/go-ieproxy"
"github.com/minio/madmin-go/v3"
"github.com/minio/mc/pkg/httptracer"
"github.com/minio/mc/pkg/limiter"
"github.com/minio/mc/pkg/probe"
"github.com/minio/minio-go/v7/pkg/credentials"
)
Expand Down Expand Up @@ -67,8 +72,97 @@ func NewAdminFactory() func(config *Config) (*madmin.AdminClient, *probe.Error)
var api *madmin.AdminClient
var found bool
if api, found = clientCache[confSum]; !found {

var transport http.RoundTripper

if config.Transport != nil {
transport = config.Transport
} else {
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: newCustomDialContext(config),
MaxIdleConnsPerHost: 1024,
WriteBufferSize: 32 << 10, // 32KiB moving up from 4KiB default
ReadBufferSize: 32 << 10, // 32KiB moving up from 4KiB default
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 10 * time.Second,
DisableCompression: true,
// Set this value so that the underlying transport round-tripper
// doesn't try to auto decode the body of objects with
// content-encoding set to `gzip`.
//
// Refer:
// https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
}
if useTLS {
// Keep TLS config.
tlsConfig := &tls.Config{
RootCAs: globalRootCAs,
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
}
if config.Insecure {
tlsConfig.InsecureSkipVerify = true
}
tr.TLSClientConfig = tlsConfig

// Because we create a custom TLSClientConfig, we have to opt-in to HTTP/2.
// See https://github.com/golang/go/issues/14275
//
// TODO: Enable http2.0 when upstream issues related to HTTP/2 are fixed.
//
// if e = http2.ConfigureTransport(tr); e != nil {
// return nil, probe.NewError(e)
// }
}
transport = tr
}

transport = limiter.New(config.UploadLimit, config.DownloadLimit, transport)

if config.Debug {
transport = httptracer.GetNewTraceTransport(newTraceV4(), transport)
}

transport = gzhttp.Transport(transport)

var credsChain []credentials.Provider

// if an STS endpoint is set, we will add that to the chain
if stsEndpoint := env.Get("MC_STS_ENDPOINT", ""); stsEndpoint != "" {
// set AWS_WEB_IDENTITY_TOKEN_FILE is MC_WEB_IDENTITY_TOKEN_FILE is set
if val := env.Get("MC_WEB_IDENTITY_TOKEN_FILE", ""); val != "" {
os.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", val)
}

stsEndpointURL, err := url.Parse(stsEndpoint)
if err != nil {
return nil, probe.NewError(fmt.Errorf("Error parsing sts endpoint: %v", err))
}
credsSts := &credentials.IAM{
Client: &http.Client{
Transport: transport,
},
Endpoint: stsEndpointURL.String(),
}
credsChain = append(credsChain, credsSts)
}

// V4 Credentials
credsV4 := &credentials.Static{
Value: credentials.Value{
AccessKeyID: config.AccessKey,
SecretAccessKey: config.SecretKey,
SessionToken: config.SessionToken,
SignerType: credentials.SignatureV4,
},
}
credsChain = append(credsChain, credsV4)
// Admin API only supports signature v4.
creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, config.SessionToken)
creds := credentials.NewChainCredentials(credsChain)

// Not found. Instantiate a new MinIO
var e error
Expand All @@ -80,34 +174,6 @@ func NewAdminFactory() func(config *Config) (*madmin.AdminClient, *probe.Error)
return nil, probe.NewError(e)
}

// Keep TLS config.
tlsConfig := &tls.Config{
RootCAs: globalRootCAs,
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
}
if config.Insecure {
tlsConfig.InsecureSkipVerify = true
}

var transport http.RoundTripper = &http.Transport{
Proxy: ieproxy.GetProxyFunc(),
DialContext: newCustomDialContext(config),
MaxIdleConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 10 * time.Second,
TLSClientConfig: tlsConfig,
DisableCompression: true,
}
transport = gzhttp.Transport(transport)

if config.Debug {
transport = httptracer.GetNewTraceTransport(newTraceV4(), transport)
}

// Set custom transport.
api.SetCustomTransport(transport)

Expand Down
127 changes: 127 additions & 0 deletions cmd/client-admin_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// # This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
"bytes"
"context"
"io"
"log"
"net/http"
"net/http/httptest"
"os"
"strconv"

checkv1 "gopkg.in/check.v1"
)

type adminPolicyHandler struct {
endpoint string
name string
policy []byte
}

func (h adminPolicyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if ak := r.Header.Get("Authorization"); len(ak) == 0 {
w.WriteHeader(http.StatusForbidden)
return
}
switch {
case r.Method == "PUT":
length, e := strconv.Atoi(r.Header.Get("Content-Length"))
if e != nil {
w.WriteHeader(http.StatusBadRequest)
return
}

var buffer bytes.Buffer
if _, e = io.CopyN(&buffer, r.Body, int64(length)); e != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}

if len(h.policy) != buffer.Len() {
w.WriteHeader(http.StatusBadRequest)
return
}

w.Header().Set("Content-Length", "0")
w.WriteHeader(http.StatusOK)

default:
w.WriteHeader(http.StatusForbidden)
}
}

func (s *TestSuite) TestAdminSTSOperation(c *checkv1.C) {
sts := stsHandler{
endpoint: "/",
jwt: []byte("eyJhbGciOiJSUzI1NiIsImtpZCI6Inc0dFNjMEc5Tk0wQWhGaWJYaWIzbkpRZkRKeDc1dURRTUVpOTNvTHJ0OWcifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNzMxMTg3NzEwLCJpYXQiOjE2OTk2NTE3MTAsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJtaW5pby10ZW5hbnQtMSIsInBvZCI6eyJuYW1lIjoic2V0dXAtYnVja2V0LXQ4eGdjIiwidWlkIjoiNjZhYjlkZWItNzkwMC00YTFlLTgzMDgtMTkwODIwZmQ3NDY5In0sInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJtYy1qb2Itc2EiLCJ1aWQiOiI3OTc4NzJjZC1kMjkwLTRlM2EtYjYyMC00ZGFkYzZhNzUyMTYifSwid2FybmFmdGVyIjoxNjk5NjU1MzE3fSwibmJmIjoxNjk5NjUxNzEwLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWluaW8tdGVuYW50LTE6bWMtam9iLXNhIn0.rY7dpAh8GBTViH9Ges7tRhgyihdFWEN0DwXchelmZg58VOI526S-YfbCqrxksTs8Iu0fp1rmk1cUj7FGDh3AOv2RphHjoWci1802zKkHgH0iOEbKMp3jHXwfyHda8CyrSCPycGzClueCf1ae91wd_0lgK9lOR1qqY1HuDeXqSEAUIGrfh1VcP2n95Zc07EY-Uh3XjJE4drtgusACEK5n3P3WtN9s0m0GomEGQzF5ZJczxLGpHBKMQ5VDhMksVKdBAsx9xHgSx84aUhKQViYilAL-8PRj-RZA9s_IpEymAh5R37dKzAO8Fqq0nG7fVbH_ifzw3xhHiG92BhHldBDqEQ"),
}

tmpfile, errFs := os.CreateTemp("", "jwt")
if errFs != nil {
log.Fatal(errFs)
}
defer os.Remove(tmpfile.Name()) // clean up

if _, errFs := tmpfile.Write(sts.jwt); errFs != nil {
log.Fatal(errFs)
}
if errFs := tmpfile.Close(); errFs != nil {
log.Fatal(errFs)
}

stsServer := httptest.NewServer(sts)
defer stsServer.Close()
os.Setenv("MC_STS_ENDPOINT", stsServer.URL+sts.endpoint)
os.Setenv("MC_WEB_IDENTITY_TOKEN_FILE", tmpfile.Name())
handler := adminPolicyHandler{
endpoint: "/minio/admin/v3/add-canned-policy?name=",
name: "test",
policy: []byte(`
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": [
"arn:aws:s3:::test-bucket",
"arn:aws:s3:::test-bucket/*"
]
}
]
}`),
}
server := httptest.NewServer(handler)
defer server.Close()

conf := new(Config)
conf.Debug = true
conf.Insecure = true
conf.HostURL = server.URL + handler.endpoint + handler.name
s3c, err := s3AdminNew(conf)
c.Assert(err, checkv1.IsNil)

policyErr := s3c.AddCannedPolicy(context.Background(), handler.name, handler.policy)
c.Assert(policyErr, checkv1.IsNil)
}
15 changes: 8 additions & 7 deletions cmd/client-s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,13 +132,19 @@ func newFactory() func(config *Config) (Client, *probe.Error) {
useTLS = false
}

// Save if target supports virtual host style.
hostName := targetURL.Host

// Generate a hash out of s3Conf.
confHash := fnv.New32a()
confHash.Write([]byte(hostName + config.AccessKey + config.SecretKey + config.SessionToken))
confSum := confHash.Sum32()

// Instantiate s3
s3Clnt := &S3Client{}
// Save the target URL.
s3Clnt.targetURL = targetURL

// Save if target supports virtual host style.
hostName := targetURL.Host
s3Clnt.virtualStyle = isVirtualHostStyle(hostName, config.Lookup)
isS3AcceleratedEndpoint := isAmazonAccelerated(hostName)

Expand All @@ -149,11 +155,6 @@ func newFactory() func(config *Config) (Client, *probe.Error) {
}
}

// Generate a hash out of s3Conf.
confHash := fnv.New32a()
confHash.Write([]byte(hostName + config.AccessKey + config.SecretKey + config.SessionToken))
confSum := confHash.Sum32()

// Lookup previous cache by hash.
mutex.Lock()
defer mutex.Unlock()
Expand Down
Loading

0 comments on commit 4ffa93f

Please sign in to comment.