diff --git a/examples/cc/qfssample_main.cc b/examples/cc/qfssample_main.cc index e268c8fe8..af4218adc 100644 --- a/examples/cc/qfssample_main.cc +++ b/examples/cc/qfssample_main.cc @@ -68,10 +68,11 @@ main(int argc, char **argv) { string serverHost = ""; int port = -1; + bool osFlag = false; bool help = false; char optchar; - while ((optchar = getopt(argc, argv, "hp:s:")) != -1) { + while ((optchar = getopt(argc, argv, "ohp:s:")) != -1) { switch (optchar) { case 'p': port = atoi(optarg); @@ -79,6 +80,9 @@ main(int argc, char **argv) case 's': serverHost = optarg; break; + case 'o': + osFlag = true; + break; case 'h': help = true; break; @@ -90,8 +94,8 @@ main(int argc, char **argv) } if (help || (serverHost == "") || (port < 0)) { - cout << "Usage: " << argv[0] << " -s -p " - << endl; + cout << "Usage: " << argv[0] << " -s -p [-o]\n"; + cout << " [-o]: write files to object store (S3)\n"; exit(0); } @@ -121,13 +125,15 @@ main(int argc, char **argv) } // Create a simple file with default replication (at most 3) + // or with replication 0 so that it's written to S3 + int numReplicas = ( osFlag ? 0 : 3); string tempFilename = baseDir + "/foo.1"; int fd; // fd is our file-handle to the file we are creating; this // file handle should be used in subsequent I/O calls on // the file. - if ((fd = gKfsClient->Create(tempFilename.c_str())) < 0) { + if ((fd = gKfsClient->Create(tempFilename.c_str(), numReplicas)) < 0) { cout << "Create failed: " << KFS::ErrorCodeToStr(fd) << endl; exit(-1); } @@ -186,7 +192,7 @@ main(int argc, char **argv) } // Re-create the file and try a rename that should fail... - int fd1 = gKfsClient->Create(tempFilename.c_str()); + int fd1 = gKfsClient->Create(tempFilename.c_str(), numReplicas); if (!gKfsClient->Exists(tempFilename.c_str())) { cout << " After rec-create..., " << tempFilename << " doesn't exist!" << endl; diff --git a/examples/sampleservers/sample_setup.cfg b/examples/sampleservers/sample_setup.cfg index 6b0a65e58..d74d6745e 100644 --- a/examples/sampleservers/sample_setup.cfg +++ b/examples/sampleservers/sample_setup.cfg @@ -31,6 +31,12 @@ rundir = ~/qfsbase/meta clientport = 20000 chunkport = 20100 clusterkey = myTestCluster +# S3 properties: for S3 support, uncomment and +# set the correct values depending on your AWS S3 bucket +# and IAM settings. +# bucketName = +# accessKeyId = +# secretAccessKey = [chunkserver1] hostname = localhost diff --git a/examples/sampleservers/sample_setup.py b/examples/sampleservers/sample_setup.py index fd0210db9..413209d32 100755 --- a/examples/sampleservers/sample_setup.py +++ b/examples/sampleservers/sample_setup.py @@ -279,6 +279,9 @@ def parse_command_line(): parser.add_option('-u', '--auth', action='store_true', help="Configure QFS authentication.") + parser.add_option('-o', '--os', action='store_true', + help="Configure object store (S3) support.") + parser.add_option('-h', '--help', action='store_true', help="Print this help message and exit.") @@ -464,7 +467,7 @@ def check_directories(config): print 'Check directories - OK.' -def setup_config_files(config, authFlag): +def setup_config_files(config, authFlag, osFlag): if config.has_section('client'): clientDir = config.get('client', 'rundir') else: @@ -507,7 +510,13 @@ def setup_config_files(config, authFlag): metaserverClientPort = config.getint('metaserver', 'clientport') metaserverChunkPort = config.getint('metaserver', 'chunkport') clusterKey = config.get('metaserver', 'clusterkey') - + if osFlag: + bucketName = config.get('metaserver', 'bucketName') + accessKeyId = config.get('metaserver', 'accessKeyId') + secretAccessKey = config.get('metaserver', 'secretAccessKey') + if not bucketName or not accessKeyId or not secretAccessKey: + sys.exit('Configuration file must set bucket name,' + 'access key id, and secret access key.') # Metaserver. metaFile = open(metaRunDir + '/conf/MetaServer.prp', 'w') print >> metaFile, 'metaServer.clientPort = %d' % metaserverClientPort @@ -534,6 +543,12 @@ def setup_config_files(config, authFlag): print >> metaFile, 'metaServer.CSAuthentication.X509.PKeyPemFile = %s/meta.key' % certsDir print >> metaFile, 'metaServer.CSAuthentication.X509.CAFile = %s/qfs_ca/cacert.pem' % certsDir print >> metaFile, 'metaServer.CSAuthentication.blackList = none' + if osFlag: + print >> metaFile, '# S3 parameters' + print >> metaFile, 'metaServer.objectStoreEnabled = 1' + print >> metaFile, 'chunkServer.diskQueue.aws.bucketName = %s' % bucketName + print >> metaFile, 'chunkServer.diskQueue.aws.accessKeyId = %s' % accessKeyId + print >> metaFile, 'chunkServer.diskQueue.aws.secretAccessKey = %s' % secretAccessKey metaFile.close() # Chunkservers. @@ -567,6 +582,9 @@ def setup_config_files(config, authFlag): print >> chunkFile, 'chunkserver.meta.auth.X509.X509PemFile = %s/chunk%d.crt' % (certsDir, chunkClientPort) print >> chunkFile, 'chunkserver.meta.auth.X509.PKeyPemFile = %s/chunk%d.key' % (certsDir, chunkClientPort) print >> chunkFile, 'chunkserver.meta.auth.X509.CAFile = %s/qfs_ca/cacert.pem' % certsDir + if osFlag: + print >> chunkFile, '# S3 parameters' + print >> chunkFile, 'chunkServer.objectDir = s3://aws.' chunkFile.close() # Webserver. @@ -737,7 +755,7 @@ def parse_config(configFile): check_ports(config) if opts.action == 'install': setup_directories(config, opts.auth) - setup_config_files(config, opts.auth) + setup_config_files(config, opts.auth, opts.os) copy_files(config, opts.source_dir) elif opts.action == 'start': check_directories(config)