QFS-321 add s3 option to sample setup script

examples/cc/qfssample_main.cc

@@ -68,17 +68,21 @@ main(int argc, char **argv)
 {
     string serverHost = "";
     int port = -1;
+    bool osFlag = false;
     bool help = false;
     char optchar;
 
-    while ((optchar = getopt(argc, argv, "hp:s:")) != -1) {
+    while ((optchar = getopt(argc, argv, "ohp:s:")) != -1) {
         switch (optchar) {
             case 'p':
                 port = atoi(optarg);
                 break;
             case 's':
                 serverHost = optarg;
                 break;
+            case 'o':
+                osFlag = true;
+                break;
             case 'h':
                 help = true;
                 break;
@@ -90,8 +94,8 @@ main(int argc, char **argv)
     }
 
     if (help || (serverHost == "") || (port < 0)) {
-        cout << "Usage: " << argv[0] << " -s <meta server name> -p <port> "
-             << endl;
+        cout << "Usage: " << argv[0] << " -s <meta server name> -p <port> [-o]\n";
+        cout << " [-o]: write files to object store (S3)\n";
         exit(0);
     }
 
@@ -121,13 +125,15 @@ main(int argc, char **argv)
     }
 
     // Create a simple file with default replication (at most 3)
+    // or with replication 0 so that it's written to S3
+    int numReplicas = ( osFlag ? 0 : 3);
     string tempFilename = baseDir + "/foo.1";
     int fd;
 
     // fd is our file-handle to the file we are creating; this
     // file handle should be used in subsequent I/O calls on
     // the file.
-    if ((fd = gKfsClient->Create(tempFilename.c_str())) < 0) {
+    if ((fd = gKfsClient->Create(tempFilename.c_str(), numReplicas)) < 0) {
         cout << "Create failed: " << KFS::ErrorCodeToStr(fd) << endl;
         exit(-1);
     }
@@ -186,7 +192,7 @@ main(int argc, char **argv)
     }
 
     // Re-create the file and try a rename that should fail...
-    int fd1 = gKfsClient->Create(tempFilename.c_str());
+    int fd1 = gKfsClient->Create(tempFilename.c_str(), numReplicas);
 
     if (!gKfsClient->Exists(tempFilename.c_str())) {
         cout << " After rec-create..., " << tempFilename << " doesn't exist!" << endl;

examples/sampleservers/sample_setup.cfg

@@ -31,6 +31,12 @@ rundir      = ~/qfsbase/meta
 clientport  = 20000
 chunkport   = 20100
 clusterkey  = myTestCluster
+# S3 properties: for S3 support, uncomment and
+# set the correct values depending on your AWS S3 bucket
+# and IAM settings.
+# bucketName = 
+# accessKeyId = 
+# secretAccessKey = 
 
 [chunkserver1]
 hostname    = localhost

examples/sampleservers/sample_setup.py

@@ -279,6 +279,9 @@ def parse_command_line():
     parser.add_option('-u', '--auth', action='store_true',
         help="Configure QFS authentication.")
 
+    parser.add_option('-o', '--os', action='store_true',
+        help="Configure object store (S3) support.")
+
     parser.add_option('-h', '--help', action='store_true',
         help="Print this help message and exit.")
 
@@ -464,7 +467,7 @@ def check_directories(config):
     print 'Check directories - OK.'
 
 
-def setup_config_files(config, authFlag):
+def setup_config_files(config, authFlag, osFlag):
     if config.has_section('client'):
         clientDir = config.get('client', 'rundir')
     else:
@@ -507,7 +510,13 @@ def setup_config_files(config, authFlag):
     metaserverClientPort = config.getint('metaserver', 'clientport')
     metaserverChunkPort = config.getint('metaserver', 'chunkport')
     clusterKey = config.get('metaserver', 'clusterkey')
-
+    if osFlag:
+        bucketName = config.get('metaserver', 'bucketName')
+        accessKeyId = config.get('metaserver', 'accessKeyId')
+        secretAccessKey = config.get('metaserver', 'secretAccessKey')
+        if not bucketName or not accessKeyId or not secretAccessKey:
+            sys.exit('Configuration file must set bucket name,'
+                'access key id, and secret access key.')
     # Metaserver.
     metaFile = open(metaRunDir + '/conf/MetaServer.prp', 'w')
     print >> metaFile, 'metaServer.clientPort = %d' % metaserverClientPort
@@ -534,6 +543,12 @@ def setup_config_files(config, authFlag):
         print >> metaFile, 'metaServer.CSAuthentication.X509.PKeyPemFile     = %s/meta.key' % certsDir
         print >> metaFile, 'metaServer.CSAuthentication.X509.CAFile          = %s/qfs_ca/cacert.pem' % certsDir
         print >> metaFile, 'metaServer.CSAuthentication.blackList            = none'
+    if osFlag:
+        print >> metaFile, '# S3 parameters'
+        print >> metaFile, 'metaServer.objectStoreEnabled = 1'
+        print >> metaFile, 'chunkServer.diskQueue.aws.bucketName = %s' % bucketName
+        print >> metaFile, 'chunkServer.diskQueue.aws.accessKeyId = %s' % accessKeyId
+        print >> metaFile, 'chunkServer.diskQueue.aws.secretAccessKey = %s' % secretAccessKey
     metaFile.close()
 
     # Chunkservers.
@@ -567,6 +582,9 @@ def setup_config_files(config, authFlag):
                     print >> chunkFile, 'chunkserver.meta.auth.X509.X509PemFile = %s/chunk%d.crt' % (certsDir, chunkClientPort)
                     print >> chunkFile, 'chunkserver.meta.auth.X509.PKeyPemFile = %s/chunk%d.key' % (certsDir, chunkClientPort)
                     print >> chunkFile, 'chunkserver.meta.auth.X509.CAFile      = %s/qfs_ca/cacert.pem' % certsDir
+                if osFlag:
+                    print >> chunkFile, '# S3 parameters'
+                    print >> chunkFile, 'chunkServer.objectDir = s3://aws.'
                 chunkFile.close()
 
     # Webserver.
@@ -737,7 +755,7 @@ def parse_config(configFile):
     check_ports(config)
     if opts.action == 'install':
         setup_directories(config, opts.auth)
-        setup_config_files(config, opts.auth)
+        setup_config_files(config, opts.auth, opts.os)
         copy_files(config, opts.source_dir)
     elif opts.action == 'start':
         check_directories(config)