forked from apache/spark
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SPARK-542] Add and document Driver<->Executor TLS support. (apache#188)
* add env var decoding and documentation about driver-to-executor tls support * Switched to file-based secrets. * Reverted spark-env.sh change. Added TLS integration test (currently disabled). * Updated docs, incorporating suggestions from Suzanne. * Removed mention of driver labels in docs. * Consolidated secrets creation code, removed SecretHandler. * Simplified the options in the run command. * Simplified the interface by automatically inferring some of the TLS config properties. * Better --help and error messages, updated docs. * Make sure user-defined secrets are preserved. Updated test and docs. * Converted passwords to "magic" options. Updated test and docs. * Added blurb in limitations docs section, moved setting of password configs.
- Loading branch information
1 parent
28ea345
commit 022f332
Showing
8 changed files
with
269 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
import os | ||
import sys | ||
from random import random | ||
from operator import add | ||
|
||
from pyspark.sql import SparkSession | ||
|
||
def check_secret(secret_name, secret_content): | ||
''' | ||
Make sure the extra secret envvar and secret file show up in driver. | ||
''' | ||
envvar_content = os.environ.get(secret_name) | ||
if envvar_content != secret_content: | ||
print("Unexpected contents in secret envvar, found: {} expected: {}".format(envvar_content, secret_content)) | ||
exit(1) | ||
|
||
file_content = open(secret_name, 'r').read() | ||
if file_content != secret_content: | ||
print("Unexpected contents in secret file, found: {} expected: {}".format(file_content, secret_content)) | ||
exit(1) | ||
|
||
|
||
if __name__ == "__main__": | ||
""" | ||
Usage: pi [partitions] [secret] [secret content] | ||
Checks for the given env-based and file-based driver secret. | ||
Then calculates the value of pi. | ||
""" | ||
|
||
check_secret(sys.argv[2], sys.argv[3]) | ||
|
||
spark = SparkSession \ | ||
.builder \ | ||
.appName("PythonPi") \ | ||
.getOrCreate() | ||
|
||
partitions = int(sys.argv[1]) | ||
n = 100000 * partitions | ||
|
||
def f(_): | ||
x = random() * 2 - 1 | ||
y = random() * 2 - 1 | ||
return 1 if x ** 2 + y ** 2 < 1 else 0 | ||
|
||
count = spark.sparkContext.parallelize(range(1, n + 1), partitions).map(f).reduce(add) | ||
print("Pi is roughly %f" % (4.0 * count / n)) | ||
|
||
spark.stop() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
/u3+7QAAAAIAAAACAAAAAgAGY2Fyb290AAABXxZdYVoABVguNTA5AAACtjCCArIwggIboAMCAQICCQCzNKodBAdsRzANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCAXDTE3MTAxMzE1Mjg0MloYDzIxMTcwOTE5MTUyODQyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDPdPSuUvDBpt6+1nV6YlTaaNZjd6OsmWm1tRllnpeJfnlXai3HkmQmRuTtbs2Foru5JVtHynPG1vWS/VeOMSBZYeDBHA2s2yLhWcWLFutPKZbNJ3Cf8OjGFf8wqfVBF8xGjUiaQUBTsrHvYlwRkX+bUo7074f5FgLOz15mdomZCQIDAQABo4GnMIGkMB0GA1UdDgQWBBQRjmxtDBg7vrMDMAH4y78lVhPtRjB1BgNVHSMEbjBsgBQRjmxtDBg7vrMDMAH4y78lVhPtRqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJALM0qh0EB2xHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEANqti2ozB3tlwTVyiPidrVK7n5RMLZQNsDJ68WMl1p7DhtSi4dU87kkNSAFIwHgndVr10CWhfa6TCFW4eVM/bGCEDK5ay+zhZRp+B+YYAKKQZatiMC/EXTdJ2pc+vEcEDF3vAbZ1hzUBbNEWZKkYNDnHOB9dfeQVv6w21P9wdc4gAAAABAAlsb2NhbGhvc3QAAAFfFl2Z7wAAAY8wggGLMA4GCisGAQQBKgIRAQEFAASCAXf8ccU6jYrFx5kW4S+fBJXuQOcCedY9TzOS2bPyouSwPCs42Zo6p0+CSbvv+JqYfMDTR5YbsjMR4xDwQ6B2iW/ldXX6ud0m1TtWroiQV0WSkmqI2s+yai40zjH8Z6/HdcEPSInoC+aXvTvQeK2bstZvPHUsa8H9ZwvKV2tsLjeg7kgoMFQAiMmSvnHawGO/icPwkJTctKEQnL9Jd6yGR2kc0FAAfJusuxYuIIsSXcw4Zisj7d5uRwPv4ecjOwCut1M8U/qs+Dz+LK8sFqdzVY2DyU0BhK8v7WLd2tAyNdcTnUVAQFrhdF4YX8EeMPA7Suyg5IsMo8/GD6zWtKZMDHDba1GelTnfCj7SQyT0dFRGYL/s0jC0DRQiX/2NYPaiNd43eo0yinxOe63L47XIQdjRjfFuBasrONYO4/ccjM84YB/YvbC6Lt0ppkkgGahFgqxhzG4ZSwAEdo9mRHWK1ORyZG8sH8qWdk7o+37yB8j7zIJcsU5EugIAAAACAAVYLjUwOQAAA0gwggNEMIICrQIJALjUH5yBVyi+MA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMTcxMDEzMTUzMTMwWhgPMjExNzA5MTkxNTMxMzBaMGwxEDAOBgNVBAYTB1Vua25vd24xEDAOBgNVBAgTB1Vua25vd24xEDAOBgNVBAcTB1Vua25vd24xEDAOBgNVBAoTB1Vua25vd24xEDAOBgNVBAsTB1Vua25vd24xEDAOBgNVBAMTB1Vua25vd24wggG4MIIBLAYHKoZIzjgEATCCAR8CgYEA/X9TgR11EilS30qcLuzk5/YRt1I870QAwx4/gLZRJmlFXUAiUftZPY1Y+r/F9bow9subVWzXgTuAHTRv8mZgt2uZUKWkn5/oBHsQIsJPu6nX/rfGG/g7V+fGqKYVDwT7g/bTxR7DAjVUE1oWkTL2dfOuK2HXKu/yIgMZndFIAccCFQCXYFCPFSMLzLKSuYKi64QL8Fgc9QKBgQD34aCF1ps93su8q1w2uFe5eZSvu/o66oL5V0wLPQeCZ1FZV4661FlP5nEHEIGAtEkWcSPoTCgWE7fPCTKMyKbhPBZ6i1R8jSjgo64eK7OmdZFuo38L+iE1YvH7YnoBJDvMpPG+qFGQiaiD3+Fa5Z8GkotmXoB7VSVkAUw7/s9JKgOBhQACgYEAu0U7jSWyAsVcIzCFBt9harR2Ajl9RDB/4nmSYOGGL52uI6kNH9p7PPeKWtgwiCSoIpWypo+GovjVubFKabKM9EbbRRjK8XJyU2TEtXVy3iiEAJM3mVBaSEA8xfmoQz4+ZjfI+fPOvZs3e7xUHwm3Kpu6WgidORSlnKL44MGQfZEwDQYJKoZIhvcNAQEFBQADgYEAbp/wgDQOmdBWKjxtu2N2k2adlY3X8V/sJWG3s/ewBRLdeEiX/qTiBGZQ4S/o3PrkmvqAkIMk4ZYIF+et/O8P0T8YBijA82H8zh6bH9HfT6n/88M0GCTtLqg1hmTeUthRyKKwKkDMzcjVT7NXi3zdf1Upnqzk9IgUNL9VrxO1s1wABVguNTA5AAACtjCCArIwggIboAMCAQICCQCzNKodBAdsRzANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCAXDTE3MTAxMzE1Mjg0MloYDzIxMTcwOTE5MTUyODQyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDPdPSuUvDBpt6+1nV6YlTaaNZjd6OsmWm1tRllnpeJfnlXai3HkmQmRuTtbs2Foru5JVtHynPG1vWS/VeOMSBZYeDBHA2s2yLhWcWLFutPKZbNJ3Cf8OjGFf8wqfVBF8xGjUiaQUBTsrHvYlwRkX+bUo7074f5FgLOz15mdomZCQIDAQABo4GnMIGkMB0GA1UdDgQWBBQRjmxtDBg7vrMDMAH4y78lVhPtRjB1BgNVHSMEbjBsgBQRjmxtDBg7vrMDMAH4y78lVhPtRqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJALM0qh0EB2xHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEANqti2ozB3tlwTVyiPidrVK7n5RMLZQNsDJ68WMl1p7DhtSi4dU87kkNSAFIwHgndVr10CWhfa6TCFW4eVM/bGCEDK5ay+zhZRp+B+YYAKKQZatiMC/EXTdJ2pc+vEcEDF3vAbZ1hzUBbNEWZKkYNDnHOB9dfeQVv6w21P9wdc4imd0svG9i4vIb3CuGKLwgRGWtmYQ== |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
/u3+7QAAAAIAAAABAAAAAgAGY2Fyb290AAABXxZa5REABVguNTA5AAACtjCCArIwggIboAMCAQICCQCzNKodBAdsRzANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCAXDTE3MTAxMzE1Mjg0MloYDzIxMTcwOTE5MTUyODQyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDPdPSuUvDBpt6+1nV6YlTaaNZjd6OsmWm1tRllnpeJfnlXai3HkmQmRuTtbs2Foru5JVtHynPG1vWS/VeOMSBZYeDBHA2s2yLhWcWLFutPKZbNJ3Cf8OjGFf8wqfVBF8xGjUiaQUBTsrHvYlwRkX+bUo7074f5FgLOz15mdomZCQIDAQABo4GnMIGkMB0GA1UdDgQWBBQRjmxtDBg7vrMDMAH4y78lVhPtRjB1BgNVHSMEbjBsgBQRjmxtDBg7vrMDMAH4y78lVhPtRqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJALM0qh0EB2xHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEANqti2ozB3tlwTVyiPidrVK7n5RMLZQNsDJ68WMl1p7DhtSi4dU87kkNSAFIwHgndVr10CWhfa6TCFW4eVM/bGCEDK5ay+zhZRp+B+YYAKKQZatiMC/EXTdJ2pc+vEcEDF3vAbZ1hzUBbNEWZKkYNDnHOB9dfeQVv6w21P9wdc4iAVWlBXqa86mJGXpe7H0vZZ/++aA== |
Oops, something went wrong.