From 4e58af6d5b7ff9d896d7905f0b4de64920ffe8c1 Mon Sep 17 00:00:00 2001 From: welpaolo Date: Tue, 9 Apr 2024 18:27:51 +0200 Subject: [PATCH] [DPE-3725] Limit secret access and add support for Configuration Hub secret (#77) --- pyproject.toml | 2 +- requirements.txt | 265 ++++++++--------- spark8t/cli/params.py | 15 + spark8t/cli/pyspark.py | 14 +- spark8t/cli/service_account_registry.py | 7 +- spark8t/cli/spark_shell.py | 14 +- spark8t/cli/spark_sql.py | 14 +- spark8t/cli/spark_submit.py | 14 +- spark8t/domain.py | 5 +- spark8t/literals.py | 1 + spark8t/resources/templates/role_yaml.tmpl | 22 +- spark8t/services.py | 275 +++++++++++++---- tests/integration/conftest.py | 13 + .../integration/test_cli_service_accounts.py | 279 ++++++++++++++---- tests/integration/test_kube_interface.py | 76 ++++- tests/unittest/test_services.py | 21 +- tox.ini | 4 +- 17 files changed, 758 insertions(+), 283 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 44367b8..9a536aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ ignore_missing_imports = true [tool.poetry] name = "spark8t" -version = "0.0.5" +version = "0.0.6" description = "This project provides some utilities function and CLI commands to run Spark on K8s." authors = [ "Canonical Data Platform " diff --git a/requirements.txt b/requirements.txt index e0dec33..51e5696 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,149 +1,138 @@ -anyio==3.7.1; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ - --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 -certifi==2023.11.17; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1 \ - --hash=sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474 +anyio==3.7.0 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce \ + --hash=sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0 +certifi==2023.5.7 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ + --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 envyaml==1.10.211231 ; python_full_version > "3.8.0" and python_version < "4.0" \ --hash=sha256:88f8a076159e3c317d3450a5f404132b6ac91aecee4934ea72eac65f911f1244 \ --hash=sha256:8d7a7a6be12587cc5da32a587067506b47b849f4643981099ad148015a72de52 -exceptiongroup==1.2.0; python_full_version > "3.8.0" and python_version < "3.11" \ - --hash=sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14 \ - --hash=sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68 +exceptiongroup==1.1.1 ; python_full_version > "3.8.0" and python_version < "3.11" \ + --hash=sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e \ + --hash=sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785 h11==0.14.0 ; python_full_version > "3.8.0" and python_version < "4.0" \ --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \ --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761 -httpcore==0.18.0; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9 \ - --hash=sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced -httpx==0.27.0; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5 \ - --hash=sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5 -idna==3.6; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \ - --hash=sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f +httpcore==0.17.2 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:125f8375ab60036db632f34f4b627a9ad085048eef7cb7d2616fea0f739f98af \ + --hash=sha256:5581b9c12379c4288fe70f43c710d16060c10080617001e6b22a3b6dbcbefd36 +httpx==0.24.1 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd \ + --hash=sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd +idna==3.4 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 jinja2==3.1.3 ; python_full_version > "3.8.0" and python_version < "4.0" \ --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90 -lightkube-models==1.29.0.7; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:36ab61e1baccb0d7d2306e909b4f6b4c3cd49c26d74e1a3f61f3b67ceac93d63 \ - --hash=sha256:b86ea48a9a1f36abe038e1d69ad7a7bab3a875102d3e8771431899a1bbda8ba5 -lightkube==0.15.2; python_full_version > "3.8.0" and python_version < "4.0" \ +lightkube-models==1.27.1.4 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:206abb6d184a07ed84c20fbabe494847e892de2c38ed302f39a87b4ed1f7bcd9 \ + --hash=sha256:5caa97ed46bde5ae8a4313ebf3fa3d0135388d0aea0752261d4720e24d982fb0 +lightkube==0.15.2 ; python_full_version > "3.8.0" and python_version < "4.0" \ --hash=sha256:1297a3c6ebe873debf73cd584e288534a1f2fd643bbf8285dfbd76550c5076b0 \ --hash=sha256:54c509f71d56f4977f70c2bc1fdf0c79e9898b55cec4f3cbc6051f8d9961a75b -MarkupSafe==2.1.5; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ - --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ - --hash=sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f \ - --hash=sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3 \ - --hash=sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532 \ - --hash=sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f \ - --hash=sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617 \ - --hash=sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df \ - --hash=sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4 \ - --hash=sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906 \ - --hash=sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f \ - --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ - --hash=sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8 \ - --hash=sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371 \ - --hash=sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2 \ - --hash=sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465 \ - --hash=sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52 \ - --hash=sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6 \ - --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ - --hash=sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad \ - --hash=sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2 \ - --hash=sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0 \ - --hash=sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029 \ - --hash=sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f \ - --hash=sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a \ - --hash=sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced \ - --hash=sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5 \ - --hash=sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c \ - --hash=sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf \ - --hash=sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9 \ - --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ - --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ - --hash=sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3 \ - --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ - --hash=sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46 \ - --hash=sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc \ - --hash=sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a \ - --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ - --hash=sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900 \ - --hash=sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5 \ - --hash=sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea \ - --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ - --hash=sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5 \ - --hash=sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e \ - --hash=sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a \ - --hash=sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f \ - --hash=sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50 \ - --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ - --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ - --hash=sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4 \ - --hash=sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff \ - --hash=sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2 \ - --hash=sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46 \ - --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ - --hash=sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf \ - --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 \ - --hash=sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5 \ - --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ - --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ - --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 -PyYAML==6.0.1; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ - --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ - --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ - --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ - --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ - --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ - --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ - --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ - --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ - --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ - --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ - --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ - --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ - --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ - --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ - --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ - --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ - --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ - --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ - --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ - --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ - --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ - --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ - --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ - --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ - --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ - --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ - --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ - --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ - --hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \ - --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ - --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ - --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ - --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ - --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ - --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ - --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ - --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ - --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ - --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ - --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ - --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ - --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ - --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ - --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ - --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ - --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ - --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ - --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ - --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ - --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f -sniffio==1.3.1; python_full_version > "3.8.0" and python_version < "4.0" \ - --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ - --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc +markupsafe==2.1.3 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ + --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ + --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ + --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ + --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ + --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ + --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ + --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ + --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ + --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ + --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ + --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ + --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ + --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ + --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ + --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ + --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ + --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ + --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ + --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ + --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 +pyyaml==6.0 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 +sniffio==1.3.0 ; python_full_version > "3.8.0" and python_version < "4.0" \ + --hash=sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101 \ + --hash=sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384 diff --git a/spark8t/cli/params.py b/spark8t/cli/params.py index 7180930..be930bd 100644 --- a/spark8t/cli/params.py +++ b/spark8t/cli/params.py @@ -44,6 +44,21 @@ def add_logging_arguments(parser: ArgumentParser) -> ArgumentParser: return parser +def add_ignore_configuration_hub(parser: ArgumentParser) -> ArgumentParser: + """ + Add option to exclude the configuration provided by the Spark Configuration Hub + + :param parser: Input parser to decorate with parsing support for logging args. + """ + parser.add_argument( + "--ignore-configuration-hub", + action="store_true", + help="Ignore the configuration provided by Spark Configuration Hub Charm.", + ) + + return parser + + def spark_user_parser(parser: ArgumentParser) -> ArgumentParser: """ Add Spark user related argument parsing to the existing parser context diff --git a/spark8t/cli/pyspark.py b/spark8t/cli/pyspark.py index 4bdf9d6..f211714 100644 --- a/spark8t/cli/pyspark.py +++ b/spark8t/cli/pyspark.py @@ -7,6 +7,7 @@ from spark8t.cli.params import ( add_config_arguments, + add_ignore_configuration_hub, add_logging_arguments, defaults, get_kube_interface, @@ -14,7 +15,7 @@ parse_arguments_with, spark_user_parser, ) -from spark8t.domain import ServiceAccount +from spark8t.domain import PropertyFile, ServiceAccount from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound from spark8t.services import K8sServiceAccountRegistry, SparkInterface from spark8t.utils import setup_logging @@ -40,6 +41,9 @@ def main(args: Namespace, logger: Logger): args.username ) if args.username else PrimaryAccountNotFound() + if args.ignore_configuration_hub: + service_account.configuration_hub_confs = PropertyFile.empty() + SparkInterface( service_account=service_account, kube_interface=kube_interface, @@ -49,7 +53,13 @@ def main(args: Namespace, logger: Logger): if __name__ == "__main__": args, extra_args = parse_arguments_with( - [add_logging_arguments, k8s_parser, spark_user_parser, add_config_arguments] + [ + add_logging_arguments, + k8s_parser, + spark_user_parser, + add_config_arguments, + add_ignore_configuration_hub, + ] ).parse_known_args() logger = setup_logging(args.log_level, args.log_conf_file, "spark8t.cli.pyspark") diff --git a/spark8t/cli/service_account_registry.py b/spark8t/cli/service_account_registry.py index 1a5eda7..c39c452 100644 --- a/spark8t/cli/service_account_registry.py +++ b/spark8t/cli/service_account_registry.py @@ -83,8 +83,11 @@ def create_service_account_registry_parser(parser: ArgumentParser): parse_arguments_with( [spark_user_parser], subparsers.add_parser(Actions.GET_CONFIG.value, parents=[base_parser]), + ).add_argument( + "--ignore-configuration-hub", + action="store_true", + help="Boolean to ignore configuration hub generated options.", ) - # subparser for sa-conf-del parse_arguments_with( [spark_user_parser], @@ -166,6 +169,8 @@ def main(args: Namespace, logger: Logger): if maybe_service_account is None: raise AccountNotFound(input_service_account.id) + if args.ignore_configuration_hub: + maybe_service_account.configuration_hub_confs = PropertyFile.empty() maybe_service_account.configurations.log(print) elif args.action == Actions.CLEAR_CONFIG: diff --git a/spark8t/cli/spark_shell.py b/spark8t/cli/spark_shell.py index 25c299e..4f0fe62 100644 --- a/spark8t/cli/spark_shell.py +++ b/spark8t/cli/spark_shell.py @@ -7,6 +7,7 @@ from spark8t.cli.params import ( add_config_arguments, + add_ignore_configuration_hub, add_logging_arguments, defaults, get_kube_interface, @@ -14,7 +15,7 @@ parse_arguments_with, spark_user_parser, ) -from spark8t.domain import ServiceAccount +from spark8t.domain import PropertyFile, ServiceAccount from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound from spark8t.services import K8sServiceAccountRegistry, SparkInterface from spark8t.utils import setup_logging @@ -40,6 +41,9 @@ def main(args: Namespace, logger: Logger): args.username ) if args.username else PrimaryAccountNotFound() + if args.ignore_configuration_hub: + service_account.configuration_hub_confs = PropertyFile.empty() + SparkInterface( service_account=service_account, kube_interface=kube_interface, @@ -49,7 +53,13 @@ def main(args: Namespace, logger: Logger): if __name__ == "__main__": args, extra_args = parse_arguments_with( - [add_logging_arguments, k8s_parser, spark_user_parser, add_config_arguments] + [ + add_logging_arguments, + k8s_parser, + spark_user_parser, + add_config_arguments, + add_ignore_configuration_hub, + ] ).parse_known_args() logger = setup_logging( diff --git a/spark8t/cli/spark_sql.py b/spark8t/cli/spark_sql.py index 48112ef..8ebfefe 100644 --- a/spark8t/cli/spark_sql.py +++ b/spark8t/cli/spark_sql.py @@ -7,6 +7,7 @@ from spark8t.cli.params import ( add_config_arguments, + add_ignore_configuration_hub, add_logging_arguments, defaults, get_kube_interface, @@ -14,7 +15,7 @@ parse_arguments_with, spark_user_parser, ) -from spark8t.domain import ServiceAccount +from spark8t.domain import PropertyFile, ServiceAccount from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound from spark8t.services import K8sServiceAccountRegistry, SparkInterface from spark8t.utils import setup_logging @@ -40,6 +41,9 @@ def main(args: Namespace, logger: Logger): args.username ) if args.username else PrimaryAccountNotFound() + if args.ignore_configuration_hub: + service_account.configuration_hub_confs = PropertyFile.empty() + SparkInterface( service_account=service_account, kube_interface=kube_interface, @@ -49,7 +53,13 @@ def main(args: Namespace, logger: Logger): if __name__ == "__main__": args, extra_args = parse_arguments_with( - [add_logging_arguments, k8s_parser, spark_user_parser, add_config_arguments] + [ + add_logging_arguments, + k8s_parser, + spark_user_parser, + add_config_arguments, + add_ignore_configuration_hub, + ] ).parse_known_args() logger = setup_logging(args.log_level, args.log_conf_file, "spark8t.cli.spark_sql") diff --git a/spark8t/cli/spark_submit.py b/spark8t/cli/spark_submit.py index bac1459..c819c4f 100644 --- a/spark8t/cli/spark_submit.py +++ b/spark8t/cli/spark_submit.py @@ -8,6 +8,7 @@ from spark8t.cli.params import ( add_config_arguments, add_deploy_arguments, + add_ignore_configuration_hub, add_logging_arguments, defaults, get_kube_interface, @@ -15,7 +16,7 @@ parse_arguments_with, spark_user_parser, ) -from spark8t.domain import ServiceAccount +from spark8t.domain import PropertyFile, ServiceAccount from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound from spark8t.services import K8sServiceAccountRegistry, SparkInterface from spark8t.utils import setup_logging @@ -41,11 +42,19 @@ def main(args: Namespace, logger: Logger): args.username ) if args.username else PrimaryAccountNotFound() + if args.ignore_configuration_hub: + service_account.configuration_hub_confs = PropertyFile.empty() + SparkInterface( service_account=service_account, kube_interface=kube_interface, defaults=defaults, - ).spark_submit(args.deploy_mode, args.conf, args.properties_file, extra_args) + ).spark_submit( + args.deploy_mode, + args.conf, + args.properties_file, + extra_args, + ) if __name__ == "__main__": @@ -56,6 +65,7 @@ def main(args: Namespace, logger: Logger): spark_user_parser, add_deploy_arguments, add_config_arguments, + add_ignore_configuration_hub, ] ).parse_known_args() diff --git a/spark8t/domain.py b/spark8t/domain.py index 889b190..bd03dda 100644 --- a/spark8t/domain.py +++ b/spark8t/domain.py @@ -300,6 +300,7 @@ class ServiceAccount: api_server: str primary: bool = False extra_confs: PropertyFile = PropertyFile.empty() + configuration_hub_confs: PropertyFile = PropertyFile.empty() @property def id(self): @@ -318,7 +319,9 @@ def _k8s_configurations(self): @property def configurations(self) -> PropertyFile: """Return the service account configuration, associated to a given spark service account.""" - return self.extra_confs + self._k8s_configurations + return ( + self.extra_confs + self.configuration_hub_confs + self._k8s_configurations + ) class KubernetesResourceType(str, Enum): diff --git a/spark8t/literals.py b/spark8t/literals.py index e6474cb..e67cb00 100644 --- a/spark8t/literals.py +++ b/spark8t/literals.py @@ -1,3 +1,4 @@ MANAGED_BY_LABELNAME = "app.kubernetes.io/managed-by" PRIMARY_LABELNAME = "app.kubernetes.io/spark8t-primary" SPARK8S_LABEL = "spark8t" +CONFIGURATION_HUB_LABEL = "configuration-hub-conf" diff --git a/spark8t/resources/templates/role_yaml.tmpl b/spark8t/resources/templates/role_yaml.tmpl index 2212d74..2e25825 100644 --- a/spark8t/resources/templates/role_yaml.tmpl +++ b/spark8t/resources/templates/role_yaml.tmpl @@ -13,10 +13,30 @@ rules: - configmaps - services - serviceaccounts - - secrets verbs: - create - get - list - watch - delete + - deletecollection + - update + - patch +- apiGroups: + - "" + resources: + - secrets + resourceNames: + - spark8t-sa-conf-{{username}} + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - secrets + resourceNames: + - configuration-hub-conf-{{username}} + verbs: + - get \ No newline at end of file diff --git a/spark8t/services.py b/spark8t/services.py index d69df24..7c1a4c8 100644 --- a/spark8t/services.py +++ b/spark8t/services.py @@ -2,6 +2,7 @@ import base64 import io +import json import os import socket import subprocess @@ -33,7 +34,12 @@ K8sResourceNotFound, ResourceAlreadyExists, ) -from spark8t.literals import MANAGED_BY_LABELNAME, PRIMARY_LABELNAME, SPARK8S_LABEL +from spark8t.literals import ( + CONFIGURATION_HUB_LABEL, + MANAGED_BY_LABELNAME, + PRIMARY_LABELNAME, + SPARK8S_LABEL, +) from spark8t.utils import ( PercentEncodingSerializer, WithLogging, @@ -218,6 +224,31 @@ def delete( """ pass + def delete_secret_content( + self, secret_name: str, namespace: Optional[str] = None + ) -> None: + """Delete the content of the specified secret. + + Args: + secret_name: name of the secret + namespace: namespace where the secret is contained + """ + pass + + def add_secret_content( + self, + secret_name: str, + namespace: Optional[str] = None, + configurations: PropertyFile = PropertyFile.empty(), + ) -> None: + """Delete the content of the specified secret. + + Args: + secret_name: name of the secret + namespace: namespace where the secret is contained + """ + pass + @abstractmethod def exists( self, @@ -389,14 +420,49 @@ def get_secret( secret = yaml.safe_load(buffer) result = dict() - for k, v in secret["data"].items(): - result[k] = base64.b64decode(v).decode("utf-8") + if "data" in secret: + for k, v in secret["data"].items(): + result[k] = base64.b64decode(v).decode("utf-8") secret["data"] = result return secret except Exception: raise K8sResourceNotFound(secret_name, KubernetesResourceType.SECRET) + def delete_secret_content( + self, secret_name: str, namespace: Optional[str] = None + ) -> None: + if len(self.get_secret(secret_name, namespace)["data"]) == 0: + self.logger.debug( + f"Secret: {secret_name} is already empty, no need to delete its content." + ) + return + + patch = [{"op": "remove", "path": "/data"}] + self.client.patch( + res=Secret, + namespace=namespace, + name=secret_name, + obj=patch, + patch_type=PatchType.JSON, + ) + + def add_secret_content( + self, + secret_name: str, + namespace: Optional[str] = None, + configurations: PropertyFile = PropertyFile.empty(), + ) -> None: + """Delete the content of the specified secret. + + Args: + secret_name: name of the secret + namespace: namespace where the secret is contained + configurations: the desired configuration to insert + """ + patch = {"op": "add", "stringData": configurations.props} + self.client.patch(res=Secret, namespace=namespace, name=secret_name, obj=patch) + def set_label( self, resource_type: KubernetesResourceType, @@ -557,9 +623,6 @@ def create( "apiVersion": "v1", "kind": "Secret", "metadata": {"name": resource_name, "namespace": namespace}, - "stringData": self.create_property_file_entries( - extra_args["from-env-file"] - ), } ) ) @@ -719,6 +782,79 @@ def get_service_account( return service_account_raw + def delete_secret_content( + self, secret_name: str, namespace: Optional[str] = None + ) -> None: + """Delete the content of the secret name entry. + + Args: + secret_name: name of the secret. + namespace: namespace where to look for the service account. Default is 'default' + """ + if len(self.get_secret(secret_name, namespace)["data"]) == 0: + self.logger.debug( + f"Secret: {secret_name} is already empty, no need to delete its content." + ) + return + + cmd = f'patch secret {secret_name} --type=json -p=\'[{{"op": "remove", "path": "/data" }}]\'' + + try: + service_account_raw = self.exec(cmd, namespace=namespace) + except subprocess.CalledProcessError as e: + if "NotFound" in e.stdout.decode("utf-8"): + raise K8sResourceNotFound( + secret_name, KubernetesResourceType.SERVICEACCOUNT + ) + raise e + + if isinstance(service_account_raw, str): + raise ValueError( + f"Error deleting secret content of {secret_name} in namespace {namespace}" + ) + + self.logger.debug(service_account_raw) + + def add_secret_content( + self, + secret_name: str, + namespace: Optional[str] = None, + configurations: PropertyFile = PropertyFile.empty(), + ) -> None: + """Add the content of the specified secret. + + Args: + secret_name: name of the secret + namespace: namespace where the secret is contained + configurations: the configuration parameters to add in + """ + """Add the content of the secret name entry. + + Args: + secret_name: name of the secret. + namespace: namespace where to look for the service account. Default is 'default' + """ + if len(configurations.props.keys()) == 0: + self.logger.debug("Empty configuration! Nothing to write") + return + cmd = f"patch secret {secret_name} -p='{{\"stringData\": {json.dumps(configurations.props)} }}'" + + try: + service_account_raw = self.exec(cmd, namespace=namespace) + except subprocess.CalledProcessError as e: + if "NotFound" in e.stdout.decode("utf-8"): + raise K8sResourceNotFound( + secret_name, KubernetesResourceType.SERVICEACCOUNT + ) + raise e + + if isinstance(service_account_raw, str): + raise ValueError( + f"Error deleting secret content of {secret_name} in namespace {namespace}" + ) + + self.logger.debug(service_account_raw) + def get_service_accounts( self, namespace: Optional[str] = None, labels: Optional[List[str]] = None ) -> List[Dict[str, Any]]: @@ -768,10 +904,10 @@ def get_secret( raise K8sResourceNotFound(secret_name, KubernetesResourceType.SECRET) result = dict() - for k, v in secret["data"].items(): - # k1 = k.replace(".", "\\.") - # value = self.kube_interface.exec(f"get secret {secret_name}", output=f"jsonpath='{{.data.{k1}}}'") - result[k] = base64.b64decode(v).decode("utf-8") + # handle empty secret + if "data" in secret: + for k, v in secret["data"].items(): + result[k] = base64.b64decode(v).decode("utf-8") secret["data"] = result return secret @@ -830,6 +966,26 @@ def create( self.exec( f"create {resource_type} {resource_name}", namespace=None, output="name" ) + elif resource_type == KubernetesResourceType.ROLE: + with open(self.defaults.template_role) as f: + res = codecs.load_all_yaml( + f, + context=filter_none( + { + "resourcename": resource_name, + "namespace": namespace, + } + | extra_args + ), + ) + with umask_named_temporary_file( + mode="w", + prefix="role-", + suffix=".yaml", + dir=os.path.expanduser("~"), + ) as t: + codecs.dump_all_yaml(res, t) + self.exec(f"apply -f {t.name}", namespace=namespace, output="name") else: # NOTE: removing 'username' to avoid interference with KUBECONFIG # ERROR: more than one authentication method found for admin; found [token basicAuth], only one is allowed @@ -1000,11 +1156,13 @@ def all(self, namespace: Optional[str] = None) -> List["ServiceAccount"]: def _get_secret_name(name): return f"{SPARK8S_LABEL}-sa-conf-{name}" - def _retrieve_account_configurations( - self, name: str, namespace: str - ) -> PropertyFile: - secret_name = self._get_secret_name(name) + @staticmethod + def _get_configuration_hub_secret_name(name): + return f"{CONFIGURATION_HUB_LABEL}-{name}" + def _retrieve_secret_configurations( + self, name: str, namespace: str, secret_name: str + ) -> PropertyFile: try: secret = self.kube_interface.get_secret(secret_name, namespace=namespace)[ "data" @@ -1024,12 +1182,20 @@ def _build_service_account_from_raw(self, metadata: Dict[str, Any]): namespace = metadata["namespace"] primary = PRIMARY_LABELNAME in metadata["labels"] + account_secret_name = self._get_secret_name(name) + configuration_hub_secret_name = self._get_configuration_hub_secret_name(name) + return ServiceAccount( name=name, namespace=namespace, primary=primary, api_server=self.kube_interface.api_server, - extra_confs=self._retrieve_account_configurations(name, namespace), + extra_confs=self._retrieve_secret_configurations( + name, namespace, account_secret_name + ), + configuration_hub_confs=self._retrieve_secret_configurations( + name, namespace, configuration_hub_secret_name + ), ) def set_primary( @@ -1131,16 +1297,7 @@ def create(self, service_account: ServiceAccount) -> str: KubernetesResourceType.ROLE, rolename, namespace=service_account.namespace, - **{ - "resource": [ - "pods", - "configmaps", - "services", - "serviceaccounts", - "secrets", - ], - "verb": ["create", "get", "list", "watch", "delete"], - }, + **{"username": username}, ) self.kube_interface.create( KubernetesResourceType.ROLEBINDING, @@ -1173,45 +1330,49 @@ def create(self, service_account: ServiceAccount) -> str: if service_account.primary is True: self.set_primary(serviceaccount, service_account.namespace) + self._create_account_secret(service_account) + if len(service_account.extra_confs) > 0: self.set_configurations(serviceaccount, service_account.extra_confs) return serviceaccount - def _create_account_configuration(self, service_account: ServiceAccount): + def _create_account_secret(self, service_account: ServiceAccount): + """This function create the secret that will contain the user configurations.""" secret_name = self._get_secret_name(service_account.name) - try: - self.kube_interface.delete( - KubernetesResourceType.SECRET, - secret_name, - namespace=service_account.namespace, - ) - except Exception: - pass + self.kube_interface.create( + KubernetesResourceType.SECRET_GENERIC, + secret_name, + namespace=service_account.namespace, + ) - with umask_named_temporary_file( - mode="w", prefix="spark-dynamic-conf-k8s-", suffix=".conf" - ) as t: - self.logger.debug( - f"Spark dynamic props available for reference at {t.name}\n" - ) + def _add_account_configuration( + self, + service_account: ServiceAccount, + ): + """Add service account configuration to the service account.""" + secret_name = self._get_secret_name(service_account.name) - PropertyFile( - { - self._kubernetes_key_serializer.serialize(key): value - for key, value in service_account.extra_confs.props.items() - } - ).write(t.file) + properties = PropertyFile( + { + self._kubernetes_key_serializer.serialize(key): value + for key, value in service_account.extra_confs.props.items() + } + ) - t.flush() + # delete secret content + self.kube_interface.delete_secret_content( + secret_name, + namespace=service_account.namespace, + ) - self.kube_interface.create( - KubernetesResourceType.SECRET_GENERIC, - secret_name, - namespace=service_account.namespace, - **{"from-env-file": str(t.name)}, - ) + # add configurations to secrets + self.kube_interface.add_secret_content( + secret_name, + service_account.namespace, + properties, + ) def set_configurations(self, account_id: str, configurations: PropertyFile) -> str: """Set a new service account configuration for the provided service account id. @@ -1222,8 +1383,7 @@ def set_configurations(self, account_id: str, configurations: PropertyFile) -> s """ namespace, name = account_id.split(":") - - self._create_account_configuration( + self._add_account_configuration( ServiceAccount( name=name, namespace=namespace, @@ -1615,7 +1775,10 @@ def pyspark_shell( os.system(submit_cmd) def spark_sql( - self, confs: List[str], cli_property: Optional[str], extra_args: List[str] + self, + confs: List[str], + cli_property: Optional[str], + extra_args: List[str], ): """Start an interactive Spark SQL shell. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 56791a8..86de3f3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,4 +1,6 @@ import os +import subprocess +import uuid import pytest from lightkube.resources.core_v1 import Namespace @@ -72,3 +74,14 @@ def lightkube_registry(lightkubeinterface): registry = K8sServiceAccountRegistry(lightkubeinterface) yield registry _clearnup_registry(registry) + + +@pytest.fixture +def namespace(): + """A temporary K8S namespace gets cleaned up automatically""" + namespace_name = str(uuid.uuid4()) + create_command = ["kubectl", "create", "namespace", namespace_name] + subprocess.run(create_command, check=True) + yield namespace_name + destroy_command = ["kubectl", "delete", "namespace", namespace_name] + subprocess.run(destroy_command, check=True) diff --git a/tests/integration/test_cli_service_accounts.py b/tests/integration/test_cli_service_accounts.py index 811a36a..2f0c3c5 100644 --- a/tests/integration/test_cli_service_accounts.py +++ b/tests/integration/test_cli_service_accounts.py @@ -1,4 +1,5 @@ import json +import os import subprocess import uuid from collections import defaultdict @@ -6,7 +7,14 @@ import pytest -from spark8t.literals import MANAGED_BY_LABELNAME, PRIMARY_LABELNAME, SPARK8S_LABEL +from spark8t.domain import KubernetesResourceType, PropertyFile +from spark8t.literals import ( + CONFIGURATION_HUB_LABEL, + MANAGED_BY_LABELNAME, + PRIMARY_LABELNAME, + SPARK8S_LABEL, +) +from spark8t.utils import umask_named_temporary_file VALID_BACKENDS = [ "kubectl", @@ -14,21 +22,51 @@ ] ALLOWED_PERMISSIONS = { - "pods": ["create", "get", "list", "watch", "delete"], - "configmaps": ["create", "get", "list", "watch", "delete"], - "services": ["create", "get", "list", "watch", "delete"], + "pods": [ + "create", + "get", + "list", + "watch", + "delete", + "deletecollection", + "patch", + "update", + ], + "configmaps": [ + "create", + "get", + "list", + "watch", + "delete", + "deletecollection", + "patch", + "update", + ], + "services": [ + "create", + "get", + "list", + "watch", + "delete", + "deletecollection", + "patch", + "update", + ], } +ALL_ACTIONS = [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch", +] -@pytest.fixture -def namespace(): - """A temporary K8S namespace gets cleaned up automatically""" - namespace_name = str(uuid.uuid4()) - create_command = ["kubectl", "create", "namespace", namespace_name] - subprocess.run(create_command, check=True) - yield namespace_name - destroy_command = ["kubectl", "delete", "namespace", namespace_name] - subprocess.run(destroy_command, check=True) +ALLOWED_PERMISSIONS_USER_SECRET = ["get", "patch", "update"] +ALLOWED_PERMISSIONS_HUB_SECRET = ["get"] def run_service_account_registry(*args): @@ -46,16 +84,6 @@ def run_service_account_registry(*args): return e.stdout.decode(), e.stderr.decode(), e.returncode -def parameterize(permissions): - """ - A utility function to parameterize combinations of actions and RBAC permissions. - """ - parameters = [] - for resource, actions in permissions.items(): - parameters.extend([(action, resource) for action in actions]) - return parameters - - @pytest.fixture(params=VALID_BACKENDS) def service_account(namespace, request): """A temporary service account that gets cleaned up automatically.""" @@ -91,9 +119,8 @@ def multiple_namespaces_and_service_accounts(): @pytest.mark.parametrize("backend", VALID_BACKENDS) -@pytest.mark.parametrize("action, resource", parameterize(ALLOWED_PERMISSIONS)) @pytest.mark.parametrize("primary", [True, False]) -def test_create_service_account(namespace, backend, action, resource, primary): +def test_create_service_account(namespace, backend, primary): """Test creation of service account using the CLI. Verify that the serviceaccount, role and rolebinding resources are created @@ -104,6 +131,8 @@ def test_create_service_account(namespace, backend, action, resource, primary): username = "foobar" role_name = f"{username}-role" role_binding_name = f"{username}-role-binding" + secret_name = f"{SPARK8S_LABEL}-sa-conf-{username}" + hub_secret_name = f"configuration-hub-conf-{username}" create_args = [ "create", @@ -181,26 +210,99 @@ def test_create_service_account(namespace, backend, action, resource, primary): expected_labels.update({PRIMARY_LABELNAME: "True"}) assert actual_labels == expected_labels - # Check for RBAC permissions - sa_identifier = f"system:serviceaccount:{namespace}:{username}" - rbac_check = subprocess.run( + # Check secret creation + secret_result = subprocess.run( [ "kubectl", - "auth", - "can-i", - action, - resource, - "--namespace", + "get", + "secret", + secret_name, + "-n", namespace, - "--as", - sa_identifier, + "-o", + "json", ], check=True, capture_output=True, text=True, ) - assert rbac_check.returncode == 0 - assert rbac_check.stdout.strip() == "yes" + assert secret_result.returncode == 0 + + # Check for RBAC permissions + sa_identifier = f"system:serviceaccount:{namespace}:{username}" + for resource, actions in ALLOWED_PERMISSIONS.items(): + for action in actions: + rbac_check = subprocess.run( + [ + "kubectl", + "auth", + "can-i", + action, + resource, + "--namespace", + namespace, + "--as", + sa_identifier, + ], + check=True, + capture_output=True, + text=True, + ) + assert rbac_check.returncode == 0 + assert rbac_check.stdout.strip() == "yes" + + # Check for RBAC permissions for named resources + + resource_name_actions = { + secret_name: ALLOWED_PERMISSIONS_USER_SECRET, + hub_secret_name: ALLOWED_PERMISSIONS_HUB_SECRET, + } + + for resource_name, actions in resource_name_actions.items(): + for action in actions: + rbac_check = subprocess.run( + [ + "kubectl", + "auth", + "can-i", + action, + f"secret/{resource_name}", + "--namespace", + namespace, + "--as", + sa_identifier, + ], + check=True, + capture_output=True, + text=True, + ) + assert rbac_check.returncode == 0 + assert rbac_check.stdout.strip() == "yes" + + not_allowed_actions = set(ALL_ACTIONS).difference(actions) + print(not_allowed_actions) + for action in not_allowed_actions: + command = [ + "kubectl", + "auth", + "can-i", + action, + f"secret/{resource_name}", + "--namespace", + namespace, + "--as", + sa_identifier, + ] + print(" ".join(command)) + rbac_check = subprocess.run( + command, + capture_output=True, + text=True, + ) + print(f"Return code: {rbac_check.returncode}") + print(f"Return stdout: {rbac_check.stdout.strip()}") + assert rbac_check.returncode != 0 + assert rbac_check.stdout.strip() == "no" @pytest.mark.parametrize("backend", VALID_BACKENDS) @@ -222,8 +324,7 @@ def test_create_service_account_when_account_already_exists(service_account, bac @pytest.mark.parametrize("backend", VALID_BACKENDS) -@pytest.mark.parametrize("action, resource", parameterize(ALLOWED_PERMISSIONS)) -def test_delete_service_account(service_account, backend, action, resource): +def test_delete_service_account(service_account, backend): """Test deletion of service account using the CLI. Verify that the serviceaccount, role and rolebinding resources are deleted @@ -274,23 +375,25 @@ def test_delete_service_account(service_account, backend, action, resource): # Check for RBAC permissions, these should be invalid now sa_identifier = f"system:serviceaccount:{namespace}:{username}" - rbac_check = subprocess.run( - [ - "kubectl", - "auth", - "can-i", - action, - resource, - "--namespace", - namespace, - "--as", - sa_identifier, - ], - capture_output=True, - text=True, - ) - assert rbac_check.returncode != 0 - assert rbac_check.stdout.strip() == "no" + for resource, actions in ALLOWED_PERMISSIONS.items(): + for action in actions: + rbac_check = subprocess.run( + [ + "kubectl", + "auth", + "can-i", + action, + resource, + "--namespace", + namespace, + "--as", + sa_identifier, + ], + capture_output=True, + text=True, + ) + assert rbac_check.returncode != 0 + assert rbac_check.stdout.strip() == "no" @pytest.mark.parametrize("backend", VALID_BACKENDS) @@ -430,7 +533,7 @@ def test_service_accounts_listing_multiple_namespaces( @pytest.mark.parametrize("backend", VALID_BACKENDS) -def test_service_account_get_config(service_account, backend): +def test_service_account_get_config(service_account, backend, request): """Test retrieval of service account configs using the CLI. Use a fixture that creates temporary service account, then @@ -458,6 +561,67 @@ def test_service_account_get_config(service_account, backend): } assert actual_configs == expected_configs + # add configuration hub secret for the test service account + secret_name = f"{CONFIGURATION_HUB_LABEL}-{username}" + + property_file = PropertyFile({"key": "value"}) + + kubeinterface = request.getfixturevalue("kubeinterface") + + with umask_named_temporary_file( + mode="w", + prefix="spark-dynamic-conf-k8s-", + suffix=".conf", + dir=os.path.expanduser("~"), + ) as t: + property_file.write(t.file) + + t.flush() + + kubeinterface.create( + KubernetesResourceType.SECRET_GENERIC, + secret_name, + namespace=namespace, + **{"from-env-file": str(t.name)}, + ) + + assert kubeinterface.exists( + KubernetesResourceType.SECRET_GENERIC, secret_name, namespace + ) + + # check that configuration hub config is there + # Get the default configs created with a service account + stdout, stderr, ret_code = run_service_account_registry( + "get-config", + "--username", + username, + "--namespace", + namespace, + "--backend", + backend, + ) + actual_configs = set(stdout.splitlines()) + expected_configs_hub = { + f"spark.kubernetes.authenticate.driver.serviceAccountName={username}", + f"spark.kubernetes.namespace={namespace}", + "key=value", + } + + assert actual_configs == expected_configs_hub + + stdout, stderr, ret_code = run_service_account_registry( + "get-config", + "--username", + username, + "--namespace", + namespace, + "--backend", + backend, + "--ignore-configuration-hub", + ) + actual_configs = set(stdout.splitlines()) + assert actual_configs == expected_configs + @pytest.mark.parametrize("backend", VALID_BACKENDS) def test_service_account_add_config(service_account, backend): @@ -505,6 +669,7 @@ def test_service_account_add_config(service_account, backend): namespace, "--backend", backend, + "--ignore-configuration-hub", ) updated_configs = set(stdout.splitlines()) diff --git a/tests/integration/test_kube_interface.py b/tests/integration/test_kube_interface.py index 8cfbe90..bb70d82 100644 --- a/tests/integration/test_kube_interface.py +++ b/tests/integration/test_kube_interface.py @@ -1,3 +1,4 @@ +import os from time import sleep import pytest @@ -35,16 +36,18 @@ def test_create_exists_delete( @pytest.mark.parametrize( "kubeinterface_name", [("kubeinterface"), ("lightkubeinterface")] ) -def test_create_exists_delete_secret(kubeinterface_name, request): +def test_create_exists_delete_secret(kubeinterface_name, request, namespace): secret_name = "my-secret" - namespace = "default" property_file = PropertyFile({"key": "value"}) kubeinterface = request.getfixturevalue(kubeinterface_name) with umask_named_temporary_file( - mode="w", prefix="spark-dynamic-conf-k8s-", suffix=".conf" + mode="w", + prefix="spark-dynamic-conf-k8s-", + suffix=".conf", + dir=os.path.expanduser("~"), ) as t: property_file.write(t.file) @@ -66,3 +69,70 @@ def test_create_exists_delete_secret(kubeinterface_name, request): assert not kubeinterface.exists( KubernetesResourceType.SECRET_GENERIC, secret_name, namespace ) + + +@pytest.mark.parametrize( + "kubeinterface_name", [("kubeinterface"), ("lightkubeinterface")] +) +def test_delete_secret_content(kubeinterface_name, request, namespace): + secret_name = "my-secret" + + property_file = PropertyFile({"key": "value"}) + + kubeinterface = request.getfixturevalue(kubeinterface_name) + + with umask_named_temporary_file( + mode="w", + prefix="spark-dynamic-conf-k8s-", + suffix=".conf", + dir=os.path.expanduser("~"), + ) as t: + property_file.write(t.file) + + t.flush() + + kubeinterface.create( + KubernetesResourceType.SECRET_GENERIC, + secret_name, + namespace=namespace, + **{"from-env-file": str(t.name)}, + ) + + assert kubeinterface.exists( + KubernetesResourceType.SECRET_GENERIC, secret_name, namespace + ) + + kubeinterface.delete_secret_content(secret_name, namespace) + + kubeinterface.exists(KubernetesResourceType.SECRET_GENERIC, secret_name, namespace) + + secret_content = kubeinterface.get_secret(secret_name, namespace)["data"] + + print(f"Secret content: {secret_content}") + assert len(secret_content.keys()) == 0 + + kubeinterface.add_secret_content(secret_name, namespace, property_file) + secret_content = kubeinterface.get_secret(secret_name, namespace)["data"] + + print(f"Secret content: {secret_content}") + assert len(secret_content.keys()) == 1 + + kubeinterface.add_secret_content(secret_name, namespace, property_file) + secret_content = kubeinterface.get_secret(secret_name, namespace)["data"] + + print(f"Secret content: {secret_content}") + assert len(secret_content.keys()) == 1 + + property_file_1 = PropertyFile({"key": "value", "key1": "value"}) + kubeinterface.add_secret_content(secret_name, namespace, property_file_1) + + secret_content = kubeinterface.get_secret(secret_name, namespace)["data"] + + print(f"Secret content: {secret_content}") + assert len(secret_content.keys()) == 2 + + kubeinterface.delete(KubernetesResourceType.SECRET_GENERIC, secret_name, namespace) + + assert not kubeinterface.exists( + KubernetesResourceType.SECRET_GENERIC, secret_name, namespace + ) diff --git a/tests/unittest/test_services.py b/tests/unittest/test_services.py index e8762ca..d09d46d 100644 --- a/tests/unittest/test_services.py +++ b/tests/unittest/test_services.py @@ -734,6 +734,7 @@ def side_effect(*args, **kwargs): with patch("builtins.open", mock_open(read_data=kubeconfig_yaml_str)): k = LightKube(kube_config_file=kubeconfig, defaults=defaults) + print(f"rn: {resource_name}, namespace: {namespace}") k.create( KubernetesResourceType.ROLEBINDING, resource_name, @@ -762,8 +763,7 @@ def test_lightkube_create_secret(mocker, tmp_kubeconf): "apiVersion": "v1", "kind": "Secret", "metadata": {"name": resource_name, "namespace": namespace}, - # "stringData": { label_key : base64.b64encode(label_value.encode("ascii")) }, - "stringData": {}, + "stringData": None, } ) @@ -1219,14 +1219,14 @@ def side_effect(*args, **kwargs): ) -def test_k8s_registry_retrieve_account_configurations(mocker): +def test_k8s_registry_secret_account_configurations(mocker): mock_kube_interface = mocker.patch("spark8t.services.KubeInterface") data = {"k": "v"} mock_kube_interface.get_secret.return_value = {"data": data} registry = K8sServiceAccountRegistry(mock_kube_interface) assert ( - registry._retrieve_account_configurations( - str(uuid.uuid4()), str(uuid.uuid4()) + registry._retrieve_secret_configurations( + str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4()) ).props == data ) @@ -1409,16 +1409,7 @@ def test_k8s_registry_create(mocker): "role", f"{name3}-role", namespace=namespace3, - **{ - "resource": [ - "pods", - "configmaps", - "services", - "serviceaccounts", - "secrets", - ], - "verb": ["create", "get", "list", "watch", "delete"], - }, + **{"username": f"{name3}"}, ) mock_kube_interface.create.assert_any_call( diff --git a/tox.ini b/tox.ini index b712c2c..5d90e9d 100644 --- a/tox.ini +++ b/tox.ini @@ -45,7 +45,7 @@ description = Run unit tests commands = poetry install --with unit --sync --no-cache poetry export -f requirements.txt -o requirements.txt - poetry run pytest tests/unittest + poetry run pytest -vv tests/unittest [testenv:integration] description = Run integration tests @@ -55,7 +55,7 @@ setenv = commands = poetry install --with integration --sync --no-cache poetry export -f requirements.txt -o requirements.txt - poetry run pytest tests/integration + poetry run pytest -vv tests/integration [testenv:all-tests] description = Run unit tests