From 7dad124c2931f3a09231c731cd39983ba81e9296 Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Thu, 30 Dec 2021 22:21:49 +0200 Subject: [PATCH 1/4] Adding support for Cassandra and Scylla This add the support for those cassandra based dbs and their drivers, cassandra-driver, scylla-driver Ref: https://cassandra.apache.org/ Ref: https://www.scylladb.com/ Ref: https://pypi.org/project/cassandra-driver/ Ref: https://pypi.org/project/scylla-driver/ --- core/testcontainers/cassandra.py | 47 +++++++++++++++++++++++++++++++ core/testcontainers/scylla.py | 48 ++++++++++++++++++++++++++++++++ tests/test_cassandra.py | 18 ++++++++++++ tests/test_scylla.py | 18 ++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 core/testcontainers/cassandra.py create mode 100644 core/testcontainers/scylla.py create mode 100644 tests/test_cassandra.py create mode 100644 tests/test_scylla.py diff --git a/core/testcontainers/cassandra.py b/core/testcontainers/cassandra.py new file mode 100644 index 00000000..696a0542 --- /dev/null +++ b/core/testcontainers/cassandra.py @@ -0,0 +1,47 @@ +from testcontainers.core.config import MAX_TRIES +from testcontainers.core.generic import DockerContainer +from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs + + +class CassandraContainer(DockerContainer): + """ + Cassandra database container. + + Example + ------- + .. doctest:: + + >>> from testcontainers.cassandra import CassandraContainer + + >>> with CassandraContainer() as cassandra: + ... cluster = cassandra.get_cluster() + ... with cluster.connect() as session: + ... result = session.execute( + ... "CREATE KEYSPACE keyspace1 WITH replication = " + ... "{'class': 'SimpleStrategy', 'replication_factor': '1'};") + """ + def __init__(self, image="rinscy/cassandra:latest", ports_to_expose=[9042]): + super(CassandraContainer, self).__init__(image) + self.ports_to_expose = ports_to_expose + self.with_exposed_ports(*self.ports_to_expose) + + @wait_container_is_ready() + def _connect(self): + wait_for_logs( + self, + predicate="Starting listening for CQL clients", + timeout=MAX_TRIES) + cluster = self.get_cluster() + cluster.connect() + + def start(self): + super(CassandraContainer, self).start() + self._connect() + return self + + def get_cluster(self, **kwargs): + from cassandra.cluster import Cluster + container = self.get_wrapped_container() + container.reload() + hostname = container.attrs['NetworkSettings']['IPAddress'] + return Cluster(contact_points=[hostname], **kwargs) diff --git a/core/testcontainers/scylla.py b/core/testcontainers/scylla.py new file mode 100644 index 00000000..f3e9c9cc --- /dev/null +++ b/core/testcontainers/scylla.py @@ -0,0 +1,48 @@ +from testcontainers.core.config import MAX_TRIES +from testcontainers.core.generic import DockerContainer +from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs + + +class ScyllaContainer(DockerContainer): + """ + Scylla database container. + + Example + ------- + .. doctest:: + + >>> from testcontainers.scylla import ScyllaContainer + + >>> with ScyllaContainer() as scylla: + ... cluster = scylla.get_cluster() + ... with cluster.connect() as session: + ... result = session.execute( + ... "CREATE KEYSPACE keyspace1 WITH replication " + ... "= {'class': 'SimpleStrategy', 'replication_factor': '1'};") + """ + def __init__(self, image="scylladb/scylla:latest", ports_to_expose=[9042]): + super(ScyllaContainer, self).__init__(image) + self.ports_to_expose = ports_to_expose + self.with_exposed_ports(*self.ports_to_expose) + self.with_command("--skip-wait-for-gossip-to-settle=0") + + @wait_container_is_ready() + def _connect(self): + wait_for_logs( + self, + predicate="Starting listening for CQL clients", + timeout=MAX_TRIES) + cluster = self.get_cluster() + cluster.connect() + + def start(self): + super(ScyllaContainer, self).start() + self._connect() + return self + + def get_cluster(self, **kwargs): + from cassandra.cluster import Cluster + container = self.get_wrapped_container() + container.reload() + hostname = container.attrs['NetworkSettings']['IPAddress'] + return Cluster(contact_points=[hostname], **kwargs) diff --git a/tests/test_cassandra.py b/tests/test_cassandra.py new file mode 100644 index 00000000..bf2aca01 --- /dev/null +++ b/tests/test_cassandra.py @@ -0,0 +1,18 @@ +from testcontainers.cassandra import CassandraContainer + + +def test_docker_run_cassandra(): + with CassandraContainer() as cassandra: + cluster = cassandra.get_cluster() + with cluster.connect() as session: + session.execute( + "CREATE KEYSPACE keyspace1 WITH replication = " + "{'class': 'SimpleStrategy', 'replication_factor': '1'};") + session.execute( + "CREATE TABLE keyspace1.table1 (key1 int, key2 int, PRIMARY KEY (key1));") + session.execute("INSERT INTO keyspace1.table1 (key1,key2) values (1,2);") + + response = session.execute("SELECT * FROM keyspace1.table1") + + assert response.one().key1 == 1 + assert response.one().key2 == 2 diff --git a/tests/test_scylla.py b/tests/test_scylla.py new file mode 100644 index 00000000..a8515e93 --- /dev/null +++ b/tests/test_scylla.py @@ -0,0 +1,18 @@ +from testcontainers.scylla import ScyllaContainer + + +def test_docker_run_scylla(): + with ScyllaContainer() as scylla: + cluster = scylla.get_cluster() + with cluster.connect() as session: + session.execute( + "CREATE KEYSPACE keyspace1 WITH replication = " + "{'class': 'SimpleStrategy', 'replication_factor': '1'};") + session.execute( + "CREATE TABLE keyspace1.table1 (key1 int, key2 int, PRIMARY KEY (key1));") + session.execute("INSERT INTO keyspace1.table1 (key1,key2) values (1,2);") + + response = session.execute("SELECT * FROM keyspace1.table1") + + assert response.one().key1 == 1 + assert response.one().key2 == 2 From b945314d692561d8431aa78a367827465df05c18 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Wed, 13 Mar 2024 23:30:45 -0400 Subject: [PATCH 2/4] move files to new locations --- core/testcontainers/cassandra.py | 47 ------------------- modules/scylla/README.rst | 2 + .../scylla/testcontainers/scylla/__init__.py | 0 .../scylla/tests}/test_scylla.py | 0 tests/test_cassandra.py | 18 ------- 5 files changed, 2 insertions(+), 65 deletions(-) delete mode 100644 core/testcontainers/cassandra.py create mode 100644 modules/scylla/README.rst rename core/testcontainers/scylla.py => modules/scylla/testcontainers/scylla/__init__.py (100%) rename {tests => modules/scylla/tests}/test_scylla.py (100%) delete mode 100644 tests/test_cassandra.py diff --git a/core/testcontainers/cassandra.py b/core/testcontainers/cassandra.py deleted file mode 100644 index 696a0542..00000000 --- a/core/testcontainers/cassandra.py +++ /dev/null @@ -1,47 +0,0 @@ -from testcontainers.core.config import MAX_TRIES -from testcontainers.core.generic import DockerContainer -from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs - - -class CassandraContainer(DockerContainer): - """ - Cassandra database container. - - Example - ------- - .. doctest:: - - >>> from testcontainers.cassandra import CassandraContainer - - >>> with CassandraContainer() as cassandra: - ... cluster = cassandra.get_cluster() - ... with cluster.connect() as session: - ... result = session.execute( - ... "CREATE KEYSPACE keyspace1 WITH replication = " - ... "{'class': 'SimpleStrategy', 'replication_factor': '1'};") - """ - def __init__(self, image="rinscy/cassandra:latest", ports_to_expose=[9042]): - super(CassandraContainer, self).__init__(image) - self.ports_to_expose = ports_to_expose - self.with_exposed_ports(*self.ports_to_expose) - - @wait_container_is_ready() - def _connect(self): - wait_for_logs( - self, - predicate="Starting listening for CQL clients", - timeout=MAX_TRIES) - cluster = self.get_cluster() - cluster.connect() - - def start(self): - super(CassandraContainer, self).start() - self._connect() - return self - - def get_cluster(self, **kwargs): - from cassandra.cluster import Cluster - container = self.get_wrapped_container() - container.reload() - hostname = container.attrs['NetworkSettings']['IPAddress'] - return Cluster(contact_points=[hostname], **kwargs) diff --git a/modules/scylla/README.rst b/modules/scylla/README.rst new file mode 100644 index 00000000..fd1ea03f --- /dev/null +++ b/modules/scylla/README.rst @@ -0,0 +1,2 @@ +.. autoclass:: testcontainers.scylla.ScyllaContainer +.. title:: testcontainers.scylla.ScyllaContainer diff --git a/core/testcontainers/scylla.py b/modules/scylla/testcontainers/scylla/__init__.py similarity index 100% rename from core/testcontainers/scylla.py rename to modules/scylla/testcontainers/scylla/__init__.py diff --git a/tests/test_scylla.py b/modules/scylla/tests/test_scylla.py similarity index 100% rename from tests/test_scylla.py rename to modules/scylla/tests/test_scylla.py diff --git a/tests/test_cassandra.py b/tests/test_cassandra.py deleted file mode 100644 index bf2aca01..00000000 --- a/tests/test_cassandra.py +++ /dev/null @@ -1,18 +0,0 @@ -from testcontainers.cassandra import CassandraContainer - - -def test_docker_run_cassandra(): - with CassandraContainer() as cassandra: - cluster = cassandra.get_cluster() - with cluster.connect() as session: - session.execute( - "CREATE KEYSPACE keyspace1 WITH replication = " - "{'class': 'SimpleStrategy', 'replication_factor': '1'};") - session.execute( - "CREATE TABLE keyspace1.table1 (key1 int, key2 int, PRIMARY KEY (key1));") - session.execute("INSERT INTO keyspace1.table1 (key1,key2) values (1,2);") - - response = session.execute("SELECT * FROM keyspace1.table1") - - assert response.one().key1 == 1 - assert response.one().key2 == 2 From 88c0c271c905831642a6aa8fd0e426cb68b8b6b6 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Wed, 13 Mar 2024 23:31:14 -0400 Subject: [PATCH 3/4] dependency housekeeping + list it on modules list --- poetry.lock | 10 +++++----- pyproject.toml | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 228c9c48..f35433bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -331,7 +331,7 @@ files = [ name = "cassandra-driver" version = "3.29.1" description = "DataStax Driver for Apache Cassandra" -optional = false +optional = true python-versions = "*" files = [ {file = "cassandra-driver-3.29.1.tar.gz", hash = "sha256:38e9c2a2f2a9664bb03f1f852d5fccaeff2163942b5db35dffcf8bf32a51cfe5"}, @@ -588,7 +588,7 @@ typing-extensions = ">=4.5.0" name = "click" version = "8.1.7" description = "Composable command line interface toolkit" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -1001,7 +1001,7 @@ typing = ["typing-extensions (>=4.8)"] name = "geomet" version = "0.2.1.post1" description = "GeoJSON <-> WKT/WKB conversion utilities" -optional = false +optional = true python-versions = ">2.6, !=3.3.*, <4" files = [ {file = "geomet-0.2.1.post1-py3-none-any.whl", hash = "sha256:a41a1e336b381416d6cbed7f1745c848e91defaa4d4c1bdc1312732e46ffad2b"}, @@ -1968,7 +1968,6 @@ python-versions = ">=3.7" files = [ {file = "milvus_lite-2.4.7-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:c828190118b104b05b8c8e0b5a4147811c86b54b8fb67bc2e726ad10fc0b544e"}, {file = "milvus_lite-2.4.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e1537633c39879714fb15082be56a4b97f74c905a6e98e302ec01320561081af"}, - {file = "milvus_lite-2.4.7-py3-none-manylinux2014_aarch64.whl", hash = "sha256:fcb909d38c83f21478ca9cb500c84264f988c69f62715ae9462e966767fb76dd"}, {file = "milvus_lite-2.4.7-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f016474d663045787dddf1c3aad13b7d8b61fd329220318f858184918143dcbf"}, ] @@ -4667,6 +4666,7 @@ qdrant = ["qdrant-client"] rabbitmq = ["pika"] redis = ["redis"] registry = ["bcrypt"] +scylla = ["cassandra-driver"] selenium = ["selenium"] sftp = ["cryptography"] test-module-import = ["httpx"] @@ -4677,4 +4677,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "18a5763385d12114513ef5d65268de3ea6567e79b21049b6d58d1803f4257306" +content-hash = "de6e3fcb9a3c1a402682f9681ba2c8270a9a0e8882f82b9835a2f50acb1e37d5" diff --git a/pyproject.toml b/pyproject.toml index 3bccf880..b80c7e41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ packages = [ { include = "testcontainers", from = "modules/registry" }, { include = "testcontainers", from = "modules/sftp" }, { include = "testcontainers", from = "modules/selenium" }, + { include = "testcontainers", from = "modules/scylla" }, { include = "testcontainers", from = "modules/trino" }, { include = "testcontainers", from = "modules/vault" }, { include = "testcontainers", from = "modules/weaviate" }, @@ -86,6 +87,7 @@ typing-extensions = "*" # community modules python-arango = { version = "^7.8", optional = true } azure-storage-blob = { version = "^12.19", optional = true } +cassandra-driver = { version = "3.29.1", optional = true } clickhouse-driver = { version = "*", optional = true } google-cloud-pubsub = { version = ">=2", optional = true } google-cloud-datastore = { version = ">=2", optional = true } @@ -156,6 +158,7 @@ rabbitmq = ["pika"] redis = ["redis"] registry = ["bcrypt"] selenium = ["selenium"] +scylla = ["cassandra-driver"] sftp = ["cryptography"] vault = [] weaviate = ["weaviate-client"] @@ -175,7 +178,6 @@ psycopg2-binary = "2.9.9" pg8000 = "1.30.5" sqlalchemy = "2.0.28" psycopg = "3.1.18" -cassandra-driver = "3.29.1" pytest-asyncio = "0.23.5" kafka-python-ng = "^2.2.0" hvac = "2.1.0" From 0b0711e2a5489c24e8a71b94e6b559a377e23776 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Wed, 13 Mar 2024 23:32:28 -0400 Subject: [PATCH 4/4] lint --- modules/scylla/testcontainers/scylla/__init__.py | 15 +++++++-------- modules/scylla/tests/test_scylla.py | 6 +++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/modules/scylla/testcontainers/scylla/__init__.py b/modules/scylla/testcontainers/scylla/__init__.py index f3e9c9cc..ca0f44af 100644 --- a/modules/scylla/testcontainers/scylla/__init__.py +++ b/modules/scylla/testcontainers/scylla/__init__.py @@ -20,29 +20,28 @@ class ScyllaContainer(DockerContainer): ... "CREATE KEYSPACE keyspace1 WITH replication " ... "= {'class': 'SimpleStrategy', 'replication_factor': '1'};") """ - def __init__(self, image="scylladb/scylla:latest", ports_to_expose=[9042]): - super(ScyllaContainer, self).__init__(image) + + def __init__(self, image="scylladb/scylla:latest", ports_to_expose=(9042,)): + super().__init__(image) self.ports_to_expose = ports_to_expose self.with_exposed_ports(*self.ports_to_expose) self.with_command("--skip-wait-for-gossip-to-settle=0") @wait_container_is_ready() def _connect(self): - wait_for_logs( - self, - predicate="Starting listening for CQL clients", - timeout=MAX_TRIES) + wait_for_logs(self, predicate="Starting listening for CQL clients", timeout=MAX_TRIES) cluster = self.get_cluster() cluster.connect() def start(self): - super(ScyllaContainer, self).start() + super().start() self._connect() return self def get_cluster(self, **kwargs): from cassandra.cluster import Cluster + container = self.get_wrapped_container() container.reload() - hostname = container.attrs['NetworkSettings']['IPAddress'] + hostname = container.attrs["NetworkSettings"]["IPAddress"] return Cluster(contact_points=[hostname], **kwargs) diff --git a/modules/scylla/tests/test_scylla.py b/modules/scylla/tests/test_scylla.py index a8515e93..3d1ecf44 100644 --- a/modules/scylla/tests/test_scylla.py +++ b/modules/scylla/tests/test_scylla.py @@ -7,9 +7,9 @@ def test_docker_run_scylla(): with cluster.connect() as session: session.execute( "CREATE KEYSPACE keyspace1 WITH replication = " - "{'class': 'SimpleStrategy', 'replication_factor': '1'};") - session.execute( - "CREATE TABLE keyspace1.table1 (key1 int, key2 int, PRIMARY KEY (key1));") + "{'class': 'SimpleStrategy', 'replication_factor': '1'};" + ) + session.execute("CREATE TABLE keyspace1.table1 (key1 int, key2 int, PRIMARY KEY (key1));") session.execute("INSERT INTO keyspace1.table1 (key1,key2) values (1,2);") response = session.execute("SELECT * FROM keyspace1.table1")