Skip to content

Commit

Permalink
Additional sinks (#231)
Browse files Browse the repository at this point in the history 
* mysql sink checkpoint

* use gorm for postgres too

* checkpoint

* get direct materialize sink working

* docs and spin up devel destinations faster

* Adds ability to create multiple db-based sinks

* starts clickhouse sink, taking a break

* Get clickhouse running alongside other sinks

* ff minor since so much functionality

* better defaults, quickstart devel, etc

* bump compose

* devel tracker domain

* clean up setup for now

* Adds mongo and mongo-express to devel env

* toss kowl and kafka and whatnot into the party
  • Loading branch information
@jakthom
jakthom authored Apr 15, 2022
1 parent 660cba4 commit 9ca6e45
Show file tree
Hide file tree
Showing 29 changed files with 1,161 additions and 84 deletions.
2 changes: 1 addition & 1 deletion .VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v0.1.45
v0.5.0
12 changes: 12 additions & 0 deletions examples/devel/clickhouse/conf.d/server.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<clickhouse>
<!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>

<!--
<logger>
<console>1</console>
</logger>
-->
</clickhouse>
133 changes: 133 additions & 0 deletions examples/devel/clickhouse/conf.d/users.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
<?xml version="1.0"?>
<clickhouse>
<!-- See also the files in users.d directory where the settings can be overridden. -->

<!-- Profiles of settings. -->
<profiles>
<!-- Default settings. -->
<default>
<!-- Maximum memory usage for processing single query, in bytes. -->
<max_memory_usage>10000000000</max_memory_usage>

<!-- How to choose between replicas during distributed query processing.
random - choose random replica from set of replicas with minimum number of errors
nearest_hostname - from set of replicas with minimum number of errors, choose replica
with minimum number of different symbols between replica's hostname and local hostname
(Hamming distance).
in_order - first live replica is chosen in specified order.
first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
-->
<load_balancing>random</load_balancing>
</default>

<!-- Profile that allows only read queries. -->
<readonly>
<readonly>1</readonly>
</readonly>
</profiles>

<!-- Users and ACL. -->
<users>
<!-- If user name was not specified, 'default' user is used. -->
<default>
<!-- See also the files in users.d directory where the password can be overridden.
Password could be specified in plaintext or in SHA256 (in hex format).
If you want to specify password in plaintext (not recommended), place it in 'password' element.
Example: <password>qwerty</password>.
Password could be empty.
If you want to specify SHA256, place it in 'password_sha256_hex' element.
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
Example: <password_double_sha1_hex>e395796d6546b1b65db9d665cd43f0e858dd4303</password_double_sha1_hex>
If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication,
place its name in 'server' element inside 'ldap' element.
Example: <ldap><server>my_ldap_server</server></ldap>
If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config),
place 'kerberos' element instead of 'password' (and similar) elements.
The name part of the canonical principal name of the initiator must match the user name for authentication to succeed.
You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests
whose initiator's realm matches it.
Example: <kerberos />
Example: <kerberos><realm>EXAMPLE.COM</realm></kerberos>
How to generate decent password:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
In first line will be password and in second - corresponding SHA256.
How to generate double SHA1:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
In first line will be password and in second - corresponding double SHA1.
-->
<password></password>

<!-- List of networks with open access.
To open access from everywhere, specify:
<ip>::/0</ip>
To open access only from localhost, specify:
<ip>::1</ip>
<ip>127.0.0.1</ip>
Each element of list has one of the following forms:
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
<host> Hostname. Example: server01.yandex.ru.
To check access, DNS query is performed, and all received addresses compared to peer address.
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
To check access, DNS PTR query is performed for peer address and then regexp is applied.
Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
Strongly recommended that regexp is ends with $
All results of DNS requests are cached till server restart.
-->
<networks>
<ip>::/0</ip>
</networks>

<!-- Settings profile for user. -->
<profile>default</profile>

<!-- Quota for user. -->
<quota>default</quota>

<!-- User can create other users and grant rights to them. -->
<access_management>1</access_management>
</default>

<honeypot>
<password>honeypot</password>
<networks>
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
<access_management>1</access_management>
</honeypot>
</users>

<!-- Quotas. -->
<quotas>
<!-- Name of quota. -->
<default>
<!-- Limits for time interval. You could specify many intervals with different limits. -->
<interval>
<!-- Length of interval. -->
<duration>3600</duration>

<!-- No limits. Just calculate resource usage for time interval. -->
<queries>0</queries>
<errors>0</errors>
<result_rows>0</result_rows>
<read_rows>0</read_rows>
<execution_time>0</execution_time>
</interval>
</default>
</quotas>
</clickhouse>
130 changes: 130 additions & 0 deletions examples/devel/docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
version: "3.9"

services:
honeypot:
container_name: honeypot
image: honeypot:v0.5.0 # Note! use latest :)
volumes:
- type: bind
source: ./honeypot/devel.conf.yml
target: /etc/honeypot/config.yml
- type: bind
source: ../../schemas
target: /schemas/
- type: bind
source: ../../site/index.html
target: /app/site/index.html
environment:
- HONEYPOT_CONFIG_PATH=/etc/honeypot/config.yml
ports:
- 8080:8080
deploy:
restart_policy:
condition: on-failure
healthcheck:
test: curl -f localhost:8080/health
interval: 15s

mysql:
container_name: mysql
image: mysql/mysql-server:8.0
ports:
- 3306:3306
environment:
- MYSQL_ROOT_PASSWORD=honeypot
- MYSQL_DATABASE=honeypot
- MYSQL_USER=honeypot
- MYSQL_PASSWORD=honeypot

postgres:
container_name: postgres
image: postgres:14.2
ports:
- 5432:5432
environment:
- POSTGRES_DB=honeypot
- POSTGRES_USER=honeypot
- POSTGRES_PASSWORD=honeypot

clickhouse:
container_name: clickhouse
image: yandex/clickhouse-server
volumes:
- type: bind
source: ./clickhouse/conf.d
target: /etc/clickhouse-server/config.d/
- type: bind
source: ./clickhouse/conf.d/users.xml
target: /etc/clickhouse-server/users.xml
ports:
- 8123:8123
- 9000:9000

mongo:
image: mongo
container_name: mongo
restart: always
environment:
MONGO_INITDB_ROOT_USERNAME: honeypot
MONGO_INITDB_ROOT_PASSWORD: honeypot
ports:
- 27017:27017

mongo-express:
container_name: mongo-express
image: mongo-express
restart: always
ports:
- 8084:8081
environment:
ME_CONFIG_MONGODB_ADMINUSERNAME: honeypot
ME_CONFIG_MONGODB_ADMINPASSWORD: honeypot
ME_CONFIG_MONGODB_URL: mongodb://honeypot:honeypot@mongo:27017/

materialized:
container_name: materialized
image: materialize/materialized # Note! Use the latest version :)
init: true
command: -w2 --disable-telemetry
ports:
- 6875:6875

redpanda-1:
image: docker.vectorized.io/vectorized/redpanda:v21.11.9 # Note! Use the latest version :)
container_name: redpanda-1
command:
- redpanda start
- --overprovisioned
- --check=false
- --smp 1
- --memory 1G
- --reserve-memory 0M
- --node-id 0
- --kafka-addr
- INSIDE://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
- --advertise-kafka-addr
- INSIDE://redpanda-1:29092,OUTSIDE://127.0.0.1:9092
- --rpc-addr 0.0.0.0:33145
- --advertise-rpc-addr redpanda-1:33145
- --set redpanda.enable_transactions=true
- --set redpanda.enable_idempotence=true
ports:
- 9092:9092
- 9081:8081
healthcheck:
test: curl -f localhost:9644/v1/status/ready
interval: 1s
start_period: 30s

kowl:
image: quay.io/cloudhut/kowl:v1.5.0
restart: on-failure
container_name: kowl
volumes:
- ./kowl/devel.yml:/etc/kowl/config.yml
ports:
- 8081:8081
entrypoint: ./kowl --config.filepath=/etc/kowl/config.yml
depends_on:
redpanda-1:
condition: service_healthy
Loading

0 comments on commit 9ca6e45

Please sign in to comment.