diff --git a/src/server/pom.xml b/src/server/pom.xml index 4ae81bfa5..253eab12f 100644 --- a/src/server/pom.xml +++ b/src/server/pom.xml @@ -165,7 +165,7 @@ org.cognitor.cassandra cassandra-migration - 2.2.1 + 2.3.0 org.flywaydb diff --git a/src/server/src/main/java/io/cassandrareaper/storage/CassandraStorage.java b/src/server/src/main/java/io/cassandrareaper/storage/CassandraStorage.java index f1e77f7cb..953ef1bea 100644 --- a/src/server/src/main/java/io/cassandrareaper/storage/CassandraStorage.java +++ b/src/server/src/main/java/io/cassandrareaper/storage/CassandraStorage.java @@ -41,7 +41,6 @@ import io.cassandrareaper.storage.cassandra.DateTimeCodec; import io.cassandrareaper.storage.cassandra.Migration016; import io.cassandrareaper.storage.cassandra.Migration021; -import io.cassandrareaper.storage.cassandra.SchemaMigrationLock; import java.io.IOException; import java.math.BigInteger; @@ -239,16 +238,19 @@ private static void initializeAndUpgradeSchema( 0 >= VersionNumber.parse("2.1").compareTo(version), "All Cassandra nodes in Reaper's backend storage must be running version 2.1+"); - try ( - SchemaMigrationLock schemaMigrationLock = new SchemaMigrationLock(version, session, config); - Database database = new Database(cassandra, config.getCassandraFactory().getKeyspace())) { + try (Database database = new Database(cassandra, config.getCassandraFactory().getKeyspace())) { int currentVersion = database.getVersion(); + Preconditions.checkState( + currentVersion == 0 || currentVersion >= 15, + "You need to upgrade from Reaper 1.2.2 at least in order to run this version. " + + "Please upgrade to 1.2.2, or greater, before performing this upgrade."); + MigrationRepository migrationRepo = new MigrationRepository("db/cassandra"); if (currentVersion < migrationRepo.getLatestVersion()) { LOG.warn("Starting db migration from {} to {}…", currentVersion, migrationRepo.getLatestVersion()); - if (4 <= currentVersion) { + if (15 <= currentVersion) { List otherRunningReapers = session.execute("SELECT reaper_instance_host FROM running_reapers").all() .stream() .map((row) -> row.getString("reaper_instance_host")) @@ -261,7 +263,9 @@ private static void initializeAndUpgradeSchema( StringUtils.join(otherRunningReapers)); } - migrate(database.getVersion(), migrationRepo, session); + // We now only support migrations starting at version 15 (Reaper 1.2.2) + int startVersion = database.getVersion() == 0 ? 15 : database.getVersion(); + migrate(startVersion, migrationRepo, session); // some migration steps depend on the Cassandra version, so must be rerun every startup Migration016.migrate(session, config.getCassandraFactory().getKeyspace()); // Switch metrics table to TWCS if possible, this is intentionally executed every startup @@ -295,7 +299,7 @@ public List getMigrationsSinceVersion(int version) { }; try (Database database = new Database(session.getCluster(), session.getLoggedKeyspace())) { - MigrationTask migration = new MigrationTask(database, migrationRepo); + MigrationTask migration = new MigrationTask(database, migrationRepo, true); migration.migrate(); // after the script execute any MigrationXXX class that exists with the same version number Class.forName("io.cassandrareaper.storage.cassandra.Migration" + String.format("%03d", nextVersion)) diff --git a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration003.java b/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration003.java deleted file mode 100644 index 8837502bc..000000000 --- a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration003.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2017-2017 Spotify AB - * Copyright 2017-2018 The Last Pickle Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.cassandrareaper.storage.cassandra; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import com.datastax.driver.core.KeyspaceMetadata; -import com.datastax.driver.core.PreparedStatement; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import com.datastax.driver.core.utils.UUIDs; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public final class Migration003 { - - private static final Logger LOG = LoggerFactory.getLogger(Migration003.class); - - private Migration003() { - } - - /** - * migrate over the repair_schedule table * - */ - public static void migrate(Session session) { - KeyspaceMetadata metadata = session.getCluster().getMetadata().getKeyspace(session.getLoggedKeyspace()); - if (null != metadata.getTable("repair_unit")) { - - LOG.warn("Migrating repair_unit and repair_schedule tables. This may take some minutes…"); - - PreparedStatement insertRprUnit = session.prepare( - "INSERT INTO repair_unit_v1 (id, cluster_name, keyspace_name, column_families, incremental_repair) " - + "VALUES(?, ?, ?, ?, ?)"); - - PreparedStatement insertRprSched = session.prepare( - "INSERT INTO repair_schedule_v1 (id, repair_unit_id, state, days_between, next_activation, run_history, " - + "segment_count, repair_parallelism, intensity, creation_time, owner, pause_time) " - + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - - PreparedStatement insertRprSchedIdx = session.prepare( - "INSERT INTO repair_schedule_by_cluster_and_keyspace(cluster_name, keyspace_name, repair_schedule_id) " - + "VALUES(?, ?, ?)"); - - Map repairUnitIds = new HashMap<>(); - Map repairUnitClusters = new HashMap<>(); - Map repairUnitKeyspaces = new HashMap<>(); - - for (Row row : session.execute(QueryBuilder.select().from("repair_unit"))) { - UUID uuid = UUIDs.timeBased(); - repairUnitIds.put(row.getLong("id"), uuid); - repairUnitClusters.put(row.getLong("id"), row.getString("cluster_name")); - repairUnitKeyspaces.put(row.getLong("id"), row.getString("keyspace_name")); - - session.execute( - insertRprUnit.bind( - uuid, - row.getString("cluster_name"), - row.getString("keyspace_name"), - row.getSet("column_families", String.class), - row.getBool("incremental_repair"))); - } - session.executeAsync("DROP TABLE repair_unit"); - - for (Row row : session.execute(QueryBuilder.select().from("repair_schedule"))) { - UUID uuid = UUIDs.timeBased(); - long repairUnitId = row.getLong("repair_unit_id"); - - session.execute( - insertRprSched.bind( - uuid, - repairUnitIds.get(repairUnitId), - row.getString("state"), - row.getInt("days_between"), - row.getTimestamp("next_activation"), - Collections.emptySet(), - row.getInt("segment_count"), - row.getString("repair_parallelism"), - row.getDouble("intensity"), - row.getTimestamp("creation_time"), - row.getString("owner"), - row.getTimestamp("pause_time"))); - - session.executeAsync( - insertRprSchedIdx.bind(repairUnitClusters.get(repairUnitId), repairUnitKeyspaces.get(repairUnitId), uuid)); - - session.executeAsync(insertRprSchedIdx.bind(repairUnitClusters.get(repairUnitId), " ", uuid)); - session.executeAsync(insertRprSchedIdx.bind(" ", repairUnitKeyspaces.get(repairUnitId), uuid)); - } - - session.executeAsync("DROP TABLE repair_schedule"); - - LOG.warn("Migration of repair_unit and repair_schedule tables completed."); - } - } -} diff --git a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration008.java b/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration008.java deleted file mode 100644 index a38a59c1c..000000000 --- a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration008.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2014-2017 Spotify AB - * Copyright 2016-2019 The Last Pickle Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.cassandrareaper.storage.cassandra; - - -import com.datastax.driver.core.Session; - -public final class Migration008 { - - private Migration008() { - } - - /** - * fix segment start and end times in the repair_run table. - * delegates to FixRepairSegmentTimestamps as that does everything already. - */ - public static void migrate(Session session) { - FixRepairSegmentTimestamps.migrate(session); - } -} diff --git a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration011.java b/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration011.java deleted file mode 100644 index 31a47c403..000000000 --- a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration011.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2014-2017 Spotify AB - * Copyright 2016-2019 The Last Pickle Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.cassandrareaper.storage.cassandra; - - -import com.datastax.driver.core.Session; - -public final class Migration011 { - - private Migration011() { - } - - /** - * fix segment start and end times in the repair_run table. - * delegates to FixRepairSegmentTimestamps as that does everything already. - */ - public static void migrate(Session session) { - FixRepairSegmentTimestamps.migrate(session); - } -} diff --git a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration012.java b/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration012.java deleted file mode 100644 index e693804f7..000000000 --- a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/Migration012.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2014-2017 Spotify AB - * Copyright 2016-2019 The Last Pickle Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.cassandrareaper.storage.cassandra; - - -import com.datastax.driver.core.Session; - -public final class Migration012 { - - private Migration012() { - } - - /** - * fix segment start and end times in the repair_run table. - * delegates to FixRepairSegmentTimestamps as that does everything already. - */ - public static void migrate(Session session) { - FixRepairSegmentTimestamps.migrate(session); - } -} diff --git a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/SchemaMigrationLock.java b/src/server/src/main/java/io/cassandrareaper/storage/cassandra/SchemaMigrationLock.java deleted file mode 100644 index f6362d0a9..000000000 --- a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/SchemaMigrationLock.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2019-2019 The Last Pickle Ltd - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.cassandrareaper.storage.cassandra; - -import io.cassandrareaper.ReaperApplicationConfiguration; - -import java.util.UUID; - -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.VersionNumber; -import com.datastax.driver.core.utils.UUIDs; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public final class SchemaMigrationLock implements AutoCloseable { - - private static final UUID SCHEMA_MIGRATION_LOCK_ID = UUIDs.startOf(1); - private static final Logger LOG = LoggerFactory.getLogger(SchemaMigrationLock.class); - private final Session session; - private final boolean locked; - - public SchemaMigrationLock(VersionNumber version, Session session, ReaperApplicationConfiguration config) { - this.session = session; - - this.locked = !config.getEnableConcurrentMigrations() - && migrationConsensusAvailable(config, version) - && lockSchemaMigration(); - } - - @Override - public void close() { - if (locked) { - unlockSchemaMigration(); - } - } - - private boolean lockSchemaMigration() { - while (true) { - LOG.debug("Trying to take lead on schema migrations"); - - ResultSet lwtResult = session.execute( - "INSERT INTO system_distributed.parent_repair_history (parent_id, started_at) VALUES (" - + SCHEMA_MIGRATION_LOCK_ID + ", dateOf(now())) IF NOT EXISTS USING TTL 300"); - - if (lwtResult.wasApplied()) { - LOG.debug("Took lead on schema migrations"); - return true; - } - - LOG.info("Cannot grab the lock on schema migration. Waiting for it to be released..."); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new IllegalStateException(e); - } - } - } - - private synchronized boolean unlockSchemaMigration() { - LOG.debug("Trying to take lead on schema migrations"); - - ResultSet lwtResult = session.execute( - "DELETE FROM system_distributed.parent_repair_history WHERE parent_id = " + SCHEMA_MIGRATION_LOCK_ID - + " IF EXISTS"); - - if (lwtResult.wasApplied()) { - LOG.debug("Released lead on schema migrations"); - return true; - } - // Another instance took the lead on the segment - LOG.error("Could not release lead on schema migrations"); - return false; - } - - private static boolean migrationConsensusAvailable(ReaperApplicationConfiguration config, VersionNumber version) { - return VersionNumber.parse("2.2").compareTo(version) <= 0; - } -} diff --git a/src/server/src/main/resources/db/cassandra/001_Initialize_db.cql b/src/server/src/main/resources/db/cassandra/001_Initialize_db.cql deleted file mode 100644 index 995cd1923..000000000 --- a/src/server/src/main/resources/db/cassandra/001_Initialize_db.cql +++ /dev/null @@ -1,114 +0,0 @@ --- --- Copyright 2016-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Cassandra schema for cassandra-reaper database - --- CREATE KEYSPACE IF NOT EXISTS reaper_db WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':3}; - --- use reaper_db; - -CREATE TABLE IF NOT EXISTS cluster ( - name text PRIMARY KEY, - partitioner text, - seed_hosts set -); - --- Repair unit is basically a keyspace with a set of column families. --- Cassandra supports repairing multiple column families in one go. - -CREATE TABLE IF NOT EXISTS repair_unit ( - id bigint PRIMARY KEY, - cluster_name text, - keyspace_name text, - column_families set, - incremental_repair boolean -); - -CREATE TABLE IF NOT EXISTS repair_run ( - id bigint PRIMARY KEY, - cluster_name text, - repair_unit_id bigint, - cause text, - owner text, - state text, - creation_time timestamp, - start_time timestamp, - end_time timestamp, - pause_time timestamp, - intensity double , - last_event text , - segment_count int , - repair_parallelism text -); - -CREATE TABLE IF NOT EXISTS repair_run_by_cluster( - cluster_name text, - id bigint, - PRIMARY KEY(cluster_name, id) -); - -CREATE TABLE IF NOT EXISTS repair_run_by_unit( - repair_unit_id bigint, - id bigint, - PRIMARY KEY(repair_unit_id, id) -); - -CREATE TABLE IF NOT EXISTS repair_segment ( - id bigint PRIMARY KEY, - repair_unit_id bigint, - run_id bigint, - start_token varint, - end_token varint, - state int , - coordinator_host text, - start_time timestamp, - end_time timestamp, - fail_count INT -); - -CREATE TABLE IF NOT EXISTS repair_segment_by_run_id ( - run_id bigint, - segment_id bigint, - PRIMARY KEY(run_id, segment_id) -); - - -CREATE TABLE IF NOT EXISTS repair_schedule ( - id bigint PRIMARY KEY, - repair_unit_id bigint, - state text , - days_between int , - next_activation timestamp, - run_history set, - segment_count int , - repair_parallelism text , - intensity double , - creation_time timestamp, - owner text , - pause_time timestamp -); - - -CREATE TABLE IF NOT EXISTS repair_schedule_by_cluster_and_keyspace( - cluster_name text, - keyspace_name text, - repair_schedule_id bigint, - PRIMARY KEY((cluster_name, keyspace_name), repair_schedule_id) -); - -CREATE TABLE IF NOT EXISTS repair_id( - id_type text PRIMARY KEY, - id bigint -); diff --git a/src/server/src/main/resources/db/cassandra/002_table_properties.cql b/src/server/src/main/resources/db/cassandra/002_table_properties.cql deleted file mode 100644 index 1ef89bd5b..000000000 --- a/src/server/src/main/resources/db/cassandra/002_table_properties.cql +++ /dev/null @@ -1,55 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Optimal table properties - -ALTER TABLE cluster - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 'ALL'}; - -ALTER TABLE repair_unit - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - -ALTER TABLE repair_run - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - -ALTER TABLE repair_run_by_cluster - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - -ALTER TABLE repair_run_by_unit - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - -ALTER TABLE repair_segment - WITH compaction = {'class': 'LeveledCompactionStrategy'}; - -ALTER TABLE repair_segment_by_run_id - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - -ALTER TABLE repair_schedule - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - -ALTER TABLE repair_schedule_by_cluster_and_keyspace - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - -ALTER TABLE repair_id - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; diff --git a/src/server/src/main/resources/db/cassandra/003_switch_to_uuids.cql b/src/server/src/main/resources/db/cassandra/003_switch_to_uuids.cql deleted file mode 100644 index dd03998a4..000000000 --- a/src/server/src/main/resources/db/cassandra/003_switch_to_uuids.cql +++ /dev/null @@ -1,126 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Cassandra schema for cassandra-reaper database - --- CREATE KEYSPACE IF NOT EXISTS reaper_db WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':3}; - --- use reaper_db; - -ALTER TABLE cluster - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 'ALL'}; - --- Repair unit is basically a keyspace with a set of column families. --- Cassandra supports repairing multiple column families in one go. - -CREATE TABLE IF NOT EXISTS repair_unit_v1 ( - id timeuuid PRIMARY KEY, - cluster_name text, - keyspace_name text, - column_families set, - incremental_repair boolean -) - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 1}; - -DROP TABLE IF EXISTS repair_run; - -CREATE TABLE IF NOT EXISTS repair_run ( - id timeuuid, - cluster_name text STATIC, - repair_unit_id timeuuid STATIC, - cause text STATIC, - owner text STATIC, - state text STATIC, - creation_time timestamp STATIC, - start_time timestamp STATIC, - end_time timestamp STATIC, - pause_time timestamp STATIC, - intensity double STATIC, - last_event text STATIC, - segment_count int STATIC, - repair_parallelism text STATIC, - segment_id timeuuid, - start_token varint, - end_token varint, - segment_state int, - coordinator_host text, - segment_start_time timestamp, - segment_end_time timestamp, - fail_count int, - PRIMARY KEY (id, segment_id) -) - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 5000}; - - -DROP TABLE IF EXISTS repair_run_by_cluster; - -CREATE TABLE IF NOT EXISTS repair_run_by_cluster ( - cluster_name text, - id timeuuid, - PRIMARY KEY(cluster_name, id) -) - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 'ALL'}; - - -DROP TABLE IF EXISTS repair_run_by_unit; - -CREATE TABLE IF NOT EXISTS repair_run_by_unit ( - repair_unit_id timeuuid, - id timeuuid, - PRIMARY KEY(repair_unit_id, id) -) - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 'ALL'}; - - - - -CREATE TABLE IF NOT EXISTS repair_schedule_v1 ( - id timeuuid PRIMARY KEY, - repair_unit_id timeuuid, - state text , - days_between int , - next_activation timestamp, - run_history set, - segment_count int , - repair_parallelism text , - intensity double , - creation_time timestamp, - owner text , - pause_time timestamp -) - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 1}; - - -DROP TABLE IF EXISTS repair_schedule_by_cluster_and_keyspace; - -CREATE TABLE IF NOT EXISTS repair_schedule_by_cluster_and_keyspace ( - cluster_name text, - keyspace_name text, - repair_schedule_id timeuuid, - PRIMARY KEY((cluster_name, keyspace_name), repair_schedule_id) -) - WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND caching = {'rows_per_partition': 10}; - - -DROP TABLE IF EXISTS repair_segment; -DROP TABLE IF EXISTS repair_segment_by_run_id; -DROP TABLE IF EXISTS repair_id; diff --git a/src/server/src/main/resources/db/cassandra/004_fault_tolerant_reaper.cql b/src/server/src/main/resources/db/cassandra/004_fault_tolerant_reaper.cql deleted file mode 100644 index 78768e1d2..000000000 --- a/src/server/src/main/resources/db/cassandra/004_fault_tolerant_reaper.cql +++ /dev/null @@ -1,51 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade for fault tolerance addons - -CREATE TABLE IF NOT EXISTS leader ( - leader_id timeuuid PRIMARY KEY, - reaper_instance_id uuid, - reaper_instance_host text, - last_heartbeat timestamp -) WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND default_time_to_live = 600 - AND gc_grace_seconds = 600; - --- --- Add table to track concurrently running Reapers - -CREATE TABLE IF NOT EXISTS running_reapers ( - reaper_instance_id uuid PRIMARY KEY, - reaper_instance_host text, - last_heartbeat timestamp -) WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND default_time_to_live = 180 - AND gc_grace_seconds = 180; - - --- --- Add table to store metrics of Cassandra nodes - -CREATE TABLE IF NOT EXISTS node_metrics ( - host_address text, - datacenter text, - pending_compactions int, - has_repair_running boolean, - active_anticompactions int, - PRIMARY KEY (host_address) - ) WITH compaction = {'class': 'LeveledCompactionStrategy'} - AND default_time_to_live = 180 - AND gc_grace_seconds = 180; diff --git a/src/server/src/main/resources/db/cassandra/005_repair_specific_nodes.cql b/src/server/src/main/resources/db/cassandra/005_repair_specific_nodes.cql deleted file mode 100644 index d89f2be86..000000000 --- a/src/server/src/main/resources/db/cassandra/005_repair_specific_nodes.cql +++ /dev/null @@ -1,20 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade to handle repair on specific nodes - -ALTER TABLE repair_unit_v1 ADD nodes set; - -ALTER TABLE repair_unit_v1 ADD "datacenters" set; diff --git a/src/server/src/main/resources/db/cassandra/006_table_blacklist.cql b/src/server/src/main/resources/db/cassandra/006_table_blacklist.cql deleted file mode 100644 index d91755731..000000000 --- a/src/server/src/main/resources/db/cassandra/006_table_blacklist.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade to handle repair on specific nodes - -ALTER TABLE repair_unit_v1 ADD blacklisted_tables set; diff --git a/src/server/src/main/resources/db/cassandra/007_segment_count_per_node.cql b/src/server/src/main/resources/db/cassandra/007_segment_count_per_node.cql deleted file mode 100644 index 02e0e9498..000000000 --- a/src/server/src/main/resources/db/cassandra/007_segment_count_per_node.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade to handle the new segment count per node - -ALTER TABLE repair_schedule_v1 ADD segment_count_per_node int; diff --git a/src/server/src/main/resources/db/cassandra/008_request_for_metrics.cql b/src/server/src/main/resources/db/cassandra/008_request_for_metrics.cql deleted file mode 100644 index 354ffaa15..000000000 --- a/src/server/src/main/resources/db/cassandra/008_request_for_metrics.cql +++ /dev/null @@ -1,36 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade to handle the metrics request mechanism - - -DROP TABLE IF EXISTS node_metrics; - -CREATE TABLE IF NOT EXISTS node_metrics_v1 ( - time_partition bigint, - run_id uuid, - node text, - cluster text, - datacenter text, - requested boolean, - pending_compactions int, - has_repair_running boolean, - active_anticompactions int, - PRIMARY KEY((run_id, time_partition), node) -) - WITH compaction = {'class':'SizeTieredCompactionStrategy', 'unchecked_tombstone_compaction':'true'} - AND caching = {'keys':'ALL', 'rows_per_partition':'ALL'} - AND gc_grace_seconds = 120 - AND default_time_to_live = 180; diff --git a/src/server/src/main/resources/db/cassandra/009_remove_nulls.cql b/src/server/src/main/resources/db/cassandra/009_remove_nulls.cql deleted file mode 100644 index a19089947..000000000 --- a/src/server/src/main/resources/db/cassandra/009_remove_nulls.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Intentionally blank --- --- Placeholder for Migration009 diff --git a/src/server/src/main/resources/db/cassandra/010_table_properties.cql b/src/server/src/main/resources/db/cassandra/010_table_properties.cql deleted file mode 100644 index d10a0a5fa..000000000 --- a/src/server/src/main/resources/db/cassandra/010_table_properties.cql +++ /dev/null @@ -1,37 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Optimal table properties - -ALTER TABLE repair_unit_v1 - WITH caching = {'keys':'ALL', 'rows_per_partition':'1'}; - -ALTER TABLE repair_schedule_by_cluster_and_keyspace - WITH caching = {'keys':'ALL', 'rows_per_partition':'10'}; - -ALTER TABLE repair_run_by_cluster - WITH caching = {'keys':'ALL', 'rows_per_partition':'ALL'}; - -ALTER TABLE repair_schedule_v1 - WITH caching = {'keys':'ALL', 'rows_per_partition':'1'}; - -ALTER TABLE cluster - WITH caching = {'keys':'ALL', 'rows_per_partition':'ALL'}; - -ALTER TABLE repair_run - WITH caching = {'keys':'ALL', 'rows_per_partition':'5000'}; - -ALTER TABLE repair_run_by_unit - WITH caching = {'keys':'ALL', 'rows_per_partition':'ALL'}; diff --git a/src/server/src/main/resources/db/cassandra/011_fix_start_times.cql b/src/server/src/main/resources/db/cassandra/011_fix_start_times.cql deleted file mode 100644 index 5c0276f48..000000000 --- a/src/server/src/main/resources/db/cassandra/011_fix_start_times.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Intentionally blank --- --- Placeholder for Migration011 diff --git a/src/server/src/main/resources/db/cassandra/012_fix_start_times.cql b/src/server/src/main/resources/db/cassandra/012_fix_start_times.cql deleted file mode 100644 index aa8136773..000000000 --- a/src/server/src/main/resources/db/cassandra/012_fix_start_times.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2017-2017 Spotify AB --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Intentionally blank --- --- Placeholder for Migration012 diff --git a/src/server/src/main/resources/db/cassandra/013_snapshots.cql b/src/server/src/main/resources/db/cassandra/013_snapshots.cql deleted file mode 100644 index 46c43af80..000000000 --- a/src/server/src/main/resources/db/cassandra/013_snapshots.cql +++ /dev/null @@ -1,29 +0,0 @@ --- --- Copyright 2018-2018 The Last Pickle Ltd --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade to handle snapshots metadata - -CREATE TABLE IF NOT EXISTS snapshot ( - cluster text, - snapshot_name text, - owner text, - cause text, - creation_time timestamp, - PRIMARY KEY((cluster, snapshot_name)) -) -WITH compaction = {'class':'LeveledCompactionStrategy'} - AND caching = {'keys':'ALL', 'rows_per_partition':'ALL'} - AND gc_grace_seconds = 864000 - AND default_time_to_live = 0; diff --git a/src/server/src/main/resources/db/cassandra/014_segment_coalescing.cql b/src/server/src/main/resources/db/cassandra/014_segment_coalescing.cql deleted file mode 100644 index d5f59b5ab..000000000 --- a/src/server/src/main/resources/db/cassandra/014_segment_coalescing.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2018-2018 The Last Pickle Ltd --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade to allow segments to contain several token ranges - -ALTER TABLE repair_run ADD token_ranges text; diff --git a/src/server/src/main/resources/db/cassandra/015_multithreaded_repair.cql b/src/server/src/main/resources/db/cassandra/015_multithreaded_repair.cql deleted file mode 100644 index ac732679f..000000000 --- a/src/server/src/main/resources/db/cassandra/015_multithreaded_repair.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2018-2018 The Last Pickle Ltd --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Upgrade to allow segments to contain several token ranges - -ALTER TABLE repair_unit_v1 ADD repair_thread_count int; diff --git a/src/server/src/main/resources/db/cassandra/016_disable_dclocal_read_repair_chance.cql b/src/server/src/main/resources/db/cassandra/016_disable_dclocal_read_repair_chance.cql deleted file mode 100644 index 660098ee3..000000000 --- a/src/server/src/main/resources/db/cassandra/016_disable_dclocal_read_repair_chance.cql +++ /dev/null @@ -1,18 +0,0 @@ --- --- Copyright 2018-2018 The Last Pickle Ltd --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Intentionally blank --- --- Placeholder for Migration014 diff --git a/src/server/src/main/resources/db/cassandra/016_init_reaper_db.cql b/src/server/src/main/resources/db/cassandra/016_init_reaper_db.cql new file mode 100644 index 000000000..15252d4c5 --- /dev/null +++ b/src/server/src/main/resources/db/cassandra/016_init_reaper_db.cql @@ -0,0 +1,175 @@ +-- +-- Copyright 2018-2019 The Last Pickle Ltd +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS reaper_db.running_reapers ( + reaper_instance_id uuid PRIMARY KEY, + last_heartbeat timestamp, + reaper_instance_host text +) WITH bloom_filter_fp_chance = 0.1 + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 180 + AND gc_grace_seconds = 180; + +CREATE TABLE IF NOT EXISTS reaper_db.repair_unit_v1 ( + id timeuuid PRIMARY KEY, + blacklisted_tables set, + cluster_name text, + column_families set, + "datacenters" set, + incremental_repair boolean, + keyspace_name text, + nodes set, + repair_thread_count int +) WITH bloom_filter_fp_chance = 0.1 + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.repair_schedule_by_cluster_and_keyspace ( + cluster_name text, + keyspace_name text, + repair_schedule_id timeuuid, + PRIMARY KEY ((cluster_name, keyspace_name), repair_schedule_id) +) WITH CLUSTERING ORDER BY (repair_schedule_id ASC) + AND bloom_filter_fp_chance = 0.1 + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.repair_run_by_cluster ( + cluster_name text, + id timeuuid, + PRIMARY KEY (cluster_name, id) +) WITH CLUSTERING ORDER BY (id ASC) + AND bloom_filter_fp_chance = 0.1 + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.repair_schedule_v1 ( + id timeuuid PRIMARY KEY, + creation_time timestamp, + days_between int, + intensity double, + next_activation timestamp, + owner text, + pause_time timestamp, + repair_parallelism text, + repair_unit_id timeuuid, + run_history set, + segment_count int, + segment_count_per_node int, + state text +) WITH bloom_filter_fp_chance = 0.1 + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.cluster ( + name text PRIMARY KEY, + partitioner text, + seed_hosts set +) WITH bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL', 'rows_per_partition':'ALL'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.snapshot ( + cluster text, + snapshot_name text, + cause text, + creation_time timestamp, + owner text, + PRIMARY KEY ((cluster, snapshot_name)) +) WITH bloom_filter_fp_chance = 0.1 + AND caching = {'keys':'ALL', 'rows_per_partition':'ALL'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.node_metrics_v1 ( + run_id uuid, + time_partition bigint, + node text, + active_anticompactions int, + cluster text, + "datacenter" text, + has_repair_running boolean, + pending_compactions int, + requested boolean, + PRIMARY KEY ((run_id, time_partition), node) +) WITH CLUSTERING ORDER BY (node ASC) + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL', 'rows_per_partition':'NONE'} + AND compaction = {'unchecked_tombstone_compaction': 'true', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} + AND default_time_to_live = 180 + AND gc_grace_seconds = 120; + +CREATE TABLE IF NOT EXISTS reaper_db.repair_run ( + id timeuuid, + segment_id timeuuid, + cause text static, + cluster_name text static, + coordinator_host text, + creation_time timestamp static, + end_time timestamp static, + end_token varint, + fail_count int, + intensity double static, + last_event text static, + owner text static, + pause_time timestamp static, + repair_parallelism text static, + repair_unit_id timeuuid static, + segment_count int static, + segment_end_time timestamp, + segment_start_time timestamp, + segment_state int, + start_time timestamp static, + start_token varint, + state text static, + token_ranges text, + PRIMARY KEY (id, segment_id) +) WITH CLUSTERING ORDER BY (segment_id ASC) + AND bloom_filter_fp_chance = 0.1 + AND caching = {'keys':'ALL', 'rows_per_partition':'5000'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.repair_run_by_unit ( + repair_unit_id timeuuid, + id timeuuid, + PRIMARY KEY (repair_unit_id, id) +) WITH CLUSTERING ORDER BY (id ASC) + AND bloom_filter_fp_chance = 0.1 + AND caching = {'keys':'ALL', 'rows_per_partition':'ALL'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000; + +CREATE TABLE IF NOT EXISTS reaper_db.leader ( + leader_id timeuuid PRIMARY KEY, + last_heartbeat timestamp, + reaper_instance_host text, + reaper_instance_id uuid +) WITH bloom_filter_fp_chance = 0.1 + AND caching = {'keys':'ALL', 'rows_per_partition':'NONE'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND default_time_to_live = 600 + AND gc_grace_seconds = 600; +