-
-
Notifications
You must be signed in to change notification settings - Fork 242
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add JSON column for each RABX column.
Include script to migrate back/forth between RABX and JSON.
- Loading branch information
Showing
40 changed files
with
434 additions
and
106 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| #!/usr/bin/env perl | ||
|
|
||
| use v5.14; | ||
| use warnings; | ||
|
|
||
| BEGIN { | ||
| use File::Basename qw(dirname); | ||
| use File::Spec; | ||
| my $d = dirname(File::Spec->rel2abs($0)); | ||
| require "$d/../setenv.pl"; | ||
| } | ||
|
|
||
| use FixMyStreet::DB; | ||
| use Getopt::Long::Descriptive; | ||
| use IO::String; | ||
| use JSON::MaybeXS; | ||
| use RABX; | ||
|
|
||
| my ($opts, $usage) = describe_options( | ||
| '%c %o', | ||
| ['commit', 'actually change the database'], | ||
| ['reverse', 'update RABX with data from JSON'], | ||
| ['help|h', "print usage message and exit" ], | ||
| ); | ||
| $usage->die if $opts->help; | ||
|
|
||
| my $DB = FixMyStreet::DB->schema->storage->dbh; | ||
| my $JSON = JSON->new->allow_nonref->canonical; | ||
|
|
||
| my @tables = qw(body contacts problem comment users moderation_original_data defect_types report_extra_fields roles token); | ||
| my %column_name = ( | ||
| token => 'data', | ||
| default => 'extra', | ||
| ); | ||
|
|
||
| foreach (@tables) { | ||
| my $rabx_column = $column_name{$_} || $column_name{default}; | ||
| my $json_column = $rabx_column . '_json'; | ||
| my $where; | ||
| if ($opts->reverse) { | ||
| $where = "($json_column IS NOT NULL AND $rabx_column IS NULL)"; | ||
| } else { | ||
| $where = "($json_column IS NULL AND $rabx_column IS NOT NULL)"; | ||
| } | ||
| if ($_ eq 'problem') { | ||
| if ($opts->reverse) { | ||
| $where .= " OR (geocode_json IS NOT NULL AND geocode IS NULL)"; | ||
| } else { | ||
| $where .= " OR (geocode_json IS NULL AND geocode IS NOT NULL)"; | ||
| } | ||
| } | ||
|
|
||
| my $total = $DB->selectrow_array("SELECT COUNT(*) FROM $_ WHERE $where"); | ||
|
|
||
| print "Migrating data for table $_ - $total rows to migrate\n"; | ||
|
|
||
| my @cols = ($rabx_column, $json_column); | ||
| if ($_ eq 'problem') { | ||
| push @cols, 'id', 'geocode', 'geocode_json'; | ||
| } elsif ($_ eq 'token') { | ||
| push @cols, 'scope', 'token'; | ||
| } else { | ||
| push @cols, 'id'; | ||
| } | ||
| my $cols = join(',', @cols); | ||
|
|
||
| my $count = 0; | ||
| my $query; | ||
| $DB->do("BEGIN"); | ||
| if ($opts->commit) { | ||
| $query = $DB->prepare("SELECT $cols FROM $_ WHERE $where LIMIT 1000 FOR UPDATE"); | ||
| } else { | ||
| $DB->do("DECLARE mycursor CURSOR FOR SELECT $cols FROM $_ WHERE $where"); | ||
| $query = $DB->prepare("FETCH 1000 FROM mycursor"); | ||
| } | ||
| while(1) { | ||
| $query->execute(); | ||
| last if $query->rows == 0; | ||
|
|
||
| while (my $r = $query->fetchrow_hashref) { | ||
| if ($opts->reverse) { | ||
| my $rabx = to_rabx(from_json($r->{$json_column})); | ||
| if ($_ eq 'problem') { | ||
| my $rabx_geocode = to_rabx(from_json($r->{geocode_json})); | ||
| update("UPDATE $_ SET $rabx_column = ?, geocode = ? WHERE id=?", $rabx, $rabx_geocode, $r->{id}); | ||
| } elsif ($_ eq 'token') { | ||
| update("UPDATE $_ SET $rabx_column = ? WHERE token=? AND scope=?", $rabx, $r->{token}, $r->{scope}); | ||
| } else { | ||
| update("UPDATE $_ SET $rabx_column = ? WHERE id=?", $rabx, $r->{id}); | ||
| } | ||
| } else { | ||
| my $json = to_json(from_rabx($r->{$rabx_column}, $_)); | ||
| if ($_ eq 'problem') { | ||
| my $json_geocode = to_json(from_rabx($r->{geocode}, 'geocode')); | ||
| update("UPDATE $_ SET $json_column = ?, geocode_json = ? WHERE id=?", $json, $json_geocode, $r->{id}); | ||
| } elsif ($_ eq 'token') { | ||
| update("UPDATE $_ SET $json_column = ? WHERE token=? AND scope=?", $json, $r->{token}, $r->{scope}); | ||
| } else { | ||
| update("UPDATE $_ SET $json_column = ? WHERE id=?", $json, $r->{id}); | ||
| } | ||
| } | ||
| $count++; | ||
| print "\r$count/$total"; | ||
| } | ||
| } | ||
| $DB->do("COMMIT"); | ||
| print "\n"; | ||
| } | ||
|
|
||
| sub update { | ||
| my $sql = shift; | ||
| return unless $opts->commit; | ||
| $DB->do($sql, undef, @_); | ||
| } | ||
|
|
||
| sub from_json { | ||
| my $ser = shift; | ||
| return $JSON->decode($ser); | ||
| } | ||
|
|
||
| sub to_json { | ||
| my $data = shift; | ||
| my $ser = $JSON->encode($data); | ||
| return $ser; | ||
| } | ||
|
|
||
| sub from_rabx { | ||
| my $ser = shift; | ||
| my $table = shift; | ||
| return undef unless defined $ser; | ||
| # Some RABX columns are text, when they should be bytea. For | ||
| # these we must re-encode the string returned from the | ||
| # database, so that it is decoded again by RABX. | ||
| utf8::encode($ser) if $table ne 'token' && $table ne 'geocode'; | ||
| my $h = new IO::String($ser); | ||
| return RABX::wire_rd($h); | ||
| } | ||
|
|
||
| sub to_rabx { | ||
| my $data = shift; | ||
| my $ser = ''; | ||
| my $h = new IO::String($ser); | ||
| RABX::wire_wr( $data, $h ); | ||
| # Decode from UTF-8 as the saving to the db will re-encode it | ||
| utf8::decode($ser); | ||
| return $ser; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| ALTER TABLE users DROP COLUMN extra_json; | ||
| ALTER TABLE body DROP COLUMN extra_json; | ||
| ALTER TABLE contacts DROP COLUMN extra_json; | ||
| ALTER TABLE problem DROP COLUMN extra_json; | ||
| ALTER TABLE problem DROP COLUMN geocode_json; | ||
| ALTER TABLE comment DROP COLUMN extra_json; | ||
| ALTER TABLE moderation_original_data DROP COLUMN extra_json; | ||
| ALTER TABLE defect_types DROP COLUMN extra_json; | ||
| ALTER TABLE report_extra_fields DROP COLUMN extra_json; | ||
| ALTER TABLE roles DROP COLUMN extra_json; | ||
| ALTER TABLE token DROP COLUMN data_json; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| ALTER TABLE users ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE body ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE contacts ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE problem ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE problem ADD COLUMN geocode_json jsonb; | ||
| ALTER TABLE comment ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE moderation_original_data ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE defect_types ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE report_extra_fields ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE roles ADD COLUMN extra_json jsonb; | ||
| ALTER TABLE token ADD COLUMN data_json jsonb; | ||
|
|
||
| ALTER TABLE token ADD CONSTRAINT token_data_not_null CHECK (data_json IS NOT NULL) NOT VALID; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package FixMyStreet::DB::JSONBColumn; | ||
|
|
||
| use strict; | ||
| use warnings; | ||
| use JSON::MaybeXS; | ||
|
|
||
| my $JSON; | ||
|
|
||
| sub register_column { | ||
| my ($self, $column, $info, @rest) = @_; | ||
|
|
||
| $self->next::method($column, $info, @rest); | ||
|
|
||
| return unless ($info->{data_type} || '') eq 'jsonb'; | ||
|
|
||
| $JSON ||= JSON->new->allow_nonref->canonical; | ||
|
|
||
| $self->filter_column( | ||
| $column => { | ||
| filter_from_storage => sub { | ||
| my ($self, $value) = @_; | ||
| return undef unless defined $value; | ||
| return $JSON->decode($value); | ||
| }, | ||
| filter_to_storage => sub { | ||
| my ($self, $value) = @_; | ||
| return $JSON->encode($value); | ||
| }, | ||
| } | ||
| ); | ||
| } | ||
|
|
||
| 1; | ||
|
|
||
| __END__ | ||
| =head1 NAME | ||
| FixMyStreet::DB::JSONBColumn | ||
| =head1 DESCRIPTION | ||
| Causes 'jsonb' type columns to automatically be JSON encoded and | ||
| decoded. | ||
| =cut |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.