Skip to content

Commit

Permalink
(WARNING: THIS IS A TEMPORARY COMMIT THAT WILL BE REPLACED WITH A FIN…
Browse files Browse the repository at this point in the history 
…AL ONE AFTER IT'S FULLY TESTED ON oshi.at)

- Added DOWNLOAD_LIMIT_PER_FILE config option to limit total download hits per file
- Added "Onion only" option for downloads to disable clearnet access for them (so all those child porn abuse reports won't bother me again but files can continue to exist on a Tor domain since nobody can blacklist it)
- Enforce UTF-8 encoding for files on downloads
  • Loading branch information
pipe committed Oct 1, 2022
1 parent 0e456c2 commit 3c416d9
Show file tree
Hide file tree
Showing 7 changed files with 92 additions and 26 deletions.
3 changes: 3 additions & 0 deletions app/config.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ CONTENT_VIEW_UNTIL_SIZE = { "text" : 5000000, "image" : 10000000, "image/svg" :
# Short URL minimal length
SHORTURL_LENGTH = 4

# Delete files after a number of downloads
DOWNLOAD_LIMIT_PER_FILE = 0

#########################################################################

# Engine admin URL path
Expand Down
16 changes: 12 additions & 4 deletions app/functions.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ sub template_vars {
USE_HTTP_HOST => $self->{conf}->{UPLOAD_LINK_USE_HOST},
INSECUREPATH => $self->{conf}->{HTTP_INSECUREPATH},
ABUSE_CAPTCHA_REQUIRED => $self->{conf}->{CAPTCHA_SHOW_FOR_ABUSE},
HASHSUMS_ENABLED => $self->{conf}->{UPLOAD_HASH_CALCULATION}
HASHSUMS_ENABLED => $self->{conf}->{UPLOAD_HASH_CALCULATION},
DOWNLOAD_LIMIT_PER_FILE => $self->{conf}->{DOWNLOAD_LIMIT_PER_FILE}
};

return $g;
Expand Down Expand Up @@ -130,7 +131,8 @@ sub checkups {
$self->{conf}->{TCP_RAW_PORT} = 7777 unless exists $self->{conf}->{TCP_RAW_PORT};
$self->{conf}->{TCP_BASE64_PORT} = 7778 unless exists $self->{conf}->{TCP_BASE64_PORT};
$self->{conf}->{TCP_HEX_PORT} = 7779 unless exists $self->{conf}->{TCP_HEX_PORT};

$self->{conf}->{DOWNLOAD_LIMIT_PER_FILE} = 0 unless exists $self->{conf}->{DOWNLOAD_LIMIT_PER_FILE};

$self->{MIMETYPE_SIZE_LIMITS} = {};

try {
Expand Down Expand Up @@ -252,7 +254,7 @@ sub process_file {

$self->{dbc}->run(sub {
my $dbh = shift;
$dbh->do("replace into uploads values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0)", undef, @values);
$dbh->do("replace into uploads values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0,0,0)", undef, @values);
});

say "[info] process_file() finished ($mpath)" if $self->{conf}->{DEBUG} > 1;
Expand Down Expand Up @@ -543,7 +545,7 @@ sub db_struct {
ftype varchar(32),
link varchar(' . ($self->{FNMAXLEN} + $storagemaxlen + $shorturlmaxlen) . '), shorturl bool,
created int unsigned, expires int unsigned, autodestroy bool, autodestroylocked bool, wasdup bool,
proto varchar(8), size bigint unsigned, hits bigint unsigned, processing smallint unsigned, scanned bool,
proto varchar(8), size bigint unsigned, hits bigint unsigned, processing smallint unsigned, scanned bool, oniononly bool, oniononlylocked bool,
primary key (mpath, urlpath)' . ($t eq'mysql'?', index(rpath), index(hashsum), index(expires), index(link)':''),

'reports' => 'time int unsigned,
Expand Down Expand Up @@ -796,6 +798,12 @@ sub db_check_tables {
my $urlpath_isfixed = $dbh->selectrow_array('select collation_name from information_schema.columns where table_name = ? and column_name = ?', undef, 'uploads', 'urlpath');
say "[info] changed collation of `urlpath` column from $urlpath_needfix to $urlpath_isfixed";
}
my $oniononly_patch = $dbh->selectrow_array('select column_name from information_schema.columns where table_name = ? and column_name = ?', undef, 'uploads', 'oniononly');
unless ($oniononly_patch) {
$dbh->do("alter table uploads add column oniononly bool default 0 after scanned, add column oniononlylocked bool default 0 after oniononly");
my $oniononly_patched = $dbh->selectrow_array('select column_name from information_schema.columns where table_name = ? and column_name = ?', undef, 'uploads', 'oniononly');
say "[info] added `oniononly` and `oniononlylocked` columns" if $oniononly_patched;
}
}

}
Expand Down
4 changes: 3 additions & 1 deletion app/templates/admin.html.ep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,10 @@
<div class="row"><div class="col text-right text-secondary">Expires</div><div class="col text-left"><%= $file->{expires} == 0 ? 'Never' : scalar localtime $file->{expires} %></div></div>
<div class="row"><div class="col text-right text-secondary">Destroy after DL</div><div class="col text-left"><%= $file->{autodestroy} == 1 ? 'Yes' : 'No' %> <form class="d-inline" method="get"><input type="hidden" name="file" value="<%= $file->{urlpath} %>"><input type="hidden" name="toggleautodestroy" value="1"><button type="submit" class="btn btn-link">[toggle]</button></form></div></div>
<div class="row"><div class="col text-right text-secondary">Destroy lock</div><div class="col text-left"><%= $file->{autodestroylocked} == 1 ? 'Yes' : 'No' %> <form class="d-inline" method="get"><input type="hidden" name="file" value="<%= $file->{urlpath} %>"><input type="hidden" name="toggleautodestroylock" value="1"><button type="submit" class="btn btn-link">[toggle]</button></form></div></div>
<div class="row"><div class="col text-right text-secondary">Onion only</div><div class="col text-left"><%= $file->{oniononly} == 1 ? 'Yes' : 'No' %> <form class="d-inline" method="get"><input type="hidden" name="file" value="<%= $file->{urlpath} %>"><input type="hidden" name="toggleoniononly" value="1"><button type="submit" class="btn btn-link">[toggle]</button></form></div></div>
<div class="row"><div class="col text-right text-secondary">Onion only lock</div><div class="col text-left"><%= $file->{oniononlylocked} == 1 ? 'Yes' : 'No' %> <form class="d-inline" method="get"><input type="hidden" name="file" value="<%= $file->{urlpath} %>"><input type="hidden" name="toggleoniononlylock" value="1"><button type="submit" class="btn btn-link">[toggle]</button></form></div></div>

<div class="row"><div class="col text-right text-secondary">Hits</div><div class="col text-left"><%= $file->{hits} %></div></div>
<div class="row"><div class="col text-right text-secondary">Hits</div><div class="col text-left"><%= $file->{hits} %><%= $DOWNLOAD_LIMIT_PER_FILE ? '/' . $DOWNLOAD_LIMIT_PER_FILE : '' %></div></div>
</div>
<p> Use this button to delete the file permanently: </p>
<form method="get">
Expand Down
4 changes: 4 additions & 0 deletions app/templates/admin_reports.html.ep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@
<input type="hidden" name="purgeall" value="<%= $i->{url} %>">
<button type="submit" class="btn btn-danger btn-sm">Purge copies</button>
</form>
<form method="get" class="d-inline" style="margin: 0; padding: 0;">
<input type="hidden" name="oniononly" value="<%= $i->{url} %>">
<button type="submit" class="btn btn-danger btn-sm">Onion only</button>
</form>
</td>
<td><%= scalar localtime $i->{time} %></td>
<td><a target="_blank" href="<%= $ADMIN_ROUTE %>?file=<%== $i->{url} %>"><%== $i->{url} %></a></td>
Expand Down
15 changes: 12 additions & 3 deletions app/templates/manage.html.ep
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
% layout 'main', title => 'Upload management';
<div class="container">
<h3> Upload management - <span class="text-secondary"><%= $file->{rpath} %></span></h3>

% if ($file->{oniononly} == 1) {
<div class="row justify-content-center text-center">
<div class="col-6">
<div class="alert alert-warning" role="alert">
This file is not available on the clearnet domain (onion only option enabled)
</div>
</div>
</div>
% }
<div class="text-center" style="background-color:#f9f9fa">
<div class="row"><div class="col text-right text-secondary">Clearnet DL</div>
<div class="col text-left"><a target="_blank" href="<%= $USE_HTTP_HOST ? $BASEURLPROTO : $MAIN_DOMAIN_PROTO %>://<%= ($USE_HTTP_HOST ? $BASEURL : $MAIN_DOMAIN) . '/' . $file->{urlpath} . ( $file->{shorturl} == 0 ? '/'.$file->{rpath} : '' ) %>">
<%= ($USE_HTTP_HOST ? $BASEURL : $MAIN_DOMAIN) . '/' . $file->{urlpath} . ( $file->{shorturl} == 0 ? '/'.$file->{rpath} : '' ) %>
</a></div>
</a><%== $file->{oniononly} == 1 ? '<span class="text-danger">[inactive]</span>' : '' %></div>
</div>
% if (defined $ONION_DOMAIN and $ONION_DOMAIN ne $MAIN_DOMAIN) {
<div class="row"><div class="col text-right text-secondary">Tor DL</div>
Expand All @@ -21,7 +29,8 @@
<div class="row"><div class="col text-right text-secondary">Created</div><div class="col text-left"><%= scalar localtime $file->{created} %></div></div>
<div class="row"><div class="col text-right text-secondary">Expires</div><div class="col text-left"><%= $file->{expires} == 0 ? 'Never' : scalar localtime $file->{expires} %></div></div>
<div class="row"><div class="col text-right text-secondary">Destroy after DL</div><div class="col text-left"><%= $file->{autodestroy} == 1 ? 'Yes' : 'No' %> <form class="d-inline" method="get"><input type="hidden" name="toggleautodestroy" value="1"><button type="submit" class="btn btn-link">[toggle]</button></form></div></div>
<div class="row"><div class="col text-right text-secondary">Hits</div><div class="col text-left"><%= $file->{hits} %></div></div>
<div class="row"><div class="col text-right text-secondary">Onion only</div><div class="col text-left"><%= $file->{oniononly} == 1 ? 'Yes' : 'No' %> <form class="d-inline" method="get"><input type="hidden" name="toggleoniononly" value="1"><button type="submit" class="btn btn-link">[toggle]</button></form></div></div>
<div class="row"><div class="col text-right text-secondary">Hits</div><div class="col text-left"><%= $file->{hits} %><%= $DOWNLOAD_LIMIT_PER_FILE ? '/' . $DOWNLOAD_LIMIT_PER_FILE : '' %></div></div>
</div>
<p> Use this button to delete the file permanently: </p>
<form method="get">
Expand Down
3 changes: 2 additions & 1 deletion app/templates/manage.txt.ep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ Hashsum: <%= $file->{hashsum} %> (SHA<%= $FILE_HASH_TYPE %>)
Created: <%= scalar localtime $file->{created} %>
Expires: <%= $file->{expires} == 0 ? 'Never' : scalar localtime $file->{expires} %>
Destroy after download: <%= $file->{autodestroy} == 1 ? 'Yes' : 'No' %>
Hits: <%= $file->{hits} %>
Onion only: <%= $file->{oniononly} == 1 ? 'Yes' : 'No' %>
Hits: <%= $file->{hits} %><%= $DOWNLOAD_LIMIT_PER_FILE ? '/' . $DOWNLOAD_LIMIT_PER_FILE : '' %>
Delete file: <%= $USE_HTTP_HOST ? $BASEURLPROTO : $MAIN_DOMAIN_PROTO %>://<%= ($USE_HTTP_HOST ? $BASEURL : $MAIN_DOMAIN) . $MANAGE_ROUTE . $file->{mpath} . '/delete' %>

% if (my $msg = stash 'ERROR') {
Expand Down
73 changes: 56 additions & 17 deletions app/webapp.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

my $main = OshiUpload->new;
app->config(hypnotoad => { listen => ['http://' . $main->{conf}->{HTTP_APP_ADDRESS}. ':' . $main->{conf}->{HTTP_APP_PORT}],
accepts => 0,
workers => ( $main->{conf}->{HTTP_APP_WORKERS} || 4 ),
pid_file => ( $main->{conf}->{HTTP_APP_PIDFILE} || '/tmp/hypnotoad_oshi.pid' )
});
Expand Down Expand Up @@ -111,6 +112,20 @@
});
return $c->redirect_to($c->req->url->path->to_abs_string . '?file=' . $row->{'urlpath'});

}elsif ( $c->param('toggleoniononly') ) {
$row->{'oniononly'} = int $row->{'oniononly'} ? 0 : 1;
$main->{dbc}->run(sub {
$_->do("update uploads set oniononly = ? where mpath = ?", undef, $row->{'oniononly'}, $row->{'mpath'} );
});
return $c->redirect_to($c->req->url->path->to_abs_string . '?file=' . $row->{'urlpath'});

}elsif ( $c->param('toggleoniononlylock') ) {
$row->{'oniononlylocked'} = int $row->{'oniononlylocked'} ? 0 : 1;
$main->{dbc}->run(sub {
$_->do("update uploads set oniononlylocked = ? where mpath = ?", undef, $row->{'oniononlylocked'}, $row->{'mpath'} );
});
return $c->redirect_to($c->req->url->path->to_abs_string . '?file=' . $row->{'urlpath'});

}elsif ( defined $expire && int $expire >= 0 ) {
my @ex = $main->expiry_check($expire, $row->{'expires'});

Expand All @@ -119,7 +134,7 @@
} else {
$row->{'expires'} = $expire == 0 ? 0 : (time+($expire*60));
$main->{dbc}->run(sub {
$_->do("update uploads set expires = ?", undef, $row->{'expires'} );
$_->do("update uploads set expires = ? where mpath = ?", undef, $row->{'expires'}, $row->{'mpath'} );
});
$c->stash(SUCCESS => "The file expiry has been updated");
}
Expand Down Expand Up @@ -164,6 +179,15 @@

return $c->redirect_to($c->req->url->path->to_abs_string);
}

my $oniononly = $c->param('oniononly');
if ( $oniononly ) {
$main->{dbc}->run(sub {
$_->do('update uploads set oniononly = 1, oniononlylocked = 1 where urlpath = ?', undef, $oniononly);
$_->do('delete from reports where url = ?', undef, $oniononly);
});
return $c->redirect_to($c->req->url->path->to_abs_string);
}

my $d;
$main->{dbc}->run(sub {
Expand Down Expand Up @@ -285,6 +309,7 @@
my $optcmd = $c->param('option');
my $pdelete = $c->param('delete');
my $ptoggleautodestroy = $c->param('toggleautodestroy');
my $ptoggleoniononly = $c->param('toggleoniononly');
my $rmethod = lc $c->req->method;
my $rnd = rand_chars ( set => 'alpha', min => 10, max => 15 );

Expand All @@ -311,6 +336,14 @@
$dbh->do("update uploads set autodestroy = ? where mpath = ?", undef, $row->{'autodestroy'}, $mpath );
});
return ($row, ['REFRESH', undef]);
}elsif ( defined $ptoggleoniononly ) {
return ($row, ['ERROR', 'This feature was disabled for your file']) if $row->{'oniononlylocked'};
$row->{'oniononly'} = int $row->{'oniononly'} ? 0 : 1;
$main->{dbc}->run(sub {
my $dbh = shift;
$dbh->do("update uploads set oniononly = ? where mpath = ?", undef, $row->{'oniononly'}, $mpath );
});
return ($row, ['REFRESH', undef]);
}elsif ( defined $expire && int $expire >= 0 ) {
my @ex = $main->expiry_check($expire, $row->{'expires'});

Expand All @@ -324,7 +357,7 @@
$row->{'expires'} = $expire == 0 ? 0 : (time+($expire*60));
$main->{dbc}->run(sub {
my $dbh = shift;
$dbh->do("update uploads set expires = ?", undef, $row->{'expires'} );
$dbh->do("update uploads set expires = ? where mpath = ?", undef, $row->{'expires'}, $mpath );
});

return ($row, ['SUCCESS', "The file expiry has been updated"]);
Expand Down Expand Up @@ -650,42 +683,48 @@
my ($subprocess, $err, $row) = @_;
return $c->reply->exception($err) if $err;
return ( $c->isconsole ? $c->render(text => "File not found\n", status => 404) : $c->reply->not_found ) unless $row;
return ( $c->isconsole ? $c->render(text => "File not found\n", status => 404) : $c->reply->not_found ) if ( $row->{'shorturl'} == 0 and ( not defined $cfilename or $cfilename ne $row->{'rpath'} ) );
return ( $c->isconsole ? $c->render(text => "File not found\n", status => 404) : $c->reply->not_found ) if ( ($row->{'shorturl'} == 0 and ( not defined $cfilename or $cfilename ne $row->{'rpath'} )) or ($row->{'oniononly'} && lc $c->req->url->to_abs->host !~ /\.onion$/) );

return $c->render(text => "File is finishing processing (calculating hashsum), please retry in some seconds") if $row->{'processing'};
return $c->render(text => $row->{'hashsum'} . " (SHA" . $main->{HASHTYPE} . ")\n") if $hashsumreq;

my $file = $row->{'type'} eq 'link' ? $row->{'link'} : $main->build_filepath( $row->{'storage'},$row->{'urlpath'},$row->{'rpath'} );

try { utf8::decode($file) };

return $c->render(text => "File not available right now (perhaps storage unmounted?)") unless -f $file;
my $dlfilename = $cfilename || $row->{'rpath'};

my $inlineview = 0;

foreach my $mimetype ( keys %{$main->{MIMETYPE_SIZE_LIMITS}} ) {
if ( $row->{'ftype'} =~ /^\Q$mimetype\E/ && $row->{'size'} <= $main->{MIMETYPE_SIZE_LIMITS}->{$mimetype} ) {
$inlineview = 1;
if ( $row->{'ftype'} =~ /^(image\/|video\/|audio\/)/ ) {
unless ( $cfilename || $urlpathext ) {
return $c->redirect_to(join('/',$row->{'urlpath'}, $dlfilename));
} else {
$c->res->headers->content_type( $row->{'ftype'} );
foreach my $mimetype ( sort {$a cmp $b} keys %{$main->{MIMETYPE_SIZE_LIMITS}} ) {
if ( $row->{'ftype'} =~ /^\Q$mimetype\E/ ) {
if ( $row->{'size'} <= $main->{MIMETYPE_SIZE_LIMITS}->{$mimetype} ) {
$inlineview = 1;
if ( $row->{'ftype'} =~ /^(image\/|video\/|audio\/)/ ) {
unless ( $cfilename || $urlpathext ) {
return $c->redirect_to(join('/',$row->{'urlpath'}, $dlfilename));
} else {
$c->res->headers->content_type( $row->{'ftype'} );
}
} elsif ( $row->{'ftype'} =~ /^text\// ) {
$c->res->headers->content_type( 'text/plain; charset=utf-8');
}elsif ( $row->{'ftype'} =~ /^application\/pdf/ ) {
$c->res->headers->content_disposition("inline; filename=$dlfilename");
}
} elsif ( $row->{'ftype'} =~ /^text\// ) {
$c->res->headers->content_type( 'text/plain; charset=utf-8');
}elsif ( $row->{'ftype'} =~ /^application\/pdf/ ) {
$c->res->headers->content_disposition("inline; filename=$dlfilename");
} else {
$inlineview = 0;
}
}
}

$c->res->headers->content_disposition("attachment; filename=$dlfilename") unless $inlineview;

$c->reply->asset(Mojo::Asset::File->new(path => $file));



if ( $row->{'autodestroy'} && ( !int $row->{'autodestroylocked'} || ( ( $row->{'hits'} + 1 ) >= $main->{RESTRICTED_FILE_HITLIMIT} ) ) ) {
if ( ( $main->{conf}->{DOWNLOAD_LIMIT_PER_FILE} > 0 && ( $row->{'hits'} + 1 ) >= $main->{conf}->{DOWNLOAD_LIMIT_PER_FILE} ) || ($row->{'autodestroy'} && ( !int $row->{'autodestroylocked'} || ( ( $row->{'hits'} + 1 ) >= $main->{RESTRICTED_FILE_HITLIMIT} ) )) ) {
$main->delete_file('mpath', $row->{'mpath'});
app->log->info('File destroyed on download') if $main->{conf}->{DEBUG} > 0;
}
Expand Down

1 comment on commit 3c416d9

@kkarhan
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mentioned in:

Apparently oshi.at got abused by pedos for CSAM?

Please sign in to comment.