harmacist/vichan
1
0
Fork 0
forked from GithubBackups/vichan
Code Pull Requests Activity

Merge pull request #785 from Zankaria/batch-maintenance

Browse Source 
Batch maintenance
This commit is contained in:
Lorenzo Yario 2024-08-11 03:05:31 -07:00 committed by GitHub
commit b3ae38da57
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 261 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
inc/anti-bot.php
View File

@ -10,7 +10,7 @@ $hidden_inputs_twig = array();
class AntiBot {
public $salt, $inputs = array(), $index = 0;
public static function randomString($length, $uppercase = false, $special_chars = false, $unicode_chars = false) {
$chars = 'abcdefghijklmnopqrstuvwxyz0123456789';
if ($uppercase)
@ -22,11 +22,11 @@ class AntiBot {
for ($n = 0; $n < $len; $n++)
$chars .= mb_convert_encoding('&#' . mt_rand(0x2600, 0x26FF) . ';', 'UTF-8', 'HTML-ENTITIES');
}
$chars = preg_split('//u', $chars, -1, PREG_SPLIT_NO_EMPTY);
$ch = array();
// fill up $ch until we reach $length
while (count($ch) < $length) {
$n = $length - count($ch);
@ -39,40 +39,40 @@ class AntiBot {
foreach ($keys as $key)
$ch[] = $chars[$key];
}
$chars = $ch;
return implode('', $chars);
}
public static function make_confusing($string) {
$chars = preg_split('//u', $string, -1, PREG_SPLIT_NO_EMPTY);
foreach ($chars as &$c) {
if (mt_rand(0, 3) != 0)
$c = utf8tohtml($c);
else
$c = mb_encode_numericentity($c, array(0, 0xffff, 0, 0xffff), 'UTF-8');
}
return implode('', $chars);
}
public function __construct(array $salt = array()) {
global $config;
if (!empty($salt)) {
// create a salted hash of the "extra salt"
$this->salt = implode(':', $salt);
} else {
} else {
$this->salt = '';
}
shuffle($config['spam']['hidden_input_names']);
$input_count = mt_rand($config['spam']['hidden_inputs_min'], $config['spam']['hidden_inputs_max']);
$hidden_input_names_x = 0;
for ($x = 0; $x < $input_count ; $x++) {
if ($hidden_input_names_x === false || mt_rand(0, 2) == 0) {
// Use an obscure name
@ -83,7 +83,7 @@ class AntiBot {
if ($hidden_input_names_x >= count($config['spam']['hidden_input_names']))
$hidden_input_names_x = false;
}
if (mt_rand(0, 2) == 0) {
// Value must be null
$this->inputs[$name] = '';
@ -96,16 +96,16 @@ class AntiBot {
}
}
}
public static function space() {
if (mt_rand(0, 3) != 0)
return ' ';
return str_repeat(' ', mt_rand(1, 3));
}
public function html($count = false) {
global $config;
$elements = array(
'<input type="hidden" name="%name%" value="%value%">',
'<input type="hidden" value="%value%" name="%name%">',
@ -119,13 +119,13 @@ class AntiBot {
'<textarea style="display:none" name="%name%">%value%</textarea>',
'<textarea name="%name%" style="display:none">%value%</textarea>'
);
$html = '';
if ($count === false) {
$count = mt_rand(1, (int)abs(count($this->inputs) / 15) + 1);
}
if ($count === true) {
// all elements
$inputs = array_slice($this->inputs, $this->index);
@ -133,7 +133,7 @@ class AntiBot {
$inputs = array_slice($this->inputs, $this->index, $count);
}
$this->index += count($inputs);
foreach ($inputs as $name => $value) {
$element = false;
while (!$element) {
@ -146,37 +146,37 @@ class AntiBot {
$element = false;
}
}
$element = str_replace('%name%', utf8tohtml($name), $element);
if (mt_rand(0, 2) == 0)
$value = $this->make_confusing($value);
else
$value = utf8tohtml($value);
if (strpos($element, 'textarea') === false)
$value = str_replace('"', '&quot;', $value);
$element = str_replace('%value%', $value, $element);
$html .= $element;
}
return $html;
}
public function reset() {
$this->index = 0;
}
public function hash() {
global $config;
// This is the tricky part: create a hash to validate it after
// First, sort the keys in alphabetical order (A-Z)
$inputs = $this->inputs;
ksort($inputs);
$hash = '';
// Iterate through each input
foreach ($inputs as $name => $value) {
@ -184,7 +184,7 @@ class AntiBot {
}
// Add a salt to the hash
$hash .= $config['cookies']['salt'];
// Use SHA1 for the hash
return sha1($hash . $this->salt);
}

230
inc/bans.php
View File

@ -4,6 +4,10 @@ use Vichan\Functions\Format;
use Lifo\IP\CIDR;
class Bans {
static private function shouldDelete(array $ban, bool $require_ban_view) {
return $ban['expires'] && ($ban['seen'] || !$require_ban_view) && $ban['expires'] < time();
}
static private function deleteBans(array $ban_ids) {
$len = count($ban_ids);
if ($len === 1) {
@ -33,6 +37,127 @@ class Bans {
}
}
static private function findSingleAutoGc(string $ip, int $ban_id, bool $require_ban_view): array|null {
// Use OR in the query to also garbage collect bans.
$query = prepare(
'SELECT ``bans``.* FROM ``bans``
WHERE ((`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC'
);
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$found_ban = null;
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if (self::shouldDelete($ban, $require_ban_view)) {
$to_delete_list[] = $ban['id'];
} elseif ($ban['id'] === $ban_id) {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
$found_ban = $ban;
}
}
self::deleteBans($to_delete_list);
return $found_ban;
}
static private function findSingleNoGc(int $ban_id): array|null {
$query = prepare(
'SELECT ``bans``.* FROM ``bans``
WHERE ``bans``.id = :id
ORDER BY `expires` IS NULL, `expires` DESC
LIMIT 1'
);
$query->bindValue(':id', $ban_id);
$query->execute() or error(db_error($query));
$ret = $query->fetch(PDO::FETCH_ASSOC);
if ($query->rowCount() == 0) {
return null;
} else {
if ($ret['post']) {
$ret['post'] = json_decode($ret['post'], true);
}
$ret['mask'] = self::range_to_string([$ret['ipstart'], $ret['ipend']]);
return $ret;
}
}
static private function findAutoGc(?string $ip, string|false $board, bool $get_mod_info, bool $require_ban_view, ?int $ban_id): array {
$query = prepare('SELECT ``bans``.*' . ($get_mod_info ? ', `username`' : '') . ' FROM ``bans``
' . ($get_mod_info ? 'LEFT JOIN ``mods`` ON ``mods``.`id` = `creator`' : '') . '
WHERE
(' . ($board !== false ? '(`board` IS NULL OR `board` = :board) AND' : '') . '
(`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC');
if ($board !== false) {
$query->bindValue(':board', $board, PDO::PARAM_STR);
}
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$ban_list = [];
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if (self::shouldDelete($ban, $require_ban_view)) {
$to_delete_list[] = $ban['id'];
} else {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
$ban_list[] = $ban;
}
}
self::deleteBans($to_delete_list);
return $ban_list;
}
static private function findNoGc(?string $ip, string|false $board, bool $get_mod_info, ?int $ban_id): array {
$query = prepare('SELECT ``bans``.*' . ($get_mod_info ? ', `username`' : '') . ' FROM ``bans``
' . ($get_mod_info ? 'LEFT JOIN ``mods`` ON ``mods``.`id` = `creator`' : '') . '
WHERE
(' . ($board !== false ? '(`board` IS NULL OR `board` = :board) AND' : '') . '
(`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
AND (`expires` IS NULL OR `expires` >= :curr_time)
ORDER BY `expires` IS NULL, `expires` DESC');
if ($board !== false) {
$query->bindValue(':board', $board, PDO::PARAM_STR);
}
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->bindValue(':curr_time', time());
$query->execute() or error(db_error($query));
$ban_list = $query->fetchAll(PDO::FETCH_ASSOC);
array_walk($ban_list, function (&$ban, $_index) {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
});
return $ban_list;
}
static public function range_to_string($mask) {
list($ipstart, $ipend) = $mask;
@ -56,7 +181,7 @@ class Bans {
$cidr = new CIDR($mask);
$range = $cidr->getRange();
return array(inet_pton($range[0]), inet_pton($range[1]));
return [ inet_pton($range[0]), inet_pton($range[1]) ];
}
public static function parse_time($str) {
@ -140,82 +265,25 @@ class Bans {
return false;
}
return array($ipstart, $ipend);
return [$ipstart, $ipend];
}
static public function findSingle(string $ip, int $ban_id, bool $require_ban_view): array|null {
/**
* Use OR in the query to also garbage collect bans. Ideally we should move the whole GC procedure to a separate
* script, but it will require a more important restructuring.
*/
$query = prepare(
'SELECT ``bans``.* FROM ``bans``
WHERE ((`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC'
);
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$found_ban = null;
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if ($ban['expires'] && ($ban['seen'] || !$require_ban_view) && $ban['expires'] < time()) {
$to_delete_list[] = $ban['id'];
} elseif ($ban['id'] === $ban_id) {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$ban['cmask'] = cloak_mask($ban['mask']);
$found_ban = $ban;
}
static public function findSingle(string $ip, int $ban_id, bool $require_ban_view, bool $auto_gc): array|null {
if ($auto_gc) {
return self::findSingleAutoGc($ip, $ban_id, $require_ban_view);
} else {
return self::findSingleNoGc($ban_id);
}
self::deleteBans($to_delete_list);
return $found_ban;
}
static public function find($ip, $board = false, $get_mod_info = false, $banid = null) {
static public function find(?string $ip, string|false $board = false, bool $get_mod_info = false, ?int $ban_id = null, bool $auto_gc = true) {
global $config;
$query = prepare('SELECT ``bans``.*' . ($get_mod_info ? ', `username`' : '') . ' FROM ``bans``
' . ($get_mod_info ? 'LEFT JOIN ``mods`` ON ``mods``.`id` = `creator`' : '') . '
WHERE
(' . ($board !== false ? '(`board` IS NULL OR `board` = :board) AND' : '') . '
(`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC');
if ($board !== false)
$query->bindValue(':board', $board, PDO::PARAM_STR);
$query->bindValue(':id', $banid);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$ban_list = array();
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if ($ban['expires'] && ($ban['seen'] || !$config['require_ban_view']) && $ban['expires'] < time()) {
$to_delete_list[] = $ban['id'];
} else {
if ($ban['post'])
$ban['post'] = json_decode($ban['post'], true);
$ban['mask'] = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$ban['cmask'] = cloak_mask($ban['mask']);
$ban_list[] = $ban;
}
if ($auto_gc) {
return self::findAutoGc($ip, $board, $get_mod_info, $config['require_ban_view'], $ban_id);
} else {
return self::findNoGc($ip, $board, $get_mod_info, $ban_id);
}
self::deleteBans($to_delete_list);
return $ban_list;
}
static public function stream_json($out = false, $filter_ips = false, $filter_staff = false, $board_access = false) {
@ -231,8 +299,7 @@ class Bans {
$end = end($bans);
foreach ($bans as &$ban) {
$uncloaked_mask = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$ban['mask'] = cloak_mask($uncloaked_mask);
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
if ($ban['post']) {
$post = json_decode($ban['post']);
@ -279,9 +346,21 @@ class Bans {
rebuildThemes('bans');
}
static public function purge() {
$query = query("DELETE FROM ``bans`` WHERE `expires` IS NOT NULL AND `expires` < " . time() . " AND `seen` = 1") or error(db_error());
rebuildThemes('bans');
static public function purge($require_seen, $moratorium) {
if ($require_seen) {
$query = prepare("DELETE FROM ``bans`` WHERE `expires` IS NOT NULL AND `expires` + :moratorium < :curr_time AND `seen` = 1");
} else {
$query = prepare("DELETE FROM ``bans`` WHERE `expires` IS NOT NULL AND `expires` + :moratorium < :curr_time");
}
$query->bindValue(':moratorium', $moratorium);
$query->bindValue(':curr_time', time());
$query->execute() or error(db_error($query));
$affected = $query->rowCount();
if ($affected > 0) {
rebuildThemes('bans');
}
return $affected;
}
static public function delete($ban_id, $modlog = false, $boards = false, $dont_rebuild = false) {
@ -299,8 +378,7 @@ class Bans {
if ($boards !== false && !in_array($ban['board'], $boards))
error($config['error']['noaccess']);
$mask = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$cloaked_mask = cloak_mask($mask);
$mask = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
modLog("Removed ban #{$ban_id} for " .
(filter_var($mask, FILTER_VALIDATE_IP) !== false ? "<a href=\"?/IP/$cloaked_mask\">$cloaked_mask</a>" : $cloaked_mask));

12
inc/config.php
View File

@ -92,6 +92,11 @@
// to the environment path (seperated by :).
$config['shell_path'] = '/usr/local/bin';
// Automatically execute some maintenance tasks when some pages are opened, which may result in higher
// latencies.
// If set to false, ensure to periodically invoke the tools/maintenance.php script.
$config['auto_maintenance'] = true;
/*
* ====================
* Database settings
@ -747,6 +752,9 @@
//);
$config['premade_ban_reasons'] = false;
// How often (minimum) to purge the ban list of expired bans (which have been seen).
$config['purge_bans'] = 60 * 60 * 12; // 12 hours
// Allow users to appeal bans through vichan.
$config['ban_appeals'] = false;
@ -1564,10 +1572,6 @@
// Enable the moving of single replies
$config['move_replies'] = false;
// How often (minimum) to purge the ban list of expired bans (which have been seen). Only works when
// $config['cache'] is enabled and working.
$config['purge_bans'] = 60 * 60 * 12; // 12 hours
// Do DNS lookups on IP addresses to get their hostname for the moderator IP pages (?/IP/x.x.x.x).
$config['mod']['dns_lookup'] = true;
// How many recent posts, per board, to show in ?/IP/x.x.x.x.

52
inc/functions.php
View File

@ -876,11 +876,13 @@ function checkBan($board = false) {
}
foreach ($ips as $ip) {
$bans = Bans::find($ip, $board, $config['show_modname']);
$bans = Bans::find($ip, $board, $config['show_modname'], null, $config['auto_maintenance']);
foreach ($bans as &$ban) {
if ($ban['expires'] && $ban['expires'] < time()) {
Bans::delete($ban['id']);
if ($config['auto_maintenance']) {
Bans::delete($ban['id']);
}
if ($config['require_ban_view'] && !$ban['seen']) {
if (!isset($_POST['json_response'])) {
displayBan($ban);
@ -900,17 +902,20 @@ function checkBan($board = false) {
}
}
// I'm not sure where else to put this. It doesn't really matter where; it just needs to be called every
// now and then to keep the ban list tidy.
if ($config['cache']['enabled'] && $last_time_purged = cache::get('purged_bans_last')) {
if (time() - $last_time_purged < $config['purge_bans'] )
return;
if ($config['auto_maintenance']) {
// I'm not sure where else to put this. It doesn't really matter where; it just needs to be called every
// now and then to keep the ban list tidy.
if ($config['cache']['enabled']) {
$last_time_purged = cache::get('purged_bans_last');
if ($last_time_purged !== false && time() - $last_time_purged > $config['purge_bans']) {
Bans::purge($config['require_ban_view'], $config['purge_bans']);
cache::set('purged_bans_last', time());
}
} else {
// Purge every time.
Bans::purge($config['require_ban_view'], $config['purge_bans']);
}
}
Bans::purge();
if ($config['cache']['enabled'])
cache::set('purged_bans_last', time());
}
function threadLocked($id) {
@ -1612,27 +1617,40 @@ function checkMute() {
}
}
function purge_old_antispam() {
$query = prepare('DELETE FROM ``antispam`` WHERE `expires` < UNIX_TIMESTAMP()');
$query->execute() or error(db_error());
return $query->rowCount();
}
function _create_antibot($board, $thread) {
global $config, $purged_old_antispam;
$antibot = new AntiBot(array($board, $thread));
$antibot = new AntiBot([$board, $thread]);
if (!isset($purged_old_antispam)) {
// Delete old expired antispam, skipping those with NULL expiration timestamps (infinite lifetime).
if (!isset($purged_old_antispam) && $config['auto_maintenance']) {
$purged_old_antispam = true;
query('DELETE FROM ``antispam`` WHERE `expires` < UNIX_TIMESTAMP()') or error(db_error());
}
if ($thread)
// Keep the now invalid timestamps around for a bit to enable users to post if they're still on an old version of
// the HTML page.
// By virtue of existing, we know that we're making a new version of the page, and the user from now on may just reload.
if ($thread) {
$query = prepare('UPDATE ``antispam`` SET `expires` = UNIX_TIMESTAMP() + :expires WHERE `board` = :board AND `thread` = :thread AND `expires` IS NULL');
else
} else {
$query = prepare('UPDATE ``antispam`` SET `expires` = UNIX_TIMESTAMP() + :expires WHERE `board` = :board AND `thread` IS NULL AND `expires` IS NULL');
}
$query->bindValue(':board', $board);
if ($thread)
if ($thread) {
$query->bindValue(':thread', $thread);
}
$query->bindValue(':expires', $config['spam']['hidden_inputs_expire']);
$query->execute() or error(db_error($query));
// Insert an antispam with infinite life as the HTML page of a thread might last well beyond the expiry date.
$query = prepare('INSERT INTO ``antispam`` VALUES (:board, :thread, :hash, UNIX_TIMESTAMP(), NULL, 0)');
$query->bindValue(':board', $board);
$query->bindValue(':thread', $thread);

8
inc/mod/pages.php
View File

@ -897,7 +897,7 @@ function mod_page_ip($cip) {
$args['token'] = make_secure_link_token('ban');
if (hasPermission($config['mod']['view_ban'])) {
$args['bans'] = Bans::find($ip, false, true);
$args['bans'] = Bans::find($ip, false, true, null, $config['auto_maintenance']);
}
if (hasPermission($config['mod']['view_notes'])) {
@ -927,7 +927,7 @@ function mod_edit_ban($ban_id) {
if (!hasPermission($config['mod']['edit_ban']))
error($config['error']['noaccess']);
$args['bans'] = Bans::find(null, false, true, $ban_id);
$args['bans'] = Bans::find(null, false, true, $ban_id, $config['auto_maintenance']);
$args['ban_id'] = $ban_id;
$args['boards'] = listBoards();
$args['current_board'] = isset($args['bans'][0]['board']) ? $args['bans'][0]['board'] : false;
@ -1044,10 +1044,6 @@ function mod_ban_appeals() {
if (!hasPermission($config['mod']['view_ban_appeals']))
error($config['error']['noaccess']);
// Remove stale ban appeals
query("DELETE FROM ``ban_appeals`` WHERE NOT EXISTS (SELECT 1 FROM ``bans`` WHERE `ban_id` = ``bans``.`id`)")
or error(db_error());
if (isset($_POST['appeal_id']) && (isset($_POST['unban']) || isset($_POST['deny']))) {
if (!hasPermission($config['mod']['ban_appeals']))
error($config['error']['noaccess']);

3
install.sql
View File

@ -294,7 +294,8 @@ CREATE TABLE IF NOT EXISTS `ban_appeals` (
`message` text NOT NULL,
`denied` tinyint(1) NOT NULL,
PRIMARY KEY (`id`),
KEY `ban_id` (`ban_id`)
KEY `ban_id` (`ban_id`),
CONSTRAINT `fk_ban_id` FOREIGN KEY (`ban_id`) REFERENCES `bans`(`id`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 AUTO_INCREMENT=1 ;
-- --------------------------------------------------------

2
post.php
View File

@ -1439,7 +1439,7 @@ if (isset($_POST['delete'])) {
$ban_id = (int)$_POST['ban_id'];
$ban = Bans::findSingle($_SERVER['REMOTE_ADDR'], $ban_id, $config['require_ban_view']);
$ban = Bans::findSingle($_SERVER['REMOTE_ADDR'], $ban_id, $config['require_ban_view'], $config['auto_maintenance']);
if (empty($ban)) {
error($config['error']['noban']);

25
tools/maintenance.php Normal file
View File

@ -0,0 +1,25 @@
<?php
/**
* Performs maintenance tasks. Invoke this periodically if the auto_maintenance configuration option is turned off.
*/
require dirname(__FILE__) . '/inc/cli.php';
echo "Clearing expired bans...\n";
$start = microtime(true);
$deleted_count = Bans::purge($config['require_ban_view'], $config['purge_bans']);
$delta = microtime(true) - $start;
echo "Deleted $deleted_count expired bans in $delta seconds!\n";
$time_tot = $delta;
$deleted_tot = $deleted_count;
echo "Clearing old antispam...\n";
$start = microtime(true);
$deleted_count = purge_old_antispam();
$delta = microtime(true) - $start;
echo "Deleted $deleted_count expired antispam in $delta seconds!\n";
$time_tot = $delta;
$deleted_tot = $deleted_count;
$time_tot = number_format((float)$time_tot, 4, '.', '');
modLog("Deleted $deleted_tot expired entries in {$time_tot}s with maintenance tool");