harmacist/vichan
1
0
Fork 0
forked from GithubBackups/vichan
Code Pull Requests Activity

Merge pull request #785 from Zankaria/batch-maintenance

Browse Source 
Batch maintenance
This commit is contained in:
Lorenzo Yario 2024-08-11 03:05:31 -07:00 committed by GitHub
commit b3ae38da57
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 261 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
inc/bans.php
View File

@ -4,6 +4,10 @@ use Vichan\Functions\Format;
use Lifo\IP\CIDR;
class Bans {
static private function shouldDelete(array $ban, bool $require_ban_view) {
return $ban['expires'] && ($ban['seen'] || !$require_ban_view) && $ban['expires'] < time();
}
static private function deleteBans(array $ban_ids) {
$len = count($ban_ids);
if ($len === 1) {
@ -33,6 +37,127 @@ class Bans {
}
}
static private function findSingleAutoGc(string $ip, int $ban_id, bool $require_ban_view): array|null {
// Use OR in the query to also garbage collect bans.
$query = prepare(
'SELECT ``bans``.* FROM ``bans``
WHERE ((`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC'
);
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$found_ban = null;
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if (self::shouldDelete($ban, $require_ban_view)) {
$to_delete_list[] = $ban['id'];
} elseif ($ban['id'] === $ban_id) {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
$found_ban = $ban;
}
}
self::deleteBans($to_delete_list);
return $found_ban;
}
static private function findSingleNoGc(int $ban_id): array|null {
$query = prepare(
'SELECT ``bans``.* FROM ``bans``
WHERE ``bans``.id = :id
ORDER BY `expires` IS NULL, `expires` DESC
LIMIT 1'
);
$query->bindValue(':id', $ban_id);
$query->execute() or error(db_error($query));
$ret = $query->fetch(PDO::FETCH_ASSOC);
if ($query->rowCount() == 0) {
return null;
} else {
if ($ret['post']) {
$ret['post'] = json_decode($ret['post'], true);
}
$ret['mask'] = self::range_to_string([$ret['ipstart'], $ret['ipend']]);
return $ret;
}
}
static private function findAutoGc(?string $ip, string|false $board, bool $get_mod_info, bool $require_ban_view, ?int $ban_id): array {
$query = prepare('SELECT ``bans``.*' . ($get_mod_info ? ', `username`' : '') . ' FROM ``bans``
' . ($get_mod_info ? 'LEFT JOIN ``mods`` ON ``mods``.`id` = `creator`' : '') . '
WHERE
(' . ($board !== false ? '(`board` IS NULL OR `board` = :board) AND' : '') . '
(`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC');
if ($board !== false) {
$query->bindValue(':board', $board, PDO::PARAM_STR);
}
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$ban_list = [];
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if (self::shouldDelete($ban, $require_ban_view)) {
$to_delete_list[] = $ban['id'];
} else {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
$ban_list[] = $ban;
}
}
self::deleteBans($to_delete_list);
return $ban_list;
}
static private function findNoGc(?string $ip, string|false $board, bool $get_mod_info, ?int $ban_id): array {
$query = prepare('SELECT ``bans``.*' . ($get_mod_info ? ', `username`' : '') . ' FROM ``bans``
' . ($get_mod_info ? 'LEFT JOIN ``mods`` ON ``mods``.`id` = `creator`' : '') . '
WHERE
(' . ($board !== false ? '(`board` IS NULL OR `board` = :board) AND' : '') . '
(`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
AND (`expires` IS NULL OR `expires` >= :curr_time)
ORDER BY `expires` IS NULL, `expires` DESC');
if ($board !== false) {
$query->bindValue(':board', $board, PDO::PARAM_STR);
}
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->bindValue(':curr_time', time());
$query->execute() or error(db_error($query));
$ban_list = $query->fetchAll(PDO::FETCH_ASSOC);
array_walk($ban_list, function (&$ban, $_index) {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
});
return $ban_list;
}
static public function range_to_string($mask) {
list($ipstart, $ipend) = $mask;
@ -56,7 +181,7 @@ class Bans {
$cidr = new CIDR($mask);
$range = $cidr->getRange();
return array(inet_pton($range[0]), inet_pton($range[1]));
return [ inet_pton($range[0]), inet_pton($range[1]) ];
}
public static function parse_time($str) {
@ -140,84 +265,27 @@ class Bans {
return false;
}
return array($ipstart, $ipend);
return [$ipstart, $ipend];
}
static public function findSingle(string $ip, int $ban_id, bool $require_ban_view): array|null {
/**
* Use OR in the query to also garbage collect bans. Ideally we should move the whole GC procedure to a separate
* script, but it will require a more important restructuring.
*/
$query = prepare(
'SELECT ``bans``.* FROM ``bans``
WHERE ((`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC'
);
$query->bindValue(':id', $ban_id);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$found_ban = null;
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if ($ban['expires'] && ($ban['seen'] || !$require_ban_view) && $ban['expires'] < time()) {
$to_delete_list[] = $ban['id'];
} elseif ($ban['id'] === $ban_id) {
if ($ban['post']) {
$ban['post'] = json_decode($ban['post'], true);
}
$ban['mask'] = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$ban['cmask'] = cloak_mask($ban['mask']);
$found_ban = $ban;
static public function findSingle(string $ip, int $ban_id, bool $require_ban_view, bool $auto_gc): array|null {
if ($auto_gc) {
return self::findSingleAutoGc($ip, $ban_id, $require_ban_view);
} else {
return self::findSingleNoGc($ban_id);
}
}
self::deleteBans($to_delete_list);
return $found_ban;
}
static public function find($ip, $board = false, $get_mod_info = false, $banid = null) {
static public function find(?string $ip, string|false $board = false, bool $get_mod_info = false, ?int $ban_id = null, bool $auto_gc = true) {
global $config;
$query = prepare('SELECT ``bans``.*' . ($get_mod_info ? ', `username`' : '') . ' FROM ``bans``
' . ($get_mod_info ? 'LEFT JOIN ``mods`` ON ``mods``.`id` = `creator`' : '') . '
WHERE
(' . ($board !== false ? '(`board` IS NULL OR `board` = :board) AND' : '') . '
(`ipstart` = :ip OR (:ip >= `ipstart` AND :ip <= `ipend`)) OR (``bans``.id = :id))
ORDER BY `expires` IS NULL, `expires` DESC');
if ($board !== false)
$query->bindValue(':board', $board, PDO::PARAM_STR);
$query->bindValue(':id', $banid);
$query->bindValue(':ip', inet_pton($ip));
$query->execute() or error(db_error($query));
$ban_list = array();
$to_delete_list = [];
while ($ban = $query->fetch(PDO::FETCH_ASSOC)) {
if ($ban['expires'] && ($ban['seen'] || !$config['require_ban_view']) && $ban['expires'] < time()) {
$to_delete_list[] = $ban['id'];
if ($auto_gc) {
return self::findAutoGc($ip, $board, $get_mod_info, $config['require_ban_view'], $ban_id);
} else {
if ($ban['post'])
$ban['post'] = json_decode($ban['post'], true);
$ban['mask'] = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$ban['cmask'] = cloak_mask($ban['mask']);
$ban_list[] = $ban;
return self::findNoGc($ip, $board, $get_mod_info, $ban_id);
}
}
self::deleteBans($to_delete_list);
return $ban_list;
}
static public function stream_json($out = false, $filter_ips = false, $filter_staff = false, $board_access = false) {
$query = query("SELECT ``bans``.*, `username` FROM ``bans``
LEFT JOIN ``mods`` ON ``mods``.`id` = `creator`
@ -231,8 +299,7 @@ class Bans {
$end = end($bans);
foreach ($bans as &$ban) {
$uncloaked_mask = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$ban['mask'] = cloak_mask($uncloaked_mask);
$ban['mask'] = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
if ($ban['post']) {
$post = json_decode($ban['post']);
@ -279,10 +346,22 @@ class Bans {
rebuildThemes('bans');
}
static public function purge() {
$query = query("DELETE FROM ``bans`` WHERE `expires` IS NOT NULL AND `expires` < " . time() . " AND `seen` = 1") or error(db_error());
static public function purge($require_seen, $moratorium) {
if ($require_seen) {
$query = prepare("DELETE FROM ``bans`` WHERE `expires` IS NOT NULL AND `expires` + :moratorium < :curr_time AND `seen` = 1");
} else {
$query = prepare("DELETE FROM ``bans`` WHERE `expires` IS NOT NULL AND `expires` + :moratorium < :curr_time");
}
$query->bindValue(':moratorium', $moratorium);
$query->bindValue(':curr_time', time());
$query->execute() or error(db_error($query));
$affected = $query->rowCount();
if ($affected > 0) {
rebuildThemes('bans');
}
return $affected;
}
static public function delete($ban_id, $modlog = false, $boards = false, $dont_rebuild = false) {
global $config;
@ -299,8 +378,7 @@ class Bans {
if ($boards !== false && !in_array($ban['board'], $boards))
error($config['error']['noaccess']);
$mask = self::range_to_string(array($ban['ipstart'], $ban['ipend']));
$cloaked_mask = cloak_mask($mask);
$mask = self::range_to_string([$ban['ipstart'], $ban['ipend']]);
modLog("Removed ban #{$ban_id} for " .
(filter_var($mask, FILTER_VALIDATE_IP) !== false ? "<a href=\"?/IP/$cloaked_mask\">$cloaked_mask</a>" : $cloaked_mask));

12
inc/config.php
View File

@ -92,6 +92,11 @@
// to the environment path (seperated by :).
$config['shell_path'] = '/usr/local/bin';
// Automatically execute some maintenance tasks when some pages are opened, which may result in higher
// latencies.
// If set to false, ensure to periodically invoke the tools/maintenance.php script.
$config['auto_maintenance'] = true;
/*
* ====================
* Database settings
@ -747,6 +752,9 @@
//);
$config['premade_ban_reasons'] = false;
// How often (minimum) to purge the ban list of expired bans (which have been seen).
$config['purge_bans'] = 60 * 60 * 12; // 12 hours
// Allow users to appeal bans through vichan.
$config['ban_appeals'] = false;
@ -1564,10 +1572,6 @@
// Enable the moving of single replies
$config['move_replies'] = false;
// How often (minimum) to purge the ban list of expired bans (which have been seen). Only works when
// $config['cache'] is enabled and working.
$config['purge_bans'] = 60 * 60 * 12; // 12 hours
// Do DNS lookups on IP addresses to get their hostname for the moderator IP pages (?/IP/x.x.x.x).
$config['mod']['dns_lookup'] = true;
// How many recent posts, per board, to show in ?/IP/x.x.x.x.

46
inc/functions.php
View File

@ -876,11 +876,13 @@ function checkBan($board = false) {
}
foreach ($ips as $ip) {
$bans = Bans::find($ip, $board, $config['show_modname']);
$bans = Bans::find($ip, $board, $config['show_modname'], null, $config['auto_maintenance']);
foreach ($bans as &$ban) {
if ($ban['expires'] && $ban['expires'] < time()) {
if ($config['auto_maintenance']) {
Bans::delete($ban['id']);
}
if ($config['require_ban_view'] && !$ban['seen']) {
if (!isset($_POST['json_response'])) {
displayBan($ban);
@ -900,17 +902,20 @@ function checkBan($board = false) {
}
}
if ($config['auto_maintenance']) {
// I'm not sure where else to put this. It doesn't really matter where; it just needs to be called every
// now and then to keep the ban list tidy.
if ($config['cache']['enabled'] && $last_time_purged = cache::get('purged_bans_last')) {
if (time() - $last_time_purged < $config['purge_bans'] )
return;
}
Bans::purge();
if ($config['cache']['enabled'])
if ($config['cache']['enabled']) {
$last_time_purged = cache::get('purged_bans_last');
if ($last_time_purged !== false && time() - $last_time_purged > $config['purge_bans']) {
Bans::purge($config['require_ban_view'], $config['purge_bans']);
cache::set('purged_bans_last', time());
}
} else {
// Purge every time.
Bans::purge($config['require_ban_view'], $config['purge_bans']);
}
}
}
function threadLocked($id) {
@ -1612,27 +1617,40 @@ function checkMute() {
}
}
function purge_old_antispam() {
$query = prepare('DELETE FROM ``antispam`` WHERE `expires` < UNIX_TIMESTAMP()');
$query->execute() or error(db_error());
return $query->rowCount();
}
function _create_antibot($board, $thread) {
global $config, $purged_old_antispam;
$antibot = new AntiBot(array($board, $thread));
$antibot = new AntiBot([$board, $thread]);
if (!isset($purged_old_antispam)) {
// Delete old expired antispam, skipping those with NULL expiration timestamps (infinite lifetime).
if (!isset($purged_old_antispam) && $config['auto_maintenance']) {
$purged_old_antispam = true;
query('DELETE FROM ``antispam`` WHERE `expires` < UNIX_TIMESTAMP()') or error(db_error());
}
if ($thread)
// Keep the now invalid timestamps around for a bit to enable users to post if they're still on an old version of
// the HTML page.
// By virtue of existing, we know that we're making a new version of the page, and the user from now on may just reload.
if ($thread) {
$query = prepare('UPDATE ``antispam`` SET `expires` = UNIX_TIMESTAMP() + :expires WHERE `board` = :board AND `thread` = :thread AND `expires` IS NULL');
else
} else {
$query = prepare('UPDATE ``antispam`` SET `expires` = UNIX_TIMESTAMP() + :expires WHERE `board` = :board AND `thread` IS NULL AND `expires` IS NULL');
}
$query->bindValue(':board', $board);
if ($thread)
if ($thread) {
$query->bindValue(':thread', $thread);
}
$query->bindValue(':expires', $config['spam']['hidden_inputs_expire']);
$query->execute() or error(db_error($query));
// Insert an antispam with infinite life as the HTML page of a thread might last well beyond the expiry date.
$query = prepare('INSERT INTO ``antispam`` VALUES (:board, :thread, :hash, UNIX_TIMESTAMP(), NULL, 0)');
$query->bindValue(':board', $board);
$query->bindValue(':thread', $thread);

8
inc/mod/pages.php
View File

@ -897,7 +897,7 @@ function mod_page_ip($cip) {
$args['token'] = make_secure_link_token('ban');
if (hasPermission($config['mod']['view_ban'])) {
$args['bans'] = Bans::find($ip, false, true);
$args['bans'] = Bans::find($ip, false, true, null, $config['auto_maintenance']);
}
if (hasPermission($config['mod']['view_notes'])) {
@ -927,7 +927,7 @@ function mod_edit_ban($ban_id) {
if (!hasPermission($config['mod']['edit_ban']))
error($config['error']['noaccess']);
$args['bans'] = Bans::find(null, false, true, $ban_id);
$args['bans'] = Bans::find(null, false, true, $ban_id, $config['auto_maintenance']);
$args['ban_id'] = $ban_id;
$args['boards'] = listBoards();
$args['current_board'] = isset($args['bans'][0]['board']) ? $args['bans'][0]['board'] : false;
@ -1044,10 +1044,6 @@ function mod_ban_appeals() {
if (!hasPermission($config['mod']['view_ban_appeals']))
error($config['error']['noaccess']);
// Remove stale ban appeals
query("DELETE FROM ``ban_appeals`` WHERE NOT EXISTS (SELECT 1 FROM ``bans`` WHERE `ban_id` = ``bans``.`id`)")
or error(db_error());
if (isset($_POST['appeal_id']) && (isset($_POST['unban']) || isset($_POST['deny']))) {
if (!hasPermission($config['mod']['ban_appeals']))
error($config['error']['noaccess']);

3
install.sql
View File

@ -294,7 +294,8 @@ CREATE TABLE IF NOT EXISTS `ban_appeals` (
`message` text NOT NULL,
`denied` tinyint(1) NOT NULL,
PRIMARY KEY (`id`),
KEY `ban_id` (`ban_id`)
KEY `ban_id` (`ban_id`),
CONSTRAINT `fk_ban_id` FOREIGN KEY (`ban_id`) REFERENCES `bans`(`id`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 AUTO_INCREMENT=1 ;
-- --------------------------------------------------------

2
post.php
View File

@ -1439,7 +1439,7 @@ if (isset($_POST['delete'])) {
$ban_id = (int)$_POST['ban_id'];
$ban = Bans::findSingle($_SERVER['REMOTE_ADDR'], $ban_id, $config['require_ban_view']);
$ban = Bans::findSingle($_SERVER['REMOTE_ADDR'], $ban_id, $config['require_ban_view'], $config['auto_maintenance']);
if (empty($ban)) {
error($config['error']['noban']);

25
tools/maintenance.php Normal file
View File

@ -0,0 +1,25 @@
<?php
/**
* Performs maintenance tasks. Invoke this periodically if the auto_maintenance configuration option is turned off.
*/
require dirname(__FILE__) . '/inc/cli.php';
echo "Clearing expired bans...\n";
$start = microtime(true);
$deleted_count = Bans::purge($config['require_ban_view'], $config['purge_bans']);
$delta = microtime(true) - $start;
echo "Deleted $deleted_count expired bans in $delta seconds!\n";
$time_tot = $delta;
$deleted_tot = $deleted_count;
echo "Clearing old antispam...\n";
$start = microtime(true);
$deleted_count = purge_old_antispam();
$delta = microtime(true) - $start;
echo "Deleted $deleted_count expired antispam in $delta seconds!\n";
$time_tot = $delta;
$deleted_tot = $deleted_count;
$time_tot = number_format((float)$time_tot, 4, '.', '');
modLog("Deleted $deleted_tot expired entries in {$time_tot}s with maintenance tool");