From 316e681bbd16d19226bc3574c9cf306e35dab802 Mon Sep 17 00:00:00 2001 From: 8n-tech <8n-tech@users.noreply.github.com> Date: Tue, 14 Apr 2015 05:36:38 +1000 Subject: [PATCH] New CLI tool for upgrade: tools/migrate_board_stats.php - Adds new coumn to `boards` for easy-access post total storage. - Migrates the AUTO_INCREMENT number to the `posts_total` column for every board. This ensures accurate post count measurement. - Adds `board_stats` table for recording histric data. - Retroactively records all boards into `board_stats`. - inc/functions.php Added handling for `posts_total` column in `boards`. - inc/functions.php Removed some bogus data from fetchBoardActivity - board-search.php Fixed issue with SFW filtering. - board-search.php Now accurately sends `posts_total` data for board sum posts. - boards-search.html Various form fixes so search now submits correctly. - boards-table.html Fixed `posts_total` value. Signed-off-by: 8n-tech <8n-tech@users.noreply.github.com> --- board-search.php | 41 ++++++---- inc/functions.php | 55 +++++-------- templates/8chan/boards-search.html | 18 ++--- templates/8chan/boards-table.html | 2 +- tools/migrate_board_stats.php | 125 +++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+), 62 deletions(-) create mode 100644 tools/migrate_board_stats.php diff --git a/board-search.php b/board-search.php index 044aaad7..5f172036 100644 --- a/board-search.php +++ b/board-search.php @@ -23,26 +23,30 @@ $languages = array( /* Determine search parameters from $_GET */ $search = array( - 'lang' => false, - 'nsfw' => true, - 'tags' => false, - 'titles' => false, + 'lang' => false, + 'nsfw' => true, + 'tags' => false, + 'title' => false, ); // Include NSFW boards? -if (isset( $_GET['nsfw'] )) { - $search['nsfw'] = (bool) $_GET['nsfw']; +if (isset( $_GET['sfw'] ) && $_GET['sfw'] != "") { + $search['nsfw'] = !$_GET['sfw']; } // Include what language (if the language is not blank and we recognize it)? -if (isset( $_GET['lang'] ) && isset($languages[$search['lang']])) { +if (isset( $_GET['lang'] ) && $_GET['lang'] != "" && isset($languages[$search['lang']])) { $search['lang'] = $_GET['lang']; } // Include what tag? -if (isset( $_GET['tags'] )) { +if (isset( $_GET['tags'] ) && $_GET['tags'] != "") { $search['tags'] = $_GET['tags']; } +// Include what in the uri / title / subtitle? +if (isset( $_GET['title'] ) && $_GET['title'] != "") { + $search['title'] = $_GET['title']; +} /* Search boards */ $boards = listBoards(); @@ -108,7 +112,6 @@ $response['tags'] = array(); // We will also be weighing and building a tag list. foreach ($response['boards'] as $boardUri => &$board) { $board['active'] = (int) $boardActivity['active'][ $boardUri ]; - $board['posts'] = (int) $boardActivity['posts'][ $boardUri ]; $board['pph'] = (int) $boardActivity['average'][ $boardUri ]; if (isset($board['tags']) && count($board['tags']) > 0) { @@ -124,13 +127,19 @@ foreach ($response['boards'] as $boardUri => &$board) { } // Sort boards by their popularity, then by their total posts. -$boardActivityValues = array(); +$boardActivityValues = array(); +$boardTotalPostsValues = array(); -foreach ($response['boards'] as $boardUri => $board) { - $boardActivityValues[$boardUri] = "{$board['active']}.{$board['posts']}"; +foreach ($response['boards'] as $boardUri => &$board) { + $boardActivityValues[$boardUri] = (int) $board['active']; + $boardTotalPostsValues[$boardUri] = (int) $board['posts_total']; } -array_multisort($boardActivityValues, SORT_DESC, $response['boards']); +array_multisort( + $boardActivityValues, SORT_DESC, SORT_NUMERIC, // Sort by number of active posters + $boardTotalPostsValues, SORT_DESC, SORT_NUMERIC, // Then, sort by total number of posts + $response['boards'] +); // Get the top most popular tags. if (count($response['tags']) > 0) { @@ -146,7 +155,7 @@ if (count($response['tags']) > 0) { /* (Please) Respond */ if (!$Included) { $json = json_encode( $response ); - + // Error Handling switch (json_last_error()) { case JSON_ERROR_NONE: @@ -171,11 +180,11 @@ if (!$Included) { $jsonError = 'Unknown error'; break; } - + if ($jsonError) { $json = "{\"error\":\"{$jsonError}\"}"; } - + // Successful output echo $json; } diff --git a/inc/functions.php b/inc/functions.php index cb2dc364..7b572dfc 100755 --- a/inc/functions.php +++ b/inc/functions.php @@ -786,9 +786,23 @@ function listBoards($just_uri = false, $indexed_only = false) { return $boards; if (!$just_uri) { - $query = query("SELECT ``boards``.`uri` uri, ``boards``.`title` title, ``boards``.`subtitle` subtitle, ``board_create``.`time` time, ``boards``.`indexed` indexed, ``boards``.`sfw` sfw FROM ``boards``" . ( $indexed_only ? " WHERE `indexed` = 1 " : "" ) . "LEFT JOIN ``board_create`` ON ``boards``.`uri` = ``board_create``.`uri` ORDER BY ``boards``.`uri`") or error(db_error()); + $query = query( + "SELECT + ``boards``.`uri` uri, + ``boards``.`title` title, + ``boards``.`subtitle` subtitle, + ``board_create``.`time` time, + ``boards``.`indexed` indexed, + ``boards``.`sfw` sfw, + ``boards``.`posts_total` posts_total + FROM ``boards``" . ( $indexed_only ? " WHERE `indexed` = 1 " : "" ) . + "LEFT JOIN ``board_create`` + ON ``boards``.`uri` = ``board_create``.`uri` + ORDER BY ``boards``.`uri`") or error(db_error()); + $boards = $query->fetchAll(PDO::FETCH_ASSOC); - } else { + } + else { $boards = array(); $query = query("SELECT `uri` FROM ``boards``" . ( $indexed_only ? " WHERE `indexed` = 1" : "" ) . " ORDER BY ``boards``.`uri`") or error(db_error()); while (true) { @@ -823,45 +837,14 @@ function fetchBoardActivity( $uris ) { global $config; $boardActivity = array(); + //$uris = "\"" . implode( (array) $uris, "\",\"" ) . "\""; - $tablePrefix = "{$config['db']['prefix']}posts_"; - $uris = "\"{$tablePrefix}" . implode( (array) $uris, "\",\"{$tablePrefix}" ) . "\""; - - /* - $tagQuery = prepare("SELECT * FROM ``board_tags`` WHERE `uri` IN ({$uris})"); - $tagQuery->execute() or error(db_error($tagQuery)); - $tagResult = $tagQuery->fetchAll(PDO::FETCH_ASSOC); - - if ($tagResult) { - foreach ($tagResult as $tagRow) { - $tag = $tagRow['tag']; - $tag = trim($tag); - $tag = strtolower($tag); - $tag = str_replace(['_', ' '], '-', $tag); - - if (!isset($boardTags[ $tagRow['uri'] ])) { - $boardTags[ $tagRow['uri'] ] = array(); - } - - $boardTags[ $tagRow['uri'] ][] = htmlentities( utf8_encode( $tag ) ); - } - } - */ - - $aiQuery = prepare("SELECT `TABLE_NAME`, `AUTO_INCREMENT` FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = \"{$config['db']['database']}\" AND TABLE_NAME IN ({$uris})"); - $aiQuery->execute() or error(db_error($aiQuery)); - $aiResult = $aiQuery->fetchAll(PDO::FETCH_ASSOC); - - foreach ($aiResult as $aiRow) { - $uri = str_replace( $tablePrefix, "", $aiRow['TABLE_NAME'] ); - $posts = $aiRow['AUTO_INCREMENT'] - 1; - - $random = rand( -1000, 1000 ); + foreach ($uris as $uri) { + $random = 0;//rand( -1000, 1000 ); if( $random < 0 ) $random = 0; $boardActivity['active'][ $uri ] = $random; $boardActivity['average'][ $uri ] = ($random * 72) / 72; - $boardActivity['posts'][ $uri ] = $posts; } return $boardActivity; diff --git a/templates/8chan/boards-search.html b/templates/8chan/boards-search.html index 1aff831c..e7519dd0 100644 --- a/templates/8chan/boards-search.html +++ b/templates/8chan/boards-search.html @@ -1,40 +1,40 @@

Global Statistics

-

{% trans %}There are currently {{boards_public}} public boards, {{boards_total}} total. Site-wide, {{posts_hour}} posts have been made in the last hour, with {{posts_total}} being made on all active boards since {{founding_date}}.{% endtrans %}

+

{% trans %}There are currently {{boards_public}} public boards, {{boards_total}} total. Site-wide, {{posts_hour}} posts have been made in the last hour, with {{posts_total}} being made on all active boards since {{founding_date}}.{% endtrans %}

{% if uptime %}

{{uptime}} without interruption

{% endif %}

This page last updated {{page_updated}}.