diff --git a/app/Jobs/UpdateWikiDailyMetricJob.php b/app/Jobs/UpdateWikiDailyMetricJob.php index fe8b8c5b..bc7c5f58 100755 --- a/app/Jobs/UpdateWikiDailyMetricJob.php +++ b/app/Jobs/UpdateWikiDailyMetricJob.php @@ -6,6 +6,7 @@ use \App\Metrics\App\WikiMetrics; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Database\DatabaseManager; use Illuminate\Foundation\Bus\Dispatchable; //This job is for the daily measurements of metrics per wikibases. @@ -18,11 +19,11 @@ class UpdateWikiDailyMetricJob extends Job implements ShouldBeUnique /** * Execute the job. */ - public function handle(): void + public function handle(DatabaseManager $manager): void { $wikis= Wiki::withTrashed()->get(); foreach ( $wikis as $wiki ) { - (new WikiMetrics())->saveMetrics($wiki); + (new WikiMetrics())->saveMetrics($wiki, $manager); } } } diff --git a/app/Metrics/App/WikiMetrics.php b/app/Metrics/App/WikiMetrics.php index e66c3c88..c3fd612d 100644 --- a/app/Metrics/App/WikiMetrics.php +++ b/app/Metrics/App/WikiMetrics.php @@ -4,37 +4,99 @@ use App\Wiki; use App\WikiDailyMetrics; +use App\WikiDb; +use Illuminate\Database\DatabaseManager; +use PDO; class WikiMetrics { - - public function saveMetrics(Wiki $wiki): void + public function saveMetrics(Wiki $wiki, DatabaseManager $manager): void { + $dailyActions = null; + $weeklyActions = null; + $monthlyActions = null; + $quarterlyActions = null; $today = now()->format('Y-m-d'); $oldRecord = WikiDailyMetrics::where('wiki_id', $wiki->id)->latest('date')->first(); $todayPageCount = $wiki->wikiSiteStats()->first()->pages ?? 0; + $number_of_actions = $this->getNumberOfActions($wiki, $manager); + if (array_key_exists('daily_actions', $number_of_actions)) { + $dailyActions = $number_of_actions['daily_actions']; + $weeklyActions = $number_of_actions['weekly_actions']; + $monthlyActions = $number_of_actions['monthly_actions']; + $quarterlyActions = $number_of_actions['quarterly_actions']; + } $isDeleted = (bool)$wiki->deleted_at; + + // compare current record to old record and only save if there is a change if ($oldRecord) { if ($oldRecord->is_deleted) { \Log::info("Wiki is deleted, no new record for WikiMetrics ID {$wiki->id}."); return; } if (!$isDeleted) { - if ($oldRecord->pages === $todayPageCount) { - \Log::info("Page count unchanged for WikiMetrics ID {$wiki->id}, no new record added."); + if ( + $oldRecord->pages === $todayPageCount /*&& + $oldRecord->daily_actions === $dailyActions && + $oldRecord->weekly_actions === $weeklyActions && + $oldRecord->monthly_actions === $monthlyActions && + $oldRecord->quarterly_actions === $quarterlyActions*/ + ) + { + \Log::info("Metrics unchanged for Wiki ID {$wiki->id}, no new record added."); return; } } } + + //save metrics WikiDailyMetrics::create([ 'id' => $wiki->id . '_' . date('Y-m-d'), 'pages' => $todayPageCount, 'is_deleted' => $isDeleted, 'date' => $today, 'wiki_id' => $wiki->id, + 'daily_actions'=> $dailyActions, + 'weekly_actions'=> $weeklyActions, + 'monthly_actions'=> $monthlyActions, + 'quarterly_actions'=> $quarterlyActions, ]); - \Log::info("New metric recorded for WikiMetrics ID {$wiki->id}"); + \Log::info("New metric recorded for Wiki ID {$wiki->id}"); } + protected function getNumberOfActions(Wiki $wiki, DatabaseManager $manager) { + $wikiDb = WikiDb::whereWikiId($wiki->id)->first(); + $result =[]; + if ($wikiDb) { + $wikiTableRcName = "{$wikiDb->name}.{$wikiDb->prefix}_recentchanges"; + $wikiTableActorName = "{$wikiDb->name}.{$wikiDb->prefix}_actor"; + $manager->purge('mw'); + $conn = $manager->connection('mw'); + if (! $conn instanceof \Illuminate\Database\Connection) { + $this->fail(new \RuntimeException('Must be run on a PDO based DB connection')); + + return; //safegaurd + } + $pdo = $conn->getPdo(); + $pdo->exec("USE {$wikiDb->name}"); + $result = $pdo->query("SELECT + SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 1 DAY), '%Y%m%d%H%i%S')) AS daily_actions, + SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 1 WEEK), '%Y%m%d%H%i%S')) AS weekly_actions, + SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 1 MONTH), '%Y%m%d%H%i%S')) AS monthly_actions, + SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 3 MONTH), '%Y%m%d%H%i%S')) AS quarterly_actions +FROM + {$wikiTableRcName} rc +INNER JOIN {$wikiTableActorName} a ON rc.rc_actor = a.actor_id +WHERE a.actor_name <> 'PlatformReservedUser' +/* +Conditions below added for consistency with Wikidata: +https://phabricator.wikimedia.org/diffusion/ADES/browse/master/src/wikidata/site_stats/sql/active_user_changes.sql +*/ + AND a.actor_user != 0 + AND rc.rc_bot = 0 + AND ( rc.rc_log_type != 'newusers' OR rc.rc_log_type IS NULL)")->fetch(PDO::FETCH_ASSOC); + } + return $result; + } } diff --git a/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php b/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php new file mode 100644 index 00000000..332733e5 --- /dev/null +++ b/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php @@ -0,0 +1,30 @@ +integer('daily_actions')->nullable(); + $table->integer('weekly_actions')->nullable(); + $table->integer('monthly_actions')->nullable(); + $table->integer('quarterly_actions')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metric', function (Blueprint $table) { + }); + } +}; diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php index d7a334c5..ac3c26f9 100644 --- a/tests/Jobs/UpdateWikiDailyMetricJobTest.php +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -2,10 +2,13 @@ namespace Tests\Jobs; +use App\Jobs\ProvisionWikiDbJob; use App\Jobs\UpdateWikiDailyMetricJob; use App\Wiki; +use App\WikiDb; use Carbon\Carbon; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; use Tests\TestCase; @@ -33,9 +36,19 @@ public function testRunJobForAllWikisIncludingDeletedWikis() $deletedWiki = Wiki::factory()->create([ 'domain' => 'deletedwiki.wikibase.cloud', ]); + $manager = $this->app->make('db'); + $job = new ProvisionWikiDbJob(null, 'deletedwikidb', 1); + $job2 = new ProvisionWikiDbJob(null, 'activewikidb', 1); + $job->handle($manager); + $job2->handle($manager); + DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('name'); + DB::table('wiki_dbs')->where(["wiki_id" => null])->update(['wiki_id' => $activeWiki->id]); + DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('name'); + DB::table('wiki_dbs')->where(["wiki_id" => null])->update(['wiki_id' => $deletedWiki->id]); + $deletedWiki->delete(); - (new UpdateWikiDailyMetricJob())->handle(); + (new UpdateWikiDailyMetricJob())->handle($manager); $this->assertDatabaseHas('wiki_daily_metrics', [ 'wiki_id' => $activeWiki->id, diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index a38f8177..82034c13 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -2,25 +2,34 @@ namespace Tests\Metrics; +use App\Jobs\ProvisionWikiDbJob; use App\Metrics\App\WikiMetrics; use App\Wiki; use App\WikiDailyMetrics; +use App\WikiDb; use Carbon\Carbon; +use Illuminate\Database\DatabaseManager; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Support\Facades\DB; use Tests\TestCase; class WikiMetricsTest extends TestCase -{ +{ // Todo: Test that the added metrics are saved in the right frequency and the right values. use RefreshDatabase; - + protected function tearDown(): void { + Wiki::query()->delete(); + WikiDb::query()->delete(); + parent::tearDown(); + } public function testSuccessfullyAddRecords() { $wiki = Wiki::factory()->create([ 'domain' => 'thisfake.wikibase.cloud' ]); - - (new WikiMetrics())->saveMetrics($wiki); + $manager = $this->app->make('db'); + $this->creatWikiDb($wiki, $manager); + (new WikiMetrics())->saveMetrics($wiki, $manager); // Assert the metric is updated in the database $this->assertDatabaseHas('wiki_daily_metrics', [ 'date' => now()->toDateString() @@ -33,6 +42,8 @@ public function testDoesNotAddDuplicateRecordsWithOnlyDateChange() $wiki = Wiki::factory()->create([ 'domain' => 'thisfake.wikibase.cloud' ]); + $manager = $this->app->make('db'); + $this->creatWikiDb($wiki, $manager); //Insert an old metric value for a wiki WikiDailyMetrics::create([ 'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(), @@ -41,7 +52,7 @@ public function testDoesNotAddDuplicateRecordsWithOnlyDateChange() 'pages' => 0, 'is_deleted' => 0 ]); - (new WikiMetrics())->saveMetrics($wiki); + (new WikiMetrics())->saveMetrics($wiki, $manager); //Assert No new record was created for today $this->assertDatabaseMissing('wiki_daily_metrics', [ @@ -55,6 +66,8 @@ public function testAddRecordsWikiIsDeleted() $wiki = Wiki::factory()->create([ 'domain' => 'thisfake.wikibase.cloud' ]); + $manager = $this->app->make('db'); + $this->creatWikiDb($wiki, $manager); //Insert an old metric value for a wiki WikiDailyMetrics::create([ 'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(), @@ -67,7 +80,7 @@ public function testAddRecordsWikiIsDeleted() $wiki->delete(); $wiki->save(); - (new WikiMetrics())->saveMetrics($wiki); + (new WikiMetrics())->saveMetrics($wiki, $manager); //Assert No new record was created for today $this->assertDatabaseMissing('wiki_daily_metrics', [ @@ -76,5 +89,26 @@ public function testAddRecordsWikiIsDeleted() 'date' => now()->toDateString() ]); } + + private function creatWikiDb(Wiki $wiki, DatabaseManager $manager){ + $job = new ProvisionWikiDbJob(null, null, 1); + $job->handle($manager); + $db = DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('name'); + $db_prefix = DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('prefix'); + DB::table('wiki_dbs')->where(["wiki_id" => null])->update(['wiki_id' => $wiki->id]); + $manager->purge('mw'); + $conn = $manager->connection('mw'); + if (! $conn instanceof \Illuminate\Database\Connection) { + $this->fail(new \RuntimeException('Must be run on a PDO based DB connection')); + + return; //safegaurd + } + $pdo = $conn->getPdo(); + $pdo->exec("USE {$db}"); + $date = Carbon::yesterday()->setTime(22, 0)->toDateString(); + $table = "{$db}.{$db_prefix}_recentchanges"; + $sql = "INSERT INTO {$table} (rc_timestamp, rc_actor, rc_comment_id) VALUES (:timestamp, '1', '2')"; + $pdo->prepare($sql)->execute([':timestamp' => $date]); + } }