diff --git a/app/Metrics/App/WikiMetrics.php b/app/Metrics/App/WikiMetrics.php index ffe93c37..8e1a85e3 100644 --- a/app/Metrics/App/WikiMetrics.php +++ b/app/Metrics/App/WikiMetrics.php @@ -4,30 +4,49 @@ use App\Wiki; use App\WikiDailyMetrics; +use Illuminate\Support\Arr; class WikiMetrics { + const INTERVAL_DAILY = 'INTERVAL 1 DAY'; + const INTERVAL_WEEKLY = ' INTERVAL 1 WEEK'; + const INTERVAL_MONTHLY = 'INTERVAL 1 MONTH'; + const INTERVAL_QUARTERLY = 'INTERVAL 3 MONTH'; + + protected $wiki; + public function saveMetrics(Wiki $wiki): void { + $this->wiki = $wiki; + $today = now()->format('Y-m-d'); $oldRecord = WikiDailyMetrics::where('wiki_id', $wiki->id)->latest('date')->first(); $todayPageCount = $wiki->wikiSiteStats()->first()->pages ?? 0; $isDeleted = (bool)$wiki->deleted_at; + $dailyActions = $this->getNumberOfActions(self::INTERVAL_DAILY); + $weeklyActions = $this->getNumberOfActions(self::INTERVAL_WEEKLY); + $monthlyActions = $this->getNumberOfActions(self::INTERVAL_MONTHLY); + $quarterlyActions = $this->getNumberOfActions(self::INTERVAL_QUARTERLY); + $dailyMetrics = new WikiDailyMetrics([ 'id' => $wiki->id . '_' . date('Y-m-d'), 'pages' => $todayPageCount, 'is_deleted' => $isDeleted, 'date' => $today, - 'wiki_id' => $wiki->id + 'wiki_id' => $wiki->id, + 'daily_actions'=> $dailyActions, + 'weekly_actions'=> $weeklyActions, + 'monthly_actions'=> $monthlyActions, + 'quarterly_actions'=> $quarterlyActions, ]); - + + // compare current record to old record and only save if there is a change if ($oldRecord) { if ($oldRecord->is_deleted) { \Log::info("Wiki is deleted, no new record for WikiMetrics ID {$wiki->id}."); return; } - if (!$isDeleted) { if ($oldRecord->areMetricsEqual($dailyMetrics)) { \Log::info("Record unchanged for WikiMetrics ID {$wiki->id}, no new record added."); @@ -38,6 +57,48 @@ public function saveMetrics(Wiki $wiki): void $dailyMetrics->save(); - \Log::info("New metric recorded for WikiMetrics ID {$wiki->id}"); + \Log::info("New metric recorded for Wiki ID {$wiki->id}"); + } + + protected function getNumberOfActions(string $interval): null|int + { + $actions = null; + + // safeguard + if (false === in_array($interval, + [ + self::INTERVAL_DAILY, + self::INTERVAL_WEEKLY, + self::INTERVAL_MONTHLY, + self::INTERVAL_QUARTERLY + ] + )) { return null; } + + $wikiDb = $this->wiki->wikiDb; + $tableRecentChanges = $wikiDb->name . '.' . $wikiDb->prefix . '_recentchanges'; + $tableActor = $wikiDb->name . '.' . $wikiDb->prefix . '_actor'; + + $query = "SELECT + SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), $interval), '%Y%m%d%H%i%S')) AS sum_actions + FROM + $tableRecentChanges AS rc + INNER JOIN $tableActor AS a ON rc.rc_actor = a.actor_id + WHERE + /* + Conditions below added for consistency with Wikidata: https://phabricator.wikimedia.org/diffusion/ADES/browse/master/src/wikidata/site_stats/sql/active_user_changes.sql + */ + a.actor_user != 0 + AND rc.rc_bot = 0 + AND ( rc.rc_log_type != 'newusers' OR rc.rc_log_type IS NULL)"; + + $manager = app()->db; + $manager->purge('mw'); + $conn = $manager->connection('mw'); + $pdo = $conn->getPdo(); + $result = $pdo->query($query)->fetch(); + + $actions = Arr::get($result, 'sum_actions', null); + + return $actions; } } diff --git a/app/WikiDailyMetrics.php b/app/WikiDailyMetrics.php index db00709f..d6f844d4 100644 --- a/app/WikiDailyMetrics.php +++ b/app/WikiDailyMetrics.php @@ -23,12 +23,20 @@ class WikiDailyMetrics extends Model 'date', 'pages', 'is_deleted', + 'daily_actions', + 'weekly_actions', + 'monthly_actions', + 'quarterly_actions', ]; // list of properties which are actual wiki metrics public static $metricNames = [ 'pages', 'is_deleted', + 'daily_actions', + 'weekly_actions', + 'monthly_actions', + 'quarterly_actions' ]; public function areMetricsEqual(WikiDailyMetrics $wikiDailyMetrics): bool diff --git a/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php b/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php new file mode 100644 index 00000000..332733e5 --- /dev/null +++ b/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php @@ -0,0 +1,30 @@ +integer('daily_actions')->nullable(); + $table->integer('weekly_actions')->nullable(); + $table->integer('monthly_actions')->nullable(); + $table->integer('quarterly_actions')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('wiki_daily_metric', function (Blueprint $table) { + }); + } +}; diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php index d7a334c5..90123c2c 100644 --- a/tests/Jobs/UpdateWikiDailyMetricJobTest.php +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -4,10 +4,12 @@ use App\Jobs\UpdateWikiDailyMetricJob; use App\Wiki; +use App\WikiDb; use Carbon\Carbon; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; use Tests\TestCase; +use App\Jobs\ProvisionWikiDbJob; class UpdateWikiDailyMetricJobTest extends TestCase { @@ -33,18 +35,41 @@ public function testRunJobForAllWikisIncludingDeletedWikis() $deletedWiki = Wiki::factory()->create([ 'domain' => 'deletedwiki.wikibase.cloud', ]); + + $manager = $this->app->make('db'); + $job = new ProvisionWikiDbJob(); + $job2 = new ProvisionWikiDbJob(); + $job->handle($manager); + $job2->handle($manager); + + $wikiDbActive = WikiDb::whereDoesntHave('wiki')->first(); + $wikiDbActive->update( ['wiki_id' => $activeWiki->id] ); + + $wikiDbDeleted = WikiDb::whereDoesntHave('wiki')->first(); + $wikiDbDeleted->update( ['wiki_id' => $deletedWiki->id] ); + $deletedWiki->delete(); + + (new UpdateWikiDailyMetricJob())->handle(); $this->assertDatabaseHas('wiki_daily_metrics', [ 'wiki_id' => $activeWiki->id, - 'date' => Carbon::today()->toDateString() + 'date' => Carbon::today()->toDateString(), + 'daily_actions' => null, + 'weekly_actions' => null, + 'monthly_actions' => null, + 'quarterly_actions' => null, ]); $this->assertDatabaseHas('wiki_daily_metrics', [ 'wiki_id' => $deletedWiki->id, - 'date' => Carbon::today()->toDateString() + 'date' => Carbon::today()->toDateString(), + 'daily_actions' => null, + 'weekly_actions' => null, + 'monthly_actions' => null, + 'quarterly_actions' => null, ]); } diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index a38f8177..04521615 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -4,7 +4,9 @@ use App\Metrics\App\WikiMetrics; use App\Wiki; +use App\WikiDb; use App\WikiDailyMetrics; +use App\Jobs\ProvisionWikiDbJob; use Carbon\Carbon; use Illuminate\Foundation\Testing\RefreshDatabase; use Tests\TestCase; @@ -13,6 +15,12 @@ class WikiMetricsTest extends TestCase { use RefreshDatabase; + public function setUp(): void { + parent::setUp(); + $manager = $this->app->make('db'); + $job = new ProvisionWikiDbJob(); + $job->handle($manager); + } public function testSuccessfullyAddRecords() { @@ -20,6 +28,9 @@ public function testSuccessfullyAddRecords() 'domain' => 'thisfake.wikibase.cloud' ]); + $wikiDb = WikiDb::first(); + $wikiDb->update( ['wiki_id' => $wiki->id] ); + (new WikiMetrics())->saveMetrics($wiki); // Assert the metric is updated in the database $this->assertDatabaseHas('wiki_daily_metrics', [ @@ -33,6 +44,10 @@ public function testDoesNotAddDuplicateRecordsWithOnlyDateChange() $wiki = Wiki::factory()->create([ 'domain' => 'thisfake.wikibase.cloud' ]); + + $wikiDb = WikiDb::first(); + $wikiDb->update( ['wiki_id' => $wiki->id] ); + //Insert an old metric value for a wiki WikiDailyMetrics::create([ 'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(), @@ -55,6 +70,10 @@ public function testAddRecordsWikiIsDeleted() $wiki = Wiki::factory()->create([ 'domain' => 'thisfake.wikibase.cloud' ]); + + $wikiDb = WikiDb::first(); + $wikiDb->update( ['wiki_id' => $wiki->id] ); + //Insert an old metric value for a wiki WikiDailyMetrics::create([ 'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(),