DNM: Metrics T383427

app/Jobs/UpdateWikiDailyMetricJob.php

@@ -6,6 +6,7 @@
 use \App\Metrics\App\WikiMetrics;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Database\DatabaseManager;
 use Illuminate\Foundation\Bus\Dispatchable;
 
 //This job is for the daily measurements of metrics per wikibases.
@@ -18,11 +19,11 @@ class UpdateWikiDailyMetricJob extends Job implements ShouldBeUnique
     /**
      * Execute the job.
      */
-    public function handle(): void
+    public function handle(DatabaseManager $manager): void
     {
         $wikis= Wiki::withTrashed()->get();
         foreach ( $wikis as $wiki ) {
-            (new WikiMetrics())->saveMetrics($wiki);
+            (new WikiMetrics())->saveMetrics($wiki, $manager);
         }
     }
 }

app/Metrics/App/WikiMetrics.php

@@ -4,37 +4,99 @@
 
 use App\Wiki;
 use App\WikiDailyMetrics;
+use App\WikiDb;
+use Illuminate\Database\DatabaseManager;
+use PDO;
 
 class WikiMetrics
 {
-
-    public function saveMetrics(Wiki $wiki): void
+    public function saveMetrics(Wiki $wiki, DatabaseManager $manager): void
     {
+        $dailyActions = null;
+        $weeklyActions = null;
+        $monthlyActions = null;
+        $quarterlyActions = null;
         $today = now()->format('Y-m-d');
         $oldRecord = WikiDailyMetrics::where('wiki_id', $wiki->id)->latest('date')->first();
         $todayPageCount = $wiki->wikiSiteStats()->first()->pages ?? 0;
+        $number_of_actions = $this->getNumberOfActions($wiki, $manager);
+        if (array_key_exists('daily_actions', $number_of_actions)) {
+            $dailyActions = $number_of_actions['daily_actions'];
+            $weeklyActions = $number_of_actions['weekly_actions'];
+            $monthlyActions = $number_of_actions['monthly_actions'];
+            $quarterlyActions = $number_of_actions['quarterly_actions'];
+        }
         $isDeleted = (bool)$wiki->deleted_at;
+
+        // compare current record to old record and only save if there is a change
         if ($oldRecord) {
             if ($oldRecord->is_deleted) {
                 \Log::info("Wiki is deleted, no new record for WikiMetrics ID {$wiki->id}.");
                 return;
             }
             if (!$isDeleted) {
-                if ($oldRecord->pages === $todayPageCount) {
-                    \Log::info("Page count unchanged for WikiMetrics ID {$wiki->id}, no new record added.");
+                if (
+                    $oldRecord->pages === $todayPageCount /*&&
+                    $oldRecord->daily_actions === $dailyActions &&
+                    $oldRecord->weekly_actions === $weeklyActions &&
+                    $oldRecord->monthly_actions === $monthlyActions &&
+                    $oldRecord->quarterly_actions === $quarterlyActions*/
+                    )
+                {
+                    \Log::info("Metrics unchanged for Wiki ID {$wiki->id}, no new record added.");
                     return;
                 }
             }
         }
+
+        //save metrics
         WikiDailyMetrics::create([
             'id' => $wiki->id . '_' . date('Y-m-d'),
             'pages' => $todayPageCount,
             'is_deleted' => $isDeleted,
             'date' => $today,
             'wiki_id' => $wiki->id,
+            'daily_actions'=> $dailyActions,
+            'weekly_actions'=> $weeklyActions,
+            'monthly_actions'=> $monthlyActions,
+            'quarterly_actions'=> $quarterlyActions,
         ]);
-        \Log::info("New metric recorded for WikiMetrics ID {$wiki->id}");
+        \Log::info("New metric recorded for Wiki ID {$wiki->id}");
     }
 
+    protected function getNumberOfActions(Wiki $wiki, DatabaseManager $manager) {
+        $wikiDb = WikiDb::whereWikiId($wiki->id)->first();
+        $result =[];
+        if ($wikiDb) {
+            $wikiTableRcName = "{$wikiDb->name}.{$wikiDb->prefix}_recentchanges";
+            $wikiTableActorName = "{$wikiDb->name}.{$wikiDb->prefix}_actor";
+            $manager->purge('mw');
+            $conn = $manager->connection('mw');
+            if (! $conn instanceof \Illuminate\Database\Connection) {
+                $this->fail(new \RuntimeException('Must be run on a PDO based DB connection'));
+
+                return; //safegaurd
+            }
+            $pdo = $conn->getPdo();
+            $pdo->exec("USE {$wikiDb->name}");
+            $result = $pdo->query("SELECT
+    SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 1 DAY), '%Y%m%d%H%i%S')) AS daily_actions,
+    SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 1 WEEK), '%Y%m%d%H%i%S')) AS weekly_actions,
+    SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 1 MONTH), '%Y%m%d%H%i%S')) AS monthly_actions,
+    SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), INTERVAL 3 MONTH), '%Y%m%d%H%i%S')) AS quarterly_actions
+FROM
+    {$wikiTableRcName} rc
+INNER JOIN {$wikiTableActorName} a ON rc.rc_actor = a.actor_id
+WHERE a.actor_name <> 'PlatformReservedUser'
+/*
+Conditions below added for consistency with Wikidata:
+https://phabricator.wikimedia.org/diffusion/ADES/browse/master/src/wikidata/site_stats/sql/active_user_changes.sql
+*/
+    AND a.actor_user != 0
+    AND rc.rc_bot = 0
+    AND ( rc.rc_log_type != 'newusers' OR rc.rc_log_type IS NULL)")->fetch(PDO::FETCH_ASSOC);
+        }
+        return $result;
+    }
 }
 

database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php

@@ -0,0 +1,30 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->integer('daily_actions')->nullable();
+            $table->integer('weekly_actions')->nullable();
+            $table->integer('monthly_actions')->nullable();
+            $table->integer('quarterly_actions')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('wiki_daily_metric', function (Blueprint $table) {
+        });
+    }
+};

tests/Jobs/UpdateWikiDailyMetricJobTest.php

@@ -2,10 +2,13 @@
 
 namespace Tests\Jobs;
 
+use App\Jobs\ProvisionWikiDbJob;
 use App\Jobs\UpdateWikiDailyMetricJob;
 use App\Wiki;
+use App\WikiDb;
 use Carbon\Carbon;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Queue;
 use Tests\TestCase;
 
@@ -33,9 +36,19 @@ public function testRunJobForAllWikisIncludingDeletedWikis()
         $deletedWiki = Wiki::factory()->create([
             'domain' => 'deletedwiki.wikibase.cloud',
         ]);
+        $manager = $this->app->make('db');
+        $job = new ProvisionWikiDbJob(null, 'deletedwikidb', 1);
+        $job2 = new ProvisionWikiDbJob(null, 'activewikidb', 1);
+        $job->handle($manager);
+        $job2->handle($manager);
+        DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('name');
+        DB::table('wiki_dbs')->where(["wiki_id" => null])->update(['wiki_id' => $activeWiki->id]);
+        DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('name');
+        DB::table('wiki_dbs')->where(["wiki_id" => null])->update(['wiki_id' => $deletedWiki->id]);
+
         $deletedWiki->delete();
 
-        (new UpdateWikiDailyMetricJob())->handle();
+        (new UpdateWikiDailyMetricJob())->handle($manager);
 
         $this->assertDatabaseHas('wiki_daily_metrics', [
             'wiki_id' => $activeWiki->id,

tests/Metrics/WikiMetricsTest.php

@@ -2,25 +2,34 @@
 
 namespace Tests\Metrics;
 
+use App\Jobs\ProvisionWikiDbJob;
 use App\Metrics\App\WikiMetrics;
 use App\Wiki;
 use App\WikiDailyMetrics;
+use App\WikiDb;
 use Carbon\Carbon;
+use Illuminate\Database\DatabaseManager;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\DB;
 use Tests\TestCase;
 
 class WikiMetricsTest extends TestCase
-{
+{ // Todo: Test that the added metrics are saved in the right frequency and the right values.
     use RefreshDatabase;
 
-
+    protected function tearDown(): void {
+        Wiki::query()->delete();
+        WikiDb::query()->delete();
+        parent::tearDown();
+    }
     public function testSuccessfullyAddRecords()
     {
         $wiki = Wiki::factory()->create([
             'domain' => 'thisfake.wikibase.cloud'
         ]);
-
-        (new WikiMetrics())->saveMetrics($wiki);
+        $manager = $this->app->make('db');
+        $this->creatWikiDb($wiki, $manager);
+        (new WikiMetrics())->saveMetrics($wiki, $manager);
         // Assert the metric is updated in the database
         $this->assertDatabaseHas('wiki_daily_metrics', [
             'date' => now()->toDateString()
@@ -33,6 +42,8 @@ public function testDoesNotAddDuplicateRecordsWithOnlyDateChange()
         $wiki = Wiki::factory()->create([
             'domain' => 'thisfake.wikibase.cloud'
         ]);
+        $manager = $this->app->make('db');
+        $this->creatWikiDb($wiki, $manager);
         //Insert an old metric value for a wiki
         WikiDailyMetrics::create([
             'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(),
@@ -41,7 +52,7 @@ public function testDoesNotAddDuplicateRecordsWithOnlyDateChange()
             'pages' => 0,
             'is_deleted' => 0
         ]);
-        (new WikiMetrics())->saveMetrics($wiki);
+        (new WikiMetrics())->saveMetrics($wiki, $manager);
 
         //Assert No new record was created for today
         $this->assertDatabaseMissing('wiki_daily_metrics', [
@@ -55,6 +66,8 @@ public function testAddRecordsWikiIsDeleted()
         $wiki = Wiki::factory()->create([
             'domain' => 'thisfake.wikibase.cloud'
         ]);
+        $manager = $this->app->make('db');
+        $this->creatWikiDb($wiki, $manager);
         //Insert an old metric value for a wiki
         WikiDailyMetrics::create([
             'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(),
@@ -67,7 +80,7 @@ public function testAddRecordsWikiIsDeleted()
         $wiki->delete();
         $wiki->save();
 
-        (new WikiMetrics())->saveMetrics($wiki);
+        (new WikiMetrics())->saveMetrics($wiki, $manager);
 
         //Assert No new record was created for today
         $this->assertDatabaseMissing('wiki_daily_metrics', [
@@ -76,5 +89,26 @@ public function testAddRecordsWikiIsDeleted()
             'date' => now()->toDateString()
         ]);
     }
+
+    private function creatWikiDb(Wiki $wiki, DatabaseManager $manager){
+        $job = new ProvisionWikiDbJob(null, null, 1);
+        $job->handle($manager);
+        $db = DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('name');
+        $db_prefix = DB::table('wiki_dbs')->where(["wiki_id" => null])->limit(1)->value('prefix');
+        DB::table('wiki_dbs')->where(["wiki_id" => null])->update(['wiki_id' => $wiki->id]);
+        $manager->purge('mw');
+        $conn = $manager->connection('mw');
+        if (! $conn instanceof \Illuminate\Database\Connection) {
+            $this->fail(new \RuntimeException('Must be run on a PDO based DB connection'));
+
+            return; //safegaurd
+        }
+        $pdo = $conn->getPdo();
+        $pdo->exec("USE {$db}");
+        $date = Carbon::yesterday()->setTime(22, 0)->toDateString();
+        $table = "{$db}.{$db_prefix}_recentchanges";
+        $sql = "INSERT INTO {$table} (rc_timestamp, rc_actor, rc_comment_id) VALUES (:timestamp, '1', '2')";
+        $pdo->prepare($sql)->execute([':timestamp' => $date]);
+    }
 }