collect action numbers metrics (#918)

deer-wmde · web-flow · commit 02efa28c250e · 2025-04-25T17:52:14.000+02:00
* add areMetricsEqual method

* use areMetricsEqual method

* add WikiDailyMetricsTest

* fix

* add db migration

* rebase de/dailyMetricsCompare

* working query

* query

* query mediawiki db

* typoe hinting

* more tyoe hinting

* improve test

* fix tests
diff --git a/app/Metrics/App/WikiMetrics.php b/app/Metrics/App/WikiMetrics.php
@@ -4,30 +4,49 @@
 
 use App\Wiki;
 use App\WikiDailyMetrics;
+use Illuminate\Support\Arr;
 
 class WikiMetrics
 {
+    const INTERVAL_DAILY = 'INTERVAL 1 DAY';
+    const INTERVAL_WEEKLY = ' INTERVAL 1 WEEK';
+    const INTERVAL_MONTHLY = 'INTERVAL 1 MONTH';
+    const INTERVAL_QUARTERLY = 'INTERVAL 3 MONTH';
+
+    protected $wiki;
+
     public function saveMetrics(Wiki $wiki): void
     {
+        $this->wiki = $wiki;
+
         $today = now()->format('Y-m-d');
         $oldRecord = WikiDailyMetrics::where('wiki_id', $wiki->id)->latest('date')->first();
         $todayPageCount = $wiki->wikiSiteStats()->first()->pages ?? 0;
         $isDeleted = (bool)$wiki->deleted_at;
 
+        $dailyActions = $this->getNumberOfActions(self::INTERVAL_DAILY);
+        $weeklyActions = $this->getNumberOfActions(self::INTERVAL_WEEKLY);
+        $monthlyActions = $this->getNumberOfActions(self::INTERVAL_MONTHLY);
+        $quarterlyActions = $this->getNumberOfActions(self::INTERVAL_QUARTERLY);
+
         $dailyMetrics = new WikiDailyMetrics([
             'id' => $wiki->id . '_' . date('Y-m-d'),
             'pages' => $todayPageCount,
             'is_deleted' => $isDeleted,
             'date' => $today,
-            'wiki_id' => $wiki->id
+            'wiki_id' => $wiki->id,
+            'daily_actions'=> $dailyActions,
+            'weekly_actions'=> $weeklyActions,
+            'monthly_actions'=> $monthlyActions,
+            'quarterly_actions'=> $quarterlyActions,
         ]);
-        
+
+        // compare current record to old record and only save if there is a change
         if ($oldRecord) {
             if ($oldRecord->is_deleted) {
                 \Log::info("Wiki is deleted, no new record for WikiMetrics ID {$wiki->id}.");
                 return;
             }
-
             if (!$isDeleted) {
                 if ($oldRecord->areMetricsEqual($dailyMetrics)) {
                     \Log::info("Record unchanged for WikiMetrics ID {$wiki->id}, no new record added.");
@@ -38,6 +57,48 @@ public function saveMetrics(Wiki $wiki): void
 
         $dailyMetrics->save();
 
-        \Log::info("New metric recorded for WikiMetrics ID {$wiki->id}");
+        \Log::info("New metric recorded for Wiki ID {$wiki->id}");
+    }
+
+    protected function getNumberOfActions(string $interval): null|int
+    {
+        $actions = null;
+
+        // safeguard
+        if (false === in_array($interval, 
+        [
+                self::INTERVAL_DAILY,
+                self::INTERVAL_WEEKLY,
+                self::INTERVAL_MONTHLY,
+                self::INTERVAL_QUARTERLY
+                ]
+        )) { return null; }
+
+        $wikiDb = $this->wiki->wikiDb;
+        $tableRecentChanges = $wikiDb->name . '.' . $wikiDb->prefix . '_recentchanges';
+        $tableActor = $wikiDb->name . '.' . $wikiDb->prefix . '_actor';
+
+        $query = "SELECT
+            SUM(rc_timestamp >= DATE_FORMAT(DATE_SUB(NOW(), $interval), '%Y%m%d%H%i%S')) AS sum_actions
+        FROM
+            $tableRecentChanges AS rc
+        INNER JOIN $tableActor AS a ON rc.rc_actor = a.actor_id
+        WHERE
+        /*
+        Conditions below added for consistency with Wikidata: https://phabricator.wikimedia.org/diffusion/ADES/browse/master/src/wikidata/site_stats/sql/active_user_changes.sql
+        */
+        a.actor_user != 0
+        AND rc.rc_bot = 0
+        AND ( rc.rc_log_type != 'newusers' OR rc.rc_log_type IS NULL)";
+
+        $manager = app()->db;
+        $manager->purge('mw');
+        $conn = $manager->connection('mw');
+        $pdo = $conn->getPdo();
+        $result = $pdo->query($query)->fetch();
+
+        $actions = Arr::get($result, 'sum_actions', null);
+
+        return $actions;
     }
 }
diff --git a/app/WikiDailyMetrics.php b/app/WikiDailyMetrics.php
@@ -23,12 +23,20 @@ class WikiDailyMetrics extends Model
         'date',
         'pages',
         'is_deleted',
+        'daily_actions',
+        'weekly_actions',
+        'monthly_actions',
+        'quarterly_actions',
     ];
 
     // list of properties which are actual wiki metrics
     public static $metricNames = [
         'pages',
         'is_deleted',
+        'daily_actions',
+        'weekly_actions',
+        'monthly_actions',
+        'quarterly_actions'
     ];
 
     public function areMetricsEqual(WikiDailyMetrics $wikiDailyMetrics): bool
diff --git a/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php b/database/migrations/2025_03_24_143626_add_columns_to_wiki_daily_metric_table.php
@@ -0,0 +1,30 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('wiki_daily_metrics', function (Blueprint $table) {
+            $table->integer('daily_actions')->nullable();
+            $table->integer('weekly_actions')->nullable();
+            $table->integer('monthly_actions')->nullable();
+            $table->integer('quarterly_actions')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('wiki_daily_metric', function (Blueprint $table) {
+        });
+    }
+};
diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php
@@ -4,10 +4,12 @@
 
 use App\Jobs\UpdateWikiDailyMetricJob;
 use App\Wiki;
+use App\WikiDb;
 use Carbon\Carbon;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Facades\Queue;
 use Tests\TestCase;
+use App\Jobs\ProvisionWikiDbJob;
 
 class UpdateWikiDailyMetricJobTest extends TestCase
 {
@@ -33,18 +35,41 @@ public function testRunJobForAllWikisIncludingDeletedWikis()
         $deletedWiki = Wiki::factory()->create([
             'domain' => 'deletedwiki.wikibase.cloud',
         ]);
+
+        $manager = $this->app->make('db');
+        $job = new ProvisionWikiDbJob();
+        $job2 = new ProvisionWikiDbJob();
+        $job->handle($manager);
+        $job2->handle($manager);
+
+        $wikiDbActive = WikiDb::whereDoesntHave('wiki')->first();
+        $wikiDbActive->update( ['wiki_id' => $activeWiki->id] );
+
+        $wikiDbDeleted = WikiDb::whereDoesntHave('wiki')->first();
+        $wikiDbDeleted->update( ['wiki_id' => $deletedWiki->id] );
+
         $deletedWiki->delete();
 
+
+
         (new UpdateWikiDailyMetricJob())->handle();
 
         $this->assertDatabaseHas('wiki_daily_metrics', [
             'wiki_id' => $activeWiki->id,
-            'date' => Carbon::today()->toDateString()
+            'date' => Carbon::today()->toDateString(),
+            'daily_actions' => null,
+            'weekly_actions' => null,
+            'monthly_actions' => null,
+            'quarterly_actions' => null,
         ]);
 
         $this->assertDatabaseHas('wiki_daily_metrics', [
             'wiki_id' => $deletedWiki->id,
-            'date' => Carbon::today()->toDateString()
+            'date' => Carbon::today()->toDateString(),
+            'daily_actions' => null,
+            'weekly_actions' => null,
+            'monthly_actions' => null,
+            'quarterly_actions' => null,
         ]);
     }
 
diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php
@@ -4,7 +4,9 @@
 
 use App\Metrics\App\WikiMetrics;
 use App\Wiki;
+use App\WikiDb;
 use App\WikiDailyMetrics;
+use App\Jobs\ProvisionWikiDbJob;
 use Carbon\Carbon;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Tests\TestCase;
@@ -13,13 +15,22 @@ class WikiMetricsTest extends TestCase
 {
     use RefreshDatabase;
 
+    public function setUp(): void      {
+        parent::setUp();
+        $manager = $this->app->make('db');
+        $job = new ProvisionWikiDbJob();
+        $job->handle($manager);
+    }
 
     public function testSuccessfullyAddRecords()
     {
         $wiki = Wiki::factory()->create([
             'domain' => 'thisfake.wikibase.cloud'
         ]);
 
+        $wikiDb = WikiDb::first();
+        $wikiDb->update( ['wiki_id' => $wiki->id] );
+
         (new WikiMetrics())->saveMetrics($wiki);
         // Assert the metric is updated in the database
         $this->assertDatabaseHas('wiki_daily_metrics', [
@@ -33,6 +44,10 @@ public function testDoesNotAddDuplicateRecordsWithOnlyDateChange()
         $wiki = Wiki::factory()->create([
             'domain' => 'thisfake.wikibase.cloud'
         ]);
+
+        $wikiDb = WikiDb::first();
+        $wikiDb->update( ['wiki_id' => $wiki->id] );
+
         //Insert an old metric value for a wiki
         WikiDailyMetrics::create([
             'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(),
@@ -55,6 +70,10 @@ public function testAddRecordsWikiIsDeleted()
         $wiki = Wiki::factory()->create([
             'domain' => 'thisfake.wikibase.cloud'
         ]);
+
+        $wikiDb = WikiDb::first();
+        $wikiDb->update( ['wiki_id' => $wiki->id] );
+
         //Insert an old metric value for a wiki
         WikiDailyMetrics::create([
             'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(),
