Merge pull request #109 from flutter-news-app-full-source-code/feat/analytics-reporting-engine

Feat/analytics reporting engine

Author: fulleni

.env.example

@@ -60,9 +60,28 @@ FIREBASE_PRIVATE_KEY="your-firebase-private-key"
 ONESIGNAL_APP_ID="your-onesignal-app-id"
 ONESIGNAL_REST_API_KEY="your-onesignal-rest-api-key"
 
+# -----------------------------------------------------------------------------
+# SECTION 4: ANALYTICS PROVIDERS (CONDITIONALLY REQUIRED)
+# Provide credentials for analytics providers to enable dashboard metrics.
+# The server will start without these, but the analytics sync worker will skip them.
+# -----------------------------------------------------------------------------
+
+# --- Google Analytics (via Firebase) ---
+# The ID of your Google Analytics 4 property.
+# Note: This requires the Firebase credentials from Section 3 to be configured
+# as it uses the same authentication mechanism.
+GOOGLE_ANALYTICS_PROPERTY_ID="your-ga4-property-id"
+
+# --- Mixpanel ---
+# The Project ID for your Mixpanel project.
+MIXPANEL_PROJECT_ID="your-mixpanel-project-id"
+# The username for your Mixpanel service account.
+MIXPANEL_SERVICE_ACCOUNT_USERNAME="your-mixpanel-service-account-username"
+# The secret for your Mixpanel service account.
+MIXPANEL_SERVICE_ACCOUNT_SECRET="your-mixpanel-service-account-secret"
 
 # -----------------------------------------------------------------------------
-# SECTION 4: API SECURITY & RATE LIMITING (OPTIONAL)
+# SECTION 5: API SECURITY & RATE LIMITING (OPTIONAL)
 # Fine-tune security settings. Defaults are provided if these are not set.
 # -----------------------------------------------------------------------------
 
@@ -80,7 +99,7 @@ ONESIGNAL_REST_API_KEY="your-onesignal-rest-api-key"
 
 
 # -----------------------------------------------------------------------------
-# SECTION 5: ADVANCED & MISCELLANEOUS CONFIGURATION (OPTIONAL)
+# SECTION 6: ADVANCED & MISCELLANEOUS CONFIGURATION (OPTIONAL)
 # -----------------------------------------------------------------------------
 
 # The duration for which a JWT is valid, in hours. Defaults to 720 (30 days).

.github/workflows/main.yaml

@@ -16,4 +16,4 @@ jobs:
   build:
     uses: VeryGoodOpenSource/very_good_workflows/.github/workflows/dart_package.yml@v1
     with:
-      min_coverage: 0
+      min_coverage: 40

README.md

@@ -5,7 +5,7 @@
 </div>
 
 <p align="center">
-<img src="https://img.shields.io/badge/coverage-_%25-red?style=for-the-badge" alt="coverage">
+<img src="https://img.shields.io/badge/coverage-53%25-green?style=for-the-badge" alt="coverage">
 <a href="https://flutter-news-app-full-source-code.github.io/docs/api-server/local-setup/"><img src="https://img.shields.io/badge/DOCUMENTATION-READ-slategray?style=for-the-badge" alt="Documentation: Read"></a>
 </p>
 <p align="center">
@@ -107,11 +107,43 @@ A complete, multi-provider notification engine empowers you to engage users with
 
 </details>
 
+<details>
+<summary><strong>📊 Insightful Analytics Engine</strong></summary>
+
+### 📈 A Unified Business Intelligence Engine
+A complete, multi-provider analytics engine that transforms raw data from both external services and your own application database into insightful, aggregated metrics for your dashboard.
+- **Dual-Source ETL:** A standalone worker process runs on a schedule to perform a full Extract, Transform, and Load (ETL) operation. It pulls behavioral data from your chosen analytics provider (Google Analytics or Mixpanel) and combines it with operational data by running direct, complex aggregations against the application's own database.
+- **High-Performance Dashboard:** The web dashboard reads this pre-aggregated data, resulting in near-instant load times for all analytics charts and metrics. This architecture avoids slow, direct, on-the-fly queries from the client to the analytics provider.
+- **Provider-Agnostic Design:** The engine is built on a provider-agnostic interface. You can switch between Google Analytics and Mixpanel via a simple configuration change, without altering any code.
+- **Extensible & Scalable:** Adding new charts or KPIs is as simple as defining a new mapping. The system is designed to be easily extended to track new metrics as your application evolves.
+> **Your Advantage:** Get a complete, production-grade BI pipeline out of the box. Deliver a fast, responsive dashboard and gain a holistic view of your business by combining user behavior analytics with real-time operational metrics—a capability that external analytics tools alone cannot provide.
+
+</details>
+
 <details>
 <summary><strong>🏗️ Architecture & Infrastructure</strong></summary>
 
 ### 🚀 High-Performance by Design
-Built on a modern, minimalist foundation to ensure low latency and excellent performance.
+- **Dart Frog Core:** Leverages the high-performance Dart Frog framework for a fast, efficient, and scalable backend.
+- **Clean, Layered Architecture:** A strict separation of concerns into distinct layers makes the codebase clean, maintainable, and easy to reason about.
+> **Your Advantage:** Your backend is built on a solid, modern foundation that is both powerful and a pleasure to work with, reducing maintenance overhead.
+
+---
+
+### 🔌 Extensible & Unlocked
+The entire application is designed with a robust dependency injection system, giving you the freedom to choose your own infrastructure.
+- **Swappable Implementations:** Easily swap out core components—like the database, email provider, or file storage service—without rewriting business logic.
+> **Your Advantage:** Avoid vendor lock-in and future-proof your application. You have the freedom to adapt and evolve your tech stack as your business needs change.
+
+---
+
+### 🔄 Automated & Traceable Database Migrations
+Say goodbye to risky manual database updates. A professional, versioned migration system ensures your database schema evolves safely and automatically.
+- **Code-Driven Schema Evolution:** The system automatically applies schema changes to your database on application startup, ensuring consistency across all environments.
+- **Traceable to Source:** Each migration is versioned and directly linked to the pull request that initiated it, providing a clear, auditable history of every change.
+> **Your Advantage:** Deploy with confidence. This robust system eliminates an entire class of deployment errors, ensuring your data models evolve gracefully and reliably with full traceability.
+
+</details>
 
 ## 🔑 Licensing
 This `Flutter News App API Server` package is an integral part of the [**Flutter News App Full Source Code Toolkit**](https://github.com/flutter-news-app-full-source-code). For comprehensive details regarding licensing, including trial and commercial options for the entire toolkit, please refer to the main toolkit organization page.

analysis_options.yaml

@@ -12,6 +12,8 @@ analyzer:
     one_member_abstracts: ignore
     cascade_invocations: ignore
     cast_nullable_to_non_nullable: ignore
+    specify_nonobvious_property_types: ignore
+    unnecessary_null_checks: ignore
   exclude:
     - build/**
 linter:

bin/analytics_sync_worker.dart

@@ -0,0 +1,35 @@
+import 'dart:io';
+
+import 'package:flutter_news_app_api_server_full_source_code/src/config/app_dependencies.dart';
+import 'package:flutter_news_app_api_server_full_source_code/src/services/analytics/analytics.dart';
+import 'package:logging/logging.dart';
+
+/// The main entry point for the standalone Analytics Sync Worker process.
+///
+/// This script initializes application dependencies, retrieves the
+/// [AnalyticsSyncService], and executes its `run()` method to perform the
+/// periodic data synchronization.
+///
+/// This executable can be compiled into a native binary and run by a scheduler
+/// (e.g., a cron job) to automate the analytics data pipeline.
+Future<void> main(List<String> args) async {
+  // Configure logger for console output.
+  Logger.root.level = Level.ALL;
+  Logger.root.onRecord.listen((record) {
+    // ignore: avoid_print
+    print('${record.level.name}: ${record.time}: ${record.message}');
+    if (record.error != null) {
+      // ignore: avoid_print
+      print('  ERROR: ${record.error}');
+    }
+    if (record.stackTrace != null) {
+      // ignore: avoid_print
+      print('  STACK TRACE: ${record.stackTrace}');
+    }
+  });
+
+  await AppDependencies.instance.init();
+  await AppDependencies.instance.analyticsSyncService.run();
+  await AppDependencies.instance.dispose();
+  exit(0);
+}

lib/src/config/app_dependencies.dart

@@ -10,10 +10,10 @@ import 'package:email_sendgrid/email_sendgrid.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/config/environment_config.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/database/migrations/all_migrations.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/rbac/permission_service.dart';
+import 'package:flutter_news_app_api_server_full_source_code/src/services/analytics/analytics.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_service.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_token_service.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/services/country_query_service.dart';
-import 'package:flutter_news_app_api_server_full_source_code/src/services/dashboard_summary_service.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/services/database_migration_service.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/services/database_seeding_service.dart';
 import 'package:flutter_news_app_api_server_full_source_code/src/services/default_user_action_limit_service.dart';
@@ -72,19 +72,22 @@ class AppDependencies {
   pushNotificationDeviceRepository;
   late final DataRepository<RemoteConfig> remoteConfigRepository;
   late final DataRepository<InAppNotification> inAppNotificationRepository;
+  late final DataRepository<KpiCardData> kpiCardDataRepository;
+  late final DataRepository<ChartCardData> chartCardDataRepository;
+  late final DataRepository<RankedListCardData> rankedListCardDataRepository;
 
   late final DataRepository<Engagement> engagementRepository;
   late final DataRepository<Report> reportRepository;
   late final DataRepository<AppReview> appReviewRepository;
   late final EmailRepository emailRepository;
 
   // Services
+  late final AnalyticsSyncService analyticsSyncService;
   late final DatabaseMigrationService databaseMigrationService;
   late final TokenBlacklistService tokenBlacklistService;
   late final AuthTokenService authTokenService;
   late final VerificationCodeStorageService verificationCodeStorageService;
   late final AuthService authService;
-  late final DashboardSummaryService dashboardSummaryService;
   late final PermissionService permissionService;
   late final UserActionLimitService userActionLimitService;
   late final RateLimitService rateLimitService;
@@ -232,6 +235,24 @@ class AppDependencies {
         logger: Logger('DataMongodb<InAppNotification>'),
       );
 
+      final kpiCardDataClient = DataMongodb<KpiCardData>(
+        connectionManager: _mongoDbConnectionManager,
+        modelName: 'kpi_card_data',
+        fromJson: KpiCardData.fromJson,
+        toJson: (item) => item.toJson(),
+      );
+      final chartCardDataClient = DataMongodb<ChartCardData>(
+        connectionManager: _mongoDbConnectionManager,
+        modelName: 'chart_card_data',
+        fromJson: ChartCardData.fromJson,
+        toJson: (item) => item.toJson(),
+      );
+      final rankedListCardDataClient = DataMongodb<RankedListCardData>(
+        connectionManager: _mongoDbConnectionManager,
+        modelName: 'ranked_list_card_data',
+        fromJson: RankedListCardData.fromJson,
+        toJson: (item) => item.toJson(),
+      );
       _log.info('Initialized data client for InAppNotification.');
 
       final engagementClient = DataMongodb<Engagement>(
@@ -349,6 +370,13 @@ class AppDependencies {
       engagementRepository = DataRepository(dataClient: engagementClient);
       reportRepository = DataRepository(dataClient: reportClient);
       appReviewRepository = DataRepository(dataClient: appReviewClient);
+      kpiCardDataRepository = DataRepository(dataClient: kpiCardDataClient);
+      chartCardDataRepository = DataRepository(
+        dataClient: chartCardDataClient,
+      );
+      rankedListCardDataRepository = DataRepository(
+        dataClient: rankedListCardDataClient,
+      );
 
       // Configure the HTTP client for SendGrid.
       // The HttpClient's AuthInterceptor will use the tokenProvider to add
@@ -394,11 +422,6 @@ class AppDependencies {
         userContentPreferencesRepository: userContentPreferencesRepository,
         log: Logger('AuthService'),
       );
-      dashboardSummaryService = DashboardSummaryService(
-        headlineRepository: headlineRepository,
-        topicRepository: topicRepository,
-        sourceRepository: sourceRepository,
-      );
       userActionLimitService = DefaultUserActionLimitService(
         remoteConfigRepository: remoteConfigRepository,
         engagementRepository: engagementRepository,
@@ -424,6 +447,69 @@ class AppDependencies {
         log: Logger('DefaultPushNotificationService'),
       );
 
+      // --- Analytics Services ---
+      final gaPropertyId = EnvironmentConfig.googleAnalyticsPropertyId;
+      final mpProjectId = EnvironmentConfig.mixpanelProjectId;
+      final mpUser = EnvironmentConfig.mixpanelServiceAccountUsername;
+      final mpSecret = EnvironmentConfig.mixpanelServiceAccountSecret;
+
+      GoogleAnalyticsDataClient? googleAnalyticsClient;
+      if (gaPropertyId != null && firebaseAuthenticator != null) {
+        final googleAnalyticsHttpClient = HttpClient(
+          baseUrl: 'https://analyticsdata.googleapis.com/v1beta',
+          tokenProvider: firebaseAuthenticator!.getAccessToken,
+          logger: Logger('GoogleAnalyticsHttpClient'),
+        );
+
+        googleAnalyticsClient = GoogleAnalyticsDataClient(
+          headlineRepository: headlineRepository,
+          propertyId: gaPropertyId,
+          firebaseAuthenticator: firebaseAuthenticator!,
+          log: Logger('GoogleAnalyticsDataClient'),
+          httpClient: googleAnalyticsHttpClient,
+        );
+      } else {
+        _log.warning(
+          'Google Analytics client could not be initialized due to missing '
+          'property ID or Firebase authenticator.',
+        );
+      }
+
+      MixpanelDataClient? mixpanelClient;
+      if (mpProjectId != null && mpUser != null && mpSecret != null) {
+        mixpanelClient = MixpanelDataClient(
+          headlineRepository: headlineRepository,
+          projectId: mpProjectId,
+          serviceAccountUsername: mpUser,
+          serviceAccountSecret: mpSecret,
+          log: Logger('MixpanelDataClient'),
+        );
+      } else {
+        _log.warning(
+          'Mixpanel client could not be initialized due to missing credentials.',
+        );
+      }
+
+      final analyticsMetricMapper = AnalyticsMetricMapper();
+
+      analyticsSyncService = AnalyticsSyncService(
+        remoteConfigRepository: remoteConfigRepository,
+        kpiCardRepository: kpiCardDataRepository,
+        chartCardRepository: chartCardDataRepository,
+        rankedListCardRepository: rankedListCardDataRepository,
+        userRepository: userRepository,
+        topicRepository: topicRepository,
+        sourceRepository: sourceRepository,
+        reportRepository: reportRepository,
+        headlineRepository: headlineRepository,
+        googleAnalyticsClient: googleAnalyticsClient,
+        mixpanelClient: mixpanelClient,
+        analyticsMetricMapper: analyticsMetricMapper,
+        engagementRepository: engagementRepository,
+        appReviewRepository: appReviewRepository,
+        log: Logger('AnalyticsSyncService'),
+      );
+
       _log.info('Application dependencies initialized successfully.');
       // Signal that initialization has completed successfully.
       _initCompleter!.complete();
@@ -459,7 +545,7 @@ class AppDependencies {
     await _mongoDbConnectionManager.close();
     tokenBlacklistService.dispose();
     rateLimitService.dispose();
-    countryQueryService.dispose(); // Dispose the new service
+    countryQueryService.dispose();
 
     // Reset the completer to allow for re-initialization (e.g., in tests).
     _initCompleter = null;

lib/src/config/environment_config.dart

@@ -219,4 +219,19 @@ abstract final class EnvironmentConfig {
   ///
   /// The value is read from the `ONESIGNAL_REST_API_KEY` environment variable, if available.
   static String? get oneSignalRestApiKey => _getEnv('ONESIGNAL_REST_API_KEY');
+
+  /// Retrieves the Google Analytics Property ID from the environment.
+  static String? get googleAnalyticsPropertyId =>
+      _getEnv('GOOGLE_ANALYTICS_PROPERTY_ID');
+
+  /// Retrieves the Mixpanel Project ID from the environment.
+  static String? get mixpanelProjectId => _getEnv('MIXPANEL_PROJECT_ID');
+
+  /// Retrieves the Mixpanel Service Account Username from the environment.
+  static String? get mixpanelServiceAccountUsername =>
+      _getEnv('MIXPANEL_SERVICE_ACCOUNT_USERNAME');
+
+  /// Retrieves the Mixpanel Service Account Secret from the environment.
+  static String? get mixpanelServiceAccountSecret =>
+      _getEnv('MIXPANEL_SERVICE_ACCOUNT_SECRET');
 }

lib/src/models/analytics/analytics.dart

@@ -0,0 +1,5 @@
+export 'analytics_query.dart';
+export 'google_analytics_request.dart';
+export 'google_analytics_response.dart';
+export 'mixpanel_request.dart';
+export 'mixpanel_response.dart';

lib/src/models/analytics/analytics_query.dart

@@ -0,0 +1,60 @@
+import 'package:core/core.dart';
+
+/// A sealed class representing a structured, provider-agnostic analytics query.
+///
+/// This replaces the fragile pattern of passing primitive strings for metrics
+/// and dimensions, centralizing query definitions into type-safe objects.
+sealed class AnalyticsQuery {
+  /// {@macro analytics_query}
+  const AnalyticsQuery();
+}
+
+/// A sealed class for queries that return a numeric metric value, such as
+/// a total count or a time series.
+sealed class MetricQuery extends AnalyticsQuery {
+  /// {@macro metric_query}
+  const MetricQuery();
+}
+
+/// A query for a simple event count.
+///
+/// This is used when the metric is the count of a specific [AnalyticsEvent].
+class EventCountQuery extends MetricQuery {
+  /// {@macro event_count_query}
+  const EventCountQuery({required this.event});
+
+  /// The core, type-safe event from the shared [AnalyticsEvent] enum.
+  final AnalyticsEvent event;
+}
+
+/// A query for a standard, provider-defined metric (e.g., 'activeUsers').
+///
+/// This is used for metrics that have a built-in name in the provider's API.
+class StandardMetricQuery extends MetricQuery {
+  /// {@macro standard_metric_query}
+  const StandardMetricQuery({required this.metric});
+
+  /// The provider-specific name for a standard metric.
+  final String metric;
+}
+
+/// A query for a ranked list of items.
+///
+/// This is used to get a "Top N" list, such as most viewed headlines.
+class RankedListQuery extends AnalyticsQuery {
+  /// {@macro ranked_list_query}
+  const RankedListQuery({
+    required this.event,
+    required this.dimension,
+    this.limit = 10,
+  });
+
+  /// The event to count for ranking (e.g., `contentViewed`).
+  final AnalyticsEvent event;
+
+  /// The property/dimension to group by (e.g., `contentId`).
+  final String dimension;
+
+  /// The number of items to return in the ranked list.
+  final int limit;
+}