diff --git a/app/config/dartlang-pub-dev.yaml b/app/config/dartlang-pub-dev.yaml index d8c3443bad..ce529735af 100644 --- a/app/config/dartlang-pub-dev.yaml +++ b/app/config/dartlang-pub-dev.yaml @@ -71,3 +71,5 @@ rateLimits: scope: package burst: 10 daily: 24 +imageProxyHmacKeyVersion: projects/dartlang-pub-dev/locations/us-central1/keyRings/image-proxy-key-ring/cryptoKeys/image-proxy-mac-key/cryptoKeyVersions/1 +imageProxyServiceBaseUrl: https://external-images-staging.pub.dev diff --git a/app/lib/service/image_proxy/backend.dart b/app/lib/service/image_proxy/backend.dart new file mode 100644 index 0000000000..da0a6af79c --- /dev/null +++ b/app/lib/service/image_proxy/backend.dart @@ -0,0 +1,98 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; +import 'package:gcloud/service_scope.dart' as ss; +import 'package:googleapis/cloudkms/v1.dart' as kms; +import 'package:googleapis_auth/auth_io.dart'; +import 'package:pub_dev/shared/configuration.dart'; +import 'package:retry/retry.dart'; + +import '../../shared/cached_value.dart'; + +/// Sets the Image proxy backend service. +void registerImageProxyBackend(ImageProxyBackend backend) => + ss.register(#_imageProxyBackend, backend); + +/// The active Youtube backend service. +ImageProxyBackend get imageProxyBackend => + ss.lookup(#_imageProxyBackend) as ImageProxyBackend; + +/// Represents the backend for the Youtube handling and related utilities. +class ImageProxyBackend { + ImageProxyBackend._(); + + static Future create() async { + final instance = ImageProxyBackend._(); + await instance._dailySecret.update(); + return instance; + } + + static Future> _getDailySecret( + DateTime day, + AuthClient client, + ) async { + return await retry(() async { + final api = kms.CloudKMSApi(client); + final response = await api + .projects + .locations + .keyRings + .cryptoKeys + .cryptoKeyVersions + .macSign( + kms.MacSignRequest() + ..dataAsBytes = utf8.encode( + DateTime( + day.year, + day.month, + day.day, + ).toUtc().toIso8601String(), + ), + activeConfiguration.imageProxyHmacKeyVersion!, + ); + return response.macAsBytes; + }); + } + + final _dailySecret = CachedValue( + name: 'image-proxy-daily-secret', + interval: Duration(minutes: 15), + maxAge: Duration(hours: 12), + timeout: Duration(hours: 12), + updateFn: () async { + final now = DateTime.now().toUtc(); + final today = DateTime(now.year, now.month, now.day); + return ( + today, + await _getDailySecret( + today, + await clientViaApplicationDefaultCredentials( + scopes: [kms.CloudKMSApi.cloudPlatformScope], + ), + ), + ); + }, + ); + + String imageProxyUrl(Uri originalUrl) { + final dailySecret = _dailySecret.value; + // TODO handle the null case more gracefully. + if (dailySecret == null) { + throw StateError('Image proxy HMAC secret is not available.'); + } + final (today, secret) = dailySecret; + + final hmac = Hmac( + sha256, + secret, + ).convert(originalUrl.toString().codeUnits).bytes; + activeConfiguration.imageProxyServiceBaseUrl; + + return '${activeConfiguration.imageProxyServiceBaseUrl}/${Uri.encodeComponent(base64Encode(hmac))}/${today.millisecondsSinceEpoch}/${Uri.encodeComponent(originalUrl.toString())}'; + } +} diff --git a/app/lib/service/services.dart b/app/lib/service/services.dart index b91627c918..0a781d248f 100644 --- a/app/lib/service/services.dart +++ b/app/lib/service/services.dart @@ -20,6 +20,7 @@ import 'package:pub_dev/package/api_export/api_exporter.dart'; import 'package:pub_dev/search/handlers.dart'; import 'package:pub_dev/service/async_queue/async_queue.dart'; import 'package:pub_dev/service/download_counts/backend.dart'; +import 'package:pub_dev/service/image_proxy/backend.dart'; import 'package:pub_dev/service/security_advisories/backend.dart'; import 'package:shelf/shelf.dart' as shelf; import 'package:shelf/shelf_io.dart'; @@ -319,6 +320,8 @@ Future _withPubServices(FutureOr Function() fn) async { registerYoutubeBackend(YoutubeBackend()); registerScopeExitCallback(youtubeBackend.close); + registerImageProxyBackend(await ImageProxyBackend.create()); + // depends on previously registered services registerPackageBackend( PackageBackend( diff --git a/app/lib/shared/configuration.dart b/app/lib/shared/configuration.dart index 2f526ad07c..2ae3922902 100644 --- a/app/lib/shared/configuration.dart +++ b/app/lib/shared/configuration.dart @@ -218,6 +218,10 @@ final class Configuration { /// The rate limits for auditable operations. final List? rateLimits; + /// The Cloud KMS HMAC key version used to sign image proxy URLs. + final String? imageProxyHmacKeyVersion; + final String? imageProxyServiceBaseUrl; + /// Load [Configuration] from YAML file at [path] substituting `{{ENV}}` for /// the value of environment variable `ENV`. factory Configuration.fromYamlFile(final String path) { @@ -285,6 +289,8 @@ final class Configuration { required this.defaultServiceBaseUrl, required this.tools, required this.rateLimits, + required this.imageProxyHmacKeyVersion, + required this.imageProxyServiceBaseUrl, }); /// Load configuration from `app/config/.yaml` where `projectId` @@ -356,6 +362,8 @@ final class Configuration { ], tools: null, rateLimits: null, + imageProxyHmacKeyVersion: null, + imageProxyServiceBaseUrl: null, ); } @@ -406,6 +414,8 @@ final class Configuration { ], tools: null, rateLimits: null, + imageProxyHmacKeyVersion: null, + imageProxyServiceBaseUrl: null, ); } diff --git a/app/lib/shared/configuration.g.dart b/app/lib/shared/configuration.g.dart index 34905ba874..e746cb1560 100644 --- a/app/lib/shared/configuration.g.dart +++ b/app/lib/shared/configuration.g.dart @@ -45,6 +45,8 @@ Configuration _$ConfigurationFromJson( 'admins', 'tools', 'rateLimits', + 'imageProxyHmacKeyVersion', + 'imageProxyServiceBaseUrl', ], ); final val = Configuration( @@ -168,6 +170,14 @@ Configuration _$ConfigurationFromJson( ?.map((e) => RateLimit.fromJson(Map.from(e as Map))) .toList(), ), + imageProxyHmacKeyVersion: $checkedConvert( + 'imageProxyHmacKeyVersion', + (v) => v as String?, + ), + imageProxyServiceBaseUrl: $checkedConvert( + 'imageProxyServiceBaseUrl', + (v) => v as String?, + ), ); return val; }); @@ -207,6 +217,8 @@ Map _$ConfigurationToJson(Configuration instance) => 'admins': instance.admins?.map((e) => e.toJson()).toList(), 'tools': instance.tools?.toJson(), 'rateLimits': instance.rateLimits?.map((e) => e.toJson()).toList(), + 'imageProxyHmacKeyVersion': instance.imageProxyHmacKeyVersion, + 'imageProxyServiceBaseUrl': instance.imageProxyServiceBaseUrl, }; AdminId _$AdminIdFromJson(Map json) => $checkedCreate('AdminId', json, ( diff --git a/app/lib/shared/markdown.dart b/app/lib/shared/markdown.dart index 7572668228..6c28c06e3b 100644 --- a/app/lib/shared/markdown.dart +++ b/app/lib/shared/markdown.dart @@ -2,12 +2,16 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; + +import 'package:googleapis/cloudkms/v1.dart' as kms; import 'package:html/dom.dart' as html; import 'package:html/dom_parsing.dart' as html_parsing; import 'package:html/parser.dart' as html_parser; import 'package:logging/logging.dart'; import 'package:markdown/markdown.dart' as m; import 'package:pub_dev/frontend/static_files.dart'; +import 'package:pub_dev/service/image_proxy/backend.dart'; import 'package:pub_dev/shared/changelog.dart'; import 'package:sanitize_html/sanitize_html.dart'; @@ -124,6 +128,7 @@ String _postProcessHtml( var root = html_parser.parseFragment(rawHtml); _RelativeUrlRewriter(urlResolverFn, relativeFrom).visit(root); + _ImageProxyRewriter().visit(root); if (isChangelog) { final oldNodes = [...root.nodes]; @@ -203,6 +208,27 @@ class _UnsafeUrlFilter extends html_parsing.TreeVisitor { } } +class _ImageProxyRewriter extends html_parsing.TreeVisitor { + @override + void visitElement(html.Element element) { + super.visitElement(element); + + if (element.localName == 'img') { + final src = element.attributes['src']; + final uri = Uri.tryParse(src ?? ''); + if (uri == null || uri.isInvalid) { + // TODO: consider removing the element entirely + return; + } + if ((uri.isScheme('http') || uri.isScheme('https')) && + !uri.isTrustedHost) { + final proxiedUrl = imageProxyBackend.imageProxyUrl(uri); + element.attributes['src'] = proxiedUrl; + } + } + } +} + /// Rewrites relative URLs with the provided [urlResolverFn]. class _RelativeUrlRewriter extends html_parsing.TreeVisitor { final UrlResolverFn? urlResolverFn;