feat: add --diff filter (#323)

Co-authored-by: Gabriel Terwesten <[email protected]>

README.md

@@ -40,11 +40,11 @@ To solve these (and many other) problems, some projects will organize their code
 To setup and use this melos mono repo locally for the purposes of contributing, clone it and run the following commands from the root of the repository:
 
 ```bash
-# Remove previous instances of melos:
-dart pub global deactivate melos
+# Install melos if it's not already installed:
+dart pub global activate melos
 
 # Activate 'melos' from path:
-dart pub global activate --source="path" . --executable="melos"
+melos activate
 
 # Confirm you now using a local development version:
 melos --help

docs/filters.mdx

@@ -52,6 +52,18 @@ Only include packages that have been changed since the specified `ref`, e.g. a c
 melos exec --since=<commit hash> -- flutter build ios
 ```
 
+## --diff
+
+Only include packages that are different between the current working tree and the specified `ref`, e.g. a commit sha or git tag, or between two different refs.
+
+```bash
+# Run `flutter build ios` on all packages that are different between current branch and the specified commit hash
+melos exec --diff=<commit hash> -- flutter build ios
+
+# Run `flutter build ios` on all packages that are different between remote `main` branch and HEAD
+melos exec --diff=origin/main...HEAD -- flutter build ios
+```
+
 ## --dir-exists
 
 Include only packages where a specific directory exists inside the package.

melos.yaml

@@ -42,4 +42,12 @@ scripts:
     run: melos exec flutter format . --set-exit-if-changed
     description: Run `flutter format` checks for all packages.
 
-  version: dart run scripts/generate_version.dart
+  version: dart run scripts/generate_version.dart
+
+  activate:
+    description: Activate the local version of melos for development.
+    run: dart pub global activate --source="path" . --executable="melos" --overwrite
+
+  activate:pub:
+    description: Activate the published version of melos.
+    run: dart pub global activate melos

packages/melos/lib/melos.dart

@@ -12,7 +12,13 @@ export 'src/common/exception.dart' show CancelledException, MelosException;
 export 'src/common/validation.dart' show MelosConfigException;
 export 'src/global_options.dart' show GlobalOptions;
 export 'src/logging.dart' show MelosLogger, ToMelosLoggerExtension;
-export 'src/package.dart' show Package, PackageFilter, PackageMap, PackageType;
+export 'src/package.dart'
+    show
+        Package,
+        PackageFilter,
+        PackageMap,
+        PackageType,
+        InvalidPackageFilterException;
 export 'src/workspace.dart' show IdeWorkspace, MelosWorkspace;
 export 'src/workspace_configs.dart'
     show

packages/melos/lib/src/command_runner/base.dart

@@ -89,7 +89,15 @@ abstract class MelosCommand extends Command<void> {
       filterOptionSince,
       valueHelp: 'ref',
       help: 'Only include packages that have been changed since the specified '
-          '`ref`, e.g. a commit sha or git tag.',
+          '`ref`, e.g. a commit sha or git tag. Cannot be used with --diff.',
+    );
+
+    argParser.addOption(
+      filterOptionDiff,
+      valueHelp: 'ref',
+      help: 'Only include packages that are different between current working '
+          'tree and the specified `ref`, e.g. a commit sha or git tag, '
+          'or between two different refs. Cannot be used with --since.',
     );
 
     argParser.addMultiOption(
@@ -149,6 +157,7 @@ abstract class MelosCommand extends Command<void> {
 
     final since =
         sinceEnabled ? argResults![filterOptionSince] as String? : null;
+    final diff = argResults![filterOptionDiff] as String?;
     final scope = argResults![filterOptionScope] as List<String>? ?? [];
     final ignore = argResults![filterOptionIgnore] as List<String>? ?? [];
 
@@ -161,6 +170,7 @@ abstract class MelosCommand extends Command<void> {
           .toList()
         ..addAll(config.ignore),
       updatedSince: since,
+      diff: diff,
       includePrivatePackages: argResults![filterOptionPrivate] as bool?,
       published: argResults![filterOptionPublished] as bool?,
       nullSafe: argResults![filterOptionNullsafety] as bool?,

packages/melos/lib/src/common/git.dart

@@ -344,3 +344,27 @@ Future<bool> gitIsBehindUpstream({
 
   return isBehind;
 }
+
+Future<bool> gitHasDiffInPackage(
+  Package package, {
+  required String diff,
+  required MelosLogger logger,
+}) async {
+  logger.trace(
+    '[GIT] Getting $diff diff for package ${package.name}.',
+  );
+
+  final processResult = await gitExecuteCommand(
+    arguments: [
+      '--no-pager',
+      'diff',
+      '--name-status',
+      diff,
+      '--',
+      '.',
+    ],
+    workingDirectory: package.path,
+    logger: logger,
+  );
+  return (processResult.stdout as String).isNotEmpty;
+}

packages/melos/lib/src/common/utils.dart

@@ -42,6 +42,7 @@ const filterOptionIgnore = 'ignore';
 const filterOptionDirExists = 'dir-exists';
 const filterOptionFileExists = 'file-exists';
 const filterOptionSince = 'since';
+const filterOptionDiff = 'diff';
 const filterOptionNullsafety = 'nullsafety';
 const filterOptionNoPrivate = 'no-private';
 const filterOptionPrivate = 'private';

packages/melos/lib/src/package.dart

@@ -27,6 +27,7 @@ import 'package:pub_semver/pub_semver.dart';
 import 'package:pubspec/pubspec.dart';
 
 import '../version.g.dart';
+import 'common/exception.dart';
 import 'common/git.dart';
 import 'common/glob.dart';
 import 'common/http.dart' as http;
@@ -129,6 +130,7 @@ class PackageFilter {
     List<String> dependsOn = const [],
     List<String> noDependsOn = const [],
     this.updatedSince,
+    this.diff,
     this.includePrivatePackages,
     this.published,
     this.nullSafe,
@@ -143,7 +145,9 @@ class PackageFilter {
         noDependsOn = [
           ...noDependsOn,
           if (flutter == false) 'flutter',
-        ];
+        ] {
+    _validate();
+  }
 
   /// A default constructor with **all** properties as requires, to ensure that
   /// copyWith functions properly copy all properties.
@@ -155,12 +159,15 @@ class PackageFilter {
     required this.dependsOn,
     required this.noDependsOn,
     required this.updatedSince,
+    required this.diff,
     required this.includePrivatePackages,
     required this.published,
     required this.nullSafe,
     required this.includeDependencies,
     required this.includeDependents,
-  });
+  }) {
+    _validate();
+  }
 
   /// Patterns for filtering packages by name.
   final List<Glob> scope;
@@ -183,6 +190,9 @@ class PackageFilter {
   /// Filter package based on whether they received changed since a specific git commit/tag ID.
   final String? updatedSince;
 
+  /// Filter package based on whether they are different between specific git commit/tag ID.
+  final String? diff;
+
   /// Include/Exclude packages with `publish_to: none`.
   final bool? includePrivatePackages;
 
@@ -202,6 +212,14 @@ class PackageFilter {
   /// This supersede other filters.
   final bool includeDependencies;
 
+  void _validate() {
+    if (updatedSince != null && diff != null) {
+      throw InvalidPackageFilterException(
+        'Cannot specify both updatedSince and diff.',
+      );
+    }
+  }
+
   Map<String, Object?> toJson() {
     return {
       if (scope.isNotEmpty)
@@ -213,6 +231,7 @@ class PackageFilter {
       if (dependsOn.isNotEmpty) filterOptionDependsOn: dependsOn,
       if (noDependsOn.isNotEmpty) filterOptionNoDependsOn: noDependsOn,
       if (updatedSince != null) filterOptionSince: updatedSince,
+      if (diff != null) filterOptionDiff: diff,
       if (includePrivatePackages != null)
         filterOptionPrivate: includePrivatePackages,
       if (published != null) filterOptionPublished: published,
@@ -234,6 +253,7 @@ class PackageFilter {
       published: published,
       scope: scope,
       updatedSince: since,
+      diff: diff,
       includeDependencies: includeDependencies,
       includeDependents: includeDependents,
     );
@@ -251,6 +271,7 @@ class PackageFilter {
       published: published,
       scope: scope,
       updatedSince: updatedSince,
+      diff: diff,
       includeDependencies: includeDependencies,
       includeDependents: includeDependents,
     );
@@ -271,7 +292,8 @@ class PackageFilter {
       const DeepCollectionEquality().equals(other.fileExists, fileExists) &&
       const DeepCollectionEquality().equals(other.dependsOn, dependsOn) &&
       const DeepCollectionEquality().equals(other.noDependsOn, noDependsOn) &&
-      other.updatedSince == updatedSince;
+      other.updatedSince == updatedSince &&
+      other.diff == diff;
 
   @override
   int get hashCode =>
@@ -287,7 +309,8 @@ class PackageFilter {
       const DeepCollectionEquality().hash(fileExists) ^
       const DeepCollectionEquality().hash(dependsOn) ^
       const DeepCollectionEquality().hash(noDependsOn) ^
-      updatedSince.hashCode;
+      updatedSince.hashCode ^
+      diff.hashCode;
 
   @override
   String toString() {
@@ -305,10 +328,20 @@ PackageFilter(
   dependsOn: $dependsOn,
   noDependsOn: $noDependsOn,
   updatedSince: $updatedSince,
+  diff: $diff,
 )''';
   }
 }
 
+class InvalidPackageFilterException extends MelosException {
+  InvalidPackageFilterException(this.message);
+
+  final String message;
+
+  @override
+  String toString() => 'Invalid package filters: $message';
+}
+
 // Not using MapView to prevent map mutation
 class PackageMap {
   PackageMap(Map<String, Package> packages, this._logger)
@@ -442,8 +475,17 @@ The packages that caused the problem are:
         .applyDependsOn(filter.dependsOn)
         .applyNoDependsOn(filter.noDependsOn)
         .filterNullSafe(nullSafe: filter.nullSafe)
-        .filterPublishedPackages(published: filter.published)
-        .then((packages) => packages.applySince(filter.updatedSince, _logger));
+        .filterPublishedPackages(published: filter.published);
+
+    final updatedSince = filter.updatedSince;
+    if (updatedSince != null) {
+      packageList = await packageList.applySince(updatedSince, _logger);
+    }
+
+    final diff = filter.diff;
+    if (diff != null) {
+      packageList = await packageList.applyDiff(diff, _logger);
+    }
 
     packageList = packageList.applyIncludeDependentsOrDependencies(
       includeDependents: filter.includeDependents,
@@ -559,6 +601,27 @@ extension on Iterable<Package> {
     return packagesFilteredWithGitCommitsSince;
   }
 
+  Future<Iterable<Package>> applyDiff(
+    String? diff,
+    MelosLogger logger,
+  ) async {
+    if (diff == null) return this;
+
+    final pool = Pool(10);
+    final packagesFilteredWithGitCommitsSince = <Package>[];
+
+    await pool.forEach<Package, void>(this, (package) {
+      return gitHasDiffInPackage(package, diff: diff, logger: logger)
+          .then((commits) async {
+        if (commits) {
+          packagesFilteredWithGitCommitsSince.add(package);
+        }
+      });
+    }).drain<void>();
+
+    return packagesFilteredWithGitCommitsSince;
+  }
+
   /// Whether to include/exclude packages that are null-safe.
   ///
   /// If `include` is true, only null-safe packages.

packages/melos/lib/src/scripts.dart

@@ -277,6 +277,12 @@ class Script {
       path: filtersPath,
     );
 
+    final diff = assertIsA<String?>(
+      value: yaml[filterOptionDiff],
+      key: filterOptionDiff,
+      path: filtersPath,
+    );
+
     final excludePrivatePackagesTmp = assertIsA<bool?>(
       value: yaml[filterOptionNoPrivate],
       key: filterOptionNoPrivate,
@@ -329,6 +335,7 @@ class Script {
       dependsOn: dependsOn,
       noDependsOn: noDependsOn,
       updatedSince: updatedSince,
+      diff: diff,
       includePrivatePackages: includePrivatePackages,
       published: published,
       nullSafe: nullSafe,