Skip to content

Commit ce7abaf

Browse files
tinder-maxwellelliotttinder-maxwellelliott
authored
Merge pull request #52 from Tinder/maxwelle/hash_all_sourcefile_targets
Add new `-a` flag to hash all source files
2 parents bfef159 + bd9165b commit ce7abaf

File tree

5 files changed

+116
-60
lines changed

5 files changed

+116
-60
lines changed

‎integration/integration_test.sh‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ starting_hashes_json="$output_dir/starting_hashes.json"
1111
final_hashes_json="$output_dir/final_hashes.json"
1212
impacted_targets_path="$output_dir/impacted_targets.txt"
1313
shared_flags="--config=verbose"
14+
command_options="--incompatible_restrict_string_escapes=false"
1415

1516
export USE_BAZEL_VERSION=last_downstream_green
1617

@@ -68,13 +69,13 @@ container_image(
6869
EOF
6970
}
7071

71-
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json
72+
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json -co $command_options
7273

73-
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $modified_filepaths_output $final_hashes_json
74+
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $modified_filepaths_output $final_hashes_json -co $command_options
7475

7576
awk '{gsub(/:StringGenerator.java": \"\w+\"/,"modifiedhash");print}' $final_hashes_json > /dev/null
7677

77-
$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)"
78+
$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)" -co $command_options
7879

7980
IFS=$'\n' read -d '' -r -a impacted_targets < $impacted_targets_path
8081
target1="//test/java/com/integration:bazel-diff-integration-test-lib"
@@ -104,15 +105,15 @@ docker_modified_filepaths_output=/tmp/docker_modified_filepaths_output.txt
104105

105106
createDockerBuildfileContent wget
106107

107-
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json
108+
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json -co $command_options
108109

109110
createDockerBuildfileContent curl
110111

111112
echo "BUILD.bazel" >> $docker_modified_filepaths_output
112113

113-
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $docker_modified_filepaths_output $final_hashes_json
114+
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $docker_modified_filepaths_output $final_hashes_json -co $command_options
114115

115-
$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)"
116+
$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)" -co $command_options
116117

117118
IFS=$'\n' read -d '' -r -a impacted_targets < $impacted_targets_path
118119
target1="//:base_packages"

‎src/main/java/com/bazel_diff/BazelClient.java‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ interface BazelClient {
2222
List<BazelTarget> queryAllTargets() throws IOException;
2323
Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery) throws IOException;
2424
Set<BazelSourceFileTarget> convertFilepathsToSourceTargets(Set<Path> filepaths) throws IOException, NoSuchAlgorithmException;
25+
Set<BazelSourceFileTarget> queryAllSourcefileTargets() throws IOException, NoSuchAlgorithmException;
2526
}
2627

2728
class BazelClientImpl implements BazelClient {
@@ -72,21 +73,32 @@ public Set<BazelSourceFileTarget> convertFilepathsToSourceTargets(Set<Path> file
7273
.stream()
7374
.map(path -> String.format("'%s'", path.toString()))
7475
.collect(Collectors.joining(" + "));
75-
List<Build.Target> targets = performBazelQuery(targetQuery);
76-
for (Build.Target target : targets) {
77-
Build.SourceFile sourceFile = target.getSourceFile();
78-
if (sourceFile != null) {
79-
MessageDigest digest = MessageDigest.getInstance("SHA-256");
80-
digest.update(sourceFile.getNameBytes().toByteArray());
81-
for (String subinclude : sourceFile.getSubincludeList()) {
82-
digest.update(subinclude.getBytes());
83-
}
84-
BazelSourceFileTargetImpl sourceFileTarget = new BazelSourceFileTargetImpl(
85-
sourceFile.getName(),
86-
digest.digest().clone()
87-
);
88-
sourceTargets.add(sourceFileTarget);
76+
sourceTargets.addAll(processBazelSourcefileTargets(performBazelQuery(targetQuery), false));
77+
}
78+
return sourceTargets;
79+
}
80+
81+
@Override
82+
public Set<BazelSourceFileTarget> queryAllSourcefileTargets() throws IOException, NoSuchAlgorithmException {
83+
return processBazelSourcefileTargets(performBazelQuery("kind('source file', deps(//...))"), true);
84+
}
85+
86+
private Set<BazelSourceFileTarget> processBazelSourcefileTargets(List<Build.Target> targets, Boolean readSourcefileTargets) throws IOException, NoSuchAlgorithmException {
87+
Set<BazelSourceFileTarget> sourceTargets = new HashSet<>();
88+
for (Build.Target target : targets) {
89+
Build.SourceFile sourceFile = target.getSourceFile();
90+
if (sourceFile != null) {
91+
MessageDigest digest = MessageDigest.getInstance("SHA-256");
92+
digest.update(sourceFile.getNameBytes().toByteArray());
93+
for (String subinclude : sourceFile.getSubincludeList()) {
94+
digest.update(subinclude.getBytes());
8995
}
96+
BazelSourceFileTargetImpl sourceFileTarget = new BazelSourceFileTargetImpl(
97+
sourceFile.getName(),
98+
digest.digest().clone(),
99+
readSourcefileTargets ? workingDirectory : null
100+
);
101+
sourceTargets.add(sourceFileTarget);
90102
}
91103
}
92104
return sourceTargets;

‎src/main/java/com/bazel_diff/BazelSourceFileTarget.java‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package com.bazel_diff;
22

3+
import java.io.ByteArrayOutputStream;
4+
import java.io.File;
5+
import java.nio.file.Path;
6+
import java.nio.file.Files;
37
import java.security.MessageDigest;
48
import java.security.NoSuchAlgorithmException;
9+
import java.io.IOException;
510

611
interface BazelSourceFileTarget {
712
String getName();
@@ -13,9 +18,22 @@ class BazelSourceFileTargetImpl implements BazelSourceFileTarget {
1318
private String name;
1419
private byte[] digest;
1520

16-
BazelSourceFileTargetImpl(String name, byte[] digest) {
21+
BazelSourceFileTargetImpl(String name, byte[] digest, Path workingDirectory) throws IOException {
1722
this.name = name;
18-
this.digest = digest;
23+
if (workingDirectory != null && name.startsWith("//")) {
24+
String filenameSubstring = name.substring(2);
25+
String filenamePath = filenameSubstring.replaceFirst(":", "/");
26+
File sourceFile = new File(workingDirectory.toString(), filenamePath);
27+
if (sourceFile.canRead()) {
28+
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
29+
outputStream.write(Files.readAllBytes(sourceFile.toPath()));
30+
outputStream.write(digest);
31+
this.digest = outputStream.toByteArray();
32+
outputStream.close();
33+
}
34+
} else {
35+
this.digest = digest;
36+
}
1937
}
2038

2139
@Override

‎src/main/java/com/bazel_diff/TargetHashingClient.java‎

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
interface TargetHashingClient {
1111
Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths) throws IOException, NoSuchAlgorithmException;
12+
Map<String, String> hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException;
1213
Set<String> getImpactedTargets(Map<String, String> startHashes, Map<String, String> endHashes, String avoidQuery) throws IOException;
1314
}
1415

@@ -22,33 +23,13 @@ class TargetHashingClientImpl implements TargetHashingClient {
2223
@Override
2324
public Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths) throws IOException, NoSuchAlgorithmException {
2425
Set<BazelSourceFileTarget> bazelSourcefileTargets = bazelClient.convertFilepathsToSourceTargets(modifiedFilepaths);
25-
List<BazelTarget> allTargets = bazelClient.queryAllTargets();
26-
Map<String, String> targetHashes = new HashMap<>();
27-
Map<String, byte[]> ruleHashes = new HashMap<>();
28-
Map<String, BazelRule> allRulesMap = new HashMap<>();
29-
for (BazelTarget target : allTargets) {
30-
String targetName = getNameForTarget(target);
31-
if (targetName == null || !target.hasRule()) {
32-
continue;
33-
}
34-
allRulesMap.put(targetName, target.getRule());
35-
}
36-
for (BazelTarget target : allTargets) {
37-
String targetName = getNameForTarget(target);
38-
if (targetName == null) {
39-
continue;
40-
}
41-
byte[] targetDigest = createDigestForTarget(
42-
target,
43-
allRulesMap,
44-
bazelSourcefileTargets,
45-
ruleHashes
46-
);
47-
if (targetDigest != null) {
48-
targetHashes.put(targetName, convertByteArrayToString(targetDigest));
49-
}
50-
}
51-
return targetHashes;
26+
return hashAllTargets(bazelSourcefileTargets);
27+
}
28+
29+
@Override
30+
public Map<String, String> hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException {
31+
Set<BazelSourceFileTarget> bazelSourcefileTargets = bazelClient.queryAllSourcefileTargets();
32+
return hashAllTargets(bazelSourcefileTargets);
5233
}
5334

5435
@Override
@@ -152,4 +133,34 @@ private String getNameForTarget(BazelTarget target) {
152133
}
153134
return null;
154135
}
136+
137+
private Map<String, String> hashAllTargets(Set<BazelSourceFileTarget> bazelSourcefileTargets) throws IOException, NoSuchAlgorithmException {
138+
List<BazelTarget> allTargets = bazelClient.queryAllTargets();
139+
Map<String, String> targetHashes = new HashMap<>();
140+
Map<String, byte[]> ruleHashes = new HashMap<>();
141+
Map<String, BazelRule> allRulesMap = new HashMap<>();
142+
for (BazelTarget target : allTargets) {
143+
String targetName = getNameForTarget(target);
144+
if (targetName == null || !target.hasRule()) {
145+
continue;
146+
}
147+
allRulesMap.put(targetName, target.getRule());
148+
}
149+
for (BazelTarget target : allTargets) {
150+
String targetName = getNameForTarget(target);
151+
if (targetName == null) {
152+
continue;
153+
}
154+
byte[] targetDigest = createDigestForTarget(
155+
target,
156+
allRulesMap,
157+
bazelSourcefileTargets,
158+
ruleHashes
159+
);
160+
if (targetDigest != null) {
161+
targetHashes.put(targetName, convertByteArrayToString(targetDigest));
162+
}
163+
}
164+
return targetHashes;
165+
}
155166
}

‎src/main/java/com/bazel_diff/main.java‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import java.util.stream.Collectors;
1919

2020
import static picocli.CommandLine.*;
21-
2221
@Command(
2322
name = "modified-filepaths",
2423
mixinStandardHelpOptions = true,
@@ -82,8 +81,16 @@ class GenerateHashes implements Callable<Integer> {
8281
@ParentCommand
8382
private BazelDiff parent;
8483

85-
@Option(names = {"-m", "--modifiedFilepaths"}, description = "The path to a file containing the list of modified filepaths in the workspace, you can use the 'modified-filepaths' command to get this list")
86-
File modifiedFilepaths;
84+
@ArgGroup(exclusive = true)
85+
Exclusive exclusive;
86+
87+
static class Exclusive {
88+
@Option(names = {"-m", "--modifiedFilepaths"}, description = "The path to a file containing the list of modified filepaths in the workspace, you can use the 'modified-filepaths' command to get this list")
89+
File modifiedFilepaths;
90+
91+
@Option(names = {"-a", "--all-sourcefiles"}, description = "Experimental: Hash all sourcefile targets (instead of relying on --modifiedFilepaths), Warning: Performance may degrade from reading all source files")
92+
Boolean hashAllSourcefiles;
93+
}
8794

8895
@Parameters(index = "0", description = "The filepath to write the resulting JSON of dictionary target => SHA-256 values")
8996
File outputPath;
@@ -96,15 +103,22 @@ public Integer call() {
96103
try {
97104
gitClient.ensureAllChangesAreCommitted();
98105
Set<Path> modifiedFilepathsSet = new HashSet<>();
99-
if (modifiedFilepaths != null) {
100-
FileReader fileReader = new FileReader(modifiedFilepaths);
101-
BufferedReader bufferedReader = new BufferedReader(fileReader);
102-
modifiedFilepathsSet = bufferedReader
103-
.lines()
104-
.map( line -> new File(line).toPath())
105-
.collect(Collectors.toSet());
106+
Map<String, String> hashes = new HashMap<>();
107+
if (exclusive != null) {
108+
if (exclusive.modifiedFilepaths != null) {
109+
FileReader fileReader = new FileReader(exclusive.modifiedFilepaths);
110+
BufferedReader bufferedReader = new BufferedReader(fileReader);
111+
modifiedFilepathsSet = bufferedReader
112+
.lines()
113+
.map( line -> new File(line).toPath())
114+
.collect(Collectors.toSet());
115+
hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet);
116+
} else if (exclusive.hashAllSourcefiles) {
117+
hashes = hashingClient.hashAllBazelTargetsAndSourcefiles();
118+
}
119+
} else {
120+
hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet);
106121
}
107-
Map<String, String> hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet);
108122
Gson gson = new GsonBuilder().setPrettyPrinting().create();
109123
FileWriter myWriter = new FileWriter(outputPath);
110124
myWriter.write(gson.toJson(hashes));

0 commit comments

Comments
 (0)