Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

avoid digesting the rsc classpath on each iteration #7837

Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 25 additions & 12 deletions src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,23 @@ def _rsc(self):
return Rsc.global_instance()

@memoized_property
def _rsc_classpath(self):
def _rsc_absolute_classpath_paths(self):
return self.tool_classpath('rsc')

@memoized_property
def _rsc_classpath_snapshot(self):
cp_rel = fast_relpath_collection(self._rsc_absolute_classpath_paths)
cp_globs = PathGlobsAndRoot(
PathGlobs(tuple(cp_rel)),
text_type(get_buildroot()))
cp_snapshot, = self.context._scheduler.capture_snapshots((cp_globs,))
# TODO: A Snapshot can have `.files` and `.dirs` in arbitrary order, which doesn't allow
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The files and dirs in a Snapshot are sorted post #7241.

# recovering the original ordering, which will change classpaths in arbitrary ways. This means
# using a `Digest`ed classpath requires separately passing along the original (relativized)
# classpath paths. Individual `ClasspathEntry`s can be created, but those only allow a single
# path per digest, which isn't possible when digesting multiple paths at once, as we do here.
return (cp_rel, cp_snapshot)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider using a ClasspathEntry for this?


# TODO: allow @memoized_method to convert lists into tuples so they can be hashed!
@memoized_property
def _nailgunnable_combined_classpath(self):
Expand All @@ -206,7 +220,7 @@ def _nailgunnable_combined_classpath(self):
#7092). We still invoke their classpaths separately when not using nailgun, however.
"""
cp = []
cp.extend(self._rsc_classpath)
cp.extend(self._rsc_absolute_classpath_paths)
# Add zinc's classpath so that it can be invoked from the same nailgun instance.
cp.extend(super(RscCompile, self).get_zinc_compiler_classpath())
return cp
Expand Down Expand Up @@ -577,37 +591,36 @@ def create_compile_context(self, target, target_workdir):
))

def _runtool_hermetic(self, main, tool_name, args, distribution, tgt=None, input_files=tuple(), input_digest=None, output_dir=None):
tool_classpath_abs = self._rsc_classpath
tool_classpath = fast_relpath_collection(tool_classpath_abs)
tool_cp_rel, tool_cp_snapshot = self._rsc_classpath_snapshot

jvm_options = self._jvm_options

if self._rsc.use_native_image:
#jvm_options = []
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(self.options_scope, safe_shlex_join(jvm_options))
)
native_image_path, native_image_snapshot = self._rsc.native_image(self.context)
additional_snapshots = [native_image_snapshot]
# The native-image already has the tool classpath baked into it, taking the place of the java
# -cp option.
tool_snapshots = [native_image_snapshot]
initial_args = [native_image_path]
else:
# TODO(#6071): Our ExecuteProcessRequest expects a specific string type for arguments,
# which py2 doesn't default to. This can be removed when we drop python 2.
str_jvm_options = [text_type(opt) for opt in self.get_options().jvm_options]
additional_snapshots = []
tool_snapshots = [tool_cp_snapshot]
initial_args = [
distribution.java,
] + str_jvm_options + [
'-cp', os.pathsep.join(tool_classpath),
'-cp', os.pathsep.join(tool_cp_rel),
main,
]

cmd = initial_args + args

pathglobs = list(tool_classpath)
pathglobs.extend(f if os.path.isfile(f) else '{}/**'.format(f) for f in input_files)
pathglobs = list(f if os.path.isfile(f) else '{}/**'.format(f) for f in input_files)

if pathglobs:
root = PathGlobsAndRoot(
Expand All @@ -619,7 +632,7 @@ def _runtool_hermetic(self, main, tool_name, args, distribution, tgt=None, input
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest,) if path_globs_input_digest else ())
+ ((input_digest,) if input_digest else ())
+ tuple(s.directory_digest for s in additional_snapshots))
+ tuple(s.directory_digest for s in tool_snapshots))

epr = ExecuteProcessRequest(
argv=tuple(cmd),
Expand Down Expand Up @@ -686,7 +699,7 @@ def _runtool(self, args, distribution,
main, tool_name, args, distribution,
tgt=tgt, input_files=input_files, input_digest=input_digest, output_dir=output_dir),
self.SUBPROCESS: lambda: self._runtool_nonhermetic(
wu, self._rsc_classpath, main, tool_name, args, distribution),
wu, self._rsc_absolute_classpath_paths, main, tool_name, args, distribution),
self.NAILGUN: lambda: self._runtool_nonhermetic(
wu, self._nailgunnable_combined_classpath, main, tool_name, args, distribution),
})()
Expand Down