fs.Digest is declared in Rust

src/python/pants/engine/fs.py

@@ -6,6 +6,7 @@
 from typing import TYPE_CHECKING, Iterable, Optional, Tuple, Union
 
 from pants.engine.collection import Collection
+from pants.engine.internals.native_engine import PyDigest
 from pants.engine.rules import QueryRule, side_effecting
 from pants.option.global_options import GlobMatchErrorBehavior as GlobMatchErrorBehavior
 from pants.util.meta import frozen_after_init
@@ -14,16 +15,12 @@
     from pants.engine.internals.scheduler import SchedulerSession
 
 
-@dataclass(frozen=True)
-class Digest:
-    """A Digest is a lightweight reference to a set of files known about by the engine.
-
-    You can use `await Get(Snapshot, Digest)` to set the file names referred to, or use `await
-    Get(DigestContents, Digest)` to see the actual file content.
-    """
+"""A Digest is a lightweight reference to a set of files known about by the engine.
 
-    fingerprint: str
-    serialized_bytes_length: int
+You can use `await Get(Snapshot, Digest)` to set the file names referred to, or use `await
+Get(DigestContents, Digest)` to see the actual file content.
+"""
+Digest = PyDigest
 
 
 @dataclass(frozen=True)

src/python/pants/engine/internals/BUILD

@@ -23,7 +23,10 @@ python_integration_tests(
 
 python_library(
   name='native',
-  sources=['native.py'],
+  sources=[
+    'native.py',
+    'native_engine.pyi',
+  ],
   dependencies=[
     ':native_engine',
   ],

src/python/pants/engine/internals/native.py

@@ -10,7 +10,7 @@
 from pants.base.exiter import ExitCode
 from pants.engine.fs import PathGlobs
 from pants.engine.internals import native_engine
-from pants.engine.internals.native_engine import (  # type: ignore[import]
+from pants.engine.internals.native_engine import (
     PyExecutionRequest,
     PyExecutionStrategyOptions,
     PyExecutor,
@@ -71,18 +71,18 @@ def generator_send(
             if isinstance(res, Get):
                 # Get.
                 return PyGeneratorResponseGet(
-                    res.output_type,
-                    res.input_type,
-                    res.input,
+                    product=res.output_type,
+                    declared_subject=res.input_type,
+                    subject=res.input,
                 )
             elif type(res) in (tuple, list):
                 # GetMulti.
                 return PyGeneratorResponseGetMulti(
-                    tuple(
+                    gets=tuple(
                         PyGeneratorResponseGet(
-                            get.output_type,
-                            get.input_type,
-                            get.input,
+                            product=get.output_type,
+                            declared_subject=get.input_type,
+                            subject=get.input,
                         )
                         for get in res
                     )
@@ -94,7 +94,7 @@ def generator_send(
                 raise
             # This was a `return` from a coroutine, as opposed to a `StopIteration` raised
             # by calling `next()` on an empty iterator.
-            return PyGeneratorResponseBreak(e.value)
+            return PyGeneratorResponseBreak(val=e.value)
 
 
 class RawFdRunner(Protocol):
@@ -211,11 +211,11 @@ def new_session(
         session_values: SessionValues,
     ) -> PySession:
         return PySession(
-            scheduler,
-            dynamic_ui,
-            build_id,
-            should_report_workunits,
-            session_values,
+            scheduler=scheduler,
+            should_render_ui=dynamic_ui,
+            build_id=build_id,
+            should_report_workunits=should_report_workunits,
+            session_values=session_values,
         )
 
     def new_scheduler(
@@ -266,20 +266,23 @@ def new_scheduler(
             local_enable_nailgun=execution_options.process_execution_local_enable_nailgun,
         )
 
-        return self.lib.scheduler_create(
-            self._executor,
-            tasks,
-            types,
-            # Project tree.
-            build_root,
-            local_store_dir,
-            local_execution_root_dir,
-            named_caches_dir,
-            ca_certs_path,
-            ignore_patterns,
-            use_gitignore,
-            remoting_options,
-            exec_stategy_opts,
+        return cast(
+            PyScheduler,
+            self.lib.scheduler_create(
+                self._executor,
+                tasks,
+                types,
+                # Project tree.
+                build_root,
+                local_store_dir,
+                local_execution_root_dir,
+                named_caches_dir,
+                ca_certs_path,
+                ignore_patterns,
+                use_gitignore,
+                remoting_options,
+                exec_stategy_opts,
+            ),
         )
 
     def set_panic_handler(self):

src/python/pants/engine/internals/native_engine.pyi

@@ -0,0 +1,48 @@
+from typing import Any
+
+# TODO: black and flake8 disagree about the content of this file:
+#   see https://github.com/psf/black/issues/1548
+# flake8: noqa: E302
+
+class PyDigest:
+    def __init__(self, fingerprint: str, serialized_bytes_length: int) -> None: ...
+    @property
+    def fingerpint(self) -> str: ...
+    @property
+    def serialized_bytes_length(self) -> int: ...
+
+class PyExecutionRequest:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyExecutionStrategyOptions:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyExecutor:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyGeneratorResponseBreak:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyGeneratorResponseGet:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyGeneratorResponseGetMulti:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyNailgunServer:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyRemotingOptions:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyScheduler:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PySession:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyTasks:
+    def __init__(self, **kwargs: Any) -> None: ...
+
+class PyTypes:
+    def __init__(self, **kwargs: Any) -> None: ...

src/python/pants/engine/internals/scheduler.py

@@ -30,7 +30,7 @@
     RemovePrefix,
     Snapshot,
 )
-from pants.engine.internals.native_engine import PyTypes  # type: ignore[import]
+from pants.engine.internals.native_engine import PyTypes
 from pants.engine.internals.nodes import Return, Throw
 from pants.engine.internals.selectors import Params
 from pants.engine.internals.session import SessionValues
@@ -130,7 +130,6 @@ def __init__(
         tasks = self._register_rules(rule_index, union_membership)
 
         types = PyTypes(
-            directory_digest=Digest,
             file_digest=FileDigest,
             snapshot=Snapshot,
             paths=Paths,

src/rust/engine/hashing/Cargo.toml

@@ -6,6 +6,7 @@ authors = [ "Pants Build <[email protected]>" ]
 publish = false
 
 [dependencies]
+byteorder = "1.3"
 digest = "0.9"
 generic-array = "0.14"
 hex = "0.3.1"

src/rust/engine/hashing/src/lib.rs

@@ -27,6 +27,7 @@
 // Arc<Mutex> can be more clear than needing to grok Orderings:
 #![allow(clippy::mutex_atomic)]
 
+use byteorder::ByteOrder;
 use digest::consts::U32;
 use generic_array::GenericArray;
 use serde::de::{MapAccess, Visitor};
@@ -85,6 +86,14 @@ impl Fingerprint {
     }
     s
   }
+
+  ///
+  /// Using the fact that a Fingerprint is computed using a strong hash function, computes a strong
+  /// but short hash value from a prefix.
+  ///
+  pub fn prefix_hash(&self) -> u64 {
+    byteorder::BigEndian::read_u64(&self.0)
+  }
 }
 
 impl fmt::Display for Fingerprint {

src/rust/engine/src/externs/fs.rs

@@ -0,0 +1,84 @@
+// Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+#![deny(warnings)]
+// Enable all clippy lints except for many of the pedantic ones. It's a shame this needs to be copied and pasted across crates, but there doesn't appear to be a way to include inner attributes from a common source.
+#![deny(
+  clippy::all,
+  clippy::default_trait_access,
+  clippy::expl_impl_clone_on_copy,
+  clippy::if_not_else,
+  clippy::needless_continue,
+  clippy::unseparated_literal_suffix,
+// TODO: Falsely triggers for async/await:
+//   see https://github.com/rust-lang/rust-clippy/issues/5360
+// clippy::used_underscore_binding
+)]
+// It is often more clear to show that nothing is being moved.
+#![allow(clippy::match_ref_pats)]
+// Subjective style.
+#![allow(
+  clippy::len_without_is_empty,
+  clippy::redundant_field_names,
+  clippy::too_many_arguments
+)]
+// Default isn't as big a deal as people seem to think it is.
+#![allow(clippy::new_without_default, clippy::new_ret_no_self)]
+// Arc<Mutex> can be more clear than needing to grok Orderings:
+#![allow(clippy::mutex_atomic)]
+// File-specific allowances to silence internal warnings of `py_class!`.
+#![allow(
+  unused_braces,
+  clippy::used_underscore_binding,
+  clippy::transmute_ptr_to_ptr,
+  clippy::zero_ptr
+)]
+
+use std::borrow::Cow;
+
+use cpython::{
+  exc, py_class, py_class_call_slot_impl_with_ref, py_class_prop_getter, CompareOp, PyErr,
+  PyObject, PyResult, Python, PythonObject, ToPyObject,
+};
+use hashing::{Digest, Fingerprint};
+
+/// TODO: See https://github.com/dgrunwald/rust-cpython/issues/242
+pub fn new_py_digest(digest: Digest) -> PyResult<PyDigest> {
+  let gil = Python::acquire_gil();
+  PyDigest::create_instance(gil.python(), digest)
+}
+
+py_class!(pub class PyDigest |py| {
+    data digest: Digest;
+    def __new__(_cls, fingerprint: Cow<str>, serialized_bytes_length: usize) -> PyResult<Self> {
+      let fingerprint = Fingerprint::from_hex_string(&fingerprint)
+        .map_err(|e| PyErr::new::<exc::Exception, _>(py, (e,)))?;
+      Self::create_instance(py, Digest(fingerprint, serialized_bytes_length))
+    }
+
+    @property def fingerprint(&self) -> PyResult<String> {
+      Ok(self.digest(py).0.to_hex())
+    }
+
+    @property def serialized_bytes_length(&self) -> PyResult<usize> {
+      Ok(self.digest(py).1)
+    }
+
+    def __richcmp__(&self, other: PyDigest, op: CompareOp) -> PyResult<PyObject> {
+      match op {
+        CompareOp::Eq => {
+          let res = self.digest(py) == other.digest(py);
+          Ok(res.to_py_object(py).into_object())
+        },
+        CompareOp::Ne => {
+          let res = self.digest(py) != other.digest(py);
+          Ok(res.to_py_object(py).into_object())
+        }
+        _ => Ok(py.NotImplemented()),
+      }
+    }
+
+    def __hash__(&self) -> PyResult<u64> {
+      Ok(self.digest(py).0.prefix_hash())
+    }
+});

src/rust/engine/src/externs/interface.rs

@@ -381,6 +381,8 @@ py_module_initializer!(native_engine, |py, m| {
   m.add_class::<externs::PyGeneratorResponseGet>(py)?;
   m.add_class::<externs::PyGeneratorResponseGetMulti>(py)?;
 
+  m.add_class::<externs::fs::PyDigest>(py)?;
+
   Ok(())
 });
 
@@ -396,7 +398,6 @@ py_class!(class PyTypes |py| {
 
   def __new__(
       _cls,
-      directory_digest: PyType,
       file_digest: PyType,
       snapshot: PyType,
       paths: PyType,
@@ -420,7 +421,7 @@ py_class!(class PyTypes |py| {
     Self::create_instance(
         py,
         RefCell::new(Some(Types {
-        directory_digest: externs::type_for(directory_digest),
+        directory_digest: externs::type_for(py.get_type::<externs::fs::PyDigest>()),
         file_digest: externs::type_for(file_digest),
         snapshot: externs::type_for(snapshot),
         paths: externs::type_for(paths),
@@ -539,14 +540,14 @@ py_class!(class PyRemotingOptions |py| {
 py_class!(class PySession |py| {
     data session: Session;
     def __new__(_cls,
-          scheduler_ptr: PyScheduler,
+          scheduler: PyScheduler,
           should_render_ui: bool,
           build_id: String,
           should_report_workunits: bool,
           session_values: PyObject
     ) -> CPyResult<Self> {
       Self::create_instance(py, Session::new(
-          scheduler_ptr.scheduler(py),
+          scheduler.scheduler(py),
           should_render_ui,
           build_id,
           should_report_workunits,

src/rust/engine/src/externs/mod.rs

@@ -9,6 +9,7 @@
 )]
 
 pub mod engine_aware;
+pub mod fs;
 mod interface;
 #[cfg(test)]
 mod interface_tests;