Merge pull request #716 from zcash/707-in-situ-dev-workflow

Add harness for `nightly -> beta -> stable` dev workflow

.github/workflows/ci.yml

@@ -4,11 +4,19 @@ on: [push, pull_request]
 
 jobs:
   test:
-    name: Test on ${{ matrix.os }}
+    name: Test on ${{ matrix.os }}${{ matrix.name_suffix }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        stage: [stable, beta, nightly]
         os: [ubuntu-latest, windows-latest, macOS-latest]
+        include:
+          - stage: beta
+            name_suffix: " with beta features"
+            extra_flags: --features beta
+          - stage: nightly
+            name_suffix: " with nightly features"
+            extra_flags: --features nightly
 
     steps:
       - uses: actions/checkout@v3
@@ -20,7 +28,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --verbose --release --workspace --features batch,dev-graph,gadget-traces,test-dependencies,unstable
+          args: --verbose --release --workspace --features batch,dev-graph,gadget-traces,test-dependencies ${{ matrix.extra_flags }}
 
   build:
     name: Build target ${{ matrix.target }}
@@ -43,7 +51,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --features batch,dev-graph,gadget-traces,unstable --target ${{ matrix.target }}
+          args: --features batch,dev-graph,gadget-traces,nightly --target ${{ matrix.target }}
 
   bitrot:
     name: Bitrot check
@@ -104,7 +112,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: tarpaulin
-          args: --features batch,dev-graph,gadget-traces,test-dependencies,unstable --timeout 600 --out Xml
+          args: --features batch,dev-graph,gadget-traces,test-dependencies,nightly --timeout 600 --out Xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3.1.1
 

book/src/SUMMARY.md

@@ -13,6 +13,8 @@
   - [Gadgets](user/gadgets.md)
   - [Tips and tricks](user/tips-and-tricks.md)
   - [WASM Guide](user/wasm-port.md)
+- [Developer Documentation](dev.md)
+  - [Feature development](dev/features.md)
 - [Design](design.md)
   - [Proving system](design/proving-system.md)
     - [Lookup argument](design/proving-system/lookup.md)

book/src/dev.md

@@ -0,0 +1,6 @@
+# Developer Documentation
+
+You want to contribute to the Halo 2 crates? Awesome!
+
+This section covers information about our development processes and review standards, and
+useful tips for maintaining and extending the codebase.

book/src/dev/features.md

@@ -0,0 +1,52 @@
+# Feature development
+
+Sometimes feature development can require iterating on a design over time. It can be
+useful to start using features in downstream crates early on to gain experience with the
+APIs and functionality, that can feed back into the feature's design prior to it being
+stabilised. To enable this, we follow a three-stage `nightly -> beta -> stable`
+development pattern inspired by (but not identical to) the Rust compiler.
+
+## Feature flags
+
+Each unstabilised feature has a default-off feature flag that enables it, of the form
+`unstable-*`. The stable API of the crates must not be affected when the feature flag is
+disabled, except for specific complex features that will be considered on a case-by-case
+basis.
+
+Two meta-flags are provided to enable all features at a particular stabilisation level:
+- `beta` enables all features at the "beta" stage (and implicitly all features at the
+  "stable" stage).
+- `nightly` enables all features at the "beta" and "nightly" stages (and implicitly all
+  features at the "stable" stage), i.e. all features are enabled.
+- When neither flag is enabled (and no feature-specific flags are enabled), then in effect
+  only features at the "stable" stage are enabled.
+
+## Feature workflow
+
+- If the maintainers have rough consensus that an experimental feature is generally
+  desired, its initial implementation can be merged into the codebase optimistically
+  behind a feature-specific feature flag with a lower standard of review. The feature's
+  flag is added to the `nightly` feature flag set.
+  - The feature will become usable by downstream published crates in the next general
+    release of the `halo2` crates.
+  - Subsequent development and refinement of the feature can be performed in-situ via
+    additional PRs, along with additional review.
+  - If the feature ends up having bad interactions with other features (in particular,
+    already-stabilised features), then it can be removed later without affecting the
+    stable or beta APIs.
+- Once the feature has had sufficient review, and is at the point where a `halo2` user
+  considers it production-ready (and is willing or planning to deploy it to production),
+  the feature's feature flag is moved to the `beta` feature flag set.
+- Once the feature has had review equivalent to the stable review policy, and there is
+  rough consensus that the feature is useful to the wider `halo2` userbase, the feature's
+  feature flag is removed and the feature becomes part of the main maintained codebase.
+
+> For more complex features, the above workflow might be augmented with `beta` and
+> `nightly` branches; this will be figured out once a feature requiring this is proposed
+> as a candidate for inclusion.
+
+## In-progress features
+
+| Feature flag | Stage | Notes |
+| --- | --- | --- |
+| `unstable-sha256-gadget` | `nightly` | The SHA-256 gadget and chip.

halo2_gadgets/Cargo.toml

@@ -55,7 +55,19 @@ test-dev-graph = [
     "plotters/bitmap_encoder",
 ]
 test-dependencies = ["proptest"]
-unstable = []
+
+# In-development features
+# See https://zcash.github.io/halo2/dev/features.html
+beta = [
+    "halo2_proofs/beta",
+]
+nightly = [
+    "beta",
+    "halo2_proofs/nightly",
+    "unstable-sha256-gadget",
+]
+unstable-sha256-gadget = []
+# Add flags for in-development features above this line.
 
 [[bench]]
 name = "primitives"
@@ -68,4 +80,4 @@ harness = false
 [[bench]]
 name = "sha256"
 harness = false
-required-features = ["unstable"]
+required-features = ["unstable-sha256-gadget"]

halo2_gadgets/src/lib.rs

@@ -23,8 +23,8 @@
 
 pub mod ecc;
 pub mod poseidon;
-#[cfg(feature = "unstable")]
-#[cfg_attr(docsrs, doc(cfg(feature = "unstable")))]
+#[cfg(feature = "unstable-sha256-gadget")]
+#[cfg_attr(docsrs, doc(cfg(feature = "unstable-sha256-gadget")))]
 pub mod sha256;
 pub mod sinsemilla;
 pub mod utilities;

halo2_proofs/Cargo.toml

@@ -78,6 +78,15 @@ gadget-traces = ["backtrace"]
 sanity-checks = []
 batch = ["rand_core/getrandom"]
 
+# In-development features
+# See https://zcash.github.io/halo2/dev/features.html
+beta = [
+]
+nightly = [
+    "beta",
+]
+# Add flags for in-development features above this line.
+
 [lib]
 bench = false