Test multi node dynamic network ci (#696)

Buildkite automation for multinode test.  This test is ignored by default because it requires a large cpu machine to run.

ci/buildkite.yml

@@ -23,6 +23,13 @@ steps:
         limit: 2
     agents:
       - "queue=cuda"
+  - command: "ci/test-large-network.sh"
+    name: "large-network [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable"
+    timeout_in_minutes: 20
+    agents:
+      - "queue=large"
   - command: "ci/pr-snap.sh"
     timeout_in_minutes: 20
     name: "snap [public]"

ci/test-large-network.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+cd "$here"/..
+export RUST_BACKTRACE=1
+export RUST_LOG=multinode=info
+set -x
+exec cargo test --release test_multi_node_dynamic_network -- --ignored