parent
df43e721e3
commit
a031b09190
|
@ -7,6 +7,7 @@ use crate::cluster_info::FULLNODE_PORT_RANGE;
|
||||||
use crate::contact_info::ContactInfo;
|
use crate::contact_info::ContactInfo;
|
||||||
use crate::service::Service;
|
use crate::service::Service;
|
||||||
use crate::streamer;
|
use crate::streamer;
|
||||||
|
use rand::{thread_rng, Rng};
|
||||||
use solana_client::thin_client::{create_client, ThinClient};
|
use solana_client::thin_client::{create_client, ThinClient};
|
||||||
use solana_sdk::pubkey::Pubkey;
|
use solana_sdk::pubkey::Pubkey;
|
||||||
use solana_sdk::signature::{Keypair, KeypairUtil};
|
use solana_sdk::signature::{Keypair, KeypairUtil};
|
||||||
|
@ -107,6 +108,7 @@ pub fn discover(
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Creates a ThinClient per valid node
|
||||||
pub fn get_clients(nodes: &[ContactInfo]) -> Vec<ThinClient> {
|
pub fn get_clients(nodes: &[ContactInfo]) -> Vec<ThinClient> {
|
||||||
nodes
|
nodes
|
||||||
.iter()
|
.iter()
|
||||||
|
@ -115,6 +117,16 @@ pub fn get_clients(nodes: &[ContactInfo]) -> Vec<ThinClient> {
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Creates a ThinClient by selecting a valid node at random
|
||||||
|
pub fn get_client(nodes: &[ContactInfo]) -> ThinClient {
|
||||||
|
let nodes: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(ContactInfo::valid_client_facing_addr)
|
||||||
|
.collect();
|
||||||
|
let select = thread_rng().gen_range(0, nodes.len());
|
||||||
|
create_client(nodes[select], FULLNODE_PORT_RANGE)
|
||||||
|
}
|
||||||
|
|
||||||
fn spy(
|
fn spy(
|
||||||
spy_ref: Arc<RwLock<ClusterInfo>>,
|
spy_ref: Arc<RwLock<ClusterInfo>>,
|
||||||
num_nodes: Option<usize>,
|
num_nodes: Option<usize>,
|
||||||
|
|
|
@ -206,8 +206,9 @@ impl Replicator {
|
||||||
&exit,
|
&exit,
|
||||||
);
|
);
|
||||||
|
|
||||||
info!("Looking for leader at {:?}", cluster_entrypoint);
|
info!("Connecting to the cluster via {:?}", cluster_entrypoint);
|
||||||
crate::gossip_service::discover_nodes(&cluster_entrypoint.gossip, 1)?;
|
let nodes = crate::gossip_service::discover_nodes(&cluster_entrypoint.gossip, 1)?;
|
||||||
|
let client = crate::gossip_service::get_client(&nodes);
|
||||||
|
|
||||||
let (storage_blockhash, storage_slot) = Self::poll_for_blockhash_and_slot(&cluster_info)?;
|
let (storage_blockhash, storage_slot) = Self::poll_for_blockhash_and_slot(&cluster_info)?;
|
||||||
|
|
||||||
|
@ -242,30 +243,33 @@ impl Replicator {
|
||||||
&Hash::default(),
|
&Hash::default(),
|
||||||
);
|
);
|
||||||
|
|
||||||
let client = create_client(cluster_entrypoint.client_facing_addr(), FULLNODE_PORT_RANGE);
|
|
||||||
Self::setup_mining_account(&client, &keypair, &storage_keypair)?;
|
Self::setup_mining_account(&client, &keypair, &storage_keypair)?;
|
||||||
|
|
||||||
let mut thread_handles =
|
let mut thread_handles =
|
||||||
create_request_processor(node.sockets.storage.unwrap(), &exit, slot);
|
create_request_processor(node.sockets.storage.unwrap(), &exit, slot);
|
||||||
|
|
||||||
// receive blobs from retransmit and drop them.
|
// receive blobs from retransmit and drop them.
|
||||||
let exit2 = exit.clone();
|
let t_retransmit = {
|
||||||
let t_retransmit = spawn(move || loop {
|
let exit = exit.clone();
|
||||||
|
spawn(move || loop {
|
||||||
let _ = retransmit_receiver.recv_timeout(Duration::from_secs(1));
|
let _ = retransmit_receiver.recv_timeout(Duration::from_secs(1));
|
||||||
if exit2.load(Ordering::Relaxed) {
|
if exit.load(Ordering::Relaxed) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
});
|
})
|
||||||
|
};
|
||||||
thread_handles.push(t_retransmit);
|
thread_handles.push(t_retransmit);
|
||||||
|
|
||||||
let exit3 = exit.clone();
|
let t_replicate = {
|
||||||
let blocktree1 = blocktree.clone();
|
let exit = exit.clone();
|
||||||
let t_replicate = spawn(move || loop {
|
let blocktree = blocktree.clone();
|
||||||
Self::wait_for_ledger_download(slot, &blocktree1, &exit3, &node_info, &cluster_info);
|
spawn(move || loop {
|
||||||
if exit3.load(Ordering::Relaxed) {
|
Self::wait_for_ledger_download(slot, &blocktree, &exit, &node_info, &cluster_info);
|
||||||
|
if exit.load(Ordering::Relaxed) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
});
|
})
|
||||||
|
};
|
||||||
|
//always push this last
|
||||||
thread_handles.push(t_replicate);
|
thread_handles.push(t_replicate);
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
@ -291,6 +295,8 @@ impl Replicator {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(&mut self) {
|
pub fn run(&mut self) {
|
||||||
|
info!("waiting for ledger download");
|
||||||
|
self.thread_handles.pop().unwrap().join().unwrap();
|
||||||
self.encrypt_ledger()
|
self.encrypt_ledger()
|
||||||
.expect("ledger encrypt not successful");
|
.expect("ledger encrypt not successful");
|
||||||
loop {
|
loop {
|
||||||
|
@ -310,7 +316,7 @@ impl Replicator {
|
||||||
node_info: &ContactInfo,
|
node_info: &ContactInfo,
|
||||||
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
||||||
) {
|
) {
|
||||||
info!("window created, waiting for ledger download done");
|
info!("window created, waiting for ledger download");
|
||||||
let mut _received_so_far = 0;
|
let mut _received_so_far = 0;
|
||||||
|
|
||||||
let mut current_slot = start_slot;
|
let mut current_slot = start_slot;
|
||||||
|
|
|
@ -36,6 +36,9 @@ else
|
||||||
program=${BASH_REMATCH[1]}
|
program=${BASH_REMATCH[1]}
|
||||||
features+="cuda,"
|
features+="cuda,"
|
||||||
fi
|
fi
|
||||||
|
if [[ $program = replicator ]]; then
|
||||||
|
features+="chacha,"
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -r "$SOLANA_ROOT/$program"/Cargo.toml ]]; then
|
if [[ -r "$SOLANA_ROOT/$program"/Cargo.toml ]]; then
|
||||||
maybe_package="--package solana-$program"
|
maybe_package="--package solana-$program"
|
||||||
|
@ -60,6 +63,7 @@ solana_gossip=$(solana_program gossip)
|
||||||
solana_keygen=$(solana_program keygen)
|
solana_keygen=$(solana_program keygen)
|
||||||
solana_ledger_tool=$(solana_program ledger-tool)
|
solana_ledger_tool=$(solana_program ledger-tool)
|
||||||
solana_wallet=$(solana_program wallet)
|
solana_wallet=$(solana_program wallet)
|
||||||
|
solana_replicator=$(solana_program replicator)
|
||||||
|
|
||||||
export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
|
export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
|
||||||
export RUST_BACKTRACE=1
|
export RUST_BACKTRACE=1
|
||||||
|
|
|
@ -15,9 +15,11 @@ fullnode_usage() {
|
||||||
echo
|
echo
|
||||||
fi
|
fi
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
|
|
||||||
|
Fullnode Usage:
|
||||||
usage: $0 [--blockstream PATH] [--init-complete-file FILE] [--label LABEL] [--stake LAMPORTS] [--no-voting] [--rpc-port port] [rsync network path to bootstrap leader configuration] [cluster entry point]
|
usage: $0 [--blockstream PATH] [--init-complete-file FILE] [--label LABEL] [--stake LAMPORTS] [--no-voting] [--rpc-port port] [rsync network path to bootstrap leader configuration] [cluster entry point]
|
||||||
|
|
||||||
Start a full node
|
Start a full node or a replicator
|
||||||
|
|
||||||
--blockstream PATH - open blockstream at this unix domain socket location
|
--blockstream PATH - open blockstream at this unix domain socket location
|
||||||
--init-complete-file FILE - create this file, if it doesn't already exist, once node initialization is complete
|
--init-complete-file FILE - create this file, if it doesn't already exist, once node initialization is complete
|
||||||
|
@ -106,9 +108,22 @@ ledger_not_setup() {
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setup_replicator_account() {
|
||||||
|
declare entrypoint_ip=$1
|
||||||
|
declare node_keypair_path=$2
|
||||||
|
declare stake=$3
|
||||||
|
|
||||||
|
if [[ -f "$node_keypair_path".configured ]]; then
|
||||||
|
echo "Replicator account has already been configured"
|
||||||
|
else
|
||||||
|
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop "$stake" || return $?
|
||||||
|
touch "$node_keypair_path".configured
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
args=()
|
args=()
|
||||||
bootstrap_leader=false
|
node_type=validator
|
||||||
stake=42 # number of lamports to assign as stake by default
|
stake=42 # number of lamports to assign as stake
|
||||||
poll_for_new_genesis_block=0
|
poll_for_new_genesis_block=0
|
||||||
label=
|
label=
|
||||||
fullnode_keypair_path=
|
fullnode_keypair_path=
|
||||||
|
@ -120,7 +135,10 @@ while [[ -n $1 ]]; do
|
||||||
label="-$2"
|
label="-$2"
|
||||||
shift 2
|
shift 2
|
||||||
elif [[ $1 = --bootstrap-leader ]]; then
|
elif [[ $1 = --bootstrap-leader ]]; then
|
||||||
bootstrap_leader=true
|
node_type=bootstrap_leader
|
||||||
|
shift
|
||||||
|
elif [[ $1 = --replicator ]]; then
|
||||||
|
node_type=replicator
|
||||||
shift
|
shift
|
||||||
elif [[ $1 = --poll-for-new-genesis-block ]]; then
|
elif [[ $1 = --poll-for-new-genesis-block ]]; then
|
||||||
poll_for_new_genesis_block=1
|
poll_for_new_genesis_block=1
|
||||||
|
@ -169,7 +187,7 @@ while [[ -n $1 ]]; do
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
if $bootstrap_leader; then
|
if [[ $node_type = bootstrap_leader ]]; then
|
||||||
if [[ ${#positional_args[@]} -ne 0 ]]; then
|
if [[ ${#positional_args[@]} -ne 0 ]]; then
|
||||||
fullnode_usage "Unknown argument: ${positional_args[0]}"
|
fullnode_usage "Unknown argument: ${positional_args[0]}"
|
||||||
fi
|
fi
|
||||||
|
@ -187,6 +205,32 @@ if $bootstrap_leader; then
|
||||||
default_arg --rpc-port 8899
|
default_arg --rpc-port 8899
|
||||||
default_arg --rpc-drone-address 127.0.0.1:9900
|
default_arg --rpc-drone-address 127.0.0.1:9900
|
||||||
default_arg --gossip-port 8001
|
default_arg --gossip-port 8001
|
||||||
|
|
||||||
|
elif [[ $node_type = replicator ]]; then
|
||||||
|
|
||||||
|
if [[ ${#positional_args[@]} -gt 2 ]]; then
|
||||||
|
fullnode_usage "Unknown arguments for replicator"
|
||||||
|
fi
|
||||||
|
|
||||||
|
read -r entrypoint entrypoint_address shift < <(find_entrypoint "${positional_args[@]}")
|
||||||
|
shift "$shift"
|
||||||
|
|
||||||
|
replicator_keypair_path=$SOLANA_CONFIG_DIR/replicator-id.json
|
||||||
|
replicator_storage_keypair_path="$SOLANA_CONFIG_DIR"/replicator-vote-id.json
|
||||||
|
ledger_config_dir=$SOLANA_CONFIG_DIR/replicator-ledger
|
||||||
|
|
||||||
|
mkdir -p "$SOLANA_CONFIG_DIR"
|
||||||
|
[[ -r "$replicator_keypair_path" ]] || $solana_keygen -o "$replicator_keypair_path"
|
||||||
|
[[ -r "$replicator_storage_keypair_path" ]] || $solana_keygen -o "$replicator_storage_keypair_path"
|
||||||
|
|
||||||
|
replicator_keypair=$($solana_keygen pubkey "$replicator_keypair_path")
|
||||||
|
replicator_storage_keypair=$($solana_keygen pubkey "$replicator_storage_keypair_path")
|
||||||
|
|
||||||
|
default_arg --entrypoint "$entrypoint_address"
|
||||||
|
default_arg --identity "$replicator_keypair_path"
|
||||||
|
default_arg --storage_id "$replicator_storage_keypair_path"
|
||||||
|
default_arg --ledger "$ledger_config_dir"
|
||||||
|
|
||||||
else
|
else
|
||||||
if [[ ${#positional_args[@]} -gt 2 ]]; then
|
if [[ ${#positional_args[@]} -gt 2 ]]; then
|
||||||
fullnode_usage "$@"
|
fullnode_usage "$@"
|
||||||
|
@ -208,10 +252,24 @@ else
|
||||||
default_arg --rpc-drone-address "${entrypoint_address%:*}:9900"
|
default_arg --rpc-drone-address "${entrypoint_address%:*}:9900"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
fullnode_keypair=$($solana_keygen pubkey "$fullnode_keypair_path")
|
|
||||||
fullnode_vote_keypair=$($solana_keygen pubkey "$fullnode_vote_keypair_path")
|
|
||||||
|
|
||||||
cat <<EOF
|
if [[ $node_type = replicator ]]; then
|
||||||
|
cat <<EOF
|
||||||
|
======================[ Replicator configuration ]======================
|
||||||
|
replicator pubkey: $replicator_keypair
|
||||||
|
storage pubkey: $replicator_storage_keypair
|
||||||
|
ledger: $ledger_config_dir
|
||||||
|
======================================================================
|
||||||
|
EOF
|
||||||
|
|
||||||
|
program=$solana_replicator
|
||||||
|
|
||||||
|
else
|
||||||
|
|
||||||
|
fullnode_keypair=$($solana_keygen pubkey "$fullnode_keypair_path")
|
||||||
|
fullnode_vote_keypair=$($solana_keygen pubkey "$fullnode_vote_keypair_path")
|
||||||
|
|
||||||
|
cat <<EOF
|
||||||
======================[ Fullnode configuration ]======================
|
======================[ Fullnode configuration ]======================
|
||||||
node pubkey: $fullnode_keypair
|
node pubkey: $fullnode_keypair
|
||||||
vote pubkey: $fullnode_vote_keypair
|
vote pubkey: $fullnode_vote_keypair
|
||||||
|
@ -220,29 +278,33 @@ accounts: $accounts_config_dir
|
||||||
======================================================================
|
======================================================================
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
|
||||||
|
default_arg --identity "$fullnode_keypair_path"
|
||||||
|
default_arg --voting-keypair "$fullnode_vote_keypair_path"
|
||||||
|
default_arg --vote-account "$fullnode_vote_keypair"
|
||||||
|
default_arg --ledger "$ledger_config_dir"
|
||||||
|
default_arg --accounts "$accounts_config_dir"
|
||||||
|
|
||||||
|
|
||||||
|
if [[ -n $SOLANA_CUDA ]]; then
|
||||||
|
program=$solana_fullnode_cuda
|
||||||
|
else
|
||||||
|
program=$solana_fullnode
|
||||||
|
fi
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -z $CI ]]; then # Skip in CI
|
if [[ -z $CI ]]; then # Skip in CI
|
||||||
# shellcheck source=scripts/tune-system.sh
|
# shellcheck source=scripts/tune-system.sh
|
||||||
source "$here"/../scripts/tune-system.sh
|
source "$here"/../scripts/tune-system.sh
|
||||||
fi
|
fi
|
||||||
|
|
||||||
default_arg --identity "$fullnode_keypair_path"
|
|
||||||
default_arg --voting-keypair "$fullnode_vote_keypair_path"
|
|
||||||
default_arg --vote-account "$fullnode_vote_keypair"
|
|
||||||
default_arg --ledger "$ledger_config_dir"
|
|
||||||
default_arg --accounts "$accounts_config_dir"
|
|
||||||
|
|
||||||
if [[ -n $SOLANA_CUDA ]]; then
|
|
||||||
program=$solana_fullnode_cuda
|
|
||||||
else
|
|
||||||
program=$solana_fullnode
|
|
||||||
fi
|
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
secs_to_next_genesis_poll=0
|
secs_to_next_genesis_poll=0
|
||||||
PS4="$(basename "$0"): "
|
PS4="$(basename "$0"): "
|
||||||
while true; do
|
while true; do
|
||||||
if [[ ! -d "$SOLANA_RSYNC_CONFIG_DIR"/ledger ]]; then
|
if [[ ! -d "$SOLANA_RSYNC_CONFIG_DIR"/ledger ]]; then
|
||||||
if $bootstrap_leader; then
|
if [[ $node_type = bootstrap_leader ]]; then
|
||||||
ledger_not_setup "$SOLANA_RSYNC_CONFIG_DIR/ledger does not exist"
|
ledger_not_setup "$SOLANA_RSYNC_CONFIG_DIR/ledger does not exist"
|
||||||
fi
|
fi
|
||||||
rsync_entrypoint_url=$(rsync_url "$entrypoint")
|
rsync_entrypoint_url=$(rsync_url "$entrypoint")
|
||||||
|
@ -256,8 +318,10 @@ while true; do
|
||||||
|
|
||||||
trap '[[ -n $pid ]] && kill "$pid" >/dev/null 2>&1 && wait "$pid"' INT TERM ERR
|
trap '[[ -n $pid ]] && kill "$pid" >/dev/null 2>&1 && wait "$pid"' INT TERM ERR
|
||||||
|
|
||||||
if ! $bootstrap_leader && ((stake)); then
|
if [[ $node_type = validator ]] && ((stake)); then
|
||||||
setup_vote_account "${entrypoint_address%:*}" "$fullnode_keypair_path" "$fullnode_vote_keypair_path" "$stake"
|
setup_vote_account "${entrypoint_address%:*}" "$fullnode_keypair_path" "$fullnode_vote_keypair_path" "$stake"
|
||||||
|
elif [[ $node_type = replicator ]] && ((stake)); then
|
||||||
|
setup_replicator_account "${entrypoint_address%:*}" "$replicator_keypair_path" "$stake"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "$PS4$program ${args[*]}"
|
echo "$PS4$program ${args[*]}"
|
||||||
|
@ -265,7 +329,7 @@ while true; do
|
||||||
pid=$!
|
pid=$!
|
||||||
oom_score_adj "$pid" 1000
|
oom_score_adj "$pid" 1000
|
||||||
|
|
||||||
if $bootstrap_leader; then
|
if [[ $node_type = bootstrap_leader ]]; then
|
||||||
wait "$pid"
|
wait "$pid"
|
||||||
sleep 1
|
sleep 1
|
||||||
else
|
else
|
||||||
|
@ -286,10 +350,10 @@ while true; do
|
||||||
|
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "############## New genesis detected, restarting fullnode ##############"
|
echo "############## New genesis detected, restarting $node_type ##############"
|
||||||
kill "$pid" || true
|
kill "$pid" || true
|
||||||
wait "$pid" || true
|
wait "$pid" || true
|
||||||
rm -rf "$ledger_config_dir" "$accounts_config_dir" "$fullnode_vote_keypair_path".configured
|
rm -rf "$ledger_config_dir" "$accounts_config_dir" "$fullnode_vote_keypair_path".configured "$replicator_storage_keypair_path".configured
|
||||||
sleep 60 # give the network time to come back up
|
sleep 60 # give the network time to come back up
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Start a relpicator
|
||||||
|
#
|
||||||
|
|
||||||
|
here=$(dirname "$0")
|
||||||
|
exec "$here"/fullnode.sh --replicator "$@"
|
||||||
|
|
||||||
|
|
|
@ -39,6 +39,15 @@ fn main() {
|
||||||
.required(true)
|
.required(true)
|
||||||
.help("use DIR as persistent ledger location"),
|
.help("use DIR as persistent ledger location"),
|
||||||
)
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("storage_keypair")
|
||||||
|
.short("s")
|
||||||
|
.long("storage_id")
|
||||||
|
.value_name("DIR")
|
||||||
|
.takes_value(true)
|
||||||
|
.required(true)
|
||||||
|
.help("File containing the storage account keypair"),
|
||||||
|
)
|
||||||
.get_matches();
|
.get_matches();
|
||||||
|
|
||||||
let ledger_path = matches.value_of("ledger").unwrap();
|
let ledger_path = matches.value_of("ledger").unwrap();
|
||||||
|
@ -51,6 +60,14 @@ fn main() {
|
||||||
} else {
|
} else {
|
||||||
Keypair::new()
|
Keypair::new()
|
||||||
};
|
};
|
||||||
|
let storage_keypair = if let Some(storage_keypair) = matches.value_of("storage_keypair") {
|
||||||
|
read_keypair(storage_keypair).unwrap_or_else(|err| {
|
||||||
|
eprintln!("{}: Unable to open keypair file: {}", err, storage_keypair);
|
||||||
|
exit(1);
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
Keypair::new()
|
||||||
|
};
|
||||||
|
|
||||||
let entrypoint_addr = matches
|
let entrypoint_addr = matches
|
||||||
.value_of("entrypoint")
|
.value_of("entrypoint")
|
||||||
|
@ -74,13 +91,12 @@ fn main() {
|
||||||
);
|
);
|
||||||
|
|
||||||
let entrypoint_info = ContactInfo::new_gossip_entry_point(&entrypoint_addr);
|
let entrypoint_info = ContactInfo::new_gossip_entry_point(&entrypoint_addr);
|
||||||
let storage_keypair = Arc::new(Keypair::new());
|
|
||||||
let mut replicator = Replicator::new(
|
let mut replicator = Replicator::new(
|
||||||
ledger_path,
|
ledger_path,
|
||||||
node,
|
node,
|
||||||
entrypoint_info,
|
entrypoint_info,
|
||||||
Arc::new(keypair),
|
Arc::new(keypair),
|
||||||
storage_keypair,
|
Arc::new(storage_keypair),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue