noisymime
/
chia-blockchain
mirror of https://github.com/noisymime/chia-blockchain.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add `chia init` and refactor out many global references to the config directory (#153) 

* Tweak subcommand template.
* Don't hack the .chia/beta-* directory when invoking `chia`.
* Get `chia init` working. Factor out all global paths.
* Reverse order of migration bases.
* Update README.
This commit is contained in:
Richard Kiss 2020-04-08 00:47:17 -07:00 committed by GitHub
parent e952579884
commit 6ded806fa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 375 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
README.md
View File

@ -12,18 +12,23 @@ For testnet most should only install harvesters, farmers, plotter and full nodes
All data is now stored in the $CHIA_ROOT environment variable. or ~/.chia/VERSION-DIR/ if unset. You can find databases, keys, plots, logs here. You can set $CHIA_ROOT to the .chia directory in your home directory with `export CHIA_ROOT=~/.chia`.
## Step 1: Install the code
## Install the code
To install chia-blockchain, follow [these install](INSTALL.md) instructions according to your operating system. This only supports 64 bit operating systems.
Remember that once you complete your install you **must be in the Python virtual environment** which you access from the chia-blockchain directory with the command `. ./activate`. (Or . ./venv/bin/activate). Both dots are critical and once executed correctly your cli prompt will look something like `(venv) username@machine:~$` with the (venv) prepended. Use `deactivate` should you want to exit the venv.
## Step 2: Generate keys
## Migrate or set up configuration files
```bash
chia init
```
## Generate keys
First, create some keys by running the following script:
```bash
chia-generate-keys
```
## Step 3a: Run a full node + wallet
## Run a full node + wallet
To run a full node on port 8444, and connect to the testnet, run the following command.
If you want to see std::out log output, modify the logging.std_out variable in ./config/config.yaml.
@ -38,7 +43,7 @@ chia-start-wallet-server &
```
And then run `chia.exe` from the unzipped `chia-win32-x64` directory in Windows (not Ubuntu/WSL 2.)
## Step 3b: Run a farmer + full node + wallet
## Run a farmer + full node + wallet
Instead of running only a full node (as in 3a), you can also run a farmer.
Farmers are entities in the network who use their hard drive space to try to create
blocks (like Bitcoin's miners), and earn block rewards. First, you must generate some hard drive plots, which
@ -60,7 +65,7 @@ chia-start-wallet-server &
And then run `chia.exe` from the unzipped `chia-win32-x64` directory in Windows (not Ubuntu/WSL 2.)
## Step 3c: Run a timelord + full node + wallet
## Run a timelord + full node + wallet
*Note*
If you want to run a timelord on Linux, see LINUX_TIMELORD.md.

5
src/cmds/check_plots.py
View File

@ -7,10 +7,9 @@ from chiapos import DiskProver, Verifier
from src.types.proof_of_space import ProofOfSpace
from src.types.sized_bytes import bytes32
from src.util.config import load_config
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.hash import std_hash
from src.util.path import path_from_root
chia_root = path_from_root()
plot_config_filename = "plots.yaml"
@ -36,7 +35,7 @@ def main():
)
if not Path(plot_filename).exists():
# Tries relative path
full_path: Path = chia_root / plot_filename
full_path: Path = DEFAULT_ROOT_PATH / plot_filename
if not full_path.exists():
# Tries absolute path
full_path = Path(plot_filename)

9
src/cmds/cli.py
View File

@ -1,11 +1,10 @@
import argparse
import importlib
from src.util.config import load_config
from src import __version__
SUBCOMMANDS = ["show", "version"]
SUBCOMMANDS = ["init", "show", "version"]
def create_parser():
@ -23,17 +22,13 @@ def create_parser():
for subcommand in SUBCOMMANDS:
mod = importlib.import_module("src.cmds.%s" % subcommand)
f = getattr(mod, "%s_parser" % subcommand)
f(subparsers.add_parser(subcommand))
mod.make_parser(subparsers.add_parser(subcommand))
parser.set_defaults(function=lambda args, parser: parser.print_help())
return parser
def chia(args, parser):
# a hack to generate the config.yaml file if missing
load_config("config.yaml")
return args.function(args, parser)

18
src/cmds/create_plots.py
View File

@ -7,7 +7,8 @@ from blspy import PrivateKey, PublicKey
from chiapos import DiskPlotter
from src.types.proof_of_space import ProofOfSpace
from src.types.sized_bytes import bytes32
from src.util.config import load_config, save_config
from src.util.config import config_path_for_filename, load_config, save_config
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.path import make_path_relative, mkdir, path_from_root
@ -15,8 +16,9 @@ def main():
"""
Script for creating plots and adding them to the plot config file.
"""
plot_config_filename = path_from_root() / "config" / "plots.yaml"
key_config_filename = path_from_root() / "config" / "keys.yaml"
root_path = DEFAULT_ROOT_PATH
plot_config_filename = config_path_for_filename(root_path, "plots.yaml")
key_config_filename = config_path_for_filename(root_path, "keys.yaml")
parser = argparse.ArgumentParser(description="Chia plotting script.")
parser.add_argument("-k", "--size", help="Plot size", type=int, default=20)
@ -36,6 +38,7 @@ def main():
)
new_plots_root = path_from_root(
root_path,
load_config("config.yaml").get("harvester", {}).get("new_plot_root", "plots")
)
parser.add_argument(
@ -99,7 +102,7 @@ def main():
print(f"Plot {filename} already exists")
# Updates the config if necessary.
plot_config = load_config(plot_config_filename)
plot_config = load_config(root_path, plot_config_filename)
plot_config_plots_new = deepcopy(plot_config.get("plots", []))
relative_path = make_path_relative(full_path)
if relative_path not in plot_config_plots_new:
@ -110,8 +113,11 @@ def main():
plot_config["plots"].update(plot_config_plots_new)
# Dumps the new config to disk.
save_config(plot_config_filename, plot_config)
tmp_dir.rmdir()
save_config(root_path, plot_config_filename, plot_config)
try:
tmp_dir.rmdir()
except Exception:
print(f"warning: couldn't delete {tmp_dir}")
if __name__ == "__main__":

19
src/cmds/generate_keys.py
View File

@ -4,11 +4,9 @@ from secrets import token_bytes
from blspy import PrivateKey, ExtendedPrivateKey
from src.consensus.coinbase import create_puzzlehash_for_pk
from src.types.BLSSignature import BLSPublicKey
from src.util.config import load_config, save_config, str2bool
from src.util.path import mkdir, path_from_root
key_config_filename = path_from_root() / "config" / "keys.yaml"
from src.util.config import config_path_for_filename, load_config, save_config, str2bool
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.path import mkdir
def main():
@ -16,6 +14,8 @@ def main():
Allows replacing keys of farmer, harvester, and pool, all default to True.
"""
root_path = DEFAULT_ROOT_PATH
keys_yaml = "keys.yaml"
parser = argparse.ArgumentParser(description="Chia key generator script.")
parser.add_argument(
"-a",
@ -46,6 +46,7 @@ def main():
)
args = parser.parse_args()
key_config_filename = config_path_for_filename(root_path, keys_yaml)
if key_config_filename.exists():
# If the file exists, warn the user
yn = input(
@ -59,7 +60,7 @@ def main():
mkdir(key_config_filename.parent)
open(key_config_filename, "a").close()
key_config = load_config(key_config_filename)
key_config = load_config(root_path, keys_yaml)
if key_config is None:
key_config = {}
@ -71,12 +72,12 @@ def main():
)
key_config["wallet_sk"] = bytes(wallet_sk).hex()
key_config["wallet_target"] = wallet_target.hex()
save_config(key_config_filename, key_config)
save_config(root_path, keys_yaml, key_config)
if args.harvester:
# Replaces the harvester's sk seed. Used to generate plot private keys, which are
# used to sign farmed blocks.
key_config["sk_seed"] = token_bytes(32).hex()
save_config(key_config_filename, key_config)
save_config(root_path, keys_yaml, key_config)
if args.pool:
# Replaces the pools keys and targes. Only useful if running a pool, or doing
# solo farming. The pool target allows spending of the coinbase.
@ -89,7 +90,7 @@ def main():
pool_target = wallet_target
key_config["pool_sks"] = [bytes(pool_sk).hex() for pool_sk in pool_sks]
key_config["pool_target"] = pool_target.hex()
save_config(key_config_filename, key_config)
save_config(root_path, keys_yaml, key_config)
if __name__ == "__main__":

101
src/cmds/init.py Normal file
View File

@ -0,0 +1,101 @@
import os
import shutil
from pathlib import Path
from src.util.config import (
config_path_for_filename,
create_default_chia_config,
load_config,
save_config,
)
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.path import make_path_relative, mkdir, path_from_root
def make_parser(parser):
parser.set_defaults(function=init)
def migrate_from(old_root, new_root, manifest):
"""
Copy all the files in "manifest" to the new config directory.
"""
if old_root == new_root:
print(f"same as new path, exiting")
return 1
if not old_root.is_dir():
print(f"{old_root} not found")
return 0
print(f"\n{old_root} found")
print(f"Copying files from {old_root} to {new_root}\n")
for f in manifest:
old_path = old_root / f
new_path = new_root / f
if old_path.is_file():
print(f"{new_path}")
mkdir(new_path.parent)
shutil.copy(old_path, new_path)
else:
print(f"{old_path} not found, skipping")
# migrate plots
# for now, we simply leave them where they are
# and make what may have been relative paths absolute
plots_config = load_config(new_root, "plots.yaml")
old_plot_paths = plots_config.get("plots", [])
if len(old_plot_paths) == 0:
print("no plots found, no plots migrated")
return 1
print("\nmigrating plots.yaml")
new_plot_paths = {}
for path, values in old_plot_paths.items():
old_path = path_from_root(old_root, path)
new_plot_path = make_path_relative(old_path, new_root)
print(f"rewriting {path}\n as {new_plot_path}")
new_plot_paths[str(new_plot_path)] = values
plots_config["plots"] = new_plot_paths
save_config(new_root, "plots.yaml", new_plot_paths)
print("\nUpdated plots.yaml to point to where your existing plots are.")
print(
"\nYour plots have not been moved so be careful deleting old preferences folders."
)
print("If you want to move your plot files, you should also modify")
print(f"{config_path_for_filename(new_root, 'plots.yaml')}")
return 1
def init(args, parser):
return chia_init()
def chia_init():
root_path = DEFAULT_ROOT_PATH
print(f"migrating to {root_path}")
if root_path.is_dir():
print(f"{root_path} already exists, no action taken")
return -1
MANIFEST = [
"config/config.yaml",
"config/plots.yaml",
"config/keys.yaml",
"wallet/db/blockchain_wallet_v4.db",
"db/blockchain_v3.db",
]
PATH_MANIFEST_LIST = [
(Path(os.path.expanduser("~/.chia/beta-%s" % _)), MANIFEST) for _ in ["1.0b2", "1.0b1"]
]
for old_path, manifest in PATH_MANIFEST_LIST:
r = migrate_from(old_path, root_path, manifest)
if r:
break
else:
create_default_chia_config(root_path)
print("Please generate your keys with chia-generate-keys")
return 0

2
src/cmds/show.py
View File

@ -12,7 +12,7 @@ from src.util.byte_types import hexstr_to_bytes
from src.util.config import str2bool
def show_parser(parser):
def make_parser(parser):
parser.add_argument(
"-b",

2
src/cmds/version.py
View File

@ -5,6 +5,6 @@ def version(args, parser):
print(__version__)
def version_parser(parser):
def make_parser(parser):
parser.set_defaults(function=version)
return parser

7
src/harvester.py
View File

@ -11,6 +11,7 @@ from src.server.outbound_message import Delivery, Message, NodeType, OutboundMes
from src.types.proof_of_space import ProofOfSpace
from src.types.sized_bytes import bytes32
from src.util.api_decorators import api_request
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.ints import uint8
from src.util.path import path_from_root
@ -58,11 +59,11 @@ class Harvester:
use any plots which don't have one of the pool keys.
"""
for partial_filename_str, plot_config in self.plot_config["plots"].items():
plot_root = path_from_root(self.config.get("plot_root", "."))
partial_filename = path_from_root(partial_filename_str, plot_root)
plot_root = path_from_root(DEFAULT_ROOT_PATH, self.config.get("plot_root", "."))
partial_filename = plot_root / partial_filename_str
potential_filenames = [
partial_filename,
path_from_root(partial_filename_str, plot_root),
path_from_root(plot_root, partial_filename_str),
]
pool_pubkey = PublicKey.from_bytes(bytes.fromhex(plot_config["pool_pk"]))

3
src/rpc/rpc_server.py
View File

@ -12,7 +12,6 @@ from src.types.peer_info import PeerInfo
from src.util.ints import uint16, uint64
from src.types.sized_bytes import bytes32
from src.util.byte_types import hexstr_to_bytes
from src.util.config import load_config
class EnhancedJSONEncoder(json.JSONEncoder):
@ -145,7 +144,7 @@ class RpcApiHandler:
port = request_data["port"]
target_node: PeerInfo = PeerInfo(host, uint16(int(port)))
config = load_config("config.yaml", "full_node")
config = self.full_node.config
if self.full_node.server is None or not (
await self.full_node.server.start_client(target_node, None, config)
):

19
src/server/server.py
View File

@ -22,17 +22,22 @@ from src.server.outbound_message import Delivery, Message, NodeType, OutboundMes
from src.types.peer_info import PeerInfo
from src.types.sized_bytes import bytes32
from src.util import partial_func
from src.util.config import load_config
from src.util.errors import Err, ProtocolError
from src.util.ints import uint16
from src.util.network import create_node_id
import traceback
config = load_config("config.yaml")
class ChiaServer:
def __init__(self, port: int, api: Any, local_type: NodeType, name: str = None):
def __init__(
self,
port: int,
api: Any,
local_type: NodeType,
ping_interval: int,
network_id: str,
name: str = None,
):
# Keeps track of all connections to and from this node.
self.global_connections: PeerConnections = PeerConnections([])
@ -46,6 +51,8 @@ class ChiaServer:
self._api = api # API module that will be called from the requests
self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet)
self._ping_interval = ping_interval
self._network_id = network_id
# (StreamReader, StreamWriter, NodeType) aiter, gets things from server and clients and
# sends them through the pipeline
self._srwt_aiter: push_aiter = push_aiter()
@ -242,7 +249,7 @@ class ChiaServer:
self.push_message(
OutboundMessage(NodeType.HARVESTER, msg, Delivery.BROADCAST)
)
await asyncio.sleep(config["ping_interval"])
await asyncio.sleep(self._ping_interval)
return asyncio.create_task(ping())
@ -362,7 +369,7 @@ class ChiaServer:
outbound_handshake = Message(
"handshake",
Handshake(
config["network_id"],
self._network_id,
protocol_version,
self._node_id,
uint16(self._port),

13
src/server/start_farmer.py
View File

@ -11,15 +11,18 @@ from src.farmer import Farmer
from src.server.outbound_message import NodeType
from src.server.server import ChiaServer
from src.types.peer_info import PeerInfo
from src.util.logging import initialize_logging
from src.util.config import load_config, load_config_cli
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.logging import initialize_logging
from src.util.setproctitle import setproctitle
async def main():
config = load_config_cli("config.yaml", "farmer")
root_path = DEFAULT_ROOT_PATH
net_config = load_config(root_path, "config.yaml")
config = load_config_cli(root_path, "config.yaml", "farmer")
try:
key_config = load_config("keys.yaml")
key_config = load_config(root_path, "keys.yaml")
except FileNotFoundError:
raise RuntimeError("Keys not generated. Run chia-generate-keys")
initialize_logging("Farmer %(name)-25s", config["logging"])
@ -34,7 +37,9 @@ async def main():
full_node_peer = PeerInfo(
config["full_node_peer"]["host"], config["full_node_peer"]["port"]
)
server = ChiaServer(config["port"], farmer, NodeType.FARMER)
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
server = ChiaServer(config["port"], farmer, NodeType.FARMER, ping_interval, network_id)
asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all)
asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all)

9
src/server/start_full_node.py
View File

@ -23,20 +23,21 @@ from src.types.peer_info import PeerInfo
from src.full_node.coin_store import CoinStore
from src.util.logging import initialize_logging
from src.util.config import load_config_cli
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.path import mkdir, path_from_root
from src.util.pip_import import pip_import
from src.util.setproctitle import setproctitle
async def main():
config = load_config_cli("config.yaml", "full_node")
config = load_config_cli(DEFAULT_ROOT_PATH, "config.yaml", "full_node")
setproctitle("chia_full_node")
initialize_logging("FullNode %(name)-23s", config["logging"])
log = logging.getLogger(__name__)
server_closed = False
db_path = path_from_root(config["database_path"])
db_path = path_from_root(DEFAULT_ROOT_PATH, config["database_path"])
mkdir(db_path.parent)
# Create the store (DB) and full node instance
@ -72,7 +73,9 @@ async def main():
log.exception(f"UPnP failed")
# Starts the full node server (which full nodes can connect to)
server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE)
ping_interval = config.get("ping_interval")
network_id = config.get("network_id")
server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE, ping_interval, network_id)
full_node._set_server(server)
_ = await server.start_server(full_node._on_connect, config)
rpc_cleanup = None

13
src/server/start_harvester.py
View File

@ -11,15 +11,18 @@ from src.harvester import Harvester
from src.server.outbound_message import NodeType
from src.server.server import ChiaServer
from src.types.peer_info import PeerInfo
from src.util.logging import initialize_logging
from src.util.config import load_config, load_config_cli
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.logging import initialize_logging
from src.util.setproctitle import setproctitle
async def main():
config = load_config_cli("config.yaml", "harvester")
root_path = DEFAULT_ROOT_PATH
net_config = load_config(root_path, "config.yaml")
config = load_config_cli(root_path, "config.yaml", "harvester")
try:
plot_config = load_config("plots.yaml")
plot_config = load_config(root_path, "plots.yaml")
except FileNotFoundError:
raise RuntimeError("Plots not generated. Run chia-create-plots")
@ -28,7 +31,9 @@ async def main():
setproctitle("chia_harvester")
harvester = Harvester(config, plot_config)
server = ChiaServer(config["port"], harvester, NodeType.HARVESTER)
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
server = ChiaServer(config["port"], harvester, NodeType.HARVESTER, ping_interval, network_id)
_ = await server.start_server(None, config)
asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all)

12
src/server/start_introducer.py
View File

@ -10,20 +10,24 @@ except ImportError:
from src.introducer import Introducer
from src.server.outbound_message import NodeType
from src.server.server import ChiaServer
from src.util.config import load_config_cli, load_config
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.logging import initialize_logging
from src.util.config import load_config_cli
from src.util.setproctitle import setproctitle
async def main():
config = load_config_cli("config.yaml", "introducer")
root_path = DEFAULT_ROOT_PATH
net_config = load_config(root_path, "config.yaml")
config = load_config_cli(root_path, "config.yaml", "introducer")
initialize_logging("Introducer %(name)-21s", config["logging"])
log = logging.getLogger(__name__)
setproctitle("chia_introducer")
introducer = Introducer(config)
server = ChiaServer(config["port"], introducer, NodeType.INTRODUCER)
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
server = ChiaServer(config["port"], introducer, NodeType.INTRODUCER, ping_interval, network_id)
introducer.set_server(server)
_ = await server.start_server(None, config)

13
src/server/start_timelord.py
View File

@ -14,20 +14,25 @@ from src.server.outbound_message import NodeType
from src.server.server import ChiaServer
from src.timelord import Timelord
from src.types.peer_info import PeerInfo
from src.util.config import load_config_cli, load_config
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.logging import initialize_logging
from src.util.config import load_config_cli
from src.util.setproctitle import setproctitle
async def main():
config = load_config_cli("config.yaml", "timelord")
root_path = DEFAULT_ROOT_PATH
net_config = load_config(root_path, "config.yaml")
config = load_config_cli(root_path, "config.yaml", "timelord")
initialize_logging("Timelord %(name)-23s", config["logging"])
log = logging.getLogger(__name__)
setproctitle("chia_timelord")
timelord = Timelord(config, constants)
server = ChiaServer(config["port"], timelord, NodeType.TIMELORD)
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
server = ChiaServer(config["port"], timelord, NodeType.TIMELORD, ping_interval, network_id)
_ = await server.start_server(None, config)
timelord_shutdown_task: Optional[asyncio.Task] = None

15
src/simulator/start_simulator.py
View File

@ -20,20 +20,23 @@ from src.server.connection import NodeType
from src.types.full_block import FullBlock
from src.full_node.coin_store import CoinStore
from src.util.logging import initialize_logging
from src.util.config import load_config_cli
from src.util.config import load_config_cli, load_config
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.setproctitle import setproctitle
from src.util.path import mkdir, path_from_root
async def main():
config = load_config_cli("config.yaml", "full_node")
root_path = DEFAULT_ROOT_PATH
net_config = load_config(root_path, "config.yaml")
config = load_config_cli(root_path, "config.yaml", "full_node")
setproctitle("chia_full_node")
initialize_logging("FullNode %(name)-23s", config["logging"])
log = logging.getLogger(__name__)
server_closed = False
db_path = path_from_root(config["simulator_database_path"])
db_path = path_from_root(DEFAULT_ROOT_PATH, config["simulator_database_path"])
mkdir(db_path.parent)
connection = await aiosqlite.connect(db_path)
@ -60,8 +63,12 @@ async def main():
override_constants=test_constants,
)
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
# Starts the full node server (which full nodes can connect to)
server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE)
server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE, ping_interval, network_id)
full_node._set_server(server)
_ = await server.start_server(full_node._on_connect, config)
rpc_cleanup = None

19
src/timelord_launcher.py
View File

@ -7,17 +7,13 @@ from src.util.logging import initialize_logging
from src.util.config import load_config
from asyncio import Lock
from typing import List
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.setproctitle import setproctitle
config = load_config("config.yaml", "timelord_launcher")
active_processes: List = []
stopped = False
lock = Lock()
initialize_logging("Launcher %(name)-23s", config["logging"])
setproctitle("chia_timelord_launcher")
log = logging.getLogger(__name__)
@ -69,7 +65,7 @@ async def spawn_process(host, port, counter):
await asyncio.sleep(0.1)
async def spawn_all_processes():
async def spawn_all_processes(config):
await asyncio.sleep(15)
port = config["port"]
process_count = config["process_count"]
@ -77,7 +73,10 @@ async def spawn_all_processes():
await asyncio.gather(*awaitables)
if __name__ == "__main__":
def main():
setproctitle("chia_timelord_launcher")
config = load_config(DEFAULT_ROOT_PATH, "config.yaml", "timelord_launcher")
initialize_logging("Launcher %(name)-23s", config["logging"])
def signal_received():
asyncio.create_task(kill_processes())
@ -88,7 +87,11 @@ if __name__ == "__main__":
loop.add_signal_handler(signal.SIGTERM, signal_received)
try:
loop.run_until_complete(spawn_all_processes())
loop.run_until_complete(spawn_all_processes(config))
finally:
log.info("Launcher fully closed.")
loop.close()
if __name__ == "__main__":
main()

62
src/util/config.py
View File

@ -1,45 +1,65 @@
import yaml
import argparse
from pathlib import Path
import pkg_resources
import sys
import yaml
from pathlib import Path
from typing import Dict, Any, Callable, Optional, Union
from src.util.path import mkdir, path_from_root
from src.util.path import mkdir
def migrate_config_file(filename: Union[str, Path], path: Path) -> None:
filepath = Path(filename)
default_config_file = pkg_resources.resource_string(
__name__, f"initial-{filepath.parts[-1]}"
).decode()
mkdir(str(path.parent))
with open(path, "w") as f:
f.write(default_config_file)
def initial_config_file(filename: Union[str, Path]) -> str:
return pkg_resources.resource_string(__name__, f"initial-{filename}").decode()
def save_config(filename: Union[str, Path], config_data):
path = path_from_root("config") / filename
def create_default_chia_config(root_path: Path) -> None:
for filename in ["config.yaml"]:
default_config_file_data = initial_config_file(filename)
path = config_path_for_filename(root_path, filename)
mkdir(path.parent)
with open(path, "w") as f:
f.write(default_config_file_data)
save_config(root_path, "plots.yaml", {})
def config_path_for_filename(root_path: Path, filename: Union[str, Path]) -> Path:
path_filename = Path(filename)
if path_filename.is_absolute():
return path_filename
return root_path / "config" / filename
def save_config(root_path: Path, filename: Union[str, Path], config_data : Any):
path = config_path_for_filename(root_path, filename)
with open(path, "w") as f:
yaml.safe_dump(config_data, f)
def load_config(filename: Union[str, Path], sub_config: Optional[str] = None) -> Dict:
path = path_from_root("config") / filename
def load_config(
root_path: Path,
filename: Union[str, Path],
sub_config: Optional[str] = None
) -> Dict:
path = config_path_for_filename(root_path, filename)
if not path.is_file():
migrate_config_file(filename, path)
print(f"can't find {path}")
print("** please run `chia init` to migrate or create new config files **")
# TODO: fix this hack
sys.exit(-1)
r = yaml.safe_load(open(path, "r"))
if sub_config is not None:
return yaml.safe_load(open(path, "r"))[sub_config]
else:
return yaml.safe_load(open(path, "r"))
r = r.get(sub_config)
return r
def load_config_cli(filename: str, sub_config: Optional[str] = None) -> Dict:
def load_config_cli(root_path: Path, filename: str, sub_config: Optional[str]) -> Dict:
"""
Loads configuration from the specified filename, in the config directory,
and then overrides any properties using the passed in command line arguments.
Nested properties in the config file can be used in the command line with ".",
for example --farmer_peer.host. Does not support lists.
"""
config = load_config(filename, sub_config)
config = load_config(root_path, filename, sub_config)
flattened_props = flatten_properties(config)
parser = argparse.ArgumentParser()

10
src/util/default_root.py Normal file
View File

@ -0,0 +1,10 @@
import os
from pathlib import Path
from src import __version__
DEFAULT_ROOT_PATH = Path(
os.path.expanduser(
os.getenv("CHIA_ROOT", "~/.chia/beta-{version}").format(version=__version__)
)
).resolve()

3
src/util/logging.py
View File

@ -2,11 +2,12 @@ import logging
import colorlog
from typing import Dict
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.path import mkdir, path_from_root
def initialize_logging(prefix: str, logging_config: Dict):
log_path = path_from_root(logging_config.get("log_filename", "log/debug.log"))
log_path = path_from_root(DEFAULT_ROOT_PATH, logging_config.get("log_filename", "log/debug.log"))
mkdir(str(log_path.parent))
if logging_config["log_stdout"]:
handler = colorlog.StreamHandler()

41
src/util/path.py
View File

@ -1,53 +1,36 @@
import os
from typing import Union
from pathlib import Path
from src import __version__
DEFAULT_ROOT_PATH = Path(
os.path.expanduser(os.getenv("CHIA_ROOT", "~/.chia/beta-%s" % __version__))
).resolve()
def path_from_root(
path_str: Union[str, Path] = ".", root_path: Path = DEFAULT_ROOT_PATH
) -> Path:
def path_from_root(root: Path, path_str: Union[str, Path]) -> Path:
"""
Return a new path from the given one, expanding "~" if present.
If path is relative, prepend $CHIA_ROOT
If path is relative, prepend root
If path is absolute, return it directly.
This lets us use "~" and other absolute paths in config files.
"""
if isinstance(path_str, Path):
path = path_str
else:
path = Path(path_str)
root = Path(os.path.expanduser(str(root)))
path = Path(path_str)
if not path.is_absolute():
path = root_path / path
return path
path = root / path
return path.resolve()
def mkdir(path_str: Union[str, Path]) -> None:
"""
Create the existing directory (and its parents) if necessary.
"""
if isinstance(path_str, Path):
path = path_str
else:
path = Path(path_str)
path = Path(path_str)
path.mkdir(parents=True, exist_ok=True)
def make_path_relative(path_str: Union[str, Path]) -> Path:
def make_path_relative(path_str: Union[str, Path], root: Path) -> Path:
"""
Try to make the given path relative, given the default root.
"""
if isinstance(path_str, Path):
path = path_str
else:
path = Path(path_str)
path = Path(path_str)
try:
path = path.relative_to(DEFAULT_ROOT_PATH)
path = path.relative_to(root)
except ValueError:
pass
return path
return path.resolve()

3
src/wallet/wallet_node.py
View File

@ -28,6 +28,7 @@ from src.types.full_block import FullBlock
from src.types.coin import Coin, hash_coin_list
from src.full_node.blockchain import ReceiveBlockResult
from src.types.mempool_inclusion_status import MempoolInclusionStatus
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.errors import Err
from src.util.path import path_from_root, mkdir
@ -81,7 +82,7 @@ class WalletNode:
else:
self.log = logging.getLogger(__name__)
path = path_from_root(config["database_path"])
path = path_from_root(DEFAULT_ROOT_PATH, config["database_path"])
mkdir(path.parent)
self.wallet_state_manager = await WalletStateManager.create(

11
src/wallet/websocket_server.py
View File

@ -29,6 +29,7 @@ from src.wallet.util.wallet_types import WalletType
from src.wallet.wallet_info import WalletInfo
from src.wallet.wallet_node import WalletNode
from src.types.mempool_inclusion_status import MempoolInclusionStatus
from src.util.default_root import DEFAULT_ROOT_PATH
from src.util.setproctitle import setproctitle
# Timeout for response from wallet/full node for sending a transaction
@ -362,7 +363,9 @@ async def start_websocket_server():
Starts WalletNode, WebSocketServer, and ChiaServer
"""
config = load_config_cli("config.yaml", "wallet")
root_path = DEFAULT_ROOT_PATH
config = load_config_cli(root_path, "config.yaml", "wallet")
initialize_logging("Wallet %(name)-25s", config["logging"])
log = logging.getLogger(__name__)
log.info(f"Config : {config}")
@ -384,8 +387,12 @@ async def start_websocket_server():
handler = WebSocketServer(wallet_node, log)
wallet_node.wallet_state_manager.set_callback(handler.state_changed_callback)
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
log.info(f"Starting wallet server on port {config['port']}.")
server = ChiaServer(config["port"], wallet_node, NodeType.WALLET)
server = ChiaServer(config["port"], wallet_node, NodeType.WALLET, ping_interval, network_id)
wallet_node.set_server(server)
_ = await server.start_server(None, config)

23
tests/block_tools.py
View File

@ -1,14 +1,18 @@
import os
import sys
import time
from typing import Any, Dict, List, Tuple, Optional
import random
import blspy
from pathlib import Path
from typing import Any, Dict, List, Tuple, Optional
from blspy import PrependSignature, PrivateKey, PublicKey
from chiavdf import prove
from chiabip158 import PyBIP158
from chiapos import DiskPlotter, DiskProver
from src import __version__
from src.cmds.init import create_default_chia_config
from src.consensus import block_rewards, pot_iterations
from src.consensus.constants import constants
from src.consensus.pot_iterations import calculate_min_iters_from_iterations
@ -26,7 +30,7 @@ from src.types.sized_bytes import bytes32
from src.util.merkle_set import MerkleSet
from src.util.ints import uint8, uint32, uint64, uint128, int512
from src.util.hash import std_hash
from src.util.path import mkdir, path_from_root
from src.util.path import mkdir
from src.util.significant_bits import truncate_to_significant_bits
# Can't go much lower than 19, since plots start having no solutions
@ -49,12 +53,21 @@ fee_target = std_hash(bytes(wallet_sk.get_public_key()))
n_wesolowski = uint8(0)
TEST_ROOT_PATH = Path(
os.path.expanduser(
os.getenv("CHIA_ROOT", "~/.chia/beta-{version}-test").format(version=__version__)
)
).resolve()
class BlockTools:
"""
Tools to generate blocks for testing.
"""
def __init__(self):
def __init__(self, root_path : Path = TEST_ROOT_PATH):
create_default_chia_config(root_path)
self.root_path = root_path
self.plot_config: Dict = {"plots": {}}
self.pool_sk = pool_sk
self.wallet_sk = wallet_sk
@ -62,7 +75,7 @@ class BlockTools:
plot_seeds: List[bytes32] = [
ProofOfSpace.calculate_plot_seed(pool_pk, plot_pk) for plot_pk in plot_pks
]
self.plot_dir = path_from_root("tests") / "plots"
self.plot_dir = self.root_path / "plots"
mkdir(self.plot_dir)
self.filenames: List[str] = [
f"genesis-plots-{k}{std_hash(int.to_bytes(i, 4, 'big')).hex()}.dat"

5
tests/full_node/test_blockchain.py
View File

@ -5,18 +5,17 @@ from pathlib import Path
import aiosqlite
import pytest
from blspy import PrivateKey, PrependSignature
from blspy import PrivateKey
from src.full_node.blockchain import Blockchain, ReceiveBlockResult
from src.full_node.store import FullNodeStore
from src.types.full_block import FullBlock
from src.types.coin import Coin
from src.types.header import Header, HeaderData
from src.types.proof_of_space import ProofOfSpace
from src.full_node.coin_store import CoinStore
from src.util.ints import uint8, uint64
from src.consensus.constants import constants as consensus_constants
from tests.block_tools import BlockTools, plot_sks
from tests.block_tools import BlockTools
from src.util.errors import Err
from src.consensus.coinbase import create_coinbase_coin_and_signature
from src.types.sized_bytes import bytes32

2
tests/full_node/test_full_sync.py
View File

@ -6,7 +6,7 @@ import pytest
from src.types.peer_info import PeerInfo
from src.protocols import full_node_protocol
from src.util.ints import uint16
from tests.setup_nodes import setup_two_nodes, setup_node_and_wallet, test_constants, bt
from tests.setup_nodes import setup_two_nodes, test_constants, bt
@pytest.fixture(scope="module")

2
tests/full_node/test_transactions.py
View File

@ -4,7 +4,7 @@ from secrets import token_bytes
import pytest
from src.protocols import full_node_protocol
from src.simulator.simulator_protocol import FarmNewBlockProtocol, ReorgProtocol
from src.simulator.simulator_protocol import FarmNewBlockProtocol
from src.types.peer_info import PeerInfo
from src.util.ints import uint16, uint32
from tests.setup_nodes import setup_simulators_and_wallets

79
tests/setup_nodes.py
View File

@ -1,13 +1,12 @@
import signal
from typing import Any, Dict, Optional, Tuple, List
from pathlib import Path
import asyncio
from typing import Any, Dict, Tuple, List
from pathlib import Path
import aiosqlite
import blspy
from secrets import token_bytes
from src.consensus.constants import constants
from src.full_node.blockchain import Blockchain
from src.full_node.mempool_manager import MempoolManager
from src.full_node.store import FullNodeStore
@ -30,8 +29,11 @@ from src.timelord import Timelord
from src.server.connection import PeerInfo
from src.util.ints import uint16, uint32
bt = BlockTools()
root_path = bt.root_path
test_constants: Dict[str, Any] = {
"DIFFICULTY_STARTING": 1,
"DISCRIMINANT_SIZE_BITS": 16,
@ -71,7 +73,12 @@ async def setup_full_node_simulator(db_name, port, introducer_port=None, dic={})
await store_1.add_block(FullBlock.from_bytes(test_constants_copy["GENESIS_BLOCK"]))
config = load_config("config.yaml", "full_node")
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
config = load_config(root_path, "config.yaml", "full_node")
if introducer_port is not None:
config["introducer_peer"]["host"] = "127.0.0.1"
config["introducer_peer"]["port"] = introducer_port
@ -85,7 +92,7 @@ async def setup_full_node_simulator(db_name, port, introducer_port=None, dic={})
test_constants_copy,
)
server_1 = ChiaServer(
port, full_node_1, NodeType.FULL_NODE, name="full-node-simulator-server"
port, full_node_1, NodeType.FULL_NODE, ping_interval, network_id, "full-node-simulator-server",
)
_ = await server_1.start_server(full_node_1._on_connect)
full_node_1._set_server(server_1)
@ -121,7 +128,11 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}):
await store_1.add_block(FullBlock.from_bytes(test_constants_copy["GENESIS_BLOCK"]))
config = load_config("config.yaml", "full_node")
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
config = load_config(root_path, "config.yaml", "full_node")
if introducer_port is not None:
config["introducer_peer"]["host"] = "127.0.0.1"
config["introducer_peer"]["port"] = introducer_port
@ -134,7 +145,7 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}):
f"full_node_{port}",
test_constants_copy,
)
server_1 = ChiaServer(port, full_node_1, NodeType.FULL_NODE)
server_1 = ChiaServer(port, full_node_1, NodeType.FULL_NODE, ping_interval, network_id)
_ = await server_1.start_server(full_node_1._on_connect)
full_node_1._set_server(server_1)
@ -148,7 +159,7 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}):
async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}):
config = load_config("config.yaml", "wallet")
config = load_config(root_path, "config.yaml", "wallet")
if "starting_height" in dic:
config["starting_height"] = dic["starting_height"]
key_config = {
@ -157,14 +168,19 @@ async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}):
test_constants_copy = test_constants.copy()
for k in dic.keys():
test_constants_copy[k] = dic[k]
db_path = "test-wallet-db" + token_bytes(32).hex() + ".db"
if Path(db_path).exists():
Path(db_path).unlink()
config["database_path"] = db_path
db_path = root_path / ("test-wallet-db%s.db" % token_bytes(32).hex())
if db_path.exists():
db_path.unlink()
config["database_path"] = str(db_path)
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
wallet = await WalletNode.create(
config, key_config, override_constants=test_constants_copy, name="wallet1",
)
server = ChiaServer(port, wallet, NodeType.WALLET, name="wallet-server")
server = ChiaServer(port, wallet, NodeType.WALLET, ping_interval, network_id, "wallet-server")
wallet.set_server(server)
yield (wallet, server)
@ -177,10 +193,15 @@ async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}):
async def setup_harvester(port, dic={}):
config = load_config("config.yaml", "harvester")
config = load_config(root_path, "config.yaml", "harvester")
harvester = Harvester(config, bt.plot_config)
server = ChiaServer(port, harvester, NodeType.HARVESTER)
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
server = ChiaServer(port, harvester, NodeType.HARVESTER, ping_interval, network_id)
_ = await server.start_server(None)
yield (harvester, server)
@ -192,7 +213,7 @@ async def setup_harvester(port, dic={}):
async def setup_farmer(port, dic={}):
config = load_config("config.yaml", "farmer")
config = load_config(root_path, "config.yaml", "farmer")
pool_sk = bt.pool_sk
pool_target = create_puzzlehash_for_pk(
BLSPublicKey(bytes(pool_sk.get_public_key()))
@ -212,8 +233,12 @@ async def setup_farmer(port, dic={}):
for k in dic.keys():
test_constants_copy[k] = dic[k]
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
farmer = Farmer(config, key_config, test_constants_copy)
server = ChiaServer(port, farmer, NodeType.FARMER)
server = ChiaServer(port, farmer, NodeType.FARMER, ping_interval, network_id)
_ = await server.start_server(farmer._on_connect)
yield (farmer, server)
@ -223,10 +248,14 @@ async def setup_farmer(port, dic={}):
async def setup_introducer(port, dic={}):
config = load_config("config.yaml", "introducer")
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
config = load_config(root_path, "config.yaml", "introducer")
introducer = Introducer(config)
server = ChiaServer(port, introducer, NodeType.INTRODUCER)
server = ChiaServer(port, introducer, NodeType.INTRODUCER, ping_interval, network_id)
_ = await server.start_server(None)
yield (introducer, server)
@ -244,12 +273,18 @@ async def setup_vdf_clients(port):
async def setup_timelord(port, dic={}):
config = load_config("config.yaml", "timelord")
config = load_config(root_path, "config.yaml", "timelord")
test_constants_copy = test_constants.copy()
for k in dic.keys():
test_constants_copy[k] = dic[k]
timelord = Timelord(config, test_constants_copy)
server = ChiaServer(port, timelord, NodeType.TIMELORD)
net_config = load_config(root_path, "config.yaml")
ping_interval = net_config.get("ping_interval")
network_id = net_config.get("network_id")
server = ChiaServer(port, timelord, NodeType.TIMELORD, ping_interval, network_id)
_ = await server.start_server(None, config)
coro = asyncio.start_server(

1
tests/test_filter.py
View File

@ -6,7 +6,6 @@ from blspy import ExtendedPrivateKey
from chiabip158 import PyBIP158
from tests.setup_nodes import test_constants, bt, setup_simulators_and_wallets
from src.util.config import load_config
@pytest.fixture(scope="module")