diff --git a/README.md b/README.md index 292bee60..c2d167b8 100644 --- a/README.md +++ b/README.md @@ -12,18 +12,23 @@ For testnet most should only install harvesters, farmers, plotter and full nodes All data is now stored in the $CHIA_ROOT environment variable. or ~/.chia/VERSION-DIR/ if unset. You can find databases, keys, plots, logs here. You can set $CHIA_ROOT to the .chia directory in your home directory with `export CHIA_ROOT=~/.chia`. -## Step 1: Install the code +## Install the code To install chia-blockchain, follow [these install](INSTALL.md) instructions according to your operating system. This only supports 64 bit operating systems. Remember that once you complete your install you **must be in the Python virtual environment** which you access from the chia-blockchain directory with the command `. ./activate`. (Or . ./venv/bin/activate). Both dots are critical and once executed correctly your cli prompt will look something like `(venv) username@machine:~$` with the (venv) prepended. Use `deactivate` should you want to exit the venv. -## Step 2: Generate keys +## Migrate or set up configuration files +```bash +chia init +``` + +## Generate keys First, create some keys by running the following script: ```bash chia-generate-keys ``` -## Step 3a: Run a full node + wallet +## Run a full node + wallet To run a full node on port 8444, and connect to the testnet, run the following command. If you want to see std::out log output, modify the logging.std_out variable in ./config/config.yaml. @@ -38,7 +43,7 @@ chia-start-wallet-server & ``` And then run `chia.exe` from the unzipped `chia-win32-x64` directory in Windows (not Ubuntu/WSL 2.) -## Step 3b: Run a farmer + full node + wallet +## Run a farmer + full node + wallet Instead of running only a full node (as in 3a), you can also run a farmer. Farmers are entities in the network who use their hard drive space to try to create blocks (like Bitcoin's miners), and earn block rewards. First, you must generate some hard drive plots, which @@ -60,7 +65,7 @@ chia-start-wallet-server & And then run `chia.exe` from the unzipped `chia-win32-x64` directory in Windows (not Ubuntu/WSL 2.) -## Step 3c: Run a timelord + full node + wallet +## Run a timelord + full node + wallet *Note* If you want to run a timelord on Linux, see LINUX_TIMELORD.md. diff --git a/src/cmds/check_plots.py b/src/cmds/check_plots.py index 0b53ecad..d4db224e 100644 --- a/src/cmds/check_plots.py +++ b/src/cmds/check_plots.py @@ -7,10 +7,9 @@ from chiapos import DiskProver, Verifier from src.types.proof_of_space import ProofOfSpace from src.types.sized_bytes import bytes32 from src.util.config import load_config +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.hash import std_hash -from src.util.path import path_from_root -chia_root = path_from_root() plot_config_filename = "plots.yaml" @@ -36,7 +35,7 @@ def main(): ) if not Path(plot_filename).exists(): # Tries relative path - full_path: Path = chia_root / plot_filename + full_path: Path = DEFAULT_ROOT_PATH / plot_filename if not full_path.exists(): # Tries absolute path full_path = Path(plot_filename) diff --git a/src/cmds/cli.py b/src/cmds/cli.py index f49503f0..717bc169 100644 --- a/src/cmds/cli.py +++ b/src/cmds/cli.py @@ -1,11 +1,10 @@ import argparse import importlib -from src.util.config import load_config from src import __version__ -SUBCOMMANDS = ["show", "version"] +SUBCOMMANDS = ["init", "show", "version"] def create_parser(): @@ -23,17 +22,13 @@ def create_parser(): for subcommand in SUBCOMMANDS: mod = importlib.import_module("src.cmds.%s" % subcommand) - f = getattr(mod, "%s_parser" % subcommand) - f(subparsers.add_parser(subcommand)) + mod.make_parser(subparsers.add_parser(subcommand)) parser.set_defaults(function=lambda args, parser: parser.print_help()) return parser def chia(args, parser): - # a hack to generate the config.yaml file if missing - load_config("config.yaml") - return args.function(args, parser) diff --git a/src/cmds/create_plots.py b/src/cmds/create_plots.py index 5fce41d9..8cc54bbb 100644 --- a/src/cmds/create_plots.py +++ b/src/cmds/create_plots.py @@ -7,7 +7,8 @@ from blspy import PrivateKey, PublicKey from chiapos import DiskPlotter from src.types.proof_of_space import ProofOfSpace from src.types.sized_bytes import bytes32 -from src.util.config import load_config, save_config +from src.util.config import config_path_for_filename, load_config, save_config +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.path import make_path_relative, mkdir, path_from_root @@ -15,8 +16,9 @@ def main(): """ Script for creating plots and adding them to the plot config file. """ - plot_config_filename = path_from_root() / "config" / "plots.yaml" - key_config_filename = path_from_root() / "config" / "keys.yaml" + root_path = DEFAULT_ROOT_PATH + plot_config_filename = config_path_for_filename(root_path, "plots.yaml") + key_config_filename = config_path_for_filename(root_path, "keys.yaml") parser = argparse.ArgumentParser(description="Chia plotting script.") parser.add_argument("-k", "--size", help="Plot size", type=int, default=20) @@ -36,6 +38,7 @@ def main(): ) new_plots_root = path_from_root( + root_path, load_config("config.yaml").get("harvester", {}).get("new_plot_root", "plots") ) parser.add_argument( @@ -99,7 +102,7 @@ def main(): print(f"Plot {filename} already exists") # Updates the config if necessary. - plot_config = load_config(plot_config_filename) + plot_config = load_config(root_path, plot_config_filename) plot_config_plots_new = deepcopy(plot_config.get("plots", [])) relative_path = make_path_relative(full_path) if relative_path not in plot_config_plots_new: @@ -110,8 +113,11 @@ def main(): plot_config["plots"].update(plot_config_plots_new) # Dumps the new config to disk. - save_config(plot_config_filename, plot_config) - tmp_dir.rmdir() + save_config(root_path, plot_config_filename, plot_config) + try: + tmp_dir.rmdir() + except Exception: + print(f"warning: couldn't delete {tmp_dir}") if __name__ == "__main__": diff --git a/src/cmds/generate_keys.py b/src/cmds/generate_keys.py index 14a50a54..3663b6c7 100644 --- a/src/cmds/generate_keys.py +++ b/src/cmds/generate_keys.py @@ -4,11 +4,9 @@ from secrets import token_bytes from blspy import PrivateKey, ExtendedPrivateKey from src.consensus.coinbase import create_puzzlehash_for_pk from src.types.BLSSignature import BLSPublicKey -from src.util.config import load_config, save_config, str2bool -from src.util.path import mkdir, path_from_root - - -key_config_filename = path_from_root() / "config" / "keys.yaml" +from src.util.config import config_path_for_filename, load_config, save_config, str2bool +from src.util.default_root import DEFAULT_ROOT_PATH +from src.util.path import mkdir def main(): @@ -16,6 +14,8 @@ def main(): Allows replacing keys of farmer, harvester, and pool, all default to True. """ + root_path = DEFAULT_ROOT_PATH + keys_yaml = "keys.yaml" parser = argparse.ArgumentParser(description="Chia key generator script.") parser.add_argument( "-a", @@ -46,6 +46,7 @@ def main(): ) args = parser.parse_args() + key_config_filename = config_path_for_filename(root_path, keys_yaml) if key_config_filename.exists(): # If the file exists, warn the user yn = input( @@ -59,7 +60,7 @@ def main(): mkdir(key_config_filename.parent) open(key_config_filename, "a").close() - key_config = load_config(key_config_filename) + key_config = load_config(root_path, keys_yaml) if key_config is None: key_config = {} @@ -71,12 +72,12 @@ def main(): ) key_config["wallet_sk"] = bytes(wallet_sk).hex() key_config["wallet_target"] = wallet_target.hex() - save_config(key_config_filename, key_config) + save_config(root_path, keys_yaml, key_config) if args.harvester: # Replaces the harvester's sk seed. Used to generate plot private keys, which are # used to sign farmed blocks. key_config["sk_seed"] = token_bytes(32).hex() - save_config(key_config_filename, key_config) + save_config(root_path, keys_yaml, key_config) if args.pool: # Replaces the pools keys and targes. Only useful if running a pool, or doing # solo farming. The pool target allows spending of the coinbase. @@ -89,7 +90,7 @@ def main(): pool_target = wallet_target key_config["pool_sks"] = [bytes(pool_sk).hex() for pool_sk in pool_sks] key_config["pool_target"] = pool_target.hex() - save_config(key_config_filename, key_config) + save_config(root_path, keys_yaml, key_config) if __name__ == "__main__": diff --git a/src/cmds/init.py b/src/cmds/init.py new file mode 100644 index 00000000..14401be3 --- /dev/null +++ b/src/cmds/init.py @@ -0,0 +1,101 @@ +import os +import shutil + +from pathlib import Path + +from src.util.config import ( + config_path_for_filename, + create_default_chia_config, + load_config, + save_config, +) +from src.util.default_root import DEFAULT_ROOT_PATH +from src.util.path import make_path_relative, mkdir, path_from_root + + +def make_parser(parser): + parser.set_defaults(function=init) + + +def migrate_from(old_root, new_root, manifest): + """ + Copy all the files in "manifest" to the new config directory. + """ + if old_root == new_root: + print(f"same as new path, exiting") + return 1 + if not old_root.is_dir(): + print(f"{old_root} not found") + return 0 + print(f"\n{old_root} found") + print(f"Copying files from {old_root} to {new_root}\n") + for f in manifest: + old_path = old_root / f + new_path = new_root / f + if old_path.is_file(): + print(f"{new_path}") + mkdir(new_path.parent) + shutil.copy(old_path, new_path) + else: + print(f"{old_path} not found, skipping") + + # migrate plots + # for now, we simply leave them where they are + # and make what may have been relative paths absolute + + plots_config = load_config(new_root, "plots.yaml") + old_plot_paths = plots_config.get("plots", []) + if len(old_plot_paths) == 0: + print("no plots found, no plots migrated") + return 1 + + print("\nmigrating plots.yaml") + new_plot_paths = {} + for path, values in old_plot_paths.items(): + old_path = path_from_root(old_root, path) + new_plot_path = make_path_relative(old_path, new_root) + print(f"rewriting {path}\n as {new_plot_path}") + new_plot_paths[str(new_plot_path)] = values + plots_config["plots"] = new_plot_paths + save_config(new_root, "plots.yaml", new_plot_paths) + print("\nUpdated plots.yaml to point to where your existing plots are.") + print( + "\nYour plots have not been moved so be careful deleting old preferences folders." + ) + print("If you want to move your plot files, you should also modify") + print(f"{config_path_for_filename(new_root, 'plots.yaml')}") + return 1 + + +def init(args, parser): + return chia_init() + + +def chia_init(): + root_path = DEFAULT_ROOT_PATH + print(f"migrating to {root_path}") + if root_path.is_dir(): + print(f"{root_path} already exists, no action taken") + return -1 + + MANIFEST = [ + "config/config.yaml", + "config/plots.yaml", + "config/keys.yaml", + "wallet/db/blockchain_wallet_v4.db", + "db/blockchain_v3.db", + ] + + PATH_MANIFEST_LIST = [ + (Path(os.path.expanduser("~/.chia/beta-%s" % _)), MANIFEST) for _ in ["1.0b2", "1.0b1"] + ] + + for old_path, manifest in PATH_MANIFEST_LIST: + r = migrate_from(old_path, root_path, manifest) + if r: + break + else: + create_default_chia_config(root_path) + print("Please generate your keys with chia-generate-keys") + + return 0 diff --git a/src/cmds/show.py b/src/cmds/show.py index 0ae5a2c8..88f636b7 100644 --- a/src/cmds/show.py +++ b/src/cmds/show.py @@ -12,7 +12,7 @@ from src.util.byte_types import hexstr_to_bytes from src.util.config import str2bool -def show_parser(parser): +def make_parser(parser): parser.add_argument( "-b", diff --git a/src/cmds/version.py b/src/cmds/version.py index 42327141..553ea58f 100644 --- a/src/cmds/version.py +++ b/src/cmds/version.py @@ -5,6 +5,6 @@ def version(args, parser): print(__version__) -def version_parser(parser): +def make_parser(parser): parser.set_defaults(function=version) return parser diff --git a/src/harvester.py b/src/harvester.py index 670d9569..a1f8e9ce 100644 --- a/src/harvester.py +++ b/src/harvester.py @@ -11,6 +11,7 @@ from src.server.outbound_message import Delivery, Message, NodeType, OutboundMes from src.types.proof_of_space import ProofOfSpace from src.types.sized_bytes import bytes32 from src.util.api_decorators import api_request +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.ints import uint8 from src.util.path import path_from_root @@ -58,11 +59,11 @@ class Harvester: use any plots which don't have one of the pool keys. """ for partial_filename_str, plot_config in self.plot_config["plots"].items(): - plot_root = path_from_root(self.config.get("plot_root", ".")) - partial_filename = path_from_root(partial_filename_str, plot_root) + plot_root = path_from_root(DEFAULT_ROOT_PATH, self.config.get("plot_root", ".")) + partial_filename = plot_root / partial_filename_str potential_filenames = [ partial_filename, - path_from_root(partial_filename_str, plot_root), + path_from_root(plot_root, partial_filename_str), ] pool_pubkey = PublicKey.from_bytes(bytes.fromhex(plot_config["pool_pk"])) diff --git a/src/rpc/rpc_server.py b/src/rpc/rpc_server.py index c54ef4af..f41797e1 100644 --- a/src/rpc/rpc_server.py +++ b/src/rpc/rpc_server.py @@ -12,7 +12,6 @@ from src.types.peer_info import PeerInfo from src.util.ints import uint16, uint64 from src.types.sized_bytes import bytes32 from src.util.byte_types import hexstr_to_bytes -from src.util.config import load_config class EnhancedJSONEncoder(json.JSONEncoder): @@ -145,7 +144,7 @@ class RpcApiHandler: port = request_data["port"] target_node: PeerInfo = PeerInfo(host, uint16(int(port))) - config = load_config("config.yaml", "full_node") + config = self.full_node.config if self.full_node.server is None or not ( await self.full_node.server.start_client(target_node, None, config) ): diff --git a/src/server/server.py b/src/server/server.py index cd7900b8..6752a364 100644 --- a/src/server/server.py +++ b/src/server/server.py @@ -22,17 +22,22 @@ from src.server.outbound_message import Delivery, Message, NodeType, OutboundMes from src.types.peer_info import PeerInfo from src.types.sized_bytes import bytes32 from src.util import partial_func -from src.util.config import load_config from src.util.errors import Err, ProtocolError from src.util.ints import uint16 from src.util.network import create_node_id import traceback -config = load_config("config.yaml") - class ChiaServer: - def __init__(self, port: int, api: Any, local_type: NodeType, name: str = None): + def __init__( + self, + port: int, + api: Any, + local_type: NodeType, + ping_interval: int, + network_id: str, + name: str = None, + ): # Keeps track of all connections to and from this node. self.global_connections: PeerConnections = PeerConnections([]) @@ -46,6 +51,8 @@ class ChiaServer: self._api = api # API module that will be called from the requests self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet) + self._ping_interval = ping_interval + self._network_id = network_id # (StreamReader, StreamWriter, NodeType) aiter, gets things from server and clients and # sends them through the pipeline self._srwt_aiter: push_aiter = push_aiter() @@ -242,7 +249,7 @@ class ChiaServer: self.push_message( OutboundMessage(NodeType.HARVESTER, msg, Delivery.BROADCAST) ) - await asyncio.sleep(config["ping_interval"]) + await asyncio.sleep(self._ping_interval) return asyncio.create_task(ping()) @@ -362,7 +369,7 @@ class ChiaServer: outbound_handshake = Message( "handshake", Handshake( - config["network_id"], + self._network_id, protocol_version, self._node_id, uint16(self._port), diff --git a/src/server/start_farmer.py b/src/server/start_farmer.py index 7111c14a..67dcf3a9 100644 --- a/src/server/start_farmer.py +++ b/src/server/start_farmer.py @@ -11,15 +11,18 @@ from src.farmer import Farmer from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.types.peer_info import PeerInfo -from src.util.logging import initialize_logging from src.util.config import load_config, load_config_cli +from src.util.default_root import DEFAULT_ROOT_PATH +from src.util.logging import initialize_logging from src.util.setproctitle import setproctitle async def main(): - config = load_config_cli("config.yaml", "farmer") + root_path = DEFAULT_ROOT_PATH + net_config = load_config(root_path, "config.yaml") + config = load_config_cli(root_path, "config.yaml", "farmer") try: - key_config = load_config("keys.yaml") + key_config = load_config(root_path, "keys.yaml") except FileNotFoundError: raise RuntimeError("Keys not generated. Run chia-generate-keys") initialize_logging("Farmer %(name)-25s", config["logging"]) @@ -34,7 +37,9 @@ async def main(): full_node_peer = PeerInfo( config["full_node_peer"]["host"], config["full_node_peer"]["port"] ) - server = ChiaServer(config["port"], farmer, NodeType.FARMER) + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + server = ChiaServer(config["port"], farmer, NodeType.FARMER, ping_interval, network_id) asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all) asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all) diff --git a/src/server/start_full_node.py b/src/server/start_full_node.py index 3ac0a486..16eaf326 100644 --- a/src/server/start_full_node.py +++ b/src/server/start_full_node.py @@ -23,20 +23,21 @@ from src.types.peer_info import PeerInfo from src.full_node.coin_store import CoinStore from src.util.logging import initialize_logging from src.util.config import load_config_cli +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.path import mkdir, path_from_root from src.util.pip_import import pip_import from src.util.setproctitle import setproctitle async def main(): - config = load_config_cli("config.yaml", "full_node") + config = load_config_cli(DEFAULT_ROOT_PATH, "config.yaml", "full_node") setproctitle("chia_full_node") initialize_logging("FullNode %(name)-23s", config["logging"]) log = logging.getLogger(__name__) server_closed = False - db_path = path_from_root(config["database_path"]) + db_path = path_from_root(DEFAULT_ROOT_PATH, config["database_path"]) mkdir(db_path.parent) # Create the store (DB) and full node instance @@ -72,7 +73,9 @@ async def main(): log.exception(f"UPnP failed") # Starts the full node server (which full nodes can connect to) - server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE) + ping_interval = config.get("ping_interval") + network_id = config.get("network_id") + server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE, ping_interval, network_id) full_node._set_server(server) _ = await server.start_server(full_node._on_connect, config) rpc_cleanup = None diff --git a/src/server/start_harvester.py b/src/server/start_harvester.py index d5136433..acd57177 100644 --- a/src/server/start_harvester.py +++ b/src/server/start_harvester.py @@ -11,15 +11,18 @@ from src.harvester import Harvester from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.types.peer_info import PeerInfo -from src.util.logging import initialize_logging from src.util.config import load_config, load_config_cli +from src.util.default_root import DEFAULT_ROOT_PATH +from src.util.logging import initialize_logging from src.util.setproctitle import setproctitle async def main(): - config = load_config_cli("config.yaml", "harvester") + root_path = DEFAULT_ROOT_PATH + net_config = load_config(root_path, "config.yaml") + config = load_config_cli(root_path, "config.yaml", "harvester") try: - plot_config = load_config("plots.yaml") + plot_config = load_config(root_path, "plots.yaml") except FileNotFoundError: raise RuntimeError("Plots not generated. Run chia-create-plots") @@ -28,7 +31,9 @@ async def main(): setproctitle("chia_harvester") harvester = Harvester(config, plot_config) - server = ChiaServer(config["port"], harvester, NodeType.HARVESTER) + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + server = ChiaServer(config["port"], harvester, NodeType.HARVESTER, ping_interval, network_id) _ = await server.start_server(None, config) asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all) diff --git a/src/server/start_introducer.py b/src/server/start_introducer.py index 92886321..f57367a3 100644 --- a/src/server/start_introducer.py +++ b/src/server/start_introducer.py @@ -10,20 +10,24 @@ except ImportError: from src.introducer import Introducer from src.server.outbound_message import NodeType from src.server.server import ChiaServer +from src.util.config import load_config_cli, load_config +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.logging import initialize_logging -from src.util.config import load_config_cli from src.util.setproctitle import setproctitle async def main(): - config = load_config_cli("config.yaml", "introducer") - + root_path = DEFAULT_ROOT_PATH + net_config = load_config(root_path, "config.yaml") + config = load_config_cli(root_path, "config.yaml", "introducer") initialize_logging("Introducer %(name)-21s", config["logging"]) log = logging.getLogger(__name__) setproctitle("chia_introducer") introducer = Introducer(config) - server = ChiaServer(config["port"], introducer, NodeType.INTRODUCER) + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + server = ChiaServer(config["port"], introducer, NodeType.INTRODUCER, ping_interval, network_id) introducer.set_server(server) _ = await server.start_server(None, config) diff --git a/src/server/start_timelord.py b/src/server/start_timelord.py index 2e2af0df..1c967ea7 100644 --- a/src/server/start_timelord.py +++ b/src/server/start_timelord.py @@ -14,20 +14,25 @@ from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.timelord import Timelord from src.types.peer_info import PeerInfo +from src.util.config import load_config_cli, load_config +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.logging import initialize_logging -from src.util.config import load_config_cli from src.util.setproctitle import setproctitle async def main(): - config = load_config_cli("config.yaml", "timelord") - + root_path = DEFAULT_ROOT_PATH + net_config = load_config(root_path, "config.yaml") + config = load_config_cli(root_path, "config.yaml", "timelord") initialize_logging("Timelord %(name)-23s", config["logging"]) log = logging.getLogger(__name__) setproctitle("chia_timelord") timelord = Timelord(config, constants) - server = ChiaServer(config["port"], timelord, NodeType.TIMELORD) + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + + server = ChiaServer(config["port"], timelord, NodeType.TIMELORD, ping_interval, network_id) _ = await server.start_server(None, config) timelord_shutdown_task: Optional[asyncio.Task] = None diff --git a/src/simulator/start_simulator.py b/src/simulator/start_simulator.py index f6a11891..0736f12c 100644 --- a/src/simulator/start_simulator.py +++ b/src/simulator/start_simulator.py @@ -20,20 +20,23 @@ from src.server.connection import NodeType from src.types.full_block import FullBlock from src.full_node.coin_store import CoinStore from src.util.logging import initialize_logging -from src.util.config import load_config_cli +from src.util.config import load_config_cli, load_config +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.setproctitle import setproctitle from src.util.path import mkdir, path_from_root async def main(): - config = load_config_cli("config.yaml", "full_node") + root_path = DEFAULT_ROOT_PATH + net_config = load_config(root_path, "config.yaml") + config = load_config_cli(root_path, "config.yaml", "full_node") setproctitle("chia_full_node") initialize_logging("FullNode %(name)-23s", config["logging"]) log = logging.getLogger(__name__) server_closed = False - db_path = path_from_root(config["simulator_database_path"]) + db_path = path_from_root(DEFAULT_ROOT_PATH, config["simulator_database_path"]) mkdir(db_path.parent) connection = await aiosqlite.connect(db_path) @@ -60,8 +63,12 @@ async def main(): override_constants=test_constants, ) + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + # Starts the full node server (which full nodes can connect to) - server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE) + + server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE, ping_interval, network_id) full_node._set_server(server) _ = await server.start_server(full_node._on_connect, config) rpc_cleanup = None diff --git a/src/timelord_launcher.py b/src/timelord_launcher.py index a1f1ca50..a53d96d0 100644 --- a/src/timelord_launcher.py +++ b/src/timelord_launcher.py @@ -7,17 +7,13 @@ from src.util.logging import initialize_logging from src.util.config import load_config from asyncio import Lock from typing import List +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.setproctitle import setproctitle -config = load_config("config.yaml", "timelord_launcher") - active_processes: List = [] stopped = False lock = Lock() -initialize_logging("Launcher %(name)-23s", config["logging"]) -setproctitle("chia_timelord_launcher") - log = logging.getLogger(__name__) @@ -69,7 +65,7 @@ async def spawn_process(host, port, counter): await asyncio.sleep(0.1) -async def spawn_all_processes(): +async def spawn_all_processes(config): await asyncio.sleep(15) port = config["port"] process_count = config["process_count"] @@ -77,7 +73,10 @@ async def spawn_all_processes(): await asyncio.gather(*awaitables) -if __name__ == "__main__": +def main(): + setproctitle("chia_timelord_launcher") + config = load_config(DEFAULT_ROOT_PATH, "config.yaml", "timelord_launcher") + initialize_logging("Launcher %(name)-23s", config["logging"]) def signal_received(): asyncio.create_task(kill_processes()) @@ -88,7 +87,11 @@ if __name__ == "__main__": loop.add_signal_handler(signal.SIGTERM, signal_received) try: - loop.run_until_complete(spawn_all_processes()) + loop.run_until_complete(spawn_all_processes(config)) finally: log.info("Launcher fully closed.") loop.close() + + +if __name__ == "__main__": + main() diff --git a/src/util/config.py b/src/util/config.py index 66ac430e..63a18fe1 100644 --- a/src/util/config.py +++ b/src/util/config.py @@ -1,45 +1,65 @@ -import yaml import argparse -from pathlib import Path import pkg_resources +import sys +import yaml + +from pathlib import Path from typing import Dict, Any, Callable, Optional, Union -from src.util.path import mkdir, path_from_root +from src.util.path import mkdir -def migrate_config_file(filename: Union[str, Path], path: Path) -> None: - filepath = Path(filename) - default_config_file = pkg_resources.resource_string( - __name__, f"initial-{filepath.parts[-1]}" - ).decode() - mkdir(str(path.parent)) - with open(path, "w") as f: - f.write(default_config_file) +def initial_config_file(filename: Union[str, Path]) -> str: + return pkg_resources.resource_string(__name__, f"initial-{filename}").decode() -def save_config(filename: Union[str, Path], config_data): - path = path_from_root("config") / filename +def create_default_chia_config(root_path: Path) -> None: + for filename in ["config.yaml"]: + default_config_file_data = initial_config_file(filename) + path = config_path_for_filename(root_path, filename) + mkdir(path.parent) + with open(path, "w") as f: + f.write(default_config_file_data) + save_config(root_path, "plots.yaml", {}) + + +def config_path_for_filename(root_path: Path, filename: Union[str, Path]) -> Path: + path_filename = Path(filename) + if path_filename.is_absolute(): + return path_filename + return root_path / "config" / filename + + +def save_config(root_path: Path, filename: Union[str, Path], config_data : Any): + path = config_path_for_filename(root_path, filename) with open(path, "w") as f: yaml.safe_dump(config_data, f) -def load_config(filename: Union[str, Path], sub_config: Optional[str] = None) -> Dict: - path = path_from_root("config") / filename +def load_config( + root_path: Path, + filename: Union[str, Path], + sub_config: Optional[str] = None +) -> Dict: + path = config_path_for_filename(root_path, filename) if not path.is_file(): - migrate_config_file(filename, path) + print(f"can't find {path}") + print("** please run `chia init` to migrate or create new config files **") + # TODO: fix this hack + sys.exit(-1) + r = yaml.safe_load(open(path, "r")) if sub_config is not None: - return yaml.safe_load(open(path, "r"))[sub_config] - else: - return yaml.safe_load(open(path, "r")) + r = r.get(sub_config) + return r -def load_config_cli(filename: str, sub_config: Optional[str] = None) -> Dict: +def load_config_cli(root_path: Path, filename: str, sub_config: Optional[str]) -> Dict: """ Loads configuration from the specified filename, in the config directory, and then overrides any properties using the passed in command line arguments. Nested properties in the config file can be used in the command line with ".", for example --farmer_peer.host. Does not support lists. """ - config = load_config(filename, sub_config) + config = load_config(root_path, filename, sub_config) flattened_props = flatten_properties(config) parser = argparse.ArgumentParser() diff --git a/src/util/default_root.py b/src/util/default_root.py new file mode 100644 index 00000000..2048fd2c --- /dev/null +++ b/src/util/default_root.py @@ -0,0 +1,10 @@ +import os +from pathlib import Path + +from src import __version__ + +DEFAULT_ROOT_PATH = Path( + os.path.expanduser( + os.getenv("CHIA_ROOT", "~/.chia/beta-{version}").format(version=__version__) + ) +).resolve() diff --git a/src/util/logging.py b/src/util/logging.py index e4d73178..b04ea31a 100644 --- a/src/util/logging.py +++ b/src/util/logging.py @@ -2,11 +2,12 @@ import logging import colorlog from typing import Dict +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.path import mkdir, path_from_root def initialize_logging(prefix: str, logging_config: Dict): - log_path = path_from_root(logging_config.get("log_filename", "log/debug.log")) + log_path = path_from_root(DEFAULT_ROOT_PATH, logging_config.get("log_filename", "log/debug.log")) mkdir(str(log_path.parent)) if logging_config["log_stdout"]: handler = colorlog.StreamHandler() diff --git a/src/util/path.py b/src/util/path.py index 3114cd77..d9fd3c45 100644 --- a/src/util/path.py +++ b/src/util/path.py @@ -1,53 +1,36 @@ import os + from typing import Union from pathlib import Path -from src import __version__ -DEFAULT_ROOT_PATH = Path( - os.path.expanduser(os.getenv("CHIA_ROOT", "~/.chia/beta-%s" % __version__)) -).resolve() - - -def path_from_root( - path_str: Union[str, Path] = ".", root_path: Path = DEFAULT_ROOT_PATH -) -> Path: +def path_from_root(root: Path, path_str: Union[str, Path]) -> Path: """ - Return a new path from the given one, expanding "~" if present. - If path is relative, prepend $CHIA_ROOT + If path is relative, prepend root If path is absolute, return it directly. - This lets us use "~" and other absolute paths in config files. """ - if isinstance(path_str, Path): - path = path_str - else: - path = Path(path_str) + root = Path(os.path.expanduser(str(root))) + path = Path(path_str) if not path.is_absolute(): - path = root_path / path - return path + path = root / path + return path.resolve() def mkdir(path_str: Union[str, Path]) -> None: """ Create the existing directory (and its parents) if necessary. """ - if isinstance(path_str, Path): - path = path_str - else: - path = Path(path_str) + path = Path(path_str) path.mkdir(parents=True, exist_ok=True) -def make_path_relative(path_str: Union[str, Path]) -> Path: +def make_path_relative(path_str: Union[str, Path], root: Path) -> Path: """ Try to make the given path relative, given the default root. """ - if isinstance(path_str, Path): - path = path_str - else: - path = Path(path_str) + path = Path(path_str) try: - path = path.relative_to(DEFAULT_ROOT_PATH) + path = path.relative_to(root) except ValueError: pass - return path + return path.resolve() diff --git a/src/wallet/wallet_node.py b/src/wallet/wallet_node.py index d193fe87..dee19ac6 100644 --- a/src/wallet/wallet_node.py +++ b/src/wallet/wallet_node.py @@ -28,6 +28,7 @@ from src.types.full_block import FullBlock from src.types.coin import Coin, hash_coin_list from src.full_node.blockchain import ReceiveBlockResult from src.types.mempool_inclusion_status import MempoolInclusionStatus +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.errors import Err from src.util.path import path_from_root, mkdir @@ -81,7 +82,7 @@ class WalletNode: else: self.log = logging.getLogger(__name__) - path = path_from_root(config["database_path"]) + path = path_from_root(DEFAULT_ROOT_PATH, config["database_path"]) mkdir(path.parent) self.wallet_state_manager = await WalletStateManager.create( diff --git a/src/wallet/websocket_server.py b/src/wallet/websocket_server.py index b6500df7..36b6f1e5 100644 --- a/src/wallet/websocket_server.py +++ b/src/wallet/websocket_server.py @@ -29,6 +29,7 @@ from src.wallet.util.wallet_types import WalletType from src.wallet.wallet_info import WalletInfo from src.wallet.wallet_node import WalletNode from src.types.mempool_inclusion_status import MempoolInclusionStatus +from src.util.default_root import DEFAULT_ROOT_PATH from src.util.setproctitle import setproctitle # Timeout for response from wallet/full node for sending a transaction @@ -362,7 +363,9 @@ async def start_websocket_server(): Starts WalletNode, WebSocketServer, and ChiaServer """ - config = load_config_cli("config.yaml", "wallet") + root_path = DEFAULT_ROOT_PATH + + config = load_config_cli(root_path, "config.yaml", "wallet") initialize_logging("Wallet %(name)-25s", config["logging"]) log = logging.getLogger(__name__) log.info(f"Config : {config}") @@ -384,8 +387,12 @@ async def start_websocket_server(): handler = WebSocketServer(wallet_node, log) wallet_node.wallet_state_manager.set_callback(handler.state_changed_callback) + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + log.info(f"Starting wallet server on port {config['port']}.") - server = ChiaServer(config["port"], wallet_node, NodeType.WALLET) + server = ChiaServer(config["port"], wallet_node, NodeType.WALLET, ping_interval, network_id) wallet_node.set_server(server) _ = await server.start_server(None, config) diff --git a/tests/block_tools.py b/tests/block_tools.py index 6c3b6c8a..332f8ce8 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -1,14 +1,18 @@ +import os import sys import time -from typing import Any, Dict, List, Tuple, Optional import random -import blspy +from pathlib import Path +from typing import Any, Dict, List, Tuple, Optional + from blspy import PrependSignature, PrivateKey, PublicKey from chiavdf import prove from chiabip158 import PyBIP158 from chiapos import DiskPlotter, DiskProver +from src import __version__ +from src.cmds.init import create_default_chia_config from src.consensus import block_rewards, pot_iterations from src.consensus.constants import constants from src.consensus.pot_iterations import calculate_min_iters_from_iterations @@ -26,7 +30,7 @@ from src.types.sized_bytes import bytes32 from src.util.merkle_set import MerkleSet from src.util.ints import uint8, uint32, uint64, uint128, int512 from src.util.hash import std_hash -from src.util.path import mkdir, path_from_root +from src.util.path import mkdir from src.util.significant_bits import truncate_to_significant_bits # Can't go much lower than 19, since plots start having no solutions @@ -49,12 +53,21 @@ fee_target = std_hash(bytes(wallet_sk.get_public_key())) n_wesolowski = uint8(0) +TEST_ROOT_PATH = Path( + os.path.expanduser( + os.getenv("CHIA_ROOT", "~/.chia/beta-{version}-test").format(version=__version__) + ) +).resolve() + + class BlockTools: """ Tools to generate blocks for testing. """ - def __init__(self): + def __init__(self, root_path : Path = TEST_ROOT_PATH): + create_default_chia_config(root_path) + self.root_path = root_path self.plot_config: Dict = {"plots": {}} self.pool_sk = pool_sk self.wallet_sk = wallet_sk @@ -62,7 +75,7 @@ class BlockTools: plot_seeds: List[bytes32] = [ ProofOfSpace.calculate_plot_seed(pool_pk, plot_pk) for plot_pk in plot_pks ] - self.plot_dir = path_from_root("tests") / "plots" + self.plot_dir = self.root_path / "plots" mkdir(self.plot_dir) self.filenames: List[str] = [ f"genesis-plots-{k}{std_hash(int.to_bytes(i, 4, 'big')).hex()}.dat" diff --git a/tests/full_node/test_blockchain.py b/tests/full_node/test_blockchain.py index cbbb4188..405e4bbf 100644 --- a/tests/full_node/test_blockchain.py +++ b/tests/full_node/test_blockchain.py @@ -5,18 +5,17 @@ from pathlib import Path import aiosqlite import pytest -from blspy import PrivateKey, PrependSignature +from blspy import PrivateKey from src.full_node.blockchain import Blockchain, ReceiveBlockResult from src.full_node.store import FullNodeStore from src.types.full_block import FullBlock -from src.types.coin import Coin from src.types.header import Header, HeaderData from src.types.proof_of_space import ProofOfSpace from src.full_node.coin_store import CoinStore from src.util.ints import uint8, uint64 from src.consensus.constants import constants as consensus_constants -from tests.block_tools import BlockTools, plot_sks +from tests.block_tools import BlockTools from src.util.errors import Err from src.consensus.coinbase import create_coinbase_coin_and_signature from src.types.sized_bytes import bytes32 diff --git a/tests/full_node/test_full_sync.py b/tests/full_node/test_full_sync.py index 88650088..06b2be76 100644 --- a/tests/full_node/test_full_sync.py +++ b/tests/full_node/test_full_sync.py @@ -6,7 +6,7 @@ import pytest from src.types.peer_info import PeerInfo from src.protocols import full_node_protocol from src.util.ints import uint16 -from tests.setup_nodes import setup_two_nodes, setup_node_and_wallet, test_constants, bt +from tests.setup_nodes import setup_two_nodes, test_constants, bt @pytest.fixture(scope="module") diff --git a/tests/full_node/test_transactions.py b/tests/full_node/test_transactions.py index 80ac3f72..dd28ffb5 100644 --- a/tests/full_node/test_transactions.py +++ b/tests/full_node/test_transactions.py @@ -4,7 +4,7 @@ from secrets import token_bytes import pytest from src.protocols import full_node_protocol -from src.simulator.simulator_protocol import FarmNewBlockProtocol, ReorgProtocol +from src.simulator.simulator_protocol import FarmNewBlockProtocol from src.types.peer_info import PeerInfo from src.util.ints import uint16, uint32 from tests.setup_nodes import setup_simulators_and_wallets diff --git a/tests/setup_nodes.py b/tests/setup_nodes.py index 269dbb8e..5300bac7 100644 --- a/tests/setup_nodes.py +++ b/tests/setup_nodes.py @@ -1,13 +1,12 @@ -import signal -from typing import Any, Dict, Optional, Tuple, List -from pathlib import Path import asyncio +from typing import Any, Dict, Tuple, List +from pathlib import Path + import aiosqlite import blspy from secrets import token_bytes -from src.consensus.constants import constants from src.full_node.blockchain import Blockchain from src.full_node.mempool_manager import MempoolManager from src.full_node.store import FullNodeStore @@ -30,8 +29,11 @@ from src.timelord import Timelord from src.server.connection import PeerInfo from src.util.ints import uint16, uint32 + bt = BlockTools() +root_path = bt.root_path + test_constants: Dict[str, Any] = { "DIFFICULTY_STARTING": 1, "DISCRIMINANT_SIZE_BITS": 16, @@ -71,7 +73,12 @@ async def setup_full_node_simulator(db_name, port, introducer_port=None, dic={}) await store_1.add_block(FullBlock.from_bytes(test_constants_copy["GENESIS_BLOCK"])) - config = load_config("config.yaml", "full_node") + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + + config = load_config(root_path, "config.yaml", "full_node") + if introducer_port is not None: config["introducer_peer"]["host"] = "127.0.0.1" config["introducer_peer"]["port"] = introducer_port @@ -85,7 +92,7 @@ async def setup_full_node_simulator(db_name, port, introducer_port=None, dic={}) test_constants_copy, ) server_1 = ChiaServer( - port, full_node_1, NodeType.FULL_NODE, name="full-node-simulator-server" + port, full_node_1, NodeType.FULL_NODE, ping_interval, network_id, "full-node-simulator-server", ) _ = await server_1.start_server(full_node_1._on_connect) full_node_1._set_server(server_1) @@ -121,7 +128,11 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}): await store_1.add_block(FullBlock.from_bytes(test_constants_copy["GENESIS_BLOCK"])) - config = load_config("config.yaml", "full_node") + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + + config = load_config(root_path, "config.yaml", "full_node") if introducer_port is not None: config["introducer_peer"]["host"] = "127.0.0.1" config["introducer_peer"]["port"] = introducer_port @@ -134,7 +145,7 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}): f"full_node_{port}", test_constants_copy, ) - server_1 = ChiaServer(port, full_node_1, NodeType.FULL_NODE) + server_1 = ChiaServer(port, full_node_1, NodeType.FULL_NODE, ping_interval, network_id) _ = await server_1.start_server(full_node_1._on_connect) full_node_1._set_server(server_1) @@ -148,7 +159,7 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}): async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}): - config = load_config("config.yaml", "wallet") + config = load_config(root_path, "config.yaml", "wallet") if "starting_height" in dic: config["starting_height"] = dic["starting_height"] key_config = { @@ -157,14 +168,19 @@ async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}): test_constants_copy = test_constants.copy() for k in dic.keys(): test_constants_copy[k] = dic[k] - db_path = "test-wallet-db" + token_bytes(32).hex() + ".db" - if Path(db_path).exists(): - Path(db_path).unlink() - config["database_path"] = db_path + db_path = root_path / ("test-wallet-db%s.db" % token_bytes(32).hex()) + if db_path.exists(): + db_path.unlink() + config["database_path"] = str(db_path) + + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + wallet = await WalletNode.create( config, key_config, override_constants=test_constants_copy, name="wallet1", ) - server = ChiaServer(port, wallet, NodeType.WALLET, name="wallet-server") + server = ChiaServer(port, wallet, NodeType.WALLET, ping_interval, network_id, "wallet-server") wallet.set_server(server) yield (wallet, server) @@ -177,10 +193,15 @@ async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}): async def setup_harvester(port, dic={}): - config = load_config("config.yaml", "harvester") + config = load_config(root_path, "config.yaml", "harvester") harvester = Harvester(config, bt.plot_config) - server = ChiaServer(port, harvester, NodeType.HARVESTER) + + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + + server = ChiaServer(port, harvester, NodeType.HARVESTER, ping_interval, network_id) _ = await server.start_server(None) yield (harvester, server) @@ -192,7 +213,7 @@ async def setup_harvester(port, dic={}): async def setup_farmer(port, dic={}): - config = load_config("config.yaml", "farmer") + config = load_config(root_path, "config.yaml", "farmer") pool_sk = bt.pool_sk pool_target = create_puzzlehash_for_pk( BLSPublicKey(bytes(pool_sk.get_public_key())) @@ -212,8 +233,12 @@ async def setup_farmer(port, dic={}): for k in dic.keys(): test_constants_copy[k] = dic[k] + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + farmer = Farmer(config, key_config, test_constants_copy) - server = ChiaServer(port, farmer, NodeType.FARMER) + server = ChiaServer(port, farmer, NodeType.FARMER, ping_interval, network_id) _ = await server.start_server(farmer._on_connect) yield (farmer, server) @@ -223,10 +248,14 @@ async def setup_farmer(port, dic={}): async def setup_introducer(port, dic={}): - config = load_config("config.yaml", "introducer") + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + + config = load_config(root_path, "config.yaml", "introducer") introducer = Introducer(config) - server = ChiaServer(port, introducer, NodeType.INTRODUCER) + server = ChiaServer(port, introducer, NodeType.INTRODUCER, ping_interval, network_id) _ = await server.start_server(None) yield (introducer, server) @@ -244,12 +273,18 @@ async def setup_vdf_clients(port): async def setup_timelord(port, dic={}): - config = load_config("config.yaml", "timelord") + config = load_config(root_path, "config.yaml", "timelord") + test_constants_copy = test_constants.copy() for k in dic.keys(): test_constants_copy[k] = dic[k] timelord = Timelord(config, test_constants_copy) - server = ChiaServer(port, timelord, NodeType.TIMELORD) + + net_config = load_config(root_path, "config.yaml") + ping_interval = net_config.get("ping_interval") + network_id = net_config.get("network_id") + + server = ChiaServer(port, timelord, NodeType.TIMELORD, ping_interval, network_id) _ = await server.start_server(None, config) coro = asyncio.start_server( diff --git a/tests/test_filter.py b/tests/test_filter.py index 58cab156..98a3f4d6 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -6,7 +6,6 @@ from blspy import ExtendedPrivateKey from chiabip158 import PyBIP158 from tests.setup_nodes import test_constants, bt, setup_simulators_and_wallets -from src.util.config import load_config @pytest.fixture(scope="module")