Add `chia init` and refactor out many global references to the config directory (#153)

* Tweak subcommand template.
* Don't hack the .chia/beta-* directory when invoking `chia`.
* Get `chia init` working. Factor out all global paths.
* Reverse order of migration bases.
* Update README.

README.md

@@ -12,18 +12,23 @@ For testnet most should only install harvesters, farmers, plotter and full nodes
 
 All data is now stored in the $CHIA_ROOT environment variable. or ~/.chia/VERSION-DIR/ if unset. You can find databases, keys, plots, logs here. You can set $CHIA_ROOT to the .chia directory in your home directory with `export CHIA_ROOT=~/.chia`.
 
-## Step 1: Install the code
+## Install the code
 To install chia-blockchain, follow [these install](INSTALL.md) instructions according to your operating system. This only supports 64 bit operating systems.
 
 Remember that once you complete your install you **must be in the Python virtual environment** which you access from the chia-blockchain directory with the command `.   ./activate`. (Or . ./venv/bin/activate). Both dots are critical and once executed correctly your cli prompt will look something like `(venv) username@machine:~$` with the (venv) prepended. Use `deactivate` should you want to exit the venv.
 
-## Step 2: Generate keys
+## Migrate or set up configuration files
+```bash
+chia init
+```
+
+## Generate keys
 First, create some keys by running the following script:
 ```bash
 chia-generate-keys
 ```
 
-## Step 3a: Run a full node + wallet
+## Run a full node + wallet
 To run a full node on port 8444, and connect to the testnet, run the following command.
 If you want to see std::out log output, modify the logging.std_out variable in ./config/config.yaml.
 
@@ -38,7 +43,7 @@ chia-start-wallet-server &
 ```
 And then run `chia.exe` from the unzipped `chia-win32-x64` directory in Windows (not Ubuntu/WSL 2.)
 
-## Step 3b: Run a farmer + full node + wallet
+## Run a farmer + full node + wallet
 Instead of running only a full node (as in 3a), you can also run a farmer.
 Farmers are entities in the network who use their hard drive space to try to create
 blocks (like Bitcoin's miners), and earn block rewards. First, you must generate some hard drive plots, which
@@ -60,7 +65,7 @@ chia-start-wallet-server &
 And then run `chia.exe` from the unzipped `chia-win32-x64` directory in Windows (not Ubuntu/WSL 2.)
 
 
-## Step 3c: Run a timelord + full node + wallet
+## Run a timelord + full node + wallet
 
 *Note*
 If you want to run a timelord on Linux, see LINUX_TIMELORD.md.

src/cmds/check_plots.py

@@ -7,10 +7,9 @@ from chiapos import DiskProver, Verifier
 from src.types.proof_of_space import ProofOfSpace
 from src.types.sized_bytes import bytes32
 from src.util.config import load_config
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.hash import std_hash
-from src.util.path import path_from_root
 
-chia_root = path_from_root()
 plot_config_filename = "plots.yaml"
 
 
@@ -36,7 +35,7 @@ def main():
         )
         if not Path(plot_filename).exists():
             # Tries relative path
-            full_path: Path = chia_root / plot_filename
+            full_path: Path = DEFAULT_ROOT_PATH / plot_filename
             if not full_path.exists():
                 # Tries absolute path
                 full_path = Path(plot_filename)

src/cmds/cli.py

@@ -1,11 +1,10 @@
 import argparse
 import importlib
 
-from src.util.config import load_config
 from src import __version__
 
 
-SUBCOMMANDS = ["show", "version"]
+SUBCOMMANDS = ["init", "show", "version"]
 
 
 def create_parser():
@@ -23,17 +22,13 @@ def create_parser():
 
     for subcommand in SUBCOMMANDS:
         mod = importlib.import_module("src.cmds.%s" % subcommand)
-        f = getattr(mod, "%s_parser" % subcommand)
+        mod.make_parser(subparsers.add_parser(subcommand))
-        f(subparsers.add_parser(subcommand))
 
     parser.set_defaults(function=lambda args, parser: parser.print_help())
     return parser
 
 
 def chia(args, parser):
-    # a hack to generate the config.yaml file if missing
-    load_config("config.yaml")
-
     return args.function(args, parser)
 
 

src/cmds/create_plots.py

@@ -7,7 +7,8 @@ from blspy import PrivateKey, PublicKey
 from chiapos import DiskPlotter
 from src.types.proof_of_space import ProofOfSpace
 from src.types.sized_bytes import bytes32
-from src.util.config import load_config, save_config
+from src.util.config import config_path_for_filename, load_config, save_config
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.path import make_path_relative, mkdir, path_from_root
 
 
@@ -15,8 +16,9 @@ def main():
     """
     Script for creating plots and adding them to the plot config file.
     """
-    plot_config_filename = path_from_root() / "config" / "plots.yaml"
+    root_path = DEFAULT_ROOT_PATH
-    key_config_filename = path_from_root() / "config" / "keys.yaml"
+    plot_config_filename = config_path_for_filename(root_path, "plots.yaml")
+    key_config_filename = config_path_for_filename(root_path, "keys.yaml")
 
     parser = argparse.ArgumentParser(description="Chia plotting script.")
     parser.add_argument("-k", "--size", help="Plot size", type=int, default=20)
@@ -36,6 +38,7 @@ def main():
     )
 
     new_plots_root = path_from_root(
+        root_path,
         load_config("config.yaml").get("harvester", {}).get("new_plot_root", "plots")
     )
     parser.add_argument(
@@ -99,7 +102,7 @@ def main():
             print(f"Plot {filename} already exists")
 
         # Updates the config if necessary.
-        plot_config = load_config(plot_config_filename)
+        plot_config = load_config(root_path, plot_config_filename)
         plot_config_plots_new = deepcopy(plot_config.get("plots", []))
         relative_path = make_path_relative(full_path)
         if relative_path not in plot_config_plots_new:
@@ -110,8 +113,11 @@ def main():
         plot_config["plots"].update(plot_config_plots_new)
 
         # Dumps the new config to disk.
-        save_config(plot_config_filename, plot_config)
+        save_config(root_path, plot_config_filename, plot_config)
+    try:
         tmp_dir.rmdir()
+    except Exception:
+        print(f"warning: couldn't delete {tmp_dir}")
 
 
 if __name__ == "__main__":

src/cmds/generate_keys.py

@@ -4,11 +4,9 @@ from secrets import token_bytes
 from blspy import PrivateKey, ExtendedPrivateKey
 from src.consensus.coinbase import create_puzzlehash_for_pk
 from src.types.BLSSignature import BLSPublicKey
-from src.util.config import load_config, save_config, str2bool
+from src.util.config import config_path_for_filename, load_config, save_config, str2bool
-from src.util.path import mkdir, path_from_root
+from src.util.default_root import DEFAULT_ROOT_PATH
-
+from src.util.path import mkdir
-
-key_config_filename = path_from_root() / "config" / "keys.yaml"
 
 
 def main():
@@ -16,6 +14,8 @@ def main():
     Allows replacing keys of farmer, harvester, and pool, all default to True.
     """
 
+    root_path = DEFAULT_ROOT_PATH
+    keys_yaml = "keys.yaml"
     parser = argparse.ArgumentParser(description="Chia key generator script.")
     parser.add_argument(
         "-a",
@@ -46,6 +46,7 @@ def main():
     )
     args = parser.parse_args()
 
+    key_config_filename = config_path_for_filename(root_path, keys_yaml)
     if key_config_filename.exists():
         # If the file exists, warn the user
         yn = input(
@@ -59,7 +60,7 @@ def main():
         mkdir(key_config_filename.parent)
         open(key_config_filename, "a").close()
 
-    key_config = load_config(key_config_filename)
+    key_config = load_config(root_path, keys_yaml)
     if key_config is None:
         key_config = {}
 
@@ -71,12 +72,12 @@ def main():
         )
         key_config["wallet_sk"] = bytes(wallet_sk).hex()
         key_config["wallet_target"] = wallet_target.hex()
-        save_config(key_config_filename, key_config)
+        save_config(root_path, keys_yaml, key_config)
     if args.harvester:
         # Replaces the harvester's sk seed. Used to generate plot private keys, which are
         # used to sign farmed blocks.
         key_config["sk_seed"] = token_bytes(32).hex()
-        save_config(key_config_filename, key_config)
+        save_config(root_path, keys_yaml, key_config)
     if args.pool:
         # Replaces the pools keys and targes. Only useful if running a pool, or doing
         # solo farming. The pool target allows spending of the coinbase.
@@ -89,7 +90,7 @@ def main():
             pool_target = wallet_target
         key_config["pool_sks"] = [bytes(pool_sk).hex() for pool_sk in pool_sks]
         key_config["pool_target"] = pool_target.hex()
-        save_config(key_config_filename, key_config)
+        save_config(root_path, keys_yaml, key_config)
 
 
 if __name__ == "__main__":

src/cmds/init.py

@@ -0,0 +1,101 @@
+import os
+import shutil
+
+from pathlib import Path
+
+from src.util.config import (
+    config_path_for_filename,
+    create_default_chia_config,
+    load_config,
+    save_config,
+)
+from src.util.default_root import DEFAULT_ROOT_PATH
+from src.util.path import make_path_relative, mkdir, path_from_root
+
+
+def make_parser(parser):
+    parser.set_defaults(function=init)
+
+
+def migrate_from(old_root, new_root, manifest):
+    """
+    Copy all the files in "manifest" to the new config directory.
+    """
+    if old_root == new_root:
+        print(f"same as new path, exiting")
+        return 1
+    if not old_root.is_dir():
+        print(f"{old_root} not found")
+        return 0
+    print(f"\n{old_root} found")
+    print(f"Copying files from {old_root} to {new_root}\n")
+    for f in manifest:
+        old_path = old_root / f
+        new_path = new_root / f
+        if old_path.is_file():
+            print(f"{new_path}")
+            mkdir(new_path.parent)
+            shutil.copy(old_path, new_path)
+        else:
+            print(f"{old_path} not found, skipping")
+
+    # migrate plots
+    # for now, we simply leave them where they are
+    # and make what may have been relative paths absolute
+
+    plots_config = load_config(new_root, "plots.yaml")
+    old_plot_paths = plots_config.get("plots", [])
+    if len(old_plot_paths) == 0:
+        print("no plots found, no plots migrated")
+        return 1
+
+    print("\nmigrating plots.yaml")
+    new_plot_paths = {}
+    for path, values in old_plot_paths.items():
+        old_path = path_from_root(old_root, path)
+        new_plot_path = make_path_relative(old_path, new_root)
+        print(f"rewriting {path}\n as {new_plot_path}")
+        new_plot_paths[str(new_plot_path)] = values
+    plots_config["plots"] = new_plot_paths
+    save_config(new_root, "plots.yaml", new_plot_paths)
+    print("\nUpdated plots.yaml to point to where your existing plots are.")
+    print(
+        "\nYour plots have not been moved so be careful deleting old preferences folders."
+    )
+    print("If you want to move your plot files, you should also modify")
+    print(f"{config_path_for_filename(new_root, 'plots.yaml')}")
+    return 1
+
+
+def init(args, parser):
+    return chia_init()
+
+
+def chia_init():
+    root_path = DEFAULT_ROOT_PATH
+    print(f"migrating to {root_path}")
+    if root_path.is_dir():
+        print(f"{root_path} already exists, no action taken")
+        return -1
+
+    MANIFEST = [
+        "config/config.yaml",
+        "config/plots.yaml",
+        "config/keys.yaml",
+        "wallet/db/blockchain_wallet_v4.db",
+        "db/blockchain_v3.db",
+    ]
+
+    PATH_MANIFEST_LIST = [
+        (Path(os.path.expanduser("~/.chia/beta-%s" % _)), MANIFEST) for _ in ["1.0b2", "1.0b1"]
+    ]
+
+    for old_path, manifest in PATH_MANIFEST_LIST:
+        r = migrate_from(old_path, root_path, manifest)
+        if r:
+            break
+    else:
+        create_default_chia_config(root_path)
+        print("Please generate your keys with chia-generate-keys")
+
+    return 0

src/cmds/show.py

@@ -12,7 +12,7 @@ from src.util.byte_types import hexstr_to_bytes
 from src.util.config import str2bool
 
 
-def show_parser(parser):
+def make_parser(parser):
 
     parser.add_argument(
         "-b",

src/cmds/version.py

@@ -5,6 +5,6 @@ def version(args, parser):
     print(__version__)
 
 
-def version_parser(parser):
+def make_parser(parser):
     parser.set_defaults(function=version)
     return parser

src/harvester.py

@@ -11,6 +11,7 @@ from src.server.outbound_message import Delivery, Message, NodeType, OutboundMes
 from src.types.proof_of_space import ProofOfSpace
 from src.types.sized_bytes import bytes32
 from src.util.api_decorators import api_request
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.ints import uint8
 from src.util.path import path_from_root
 
@@ -58,11 +59,11 @@ class Harvester:
         use any plots which don't have one of the pool keys.
         """
         for partial_filename_str, plot_config in self.plot_config["plots"].items():
-            plot_root = path_from_root(self.config.get("plot_root", "."))
+            plot_root = path_from_root(DEFAULT_ROOT_PATH, self.config.get("plot_root", "."))
-            partial_filename = path_from_root(partial_filename_str, plot_root)
+            partial_filename = plot_root / partial_filename_str
             potential_filenames = [
                 partial_filename,
-                path_from_root(partial_filename_str, plot_root),
+                path_from_root(plot_root, partial_filename_str),
             ]
             pool_pubkey = PublicKey.from_bytes(bytes.fromhex(plot_config["pool_pk"]))
 

src/rpc/rpc_server.py

@@ -12,7 +12,6 @@ from src.types.peer_info import PeerInfo
 from src.util.ints import uint16, uint64
 from src.types.sized_bytes import bytes32
 from src.util.byte_types import hexstr_to_bytes
-from src.util.config import load_config
 
 
 class EnhancedJSONEncoder(json.JSONEncoder):
@@ -145,7 +144,7 @@ class RpcApiHandler:
         port = request_data["port"]
         target_node: PeerInfo = PeerInfo(host, uint16(int(port)))
 
-        config = load_config("config.yaml", "full_node")
+        config = self.full_node.config
         if self.full_node.server is None or not (
             await self.full_node.server.start_client(target_node, None, config)
         ):

src/server/server.py

@@ -22,17 +22,22 @@ from src.server.outbound_message import Delivery, Message, NodeType, OutboundMes
 from src.types.peer_info import PeerInfo
 from src.types.sized_bytes import bytes32
 from src.util import partial_func
-from src.util.config import load_config
 from src.util.errors import Err, ProtocolError
 from src.util.ints import uint16
 from src.util.network import create_node_id
 import traceback
 
-config = load_config("config.yaml")
-
 
 class ChiaServer:
-    def __init__(self, port: int, api: Any, local_type: NodeType, name: str = None):
+    def __init__(
+        self,
+        port: int,
+        api: Any,
+        local_type: NodeType,
+        ping_interval: int,
+        network_id: str,
+        name: str = None,
+    ):
         # Keeps track of all connections to and from this node.
         self.global_connections: PeerConnections = PeerConnections([])
 
@@ -46,6 +51,8 @@ class ChiaServer:
         self._api = api  # API module that will be called from the requests
         self._local_type = local_type  # NodeType (farmer, full node, timelord, pool, harvester, wallet)
 
+        self._ping_interval = ping_interval
+        self._network_id = network_id
         # (StreamReader, StreamWriter, NodeType) aiter, gets things from server and clients and
         # sends them through the pipeline
         self._srwt_aiter: push_aiter = push_aiter()
@@ -242,7 +249,7 @@ class ChiaServer:
                 self.push_message(
                     OutboundMessage(NodeType.HARVESTER, msg, Delivery.BROADCAST)
                 )
-                await asyncio.sleep(config["ping_interval"])
+                await asyncio.sleep(self._ping_interval)
 
         return asyncio.create_task(ping())
 
@@ -362,7 +369,7 @@ class ChiaServer:
         outbound_handshake = Message(
             "handshake",
             Handshake(
-                config["network_id"],
+                self._network_id,
                 protocol_version,
                 self._node_id,
                 uint16(self._port),

src/server/start_farmer.py

@@ -11,15 +11,18 @@ from src.farmer import Farmer
 from src.server.outbound_message import NodeType
 from src.server.server import ChiaServer
 from src.types.peer_info import PeerInfo
-from src.util.logging import initialize_logging
 from src.util.config import load_config, load_config_cli
+from src.util.default_root import DEFAULT_ROOT_PATH
+from src.util.logging import initialize_logging
 from src.util.setproctitle import setproctitle
 
 
 async def main():
-    config = load_config_cli("config.yaml", "farmer")
+    root_path = DEFAULT_ROOT_PATH
+    net_config = load_config(root_path, "config.yaml")
+    config = load_config_cli(root_path, "config.yaml", "farmer")
     try:
-        key_config = load_config("keys.yaml")
+        key_config = load_config(root_path, "keys.yaml")
     except FileNotFoundError:
         raise RuntimeError("Keys not generated. Run chia-generate-keys")
     initialize_logging("Farmer %(name)-25s", config["logging"])
@@ -34,7 +37,9 @@ async def main():
     full_node_peer = PeerInfo(
         config["full_node_peer"]["host"], config["full_node_peer"]["port"]
     )
-    server = ChiaServer(config["port"], farmer, NodeType.FARMER)
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+    server = ChiaServer(config["port"], farmer, NodeType.FARMER, ping_interval, network_id)
 
     asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all)
     asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all)

src/server/start_full_node.py

@@ -23,20 +23,21 @@ from src.types.peer_info import PeerInfo
 from src.full_node.coin_store import CoinStore
 from src.util.logging import initialize_logging
 from src.util.config import load_config_cli
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.path import mkdir, path_from_root
 from src.util.pip_import import pip_import
 from src.util.setproctitle import setproctitle
 
 
 async def main():
-    config = load_config_cli("config.yaml", "full_node")
+    config = load_config_cli(DEFAULT_ROOT_PATH, "config.yaml", "full_node")
     setproctitle("chia_full_node")
     initialize_logging("FullNode %(name)-23s", config["logging"])
 
     log = logging.getLogger(__name__)
     server_closed = False
 
-    db_path = path_from_root(config["database_path"])
+    db_path = path_from_root(DEFAULT_ROOT_PATH, config["database_path"])
     mkdir(db_path.parent)
 
     # Create the store (DB) and full node instance
@@ -72,7 +73,9 @@ async def main():
             log.exception(f"UPnP failed")
 
     # Starts the full node server (which full nodes can connect to)
-    server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE)
+    ping_interval = config.get("ping_interval")
+    network_id = config.get("network_id")
+    server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE, ping_interval, network_id)
     full_node._set_server(server)
     _ = await server.start_server(full_node._on_connect, config)
     rpc_cleanup = None

src/server/start_harvester.py

@@ -11,15 +11,18 @@ from src.harvester import Harvester
 from src.server.outbound_message import NodeType
 from src.server.server import ChiaServer
 from src.types.peer_info import PeerInfo
-from src.util.logging import initialize_logging
 from src.util.config import load_config, load_config_cli
+from src.util.default_root import DEFAULT_ROOT_PATH
+from src.util.logging import initialize_logging
 from src.util.setproctitle import setproctitle
 
 
 async def main():
-    config = load_config_cli("config.yaml", "harvester")
+    root_path = DEFAULT_ROOT_PATH
+    net_config = load_config(root_path, "config.yaml")
+    config = load_config_cli(root_path, "config.yaml", "harvester")
     try:
-        plot_config = load_config("plots.yaml")
+        plot_config = load_config(root_path, "plots.yaml")
     except FileNotFoundError:
         raise RuntimeError("Plots not generated. Run chia-create-plots")
 
@@ -28,7 +31,9 @@ async def main():
     setproctitle("chia_harvester")
 
     harvester = Harvester(config, plot_config)
-    server = ChiaServer(config["port"], harvester, NodeType.HARVESTER)
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+    server = ChiaServer(config["port"], harvester, NodeType.HARVESTER, ping_interval, network_id)
     _ = await server.start_server(None, config)
 
     asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all)

src/server/start_introducer.py

@@ -10,20 +10,24 @@ except ImportError:
 from src.introducer import Introducer
 from src.server.outbound_message import NodeType
 from src.server.server import ChiaServer
+from src.util.config import load_config_cli, load_config
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.logging import initialize_logging
-from src.util.config import load_config_cli
 from src.util.setproctitle import setproctitle
 
 
 async def main():
-    config = load_config_cli("config.yaml", "introducer")
+    root_path = DEFAULT_ROOT_PATH
-
+    net_config = load_config(root_path, "config.yaml")
+    config = load_config_cli(root_path, "config.yaml", "introducer")
     initialize_logging("Introducer %(name)-21s", config["logging"])
     log = logging.getLogger(__name__)
     setproctitle("chia_introducer")
 
     introducer = Introducer(config)
-    server = ChiaServer(config["port"], introducer, NodeType.INTRODUCER)
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+    server = ChiaServer(config["port"], introducer, NodeType.INTRODUCER, ping_interval, network_id)
     introducer.set_server(server)
     _ = await server.start_server(None, config)
 

src/server/start_timelord.py

@@ -14,20 +14,25 @@ from src.server.outbound_message import NodeType
 from src.server.server import ChiaServer
 from src.timelord import Timelord
 from src.types.peer_info import PeerInfo
+from src.util.config import load_config_cli, load_config
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.logging import initialize_logging
-from src.util.config import load_config_cli
 from src.util.setproctitle import setproctitle
 
 
 async def main():
-    config = load_config_cli("config.yaml", "timelord")
+    root_path = DEFAULT_ROOT_PATH
-
+    net_config = load_config(root_path, "config.yaml")
+    config = load_config_cli(root_path, "config.yaml", "timelord")
     initialize_logging("Timelord %(name)-23s", config["logging"])
     log = logging.getLogger(__name__)
     setproctitle("chia_timelord")
 
     timelord = Timelord(config, constants)
-    server = ChiaServer(config["port"], timelord, NodeType.TIMELORD)
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
+    server = ChiaServer(config["port"], timelord, NodeType.TIMELORD, ping_interval, network_id)
     _ = await server.start_server(None, config)
 
     timelord_shutdown_task: Optional[asyncio.Task] = None

src/simulator/start_simulator.py

@@ -20,20 +20,23 @@ from src.server.connection import NodeType
 from src.types.full_block import FullBlock
 from src.full_node.coin_store import CoinStore
 from src.util.logging import initialize_logging
-from src.util.config import load_config_cli
+from src.util.config import load_config_cli, load_config
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.setproctitle import setproctitle
 from src.util.path import mkdir, path_from_root
 
 
 async def main():
-    config = load_config_cli("config.yaml", "full_node")
+    root_path = DEFAULT_ROOT_PATH
+    net_config = load_config(root_path, "config.yaml")
+    config = load_config_cli(root_path, "config.yaml", "full_node")
     setproctitle("chia_full_node")
     initialize_logging("FullNode %(name)-23s", config["logging"])
 
     log = logging.getLogger(__name__)
     server_closed = False
 
-    db_path = path_from_root(config["simulator_database_path"])
+    db_path = path_from_root(DEFAULT_ROOT_PATH, config["simulator_database_path"])
     mkdir(db_path.parent)
     connection = await aiosqlite.connect(db_path)
 
@@ -60,8 +63,12 @@ async def main():
         override_constants=test_constants,
     )
 
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
     # Starts the full node server (which full nodes can connect to)
-    server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE)
+
+    server = ChiaServer(config["port"], full_node, NodeType.FULL_NODE, ping_interval, network_id)
     full_node._set_server(server)
     _ = await server.start_server(full_node._on_connect, config)
     rpc_cleanup = None

src/timelord_launcher.py

@@ -7,17 +7,13 @@ from src.util.logging import initialize_logging
 from src.util.config import load_config
 from asyncio import Lock
 from typing import List
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.setproctitle import setproctitle
 
-config = load_config("config.yaml", "timelord_launcher")
-
 active_processes: List = []
 stopped = False
 lock = Lock()
 
-initialize_logging("Launcher %(name)-23s", config["logging"])
-setproctitle("chia_timelord_launcher")
-
 log = logging.getLogger(__name__)
 
 
@@ -69,7 +65,7 @@ async def spawn_process(host, port, counter):
         await asyncio.sleep(0.1)
 
 
-async def spawn_all_processes():
+async def spawn_all_processes(config):
     await asyncio.sleep(15)
     port = config["port"]
     process_count = config["process_count"]
@@ -77,7 +73,10 @@ async def spawn_all_processes():
     await asyncio.gather(*awaitables)
 
 
-if __name__ == "__main__":
+def main():
+    setproctitle("chia_timelord_launcher")
+    config = load_config(DEFAULT_ROOT_PATH, "config.yaml", "timelord_launcher")
+    initialize_logging("Launcher %(name)-23s", config["logging"])
 
     def signal_received():
         asyncio.create_task(kill_processes())
@@ -88,7 +87,11 @@ if __name__ == "__main__":
     loop.add_signal_handler(signal.SIGTERM, signal_received)
 
     try:
-        loop.run_until_complete(spawn_all_processes())
+        loop.run_until_complete(spawn_all_processes(config))
     finally:
         log.info("Launcher fully closed.")
         loop.close()
+
+
+if __name__ == "__main__":
+    main()

src/util/config.py

@@ -1,45 +1,65 @@
-import yaml
 import argparse
-from pathlib import Path
 import pkg_resources
+import sys
+import yaml
+
+from pathlib import Path
 from typing import Dict, Any, Callable, Optional, Union
-from src.util.path import mkdir, path_from_root
+from src.util.path import mkdir
 
 
-def migrate_config_file(filename: Union[str, Path], path: Path) -> None:
+def initial_config_file(filename: Union[str, Path]) -> str:
-    filepath = Path(filename)
+    return pkg_resources.resource_string(__name__, f"initial-{filename}").decode()
-    default_config_file = pkg_resources.resource_string(
+
-        __name__, f"initial-{filepath.parts[-1]}"
+
-    ).decode()
+def create_default_chia_config(root_path: Path) -> None:
-    mkdir(str(path.parent))
+    for filename in ["config.yaml"]:
+        default_config_file_data = initial_config_file(filename)
+        path = config_path_for_filename(root_path, filename)
+        mkdir(path.parent)
         with open(path, "w") as f:
-        f.write(default_config_file)
+            f.write(default_config_file_data)
+    save_config(root_path, "plots.yaml", {})
 
 
-def save_config(filename: Union[str, Path], config_data):
+def config_path_for_filename(root_path: Path, filename: Union[str, Path]) -> Path:
-    path = path_from_root("config") / filename
+    path_filename = Path(filename)
+    if path_filename.is_absolute():
+        return path_filename
+    return root_path / "config" / filename
+
+
+def save_config(root_path: Path, filename: Union[str, Path], config_data : Any):
+    path = config_path_for_filename(root_path, filename)
     with open(path, "w") as f:
         yaml.safe_dump(config_data, f)
 
 
-def load_config(filename: Union[str, Path], sub_config: Optional[str] = None) -> Dict:
+def load_config(
-    path = path_from_root("config") / filename
+    root_path: Path,
+    filename: Union[str, Path],
+    sub_config: Optional[str] = None
+) -> Dict:
+    path = config_path_for_filename(root_path, filename)
     if not path.is_file():
-        migrate_config_file(filename, path)
+        print(f"can't find {path}")
+        print("** please run `chia init` to migrate or create new config files **")
+        # TODO: fix this hack
+        sys.exit(-1)
+    r = yaml.safe_load(open(path, "r"))
     if sub_config is not None:
-        return yaml.safe_load(open(path, "r"))[sub_config]
+        r = r.get(sub_config)
-    else:
+    return r
-        return yaml.safe_load(open(path, "r"))
 
 
-def load_config_cli(filename: str, sub_config: Optional[str] = None) -> Dict:
+def load_config_cli(root_path: Path, filename: str, sub_config: Optional[str]) -> Dict:
     """
     Loads configuration from the specified filename, in the config directory,
     and then overrides any properties using the passed in command line arguments.
     Nested properties in the config file can be used in the command line with ".",
     for example --farmer_peer.host. Does not support lists.
     """
-    config = load_config(filename, sub_config)
+    config = load_config(root_path, filename, sub_config)
 
     flattened_props = flatten_properties(config)
     parser = argparse.ArgumentParser()

src/util/default_root.py

@@ -0,0 +1,10 @@
+import os
+from pathlib import Path
+
+from src import __version__
+
+DEFAULT_ROOT_PATH = Path(
+    os.path.expanduser(
+        os.getenv("CHIA_ROOT", "~/.chia/beta-{version}").format(version=__version__)
+    )
+).resolve()

src/util/logging.py

@@ -2,11 +2,12 @@ import logging
 import colorlog
 from typing import Dict
 
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.path import mkdir, path_from_root
 
 
 def initialize_logging(prefix: str, logging_config: Dict):
-    log_path = path_from_root(logging_config.get("log_filename", "log/debug.log"))
+    log_path = path_from_root(DEFAULT_ROOT_PATH, logging_config.get("log_filename", "log/debug.log"))
     mkdir(str(log_path.parent))
     if logging_config["log_stdout"]:
         handler = colorlog.StreamHandler()

src/util/path.py

@@ -1,53 +1,36 @@
 import os
+
 from typing import Union
 from pathlib import Path
 
-from src import __version__
 
-DEFAULT_ROOT_PATH = Path(
+def path_from_root(root: Path, path_str: Union[str, Path]) -> Path:
-    os.path.expanduser(os.getenv("CHIA_ROOT", "~/.chia/beta-%s" % __version__))
-).resolve()
-
-
-def path_from_root(
-    path_str: Union[str, Path] = ".", root_path: Path = DEFAULT_ROOT_PATH
-) -> Path:
     """
-    Return a new path from the given one, expanding "~" if present.
+    If path is relative, prepend root
-    If path is relative, prepend $CHIA_ROOT
     If path is absolute, return it directly.
-    This lets us use "~" and other absolute paths in config files.
     """
-    if isinstance(path_str, Path):
+    root = Path(os.path.expanduser(str(root)))
-        path = path_str
-    else:
     path = Path(path_str)
     if not path.is_absolute():
-        path = root_path / path
+        path = root / path
-    return path
+    return path.resolve()
 
 
 def mkdir(path_str: Union[str, Path]) -> None:
     """
     Create the existing directory (and its parents) if necessary.
     """
-    if isinstance(path_str, Path):
-        path = path_str
-    else:
     path = Path(path_str)
     path.mkdir(parents=True, exist_ok=True)
 
 
-def make_path_relative(path_str: Union[str, Path]) -> Path:
+def make_path_relative(path_str: Union[str, Path], root: Path) -> Path:
     """
     Try to make the given path relative, given the default root.
     """
-    if isinstance(path_str, Path):
-        path = path_str
-    else:
     path = Path(path_str)
     try:
-        path = path.relative_to(DEFAULT_ROOT_PATH)
+        path = path.relative_to(root)
     except ValueError:
         pass
-    return path
+    return path.resolve()

src/wallet/wallet_node.py

@@ -28,6 +28,7 @@ from src.types.full_block import FullBlock
 from src.types.coin import Coin, hash_coin_list
 from src.full_node.blockchain import ReceiveBlockResult
 from src.types.mempool_inclusion_status import MempoolInclusionStatus
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.errors import Err
 from src.util.path import path_from_root, mkdir
 
@@ -81,7 +82,7 @@ class WalletNode:
         else:
             self.log = logging.getLogger(__name__)
 
-        path = path_from_root(config["database_path"])
+        path = path_from_root(DEFAULT_ROOT_PATH, config["database_path"])
         mkdir(path.parent)
 
         self.wallet_state_manager = await WalletStateManager.create(

src/wallet/websocket_server.py

@@ -29,6 +29,7 @@ from src.wallet.util.wallet_types import WalletType
 from src.wallet.wallet_info import WalletInfo
 from src.wallet.wallet_node import WalletNode
 from src.types.mempool_inclusion_status import MempoolInclusionStatus
+from src.util.default_root import DEFAULT_ROOT_PATH
 from src.util.setproctitle import setproctitle
 
 # Timeout for response from wallet/full node for sending a transaction
@@ -362,7 +363,9 @@ async def start_websocket_server():
     Starts WalletNode, WebSocketServer, and ChiaServer
     """
 
-    config = load_config_cli("config.yaml", "wallet")
+    root_path = DEFAULT_ROOT_PATH
+
+    config = load_config_cli(root_path, "config.yaml", "wallet")
     initialize_logging("Wallet %(name)-25s", config["logging"])
     log = logging.getLogger(__name__)
     log.info(f"Config : {config}")
@@ -384,8 +387,12 @@ async def start_websocket_server():
     handler = WebSocketServer(wallet_node, log)
     wallet_node.wallet_state_manager.set_callback(handler.state_changed_callback)
 
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
     log.info(f"Starting wallet server on port {config['port']}.")
-    server = ChiaServer(config["port"], wallet_node, NodeType.WALLET)
+    server = ChiaServer(config["port"], wallet_node, NodeType.WALLET, ping_interval, network_id)
     wallet_node.set_server(server)
 
     _ = await server.start_server(None, config)

tests/block_tools.py

@@ -1,14 +1,18 @@
+import os
 import sys
 import time
-from typing import Any, Dict, List, Tuple, Optional
 import random
 
-import blspy
+from pathlib import Path
+from typing import Any, Dict, List, Tuple, Optional
+
 from blspy import PrependSignature, PrivateKey, PublicKey
 from chiavdf import prove
 from chiabip158 import PyBIP158
 
 from chiapos import DiskPlotter, DiskProver
+from src import __version__
+from src.cmds.init import create_default_chia_config
 from src.consensus import block_rewards, pot_iterations
 from src.consensus.constants import constants
 from src.consensus.pot_iterations import calculate_min_iters_from_iterations
@@ -26,7 +30,7 @@ from src.types.sized_bytes import bytes32
 from src.util.merkle_set import MerkleSet
 from src.util.ints import uint8, uint32, uint64, uint128, int512
 from src.util.hash import std_hash
-from src.util.path import mkdir, path_from_root
+from src.util.path import mkdir
 from src.util.significant_bits import truncate_to_significant_bits
 
 # Can't go much lower than 19, since plots start having no solutions
@@ -49,12 +53,21 @@ fee_target = std_hash(bytes(wallet_sk.get_public_key()))
 n_wesolowski = uint8(0)
 
 
+TEST_ROOT_PATH = Path(
+    os.path.expanduser(
+        os.getenv("CHIA_ROOT", "~/.chia/beta-{version}-test").format(version=__version__)
+    )
+).resolve()
+
+
 class BlockTools:
     """
     Tools to generate blocks for testing.
     """
 
-    def __init__(self):
+    def __init__(self, root_path : Path = TEST_ROOT_PATH):
+        create_default_chia_config(root_path)
+        self.root_path = root_path
         self.plot_config: Dict = {"plots": {}}
         self.pool_sk = pool_sk
         self.wallet_sk = wallet_sk
@@ -62,7 +75,7 @@ class BlockTools:
         plot_seeds: List[bytes32] = [
             ProofOfSpace.calculate_plot_seed(pool_pk, plot_pk) for plot_pk in plot_pks
         ]
-        self.plot_dir = path_from_root("tests") / "plots"
+        self.plot_dir = self.root_path / "plots"
         mkdir(self.plot_dir)
         self.filenames: List[str] = [
             f"genesis-plots-{k}{std_hash(int.to_bytes(i, 4, 'big')).hex()}.dat"

tests/full_node/test_blockchain.py

@@ -5,18 +5,17 @@ from pathlib import Path
 
 import aiosqlite
 import pytest
-from blspy import PrivateKey, PrependSignature
+from blspy import PrivateKey
 
 from src.full_node.blockchain import Blockchain, ReceiveBlockResult
 from src.full_node.store import FullNodeStore
 from src.types.full_block import FullBlock
-from src.types.coin import Coin
 from src.types.header import Header, HeaderData
 from src.types.proof_of_space import ProofOfSpace
 from src.full_node.coin_store import CoinStore
 from src.util.ints import uint8, uint64
 from src.consensus.constants import constants as consensus_constants
-from tests.block_tools import BlockTools, plot_sks
+from tests.block_tools import BlockTools
 from src.util.errors import Err
 from src.consensus.coinbase import create_coinbase_coin_and_signature
 from src.types.sized_bytes import bytes32

tests/full_node/test_full_sync.py

@@ -6,7 +6,7 @@ import pytest
 from src.types.peer_info import PeerInfo
 from src.protocols import full_node_protocol
 from src.util.ints import uint16
-from tests.setup_nodes import setup_two_nodes, setup_node_and_wallet, test_constants, bt
+from tests.setup_nodes import setup_two_nodes, test_constants, bt
 
 
 @pytest.fixture(scope="module")

tests/full_node/test_transactions.py

@@ -4,7 +4,7 @@ from secrets import token_bytes
 import pytest
 
 from src.protocols import full_node_protocol
-from src.simulator.simulator_protocol import FarmNewBlockProtocol, ReorgProtocol
+from src.simulator.simulator_protocol import FarmNewBlockProtocol
 from src.types.peer_info import PeerInfo
 from src.util.ints import uint16, uint32
 from tests.setup_nodes import setup_simulators_and_wallets

tests/setup_nodes.py

@@ -1,13 +1,12 @@
-import signal
-from typing import Any, Dict, Optional, Tuple, List
-from pathlib import Path
 import asyncio
 
+from typing import Any, Dict, Tuple, List
+from pathlib import Path
+
 import aiosqlite
 import blspy
 from secrets import token_bytes
 
-from src.consensus.constants import constants
 from src.full_node.blockchain import Blockchain
 from src.full_node.mempool_manager import MempoolManager
 from src.full_node.store import FullNodeStore
@@ -30,8 +29,11 @@ from src.timelord import Timelord
 from src.server.connection import PeerInfo
 from src.util.ints import uint16, uint32
 
+
 bt = BlockTools()
 
+root_path = bt.root_path
+
 test_constants: Dict[str, Any] = {
     "DIFFICULTY_STARTING": 1,
     "DISCRIMINANT_SIZE_BITS": 16,
@@ -71,7 +73,12 @@ async def setup_full_node_simulator(db_name, port, introducer_port=None, dic={})
 
     await store_1.add_block(FullBlock.from_bytes(test_constants_copy["GENESIS_BLOCK"]))
 
-    config = load_config("config.yaml", "full_node")
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
+    config = load_config(root_path, "config.yaml", "full_node")
+
     if introducer_port is not None:
         config["introducer_peer"]["host"] = "127.0.0.1"
         config["introducer_peer"]["port"] = introducer_port
@@ -85,7 +92,7 @@ async def setup_full_node_simulator(db_name, port, introducer_port=None, dic={})
         test_constants_copy,
     )
     server_1 = ChiaServer(
-        port, full_node_1, NodeType.FULL_NODE, name="full-node-simulator-server"
+        port, full_node_1, NodeType.FULL_NODE, ping_interval, network_id, "full-node-simulator-server",
     )
     _ = await server_1.start_server(full_node_1._on_connect)
     full_node_1._set_server(server_1)
@@ -121,7 +128,11 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}):
 
     await store_1.add_block(FullBlock.from_bytes(test_constants_copy["GENESIS_BLOCK"]))
 
-    config = load_config("config.yaml", "full_node")
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
+    config = load_config(root_path, "config.yaml", "full_node")
     if introducer_port is not None:
         config["introducer_peer"]["host"] = "127.0.0.1"
         config["introducer_peer"]["port"] = introducer_port
@@ -134,7 +145,7 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}):
         f"full_node_{port}",
         test_constants_copy,
     )
-    server_1 = ChiaServer(port, full_node_1, NodeType.FULL_NODE)
+    server_1 = ChiaServer(port, full_node_1, NodeType.FULL_NODE, ping_interval, network_id)
     _ = await server_1.start_server(full_node_1._on_connect)
     full_node_1._set_server(server_1)
 
@@ -148,7 +159,7 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}):
 
 
 async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}):
-    config = load_config("config.yaml", "wallet")
+    config = load_config(root_path, "config.yaml", "wallet")
     if "starting_height" in dic:
         config["starting_height"] = dic["starting_height"]
     key_config = {
@@ -157,14 +168,19 @@ async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}):
     test_constants_copy = test_constants.copy()
     for k in dic.keys():
         test_constants_copy[k] = dic[k]
-    db_path = "test-wallet-db" + token_bytes(32).hex() + ".db"
+    db_path = root_path / ("test-wallet-db%s.db" % token_bytes(32).hex())
-    if Path(db_path).exists():
+    if db_path.exists():
-        Path(db_path).unlink()
+        db_path.unlink()
-    config["database_path"] = db_path
+    config["database_path"] = str(db_path)
+
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
     wallet = await WalletNode.create(
         config, key_config, override_constants=test_constants_copy, name="wallet1",
     )
-    server = ChiaServer(port, wallet, NodeType.WALLET, name="wallet-server")
+    server = ChiaServer(port, wallet, NodeType.WALLET, ping_interval, network_id, "wallet-server")
     wallet.set_server(server)
 
     yield (wallet, server)
@@ -177,10 +193,15 @@ async def setup_wallet_node(port, introducer_port=None, key_seed=b"", dic={}):
 
 
 async def setup_harvester(port, dic={}):
-    config = load_config("config.yaml", "harvester")
+    config = load_config(root_path, "config.yaml", "harvester")
 
     harvester = Harvester(config, bt.plot_config)
-    server = ChiaServer(port, harvester, NodeType.HARVESTER)
+
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
+    server = ChiaServer(port, harvester, NodeType.HARVESTER, ping_interval, network_id)
     _ = await server.start_server(None)
 
     yield (harvester, server)
@@ -192,7 +213,7 @@ async def setup_harvester(port, dic={}):
 
 
 async def setup_farmer(port, dic={}):
-    config = load_config("config.yaml", "farmer")
+    config = load_config(root_path, "config.yaml", "farmer")
     pool_sk = bt.pool_sk
     pool_target = create_puzzlehash_for_pk(
         BLSPublicKey(bytes(pool_sk.get_public_key()))
@@ -212,8 +233,12 @@ async def setup_farmer(port, dic={}):
     for k in dic.keys():
         test_constants_copy[k] = dic[k]
 
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
     farmer = Farmer(config, key_config, test_constants_copy)
-    server = ChiaServer(port, farmer, NodeType.FARMER)
+    server = ChiaServer(port, farmer, NodeType.FARMER, ping_interval, network_id)
     _ = await server.start_server(farmer._on_connect)
 
     yield (farmer, server)
@@ -223,10 +248,14 @@ async def setup_farmer(port, dic={}):
 
 
 async def setup_introducer(port, dic={}):
-    config = load_config("config.yaml", "introducer")
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
+    config = load_config(root_path, "config.yaml", "introducer")
 
     introducer = Introducer(config)
-    server = ChiaServer(port, introducer, NodeType.INTRODUCER)
+    server = ChiaServer(port, introducer, NodeType.INTRODUCER, ping_interval, network_id)
     _ = await server.start_server(None)
 
     yield (introducer, server)
@@ -244,12 +273,18 @@ async def setup_vdf_clients(port):
 
 
 async def setup_timelord(port, dic={}):
-    config = load_config("config.yaml", "timelord")
+    config = load_config(root_path, "config.yaml", "timelord")
+
     test_constants_copy = test_constants.copy()
     for k in dic.keys():
         test_constants_copy[k] = dic[k]
     timelord = Timelord(config, test_constants_copy)
-    server = ChiaServer(port, timelord, NodeType.TIMELORD)
+
+    net_config = load_config(root_path, "config.yaml")
+    ping_interval = net_config.get("ping_interval")
+    network_id = net_config.get("network_id")
+
+    server = ChiaServer(port, timelord, NodeType.TIMELORD, ping_interval, network_id)
     _ = await server.start_server(None, config)
 
     coro = asyncio.start_server(

tests/test_filter.py

@@ -6,7 +6,6 @@ from blspy import ExtendedPrivateKey
 from chiabip158 import PyBIP158
 
 from tests.setup_nodes import test_constants, bt, setup_simulators_and_wallets
-from src.util.config import load_config
 
 
 @pytest.fixture(scope="module")