zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix notebook

This commit is contained in:
lcaggio 2023-03-06 12:51:46 +01:00
parent 0ac6dd65cf
commit ee55127ede
1 changed files with 20 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb
View File

@ -14,6 +14,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Copyright 2023 Google LLC\n",
"#\n",
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
"# you may not use this file except in compliance with the License.\n", "# you may not use this file except in compliance with the License.\n",
"# You may obtain a copy of the License at\n", "# You may obtain a copy of the License at\n",
@ -51,8 +53,9 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import kfp\n", "import kfp\n",
"from google.cloud import aiplatform as aip\n",
"import google_cloud_pipeline_components.v1.bigquery as bqop\n", "import google_cloud_pipeline_components.v1.bigquery as bqop\n",
"\n",
"from google.cloud import aiplatform as aip\n",
"from google.cloud import bigquery" "from google.cloud import bigquery"
] ]
}, },
@ -70,18 +73,17 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"PREFIX = 'your-prefix'\n",
"PROJECT_ID = 'your-project-id'\n",
"LOCATION = 'US'\n",
"REGION = 'us-central1'\n",
"PIPELINE_NAME = 'bqml-vertex-pipeline'\n",
"MODEL_NAME = 'bqml-model'\n",
"EXPERIMENT_NAME = 'bqml-experiment'\n", "EXPERIMENT_NAME = 'bqml-experiment'\n",
"ENDPOINT_DISPLAY_NAME = 'bqml-endpoint'\n", "ENDPOINT_DISPLAY_NAME = 'bqml-endpoint'\n",
"\n", "DATASET = \"{}_data\".format(PREFIX.replace(\"-\",\"_\")) \n",
"SERVICE_ACCOUNT = f\"vertex-sa@{PROJECT_ID}.iam.gserviceaccount.com\"\n", "LOCATION = 'US'\n",
"MODEL_NAME = 'bqml-model'\n",
"PIPELINE_NAME = 'bqml-vertex-pipeline'\n",
"PIPELINE_ROOT = f\"gs://{PREFIX}-data\"\n", "PIPELINE_ROOT = f\"gs://{PREFIX}-data\"\n",
"DATASET = \"{}_data\".format(PREFIX.replace(\"-\",\"_\")) " "PREFIX = 'your-prefix'\n",
"PROJECT_ID = 'your-project-id'\n",
"REGION = 'us-central1'\n",
"SERVICE_ACCOUNT = f\"vertex-sa@{PROJECT_ID}.iam.gserviceaccount.com\""
] ]
}, },
{ {
@ -97,12 +99,12 @@
"3. Evaluate the BigQuery ML model with the standard evaluation metrics\n", "3. Evaluate the BigQuery ML model with the standard evaluation metrics\n",
"\n", "\n",
"The pipeline takes as input the following variables:\n", "The pipeline takes as input the following variables:\n",
"- ```model_name```: the display name of the BigQuery ML model\n",
"- ```split_fraction```: the percentage of data that will be used as an evaluation dataset\n",
"- ```evaluate_job_conf```: bq dict configuration to define where to store evaluation metrics\n",
"- ```dataset```: name of the dataset where the artifacts will be stored\n", "- ```dataset```: name of the dataset where the artifacts will be stored\n",
"- ```evaluate_job_conf```: bq dict configuration to define where to store evaluation metrics\n",
"- ```location```: BigQuery location\n",
"- ```model_name```: the display name of the BigQuery ML model\n",
"- ```project_id```: the project id where the GCP resources will be created\n", "- ```project_id```: the project id where the GCP resources will be created\n",
"- ```location```: BigQuery location" "- ```split_fraction```: the percentage of data that will be used as an evaluation dataset"
] ]
}, },
{ {
@ -186,7 +188,7 @@
" description='This is a new experiment to keep track of bqml trainings',\n", " description='This is a new experiment to keep track of bqml trainings',\n",
" project=PROJECT_ID,\n", " project=PROJECT_ID,\n",
" location=REGION\n", " location=REGION\n",
" )" ")"
] ]
}, },
{ {
@ -218,7 +220,6 @@
" template_path=f'{PIPELINE_NAME}.json',\n", " template_path=f'{PIPELINE_NAME}.json',\n",
" pipeline_root=PIPELINE_ROOT,\n", " pipeline_root=PIPELINE_ROOT,\n",
" enable_caching=True\n", " enable_caching=True\n",
" \n",
" )\n", " )\n",
"\n", "\n",
" pipeline.submit(service_account=SERVICE_ACCOUNT, experiment=my_experiment)" " pipeline.submit(service_account=SERVICE_ACCOUNT, experiment=my_experiment)"
@ -278,7 +279,8 @@
" 'day_of_week': 'WEEKDAY',\n", " 'day_of_week': 'WEEKDAY',\n",
" 'traffic_source': 'Facebook',\n", " 'traffic_source': 'Facebook',\n",
" 'browser': 'Firefox',\n", " 'browser': 'Firefox',\n",
" 'hour_of_day': 20}" " 'hour_of_day': 20\n",
"}"
] ]
}, },
{ {
@ -337,7 +339,7 @@
}, },
"language_info": { "language_info": {
"name": "python", "name": "python",
"version": "3.10.9" "version": "3.8.9"
}, },
"orig_nbformat": 4, "orig_nbformat": 4,
"vscode": { "vscode": {