Migrate gsutil usage to gcloud storage (#13727)

Author: bhandarivijay-png

people-and-planet-ai/land-cover-classification/e2e_test.py

@@ -57,7 +57,7 @@ def data_path(bucket_name: str) -> str:
 def model_path(bucket_name: str) -> str:
     # This is a different path than where Vertex AI saves its model.
     gcs_path = f"gs://{bucket_name}/pretrained-model"
-    conftest.run_cmd("gsutil", "-m", "cp", "-r", "./pretrained-model", gcs_path)
+    conftest.run_cmd("gcloud", "storage", "cp", "--recursive", "./pretrained-model", gcs_path)
     return gcs_path
 
 

people-and-planet-ai/weather-forecasting/notebooks/2-dataset.ipynb

@@ -700,8 +700,7 @@
    },
    "outputs": [],
    "source": [
-    "!gsutil ls -lh gs://{bucket}/weather/data-small"
-   ],
+    "!gcloud storage ls --long --readable-sizes gs://{bucket}/weather/data-small"   ],
    "id": "F43OAIlrDosG"
   },
   {

people-and-planet-ai/weather-forecasting/notebooks/3-training.ipynb

@@ -285,7 +285,7 @@
     "data_path_gcs = f\"gs://{bucket}/weather/data\"\n",
     "\n",
     "!mkdir -p data-training\n",
-    "!gsutil -m cp {data_path_gcs}/* data-training"
+    "!gcloud storage cp {data_path_gcs}/* data-training"
    ],
    "metadata": {
     "id": "h_IUpnqvO-sa"
@@ -1336,7 +1336,7 @@
    "cell_type": "code",
    "source": [
     "# Stage the `weather-model` package in Cloud Storage.\n",
-    "!gsutil cp serving/weather-model/dist/weather-model-1.0.0.tar.gz gs://{bucket}/weather/"
+    "!gcloud storage cp serving/weather-model/dist/weather-model-1.0.0.tar.gz gs://{bucket}/weather/"
    ],
    "metadata": {
     "id": "JA1k9ky02dsx"

people-and-planet-ai/weather-forecasting/notebooks/4-predictions.ipynb

@@ -336,8 +336,7 @@
     "model_path_gcs = f\"gs://{bucket}/weather/model\"\n",
     "\n",
     "!mkdir -p model\n",
-    "!gsutil cp {model_path_gcs}/* model"
-   ],
+    "!gcloud storage cp {model_path_gcs}/* model"   ],
    "metadata": {
     "id": "5w_uNjluhDMG"
    },

people-and-planet-ai/weather-forecasting/tests/predictions_tests/test_predictions.py

@@ -37,7 +37,7 @@ def test_name() -> str:
 @pytest.fixture(scope="session")
 def model_path_gcs(bucket_name: str) -> str:
     path_gcs = f"gs://{bucket_name}/model"
-    conftest.run_cmd("gsutil", "cp", "serving/model/*", path_gcs)
+    conftest.run_cmd("gcloud", "storage", "cp", "serving/model/*", path_gcs)
     return path_gcs
 
 

people-and-planet-ai/weather-forecasting/tests/training_tests/test_training.py

@@ -51,7 +51,7 @@ def data_path_gcs(bucket_name: str) -> str:
         inputs_batch = [inputs] * batch_size
         labels_batch = [labels] * batch_size
         np.savez_compressed(f, inputs=inputs_batch, labels=labels_batch)
-        conftest.run_cmd("gsutil", "cp", f.name, f"{path_gcs}/example.npz")
+        conftest.run_cmd("gcloud", "storage", "cp", f.name, f"{path_gcs}/example.npz")
     return path_gcs