From a0de75f98b2f1e89485b790d915c5163d65aaae8 Mon Sep 17 00:00:00 2001 From: Bernard Han <103850980+bernardhan33@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:08:29 -0700 Subject: [PATCH] Update README to note the new pip install command (#16) --- README.md | 4 +- demo/image-segmentation/README.md | 4 +- ...or PyTorch with Google Cloud Storage.ipynb | 159 +++++++++--------- 3 files changed, 87 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 6f09373f..da395d6d 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,7 @@ Note that the Dataflux Dataset for PyTorch library is in an early preview stage ### Installation ```shell -git clone --recurse-submodules https://github.com/GoogleCloudPlatform/dataflux-pytorch -cd dataflux-pytorch -pip install . +pip install gcs-torch-dataflux ``` ### Configuration diff --git a/demo/image-segmentation/README.md b/demo/image-segmentation/README.md index 189ca88f..9531ba6d 100644 --- a/demo/image-segmentation/README.md +++ b/demo/image-segmentation/README.md @@ -59,11 +59,11 @@ We will spin up a [Ray cluster](https://docs.ray.io/en/latest/cluster/key-concep > **_NOTE:_** It would be best to run the following steps on your workstation with a display device so you can see the Ray dashboard using a browser. -1. Clone the `dataflux-pytorch` repository and go to the working directory. +1. Clone the `dataflux-pytorch` repository, install the packages and go to the working directory. ```shell git clone --recurse-submodules https://github.com/GoogleCloudPlatform/dataflux-pytorch cd dataflux-pytorch - pip install . + pip install gcs-torch-dataflux cd demo/image-segmentation ``` 2. [Install Ray](https://docs.ray.io/en/latest/ray-overview/installation.html). diff --git a/demo/simple-walkthrough/Getting Started with Dataflux Dataset for PyTorch with Google Cloud Storage.ipynb b/demo/simple-walkthrough/Getting Started with Dataflux Dataset for PyTorch with Google Cloud Storage.ipynb index 4f4a46ff..f8c74b9f 100644 --- a/demo/simple-walkthrough/Getting Started with Dataflux Dataset for PyTorch with Google Cloud Storage.ipynb +++ b/demo/simple-walkthrough/Getting Started with Dataflux Dataset for PyTorch with Google Cloud Storage.ipynb @@ -24,85 +24,93 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, - "collapsed": true, "id": "lii_KERCS3uC", - "outputId": "d1ce253c-6a70-4804-b641-93b6b7ef8942" + "outputId": "b37e6108-0ac6-436d-b43d-06b7cf066731" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Cloning into 'dataflux-pytorch'...\n", - "remote: Enumerating objects: 168, done.\u001b[K\n", - "remote: Counting objects: 100% (39/39), done.\u001b[K\n", - "remote: Compressing objects: 100% (33/33), done.\u001b[K\n", - "remote: Total 168 (delta 16), reused 14 (delta 6), pack-reused 129\u001b[K\n", - "Receiving objects: 100% (168/168), 1.17 MiB | 13.98 MiB/s, done.\n", - "Resolving deltas: 100% (75/75), done.\n", - "Submodule 'dataflux-client-python' (https://github.com/GoogleCloudPlatform/dataflux-client-python.git) registered for path 'dataflux_client_python'\n", - "Cloning into '/content/dataflux-pytorch/dataflux_client_python'...\n", - "remote: Enumerating objects: 125, done. \n", - "remote: Counting objects: 100% (21/21), done. \n", - "remote: Compressing objects: 100% (14/14), done. \n", - "remote: Total 125 (delta 8), reused 13 (delta 7), pack-reused 104 \n", - "Receiving objects: 100% (125/125), 57.26 KiB | 3.18 MiB/s, done.\n", - "Resolving deltas: 100% (66/66), done.\n", - "Submodule path 'dataflux_client_python': checked out '2fc12b62a3942b6bb9e637482ac7d428acad1b46'\n", - "Processing /content/dataflux-pytorch\n", - " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from dataflux-pytorch==0.0.0) (2.1.0+cu121)\n", - "Requirement already satisfied: google-cloud-storage in /usr/local/lib/python3.10/dist-packages (from dataflux-pytorch==0.0.0) (2.8.0)\n", - "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from dataflux-pytorch==0.0.0) (1.4.0)\n", - "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from dataflux-pytorch==0.0.0) (1.25.2)\n", - "Requirement already satisfied: pytest in /usr/local/lib/python3.10/dist-packages (from dataflux-pytorch==0.0.0) (7.4.4)\n", - "Requirement already satisfied: google-auth<3.0dev,>=1.25.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->dataflux-pytorch==0.0.0) (2.27.0)\n", - "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->dataflux-pytorch==0.0.0) (2.11.1)\n", - "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->dataflux-pytorch==0.0.0) (2.3.3)\n", - "Requirement already satisfied: google-resumable-media>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->dataflux-pytorch==0.0.0) (2.7.0)\n", - "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->dataflux-pytorch==0.0.0) (2.31.0)\n", - "Requirement already satisfied: iniconfig in /usr/local/lib/python3.10/dist-packages (from pytest->dataflux-pytorch==0.0.0) (2.0.0)\n", - "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from pytest->dataflux-pytorch==0.0.0) (23.2)\n", - "Requirement already satisfied: pluggy<2.0,>=0.12 in /usr/local/lib/python3.10/dist-packages (from pytest->dataflux-pytorch==0.0.0) (1.4.0)\n", - "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /usr/local/lib/python3.10/dist-packages (from pytest->dataflux-pytorch==0.0.0) (1.2.0)\n", - "Requirement already satisfied: tomli>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from pytest->dataflux-pytorch==0.0.0) (2.0.1)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->dataflux-pytorch==0.0.0) (3.13.1)\n", - "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch->dataflux-pytorch==0.0.0) (4.10.0)\n", - "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->dataflux-pytorch==0.0.0) (1.12)\n", - "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->dataflux-pytorch==0.0.0) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->dataflux-pytorch==0.0.0) (3.1.3)\n", - "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->dataflux-pytorch==0.0.0) (2023.6.0)\n", - "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch->dataflux-pytorch==0.0.0) (2.1.0)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->dataflux-pytorch==0.0.0) (1.62.0)\n", - "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->dataflux-pytorch==0.0.0) (3.20.3)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->dataflux-pytorch==0.0.0) (5.3.3)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->dataflux-pytorch==0.0.0) (0.3.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->dataflux-pytorch==0.0.0) (4.9)\n", - "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-resumable-media>=2.3.2->google-cloud-storage->dataflux-pytorch==0.0.0) (1.5.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->dataflux-pytorch==0.0.0) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->dataflux-pytorch==0.0.0) (3.6)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->dataflux-pytorch==0.0.0) (2.0.7)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->dataflux-pytorch==0.0.0) (2024.2.2)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->dataflux-pytorch==0.0.0) (2.1.5)\n", - "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->dataflux-pytorch==0.0.0) (1.3.0)\n", - "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-cloud-storage->dataflux-pytorch==0.0.0) (0.5.1)\n", - "Building wheels for collected packages: dataflux-pytorch\n", - " Building wheel for dataflux-pytorch (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for dataflux-pytorch: filename=dataflux_pytorch-0.0.0-py3-none-any.whl size=20376 sha256=67e69f9980817f1bc9a9faae371ac5d5e39a56d6281aff88d238f442d22f53e5\n", - " Stored in directory: /root/.cache/pip/wheels/84/8a/05/4b6405e1961fc3e466a54f37b7cea20df8e62fe585abcd6600\n", - "Successfully built dataflux-pytorch\n", - "Installing collected packages: dataflux-pytorch\n", - "Successfully installed dataflux-pytorch-0.0.0\n" + "Collecting gcs-torch-dataflux\n", + " Downloading gcs_torch_dataflux-0.0.0-py3-none-any.whl (24 kB)\n", + "Requirement already satisfied: torch>=2.0.1 in /usr/local/lib/python3.10/dist-packages (from gcs-torch-dataflux) (2.2.1+cu121)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from gcs-torch-dataflux) (1.25.2)\n", + "Requirement already satisfied: google-cloud-storage in /usr/local/lib/python3.10/dist-packages (from gcs-torch-dataflux) (2.8.0)\n", + "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from gcs-torch-dataflux) (1.4.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->gcs-torch-dataflux) (3.13.1)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->gcs-torch-dataflux) (4.10.0)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->gcs-torch-dataflux) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->gcs-torch-dataflux) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->gcs-torch-dataflux) (3.1.3)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->gcs-torch-dataflux) (2023.6.0)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m28.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m44.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m60.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.1->gcs-torch-dataflux) (2.2.0)\n", + "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=2.0.1->gcs-torch-dataflux)\n", + " Downloading nvidia_nvjitlink_cu12-12.4.99-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m58.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: google-auth<3.0dev,>=1.25.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcs-torch-dataflux) (2.27.0)\n", + "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcs-torch-dataflux) (2.11.1)\n", + "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcs-torch-dataflux) (2.3.3)\n", + "Requirement already satisfied: google-resumable-media>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcs-torch-dataflux) (2.7.0)\n", + "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcs-torch-dataflux) (2.31.0)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->gcs-torch-dataflux) (1.63.0)\n", + "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->gcs-torch-dataflux) (3.20.3)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->gcs-torch-dataflux) (5.3.3)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->gcs-torch-dataflux) (0.3.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage->gcs-torch-dataflux) (4.9)\n", + "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-resumable-media>=2.3.2->google-cloud-storage->gcs-torch-dataflux) (1.5.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->gcs-torch-dataflux) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->gcs-torch-dataflux) (3.6)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->gcs-torch-dataflux) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage->gcs-torch-dataflux) (2024.2.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.0.1->gcs-torch-dataflux) (2.1.5)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=2.0.1->gcs-torch-dataflux) (1.3.0)\n", + "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-cloud-storage->gcs-torch-dataflux) (0.5.1)\n", + "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, gcs-torch-dataflux\n", + "Successfully installed gcs-torch-dataflux-0.0.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.99 nvidia-nvtx-cu12-12.1.105\n" ] } ], "source": [ - "! git clone --recurse-submodules https://github.com/GoogleCloudPlatform/dataflux-pytorch && cd dataflux-pytorch && pip install ." + "! pip install gcs-torch-dataflux" ] }, { @@ -116,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -172,7 +180,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "metadata": { "id": "SOb3SoE0Xsj5" }, @@ -196,7 +204,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "id": "XydFQszKZJwM" }, @@ -296,13 +304,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Ri-NBCLndPUx", - "outputId": "1068d314-eff4-4da8-b0a3-fb76732e1600" + "outputId": "f93f0f61-bfc2-4bbc-8ff6-2b9196552f4f" }, "outputs": [ { @@ -567,13 +575,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Oi_6yzy5_XCL", - "outputId": "e16ef8b0-86cf-47c4-8e40-ae84a3f9ab9c" + "outputId": "ffbf238b-2352-4b92-b2e2-87b80c5e4204" }, "outputs": [ { @@ -642,7 +650,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -679,7 +687,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -713,7 +721,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -981,7 +989,8 @@ ], "metadata": { "colab": { - "provenance": [] + "provenance": [], + "toc_visible": true }, "kernelspec": { "display_name": "Python 3",