diff --git a/Train-Runpod.ipynb b/Train-Runpod.ipynb index ccdb83a..34175a0 100644 --- a/Train-Runpod.ipynb +++ b/Train-Runpod.ipynb @@ -29,9 +29,12 @@ "metadata": {}, "outputs": [], "source": [ - "# check that you have sufficient SWAP FILE space so saving CKPTs doesn't show \"Killed\"\n", + "# check system resources, make sure your GPU actually has 24GB\n", + "# You should see \"0 MB / 24576 MB\" in the middle of the printout\n", + "# if you see 0 MB / 22000 MB find a different instance or provider...\n", "!grep Swap /proc/meminfo\n", - "!swapon -s" + "!swapon -s\n", + "!nvidia-smi" ] }, { @@ -311,7 +314,7 @@ "\n", "Use the cell below to generate links, right click and save as to download. If you use Colab, you can skip this and use the Gdrive connect instead.\n", "\n", - "If the links don't work, you can double left click the ckpt file in the file drawer on the left, then go to \"File\" menu then \"Download\".\n", + "**If the links don't work, you can double left click the ckpt file in the file drawer on the left, then go to \"File\" menu then \"Download\"** or use the Hugging Face upload below.\n", "\n", "[EveryDream Discord](https://discord.gg/uheqxU6sXN)" ] @@ -330,6 +333,72 @@ "# right click save as to download the ckpt files" ] }, + { + "cell_type": "markdown", + "id": "c13d37a1", + "metadata": {}, + "source": [ + "# HuggingFace upload\n", + "Use the cell below to upload your checkpoint to your personal HuggingFace account if you want instead of manually downloading. You should already be authorized to Huggingface by token if you used the download/token cells above.\n", + "\n", + "Make sure to fill in the three fields at the top. This will only upload one file at a time, so you will need to run the cell below for each file you want to upload.\n", + "\n", + "* You can get your account name from your [HuggingFace account page](https://huggingface.co/settings/account). Look for your \"username\" field and paste it below.\n", + "\n", + "* You only need to setup a repository one time. You can create it here: [Create New HF Dataset](https://huggingface.co/new) Make sure you write down the repo name you make for future use. You can reuse it later.\n", + "\n", + "* You need to type the name of the ckpts one at a time in the cell below, find them in the left file drawer of your Runpod/Vast/Colab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9fb962e0", + "metadata": {}, + "outputs": [], + "source": [ + "#list ckpts in root that are ready for download\n", + "import glob\n", + "\n", + "for f in glob.glob(\"*.ckpt\"):\n", + " print(f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7237ed4", + "metadata": {}, + "outputs": [], + "source": [ + "!huggingface-cli lfs-enable-largefiles\n", + "# fill in these three fields:\n", + "hfusername = \"panopstor\"\n", + "reponame = \"EveryDream\"\n", + "ckpt_name = \"epoch=01-step=00030-pruned.ckpt\"\n", + "\n", + "\n", + "target_name = ckpt_name.replace('-','').replace('=','')\n", + "import os\n", + "os.rename(ckpt_name,target_name)\n", + "repo_id=f\"{hfusername}/{reponame}\"\n", + "print(f\"uploading to HF: {repo_id}/{ckpt_name}\")\n", + "print(\"this make take a while...\")\n", + "\n", + "from huggingface_hub import HfApi\n", + "api = HfApi()\n", + "response = api.upload_file(\n", + " path_or_fileobj=target_name,\n", + " path_in_repo=target_name,\n", + " repo_id=repo_id,\n", + " repo_type=None,\n", + " create_pr=1,\n", + ")\n", + "print(response)\n", + "print(finish_msg)\n", + "print(\"go to your repo and accept the PR this created to see your file\")" + ] + }, { "cell_type": "markdown", "id": "54171a12",