EveryDream/AutoCaption.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":929,"status":"ok","timestamp":1667184580032,"user":{"displayName":"Victor Hall","userId":"00029068894644207946"},"user_tz":240},"id":"lWGx2LuU8Q_I","outputId":"d0eb4d03-f16d-460b-981d-d5f88447e85e"},"outputs":[{"name":"stdout","output_type":"stream","text":["Cloning into 'EveryDream'...\n","remote: Enumerating objects: 90, done.\u001b[K\n","remote: Counting objects: 100% (90/90), done.\u001b[K\n","remote: Compressing objects: 100% (59/59), done.\u001b[K\n","remote: Total 90 (delta 30), reused 76 (delta 18), pack-reused 0\u001b[K\n","Unpacking objects: 100% (90/90), done.\n"]}],"source":["#download repo\n","!git clone https://github.com/victorchall/EveryDream.git\n","%cd EveryDream"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4944,"status":"ok","timestamp":1667184754992,"user":{"displayName":"Victor Hall","userId":"00029068894644207946"},"user_tz":240},"id":"RJxfSai-8pkD","outputId":"0ac1b805-62a0-48aa-e0da-ee19503bb3f1"},"outputs":[{"ename":"","evalue":"","output_type":"error","traceback":["\u001b[1;31mRunning cells with 'Python 3.10.5 ('.venv': venv)' requires ipykernel package.\n","\u001b[1;31mRun the following command to install 'ipykernel' into the Python environment. \n","\u001b[1;31mCommand: 'r:/EveryDream/.venv/Scripts/python.exe -m pip install ipykernel -U --force-reinstall'"]}],"source":["!pip install torch=='1.12.1+cu113' 'torchvision==0.13.1+cu113' --extra-index-url https://download.pytorch.org/whl/cu113\n","!pip install pandas>='1.3.5'\n","!git clone https://github.com/salesforce/BLIP EveryDream/scripts/BLIP\n","!pip install timm\n","!pip install fairscale=='0.4.4'\n","!pip install transformers=='4.19.2'\n","!pip install timm\n","# pandas will fail, fixed below"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":383,"status":"ok","timestamp":1667185773878,"user":{"displayName":"Victor Hall","userId":"00029068894644207946"},"user_tz":240},"id":"ruRaJ7Cx9vhw","outputId":"f0701d3e-bfa9-45a9-a742-c3615466aad7"},"outputs":[{"name":"stdout","output_type":"stream","text":["mkdir: cannot create directory ‘EveryDream/input’: File exists\n","mkdir: cannot create directory ‘EveryDream/output’: File exists\n"]}],"source":["# make folders for input and output\n","!mkdir input\n","!mkdir output\n","!mkdir .cache"]},{"cell_type":"markdown","metadata":{"id":"sbeUIVXJ-EVf"},"source":["Add your input images into the content/EveryDream/input folder\n","![a](/doc/upload_images_caption.png)\n","![Beam vs Nucleus](../demo/beam_vs_nucleus.webp)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#import sys\n","#import os\n","#blip_path = os.path.join(os.getcwd(), \"EveryDream/scripts/BLIP\")\n","#sys.path.append(blip_path)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":18221,"status":"ok","timestamp":1667185808005,"user":{"displayName":"Victor Hall","userId":"00029068894644207946"},"user_tz":240},"id":"4TAICahl-RPn","outputId":"da7fa1a8-0855-403a-c295-4da31658d1f6"},"outputs":[{"name":"stdout","output_type":"stream","text":["starting\n","Unix detected, using default asyncio event loop policy\n","starting\n","input_dir:  /content/EveryDream/input\n","Downloading model to /content/EveryDream/.cache/model_base_caption_capfilt_large.pth... please wait\n","Model cached to: /content/EveryDream/.cache/model_base_caption_capfilt_large.pth\n","Downloading: 100% 226k/226k [00:00<00:00, 678kB/s]\n","Downloading: 100% 28.0/28.0 [00:00<00:00, 34.0kB/s]\n","Downloading: 100% 570/570 [00:00<00:00, 675kB/s]\n","load checkpoint from /content/EveryDream/.cache/model_base_caption_capfilt_large.pth\n","loading model to cuda\n","Traceback (most recent call last):\n","  File \"EveryDream/scripts/auto_caption.py\", line 178, in <module>\n","    asyncio.run(main(opt))\n","  File \"/usr/lib/python3.7/asyncio/runners.py\", line 43, in run\n","    return loop.run_until_complete(main)\n","  File \"/usr/lib/python3.7/asyncio/base_events.py\", line 587, in run_until_complete\n","    return future.result()\n","  File \"EveryDream/scripts/auto_caption.py\", line 110, in main\n","    blip_decoder = blip_decoder.to(torch.device(\"cuda\"))\n","  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 927, in to\n","    return self._apply(convert)\n","  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 579, in _apply\n","    module._apply(fn)\n","  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 579, in _apply\n","    module._apply(fn)\n","  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 579, in _apply\n","    module._apply(fn)\n","  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 602, in _apply\n","    param_applied = fn(param)\n","  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 925, in convert\n","    return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)\n","  File \"/usr/local/lib/python3.7/dist-packages/torch/cuda/__init__.py\", line 217, in _lazy_init\n","    torch._C._cuda_init()\n","RuntimeError: No CUDA GPUs are available\n"]}],"source":["!python EveryDream/scripts/auto_caption.py \\\n","--img_dir EveryDream/input \\\n","--out_dir EveryDream/output \\\n","#--min_length 34 \\ # optional longer prompts\n","#--q_factor 1.3  \\ # optional tweak for longer prompts\n","#--nucleus \\ # alternative algorithm for short captions"]},{"cell_type":"markdown","metadata":{"id":"HBrWnu1C_lN9"},"source":["Download your captioned images from /content/EveryDream/output"]}],"metadata":{"colab":{"authorship_tag":"ABX9TyN9ZSr0RyOQKdfeVsl2uOiE","collapsed_sections":[],"provenance":[{"file_id":"16QrivRfoDFvE7fAa7eLeVlxj78Q573E0","timestamp":1667185879409}]},"kernelspec":{"display_name":"Python 3.10.5 ('.venv': venv)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.5"},"vscode":{"interpreter":{"hash":"faf4a6abb601e3a9195ce3e9620411ceec233a951446de834cdf28542d2d93b4"}}},"nbformat":4,"nbformat_minor":0}