diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index 7518f6fa..24d75b34 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -16,7 +16,7 @@ "id": "TitleTop" }, "source": [ - "# Disco Diffusion v5.7 - Now with MiDaS (3D mode) not being broken\n", + "# Disco Diffusion v5.8 - Now with stsp and ltsp sampling\n", "\n", "Disco Diffusion - http://discodiffusion.com/ , https://github.com/alembics/disco-diffusion\n", "\n", @@ -44,7 +44,7 @@ "\n", "Original notebook by Katherine Crowson (https://github.com/crowsonkb, https://twitter.com/RiversHaveWings). It uses either OpenAI's 256x256 unconditional ImageNet or Katherine Crowson's fine-tuned 512x512 diffusion model (https://github.com/openai/guided-diffusion), together with CLIP (https://github.com/openai/CLIP) to connect text prompts with images.\n", "\n", - "Modified by Daniel Russell (https://github.com/russelldc, https://twitter.com/danielrussruss) to include (hopefully) optimal params for quick generations in 15-100 timesteps rather than 1000, as well as more robust augmentations.\n", + "Modified by Daniel Russell (https://github.com/russelldc, https://twitter.com/danielrussruss) to include (hopefully) optimal params for quick generations in 15-100 timess rather than 1000, as well as more robust augmentations.\n", "\n", "Further improvements from Dango233 and nshepperd helped improve the quality of diffusion in general, and especially so for shorter runs like this notebook aims to achieve.\n", "\n", @@ -76,7 +76,9 @@ "\n", "Integration of OpenCLIP models and initiation of integration of KaliYuga models by Palmweaver / Chris Scalf (https://twitter.com/ChrisScalf11)\n", "\n", - "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)" + "Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist)\n", + "\n", + "Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion)" ] }, { @@ -334,6 +336,8 @@ " v5.7 Update: Dec 31st 2022 - Steffen Moelter (with minor colab-convert integration by gandamu)\n", "\n", " Clone MiDaS v3 specifically. This fixes 3D mode. It had been broken since MiDaS v3.1 introduced an incompatibility.\n", + "\n", + " v5.8 Update: Jun 5st 2023 - Added STSP and LTSP Sampling with Splitting Numerical Methods from space-nuko repository. (integration by DrSen)\n", " '''\n", " )" ], @@ -596,7 +600,7 @@ " from guided_diffusion.script_util import create_model_and_diffusion\n", "except:\n", " if not os.path.exists(\"guided-diffusion\"):\n", - " gitclone(\"https://github.com/kostarion/guided-diffusion\")\n", + " gitclone(\"https://github.com/space-nuko/guided-diffusion.git\")\n", " sys.path.append(f'{PROJECT_DIR}/guided-diffusion')\n", "\n", "try:\n", @@ -1453,6 +1457,10 @@ " \n", " if args.diffusion_sampling_mode == 'ddim':\n", " sample_fn = diffusion.ddim_sample_loop_progressive\n", + " elif args.diffusion_sampling_mode == 'ltsp':\n", + " sample_fn = diffusion.ltsp_sample_loop_progressive\n", + " elif args.diffusion_sampling_mode == 'stsp':\n", + " sample_fn = diffusion.stsp_sample_loop_progressive\n", " else:\n", " sample_fn = diffusion.plms_sample_loop_progressive\n", "\n", @@ -1489,6 +1497,34 @@ " transformation_fn=symmetry_transformation_fn,\n", " transformation_percent=args.transformation_percent\n", " )\n", + " if args.diffusion_sampling_mode == 'stsp':\n", + " samples = sample_fn(\n", + " model,\n", + " (batch_size, 3, args.side_y, args.side_x),\n", + " clip_denoised=clip_denoised,\n", + " model_kwargs={},\n", + " cond_fn=cond_fn,\n", + " impu_fn=None,\n", + " progress=True,\n", + " skip_timesteps=skip_steps,\n", + " init_image=init,\n", + " randomize_class=randomize_class,\n", + " order=2,\n", + " )\n", + " if args.diffusion_sampling_mode == 'ltsp':\n", + " samples = sample_fn(\n", + " model,\n", + " (batch_size, 3, args.side_y, args.side_x),\n", + " clip_denoised=clip_denoised,\n", + " model_kwargs={},\n", + " cond_fn=cond_fn,\n", + " impu_fn=None,\n", + " progress=True,\n", + " skip_timesteps=skip_steps,\n", + " init_image=init,\n", + " randomize_class=randomize_class,\n", + " order=2,\n", + " )\n", " else:\n", " samples = sample_fn(\n", " model,\n", @@ -1895,7 +1931,7 @@ "diffusion_model = \"512x512_diffusion_uncond_finetune_008100\" #@param [\"256x256_diffusion_uncond\", \"512x512_diffusion_uncond_finetune_008100\", \"portrait_generator_v001\", \"pixelartdiffusion_expanded\", \"pixel_art_diffusion_hard_256\", \"pixel_art_diffusion_soft_256\", \"pixelartdiffusion4k\", \"watercolordiffusion_2\", \"watercolordiffusion\", \"PulpSciFiDiffusion\", \"custom\"]\n", "\n", "use_secondary_model = True #@param {type: 'boolean'}\n", - "diffusion_sampling_mode = 'ddim' #@param ['plms','ddim']\n", + "diffusion_sampling_mode = 'stsp' #@param ['plms','ddim','stsp','ltsp']\n", "#@markdown #####**Custom model:**\n", "custom_path = '/content/drive/MyDrive/deep_learning/ddpm/ema_0.9999_058000.pt'#@param {type: 'string'}\n", "\n", @@ -2154,7 +2190,7 @@ "source": [ "#@markdown ####**Basic Settings:**\n", "batch_name = 'TimeToDisco' #@param{type: 'string'}\n", - "steps = 250 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true}\n", + "steps = 142 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true}\n", "width_height_for_512x512_models = [1280, 768] #@param{type: 'raw'}\n", "clip_guidance_scale = 5000 #@param{type: 'number'}\n", "tv_scale = 0#@param{type: 'number'}\n", @@ -3371,7 +3407,7 @@ "FlowFns2" ], "machine_shape": "hm", - "name": "Disco Diffusion v5.7 [Now with MiDaS (3D mode) not being broken]", + "name": "Disco Diffusion v5.8 - Now with stsp and ltsp sampling", "private_outputs": true, "provenance": [], "include_colab_link": true diff --git a/disco.py b/disco.py index 65dce7a2..392c980d 100644 --- a/disco.py +++ b/disco.py @@ -12,7 +12,7 @@ # !! "id": "TitleTop" # !! }} """ -# Disco Diffusion v5.7 - Now with MiDaS (3D mode) not being broken +# Disco Diffusion v5.8 - Now with stsp and ltsp sampling Disco Diffusion - http://discodiffusion.com/ , https://github.com/alembics/disco-diffusion @@ -71,6 +71,8 @@ Integration of OpenCLIP models and initiation of integration of KaliYuga models by Palmweaver / Chris Scalf (https://twitter.com/ChrisScalf11) Integrated portrait_generator_v001 from Felipe3DArtist (https://twitter.com/Felipe3DArtist) + +Integratedion of sampling with splitting Numerical methods from sWizad (https://github.com/sWizad/split-diffusion) """ # %% @@ -323,6 +325,8 @@ v5.7 Update: Dec 31st 2022 - Steffen Moelter (with minor colab-convert integration by gandamu) Clone MiDaS v3 specifically. This fixes 3D mode. It had been broken since MiDaS v3.1 introduced an incompatibility. + + v5.8 Update: Jun 5st 2023 - Added STSP and LTSP Sampling with Splitting Numerical Methods from space-nuko repository. (integration by DrSen) ''' ) @@ -567,7 +571,7 @@ def createPath(filepath): from guided_diffusion.script_util import create_model_and_diffusion except: if not os.path.exists("guided-diffusion"): - gitclone("https://github.com/kostarion/guided-diffusion") + gitclone("https://github.com/space-nuko/guided-diffusion.git") sys.path.append(f'{PROJECT_DIR}/guided-diffusion') try: @@ -1414,6 +1418,10 @@ def cond_fn(x, t, y=None): if args.diffusion_sampling_mode == 'ddim': sample_fn = diffusion.ddim_sample_loop_progressive + elif args.diffusion_sampling_mode == 'ltsp': + sample_fn = diffusion.ltsp_sample_loop_progressive + elif args.diffusion_sampling_mode == 'stsp': + sample_fn = diffusion.stsp_sample_loop_progressive else: sample_fn = diffusion.plms_sample_loop_progressive @@ -1450,6 +1458,34 @@ def cond_fn(x, t, y=None): transformation_fn=symmetry_transformation_fn, transformation_percent=args.transformation_percent ) + if args.diffusion_sampling_mode == 'stsp': + samples = sample_fn( + model, + (batch_size, 3, args.side_y, args.side_x), + clip_denoised=clip_denoised, + model_kwargs={}, + cond_fn=cond_fn, + impu_fn=None, + progress=True, + skip_timesteps=skip_steps, + init_image=init, + randomize_class=randomize_class, + order=2, + ) + if args.diffusion_sampling_mode == 'ltsp': + samples = sample_fn( + model, + (batch_size, 3, args.side_y, args.side_x), + clip_denoised=clip_denoised, + model_kwargs={}, + cond_fn=cond_fn, + impu_fn=None, + progress=True, + skip_timesteps=skip_steps, + init_image=init, + randomize_class=randomize_class, + order=2, + ) else: samples = sample_fn( model, @@ -1845,7 +1881,7 @@ def forward(self, input, t): diffusion_model = "512x512_diffusion_uncond_finetune_008100" #@param ["256x256_diffusion_uncond", "512x512_diffusion_uncond_finetune_008100", "portrait_generator_v001", "pixelartdiffusion_expanded", "pixel_art_diffusion_hard_256", "pixel_art_diffusion_soft_256", "pixelartdiffusion4k", "watercolordiffusion_2", "watercolordiffusion", "PulpSciFiDiffusion", "custom"] use_secondary_model = True #@param {type: 'boolean'} -diffusion_sampling_mode = 'ddim' #@param ['plms','ddim'] +diffusion_sampling_mode = 'stsp' #@param ['plms','ddim','stsp','ltsp'] #@markdown #####**Custom model:** custom_path = '/content/drive/MyDrive/deep_learning/ddpm/ema_0.9999_058000.pt'#@param {type: 'string'} @@ -2092,7 +2128,7 @@ def download_model(diffusion_model_name, uri_index=0): # !! }} #@markdown ####**Basic Settings:** batch_name = 'TimeToDisco' #@param{type: 'string'} -steps = 250 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true} +steps = 142 #@param [25,50,100,150,250,500,1000]{type: 'raw', allow-input: true} width_height_for_512x512_models = [1280, 768] #@param{type: 'raw'} clip_guidance_scale = 5000 #@param{type: 'number'} tv_scale = 0#@param{type: 'number'} @@ -3264,7 +3300,7 @@ def move_files(start_num, end_num, old_folder, new_folder): # !! "FlowFns2" # !! ], # !! "machine_shape": "hm", -# !! "name": "Disco Diffusion v5.61 [Now with portrait_generator_v001]", +# !! "name": "Disco Diffusion v5.8 - Now with stsp and ltsp sampling", # !! "private_outputs": true, # !! "provenance": [], # !! "include_colab_link": true