Stable Diffusion#
Note
Install ekorpkit package first.
Set logging level to Warning, if you don’t want to see verbose logging.
If you run this notebook in Colab, set Hardware accelerator to GPU.
%pip install -U --pre ekorpkit[art]
exit()
Prepare Environment#
from ekorpkit import eKonf
if eKonf.is_colab():
eKonf.mount_google_drive()
ws = eKonf.set_workspace(
workspace="/content/drive/MyDrive/workspace",
project="ekorpkit-book",
task="aiart",
log_level="INFO"
)
print("version:", ws.version)
print("project_dir:", ws.project_dir)
Show code cell output
INFO:ekorpkit.base:Set environment variable EKORPKIT_DATA_ROOT=/content/drive/MyDrive/workspace/data
INFO:ekorpkit.base:Set environment variable CACHED_PATH_CACHE_ROOT=/content/drive/MyDrive/workspace/.cache/cached_path
version: 0.1.40.post0.dev57
project_dir: /content/drive/MyDrive/workspace/projects/ekorpkit-book
time: 934 ms (started: 2022-12-16 00:49:26 +00:00)
Create a stable diffusion instance#
To download a certain dataset or model checkpoint, you may need to provide a HuggingFace API token. You can get one from here.
# Set HuggingFace API token
ws.secrets.HUGGING_FACE_HUB_TOKEN = "YOUR_TOKEN"
# Set HuggingFace API token
# ws.secrets.HUGGING_FACE_HUB_TOKEN = "YOUR_TOKEN"
ws.secrets.HUGGING_FACE_HUB_TOKEN
SecretStr('**********')
time: 23.2 ms (started: 2022-12-16 00:43:32 +00:00)
# Set CUDA DEVICES for the model
ws.envs.CUDA_VISIBLE_DEVICES = "1,2"
INFO:ekorpkit.base:Set environment variable CUDA_VISIBLE_DEVICES=1,2
time: 21 ms (started: 2022-12-16 00:43:34 +00:00)
from ekorpkit.tasks.multi import StableDiffusion
sd = StableDiffusion()
INFO:absl:Unable to initialize backend 'tpu_driver': NOT_FOUND: Unable to find driver in registry given worker:
INFO:absl:Unable to initialize backend 'rocm': NOT_FOUND: Could not find registered platform with name: "rocm". Available platform names are: Interpreter Host CUDA
INFO:absl:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'
INFO:ekorpkit.config:No modules to load
time: 4.55 s (started: 2022-12-16 00:43:36 +00:00)
Generate images#
text_prompts = "people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation."
batch_name = "hopper-street"
sd.config.collage.ncols = 2
results = sd.generate(
text_prompts,
batch_name=batch_name,
num_samples=4,
guidance_scale=9,
width=800,
height=400,
num_inference_steps=50,
)
Prompt: people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation.
time: 30.3 s (started: 2022-12-16 00:43:45 +00:00)
Inpaint images#
base_url = "https://github.com/entelecheia/ekorpkit-book/raw/main/assets/figs"
init_image_path = f"{base_url}/chu-horse.png"
mask_image_path = f"{base_url}/chu-horse-masking-rough.png"
mask_image_exact_path = f"{base_url}/chu-horse-masking-exact.png"
init_image = eKonf.load_image(init_image_path)
mask_image = eKonf.load_image(mask_image_path)
mask_image_exact = eKonf.load_image(mask_image_exact_path)
sd.collage_images([mask_image, init_image, mask_image_exact])
Prompt: people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation.
time: 2.92 s (started: 2022-12-14 02:12:32 +00:00)
prompt = "a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning"
batch_name = "inpaint-halla"
sd.config.collage.ncols = 3
results = sd.generate(
prompt,
batch_name=batch_name,
mode="inpaint",
init_image=init_image_path,
mask_image=mask_image_path,
num_samples=3,
num_images_per_prompt=1,
inpaint_strength=10,
num_inference_steps=50,
)
Prompt: a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning
time: 30.1 s (started: 2022-12-14 02:12:35 +00:00)
Stitch images#
text_prompts = "people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation."
batch_name = "city-stitch"
results = sd.generate(
text_prompts,
mode="stitch",
batch_name=batch_name,
num_samples=6,
num_images_per_prompt=2,
guidance_scale=7,
inpaint_strength=10,
num_inference_steps=50,
)
time: 1min 17s (started: 2022-12-14 01:53:15 +00:00)
To view the panorama, upload the image to renderstuff.
Collage Images#
sd.collage_images(batch_name="hopper-street", batch_num=10, ncols=2, collage_width=800)
Prompt: people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation.
time: 1.32 s (started: 2022-12-14 01:55:14 +00:00)
Show Configuration#
sd.show_config(batch_name="hopper-street", batch_num=None)
Show code cell output
{'_target_': 'ekorpkit.tasks.multi.stable.StableDiffusion',
'about': {'author': 'Young Joon Lee',
'description': 'eKorpkit provides a flexible interface for NLP and '
'ML research pipelines such as extraction, '
'transformation, tokenization, training, and '
'visualization.',
'name': 'ekorpkit',
'version': '0.1.40.post0.dev53',
'website': 'https://entelecheia.github.io/ekorpkit-book/'},
'app_name': 'ekorpkit',
'auto': {},
'autoload': False,
'batch': {'batch_name': 'hopper-street',
'batch_num': 37,
'config_dirname': 'configs',
'config_json': 'config.json',
'config_yaml': 'config.yaml',
'device': 'cuda:0',
'num_devices': None,
'num_workers': 1,
'output_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/outputs'),
'output_extention': '',
'output_suffix': None,
'random_seed': True,
'resume_latest': False,
'resume_run': False,
'seed': 713468011,
'verbose': False},
'batch_name': 'hopper-street',
'batch_num': 37,
'collage': {'bg_color': 'black',
'clear_output': True,
'collage_width': 1200,
'crop_to_min_size': False,
'filename_offset': [5, 5],
'fontcolor': '#000',
'fontname': None,
'fontsize': 12,
'max_images': 12,
'ncols': 3,
'padding': 5,
'show_filename': False,
'show_prompt': True},
'config_group': 'task=stable.diffusion',
'config_name': 'stable.diffusion',
'config_to_save': ['batch', 'imagine'],
'debug_mode': False,
'ignore_warnings': True,
'imagine': {'NSFW_retry': 0,
'clear_output': True,
'display_collage': True,
'display_image': True,
'guidance_scale': 7.5,
'height': 512,
'image_ext': 'png',
'increase_seed_by': 1,
'init_image': None,
'init_max_pixels': 262144,
'init_strength': None,
'inpaint_strength': 7.5,
'mask_image': None,
'mode': <ImagineMode.GENERATE: 'generate'>,
'num_images_per_prompt': 1,
'num_inference_steps': 50,
'num_samples': 6,
'save_collage': True,
'save_image': True,
'save_image_config': True,
'scheduler': <SchedulerType.K_LMS: 'k_lms'>,
'seed': None,
'text_prompts': None,
'width': 768},
'model': {'generate': {'device': 'cuda:0',
'name': 'CompVis/stable-diffusion-v1-4',
'pipeline': 'StableDiffusionPipeline',
'revision': 'fp16',
'use_auth_token': True},
'inpaint': {'device': 'cuda:0',
'name': 'runwayml/stable-diffusion-inpainting',
'pipeline': 'StableDiffusionInpaintPipeline',
'revision': 'fp16',
'use_auth_token': True}},
'module': {'library_dir': 'libs', 'modules': None},
'name': 'hopper-street',
'output_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/outputs'),
'path': {'batch_name': 'hopper-street',
'root': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart',
'task_name': 'aiart',
'verbose': False},
'print_config': False,
'print_resolved_config': True,
'project': {'description': None,
'env': {'batcher': {'backend': 'joblib',
'minibatch_size': 1000,
'procs': 50,
'task_num_cpus': 1,
'task_num_gpus': 0,
'verbose': 10},
'dask': {'n_workers': 50},
'distributed_framework': {'backend': 'joblib',
'initialize': True,
'num_workers': 50},
'dotenv': None,
'dotenv_path': '/workspace/projects/ekorpkit-book/ekorpkit-book/docs/lectures/aiart/.env',
'ray': {'num_cpus': 50}},
'init_huggingface_hub': True,
'path': {'archive': '/content/drive/MyDrive/workspace/data/archive',
'cache': '/content/drive/MyDrive/workspace/.cache',
'corpus': '/content/drive/MyDrive/workspace/data/datasets/corpus',
'data': '/content/drive/MyDrive/workspace/data',
'datasets': '/content/drive/MyDrive/workspace/data/datasets',
'ekorpkit': '/workspace/projects/ekorpkit/ekorpkit',
'home': '/root',
'library': '/content/drive/MyDrive/workspace/data/libs',
'logs': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/logs',
'models': '/content/drive/MyDrive/workspace/data/models',
'outputs': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/outputs',
'project': '/content/drive/MyDrive/workspace/projects/ekorpkit-book',
'resources': '/workspace/projects/ekorpkit/ekorpkit/resources',
'runtime': '/workspace/projects/ekorpkit-book/ekorpkit-book/docs/lectures/aiart',
'tmp': '/content/drive/MyDrive/workspace/.tmp',
'workspace': '/content/drive/MyDrive/workspace'},
'project_name': 'ekorpkit-book',
'project_root': '/content/drive/MyDrive/workspace/projects/ekorpkit-book',
'task_name': 'aiart',
'use_huggingface_hub': False,
'use_wandb': False,
'verbose': False,
'version': '0.1.40.post0.dev53',
'workspace_root': '/content/drive/MyDrive/workspace'},
'root': '/content/drive/MyDrive/workspace',
'root_dir': None,
'sample_imagepaths': [],
'verbose': False,
'version': '0.0.0'}
time: 1.23 s (started: 2022-12-14 02:14:57 +00:00)