DALL·E Mini#
This is an simple way of creating DALL·E Mini artworks for generative artists.
Note
Install ekorpkit package first.
Set logging level to Warning, if you don’t want to see verbose logging.
If you run this notebook in Colab, set Hardware accelerator to GPU.
Check your jaxlib version and install the appropriate version. for example,
pip install “jax[cuda11_cudnn82]” -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
!pip install ekorpkit[art] exit()
Prepare Environment#
%config InlineBackend.figure_format='retina'
%load_ext autotime
%load_ext autoreload
%autoreload 2 # reload modules before executing user code, set to 0 to disable
from ekorpkit import eKonf
eKonf.setLogger("WARNING")
eKonf.set_cuda(device="4,5")
print("version:", eKonf.__version__)
is_notebook = eKonf.is_notebook()
is_colab = eKonf.is_colab()
print("is notebook?", is_notebook)
print("is colab?", is_colab)
if is_colab:
eKonf.mount_google_drive()
eKonf.set_workspace(workspace="/content/drive/MyDrive/workspace/", project="ekorpkit-book")
INFO:ekorpkit.base:Setting cuda device to ['A100-SXM4-40GB (id:4)', 'A100-SXM4-40GB (id:5)']
INFO:ekorpkit.base:Setting EKORPKIT_WORKSPACE_ROOT to /content/drive/MyDrive/workspace/
INFO:ekorpkit.base:Setting EKORPKIT_PROJECT to ekorpkit-book
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env
version: 0.1.40.post0.dev4
is notebook? True
is colab? False
('/content/drive/MyDrive/workspace', 'ekorpkit-book')
time: 3.27 s (started: 2022-11-02 11:40:22 +00:00)
Create a dalle-mini instance#
from ekorpkit.models.art.mini import DalleMini
# cfg = eKonf.compose("model/dalle_mini")
# dalle = eKonf.instantiate(cfg)
dalle = DalleMini()
Show code cell output
wandb: Downloading large artifact mega-1-fp16:latest, 4938.53MB. 7 files... Done. 0:0:13.4
Some of the weights of DalleBart were initialized in float16 precision from the model checkpoint at /tmp/tmpjq5ecckz:
[('lm_head', 'kernel'), ('model', 'decoder', 'embed_positions', 'embedding'), ('model', 'decoder', 'embed_tokens', 'embedding'), ('model', 'decoder', 'final_ln', 'bias'), ('model', 'decoder', 'layernorm_embedding', 'bias'), ('model', 'decoder', 'layernorm_embedding', 'scale'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_0', 'k_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_0', 'out_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_0', 'q_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_0', 'v_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_1', 'k_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_1', 'out_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_1', 'q_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'FlaxBartAttention_1', 'v_proj', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'GLU_0', 'Dense_0', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'GLU_0', 'Dense_1', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'GLU_0', 'Dense_2', 'kernel'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'GLU_0', 'LayerNorm_0', 'bias'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'GLU_0', 'LayerNorm_1', 'bias'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'LayerNorm_0', 'bias'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'LayerNorm_1', 'bias'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'LayerNorm_1', 'scale'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'LayerNorm_2', 'bias'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'LayerNorm_3', 'bias'), ('model', 'decoder', 'layers', 'FlaxBartDecoderLayers', 'LayerNorm_3', 'scale'), ('model', 'encoder', 'embed_positions', 'embedding'), ('model', 'encoder', 'embed_tokens', 'embedding'), ('model', 'encoder', 'final_ln', 'bias'), ('model', 'encoder', 'layernorm_embedding', 'bias'), ('model', 'encoder', 'layernorm_embedding', 'scale'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'FlaxBartAttention_0', 'k_proj', 'kernel'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'FlaxBartAttention_0', 'out_proj', 'kernel'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'FlaxBartAttention_0', 'q_proj', 'kernel'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'FlaxBartAttention_0', 'v_proj', 'kernel'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'GLU_0', 'Dense_0', 'kernel'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'GLU_0', 'Dense_1', 'kernel'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'GLU_0', 'Dense_2', 'kernel'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'GLU_0', 'LayerNorm_0', 'bias'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'GLU_0', 'LayerNorm_1', 'bias'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'LayerNorm_0', 'bias'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'LayerNorm_1', 'bias'), ('model', 'encoder', 'layers', 'FlaxBartEncoderLayers', 'LayerNorm_1', 'scale')]
You should probably UPCAST the model weights to float32 if this was not intended. See [`~FlaxPreTrainedModel.to_fp32`] for further information on how to do this.
wandb: Downloading large artifact mega-1-fp16:latest, 4938.53MB. 7 files... Done. 0:0:13.5
time: 51.3 s (started: 2022-11-02 11:40:25 +00:00)
Generate an image#
text_prompts = "a living tiger inside a gorgeous cave in a futuristic Jeju island, relaxing night, watercolour on the ceiling, glowing on glass, light art, dark, dark, gorgeous, dark, bright in an abandoned village in a futuristic alien nightscape"
batch_name = "cave-jeju"
time: 924 µs (started: 2022-11-02 11:41:16 +00:00)
results = dalle.imagine(
text_prompts,
batch_name=batch_name,
num_samples=8,
show_collage=False,
)
8 samples generated to /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch
text prompts: ['a living tiger inside a gorgeous cave in a futuristic Jeju island, relaxing night, watercolour on the ceiling, glowing on glass, light art, dark, dark, gorgeous, dark, bright in an abandoned village in a futuristic alien nightscape']
time: 1min 16s (started: 2022-11-02 11:41:16 +00:00)
Collage generated sample images#
dalle.collage(
batch_name=batch_name,
batch_num=0,
ncols=4,
num_images=8,
show_filename=True,
fontcolor="white",
)
time: 2.04 s (started: 2022-11-02 11:42:32 +00:00)
Configuration Management#
Load configuration#
dalle.verbose = True
eKonf.setLogger("INFO")
config = dalle.load_config(batch_name=batch_name, batch_num=0)
eKonf.print(config)
Show code cell output
INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: cave-jeju, Batch num: 0
INFO:ekorpkit.batch:Loading config from /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch/cave-jeju(0)_config.yaml
INFO:ekorpkit.batch:Merging config with the loaded config
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 2733022696
{'_target_': 'ekorpkit.models.art.mini.DalleMini',
'autoload': True,
'batch_name': 'cave-jeju',
'batch_num': 0,
'device': '4, 5',
'imagine': {'cond_scale': 10.0,
'gen_top_k': None,
'gen_top_p': None,
'num_samples': 8,
'temperature': None,
'text_prompts': ['a living tiger inside a gorgeous cave in a '
'futuristic Jeju island, relaxing night, '
'watercolour on the ceiling, glowing on glass, '
'light art, dark, dark, gorgeous, dark, bright '
'in an abandoned village in a futuristic alien '
'nightscape']},
'model': {'CLIP_COMMIT_ID': None,
'CLIP_INIT': False,
'CLIP_REPO': 'openai/clip-vit-base-patch32',
'DALLE_COMMIT_ID': None,
'DALLE_INIT': False,
'DALLE_MODEL': 'dalle-mini/dalle-mini/mega-1-fp16:latest',
'VQGAN_COMMIT_ID': 'e93a26e7707683d349bf5d5c41c5b0ef69b677a9',
'VQGAN_INIT': False,
'VQGAN_REPO': 'dalle-mini/vqgan_imagenet_f16_16384'},
'module': {'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
'modules': [{'libname': 'dalle-mini',
'liburi': 'https://github.com/borisdayma/dalle-mini',
'name': 'dalle_mini',
'specname': None,
'syspath': 'dalle-mini/src/dalle_mini'},
{'libname': 'vqgan-jax',
'liburi': 'https://github.com/patil-suraj/vqgan-jax',
'name': 'vqgan_jax.modeling_flax_vqgan',
'specname': None}]},
'name': 'dalle-mini',
'num_devices': 8,
'path': {'batch': {'base_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch'),
'name': 'mini-batch'},
'batch_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch'),
'batch_name': 'mini-batch',
'cache_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/cache'),
'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
'model_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/models'),
'name': 'disco-imagen',
'output_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini'),
'root': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen'),
'tmp_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/tmp'),
'verbose': False},
'resume_run': False,
'run_to_resume': 'latest',
'seed': 2733022696,
'set_seed': 'random_seed',
'show_collage': True,
'verbose': False}
time: 40.5 ms (started: 2022-11-02 11:42:34 +00:00)
Save configuration#
dalle.save_config()
INFO:ekorpkit.batch:Saving config to /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch/cave-jeju(0)_config.yaml
'cave-jeju(0)_config.yaml'
time: 9.95 ms (started: 2022-11-02 11:42:34 +00:00)
Show configuration#
dalle.show_config(batch_name=batch_name, batch_num=0)
Show code cell output
INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: cave-jeju, Batch num: 0
INFO:ekorpkit.batch:Loading config from /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch/cave-jeju(0)_config.yaml
INFO:ekorpkit.batch:Merging config with the loaded config
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 812519167
{'_target_': 'ekorpkit.models.art.mini.DalleMini',
'autoload': True,
'batch_name': 'cave-jeju',
'batch_num': 0,
'device': '4, 5',
'imagine': {'cond_scale': 10.0,
'gen_top_k': None,
'gen_top_p': None,
'num_samples': 8,
'temperature': None,
'text_prompts': ['a living tiger inside a gorgeous cave in a '
'futuristic Jeju island, relaxing night, '
'watercolour on the ceiling, glowing on glass, '
'light art, dark, dark, gorgeous, dark, bright '
'in an abandoned village in a futuristic alien '
'nightscape']},
'model': {'CLIP_COMMIT_ID': None,
'CLIP_INIT': False,
'CLIP_REPO': 'openai/clip-vit-base-patch32',
'DALLE_COMMIT_ID': None,
'DALLE_INIT': False,
'DALLE_MODEL': 'dalle-mini/dalle-mini/mega-1-fp16:latest',
'VQGAN_COMMIT_ID': 'e93a26e7707683d349bf5d5c41c5b0ef69b677a9',
'VQGAN_INIT': False,
'VQGAN_REPO': 'dalle-mini/vqgan_imagenet_f16_16384'},
'module': {'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
'modules': [{'libname': 'dalle-mini',
'liburi': 'https://github.com/borisdayma/dalle-mini',
'name': 'dalle_mini',
'specname': None,
'syspath': 'dalle-mini/src/dalle_mini'},
{'libname': 'vqgan-jax',
'liburi': 'https://github.com/patil-suraj/vqgan-jax',
'name': 'vqgan_jax.modeling_flax_vqgan',
'specname': None}]},
'name': 'dalle-mini',
'num_devices': 8,
'path': {'batch': {'base_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch'),
'name': 'mini-batch'},
'batch_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini/mini-batch'),
'batch_name': 'mini-batch',
'cache_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/cache'),
'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
'model_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/models'),
'name': 'disco-imagen',
'output_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/dalle-mini'),
'root': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen'),
'tmp_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/tmp'),
'verbose': False},
'resume_run': False,
'run_to_resume': 'latest',
'seed': 812519167,
'set_seed': 'random_seed',
'show_collage': True,
'verbose': False}
time: 39.1 ms (started: 2022-11-02 11:42:34 +00:00)