### Download and setup font dataset Source: https://github.com/yukistavailable/fontclip/blob/main/README.md Download and set up the font dataset using the command `python setup_data.py`. ```bash python setup_data.py ``` -------------------------------- ### Install dependencies for vector optimization Source: https://github.com/yukistavailable/fontclip/blob/main/README.md Install dependencies for vector optimization, including numpy, scikit-image, cmake, ffmpeg, and various Python packages. Also installs diffvg from source. Note that library versions must be compatible with diffvg; see the linked issue for details. ```bash conda install -y numpy scikit-image conda install -y -c anaconda cmake=3.22.1 conda install -y -c conda-forge ffmpeg pip install svgwrite svgpathtools cssutils numba torch-tools scikit-fmm easydict freetype-py shapely ttf save_svg pip install opencv-python==4.5.4.60 pip install kornia==0.6.8 pip install wandb pip install shapely # install diffvg git clone https://github.com/BachiLi/diffvg.git cd diffvg git submodule update --init --recursive python setup.py install ``` -------------------------------- ### Run font retrieval demo Source: https://github.com/yukistavailable/fontclip/blob/main/README.md Run the font retrieval demo with the command `python font_retrieval.py`. This starts the retrieval application that takes text or image input and returns top-ranked fonts based on cosine distance. ```sh python font_retrieval.py ``` -------------------------------- ### Create conda environment for finetuning CLIP Source: https://github.com/yukistavailable/fontclip/blob/main/README.md Commands to create and activate a conda environment named 'fontclip' with Python 3.8.15, install PyTorch with CUDA 12.1, and install required Python packages for finetuning CLIP. ```bash conda create -y --name fontclip python=3.8.15 conda activate fontclip conda install -y pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia pip install tqdm ftfy regex gdown pip install gradio==3.40.0 pip install httpx==0.24.1 ``` -------------------------------- ### Construct Config and Load YAML Configuration Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Builds a Config object with many optimization parameters, then loads a YAML config file, prints loss settings, and merges parent configs. It also loads a target image or generates one from a font, and displays the target and semantic concept. ```python cfg = Config( font_path, optimized_letter, None, optimized_letter, use_wandb=USE_WANDB, wandb_user=WANDB_USER, num_iter=num_iter, char_size=char_size, size=size, do_preprocess=do_preprocess, lr=lr, use_aug=use_aug, use_visual_encoder=use_visual_encoder, fclip_loss=use_fclip, fclip_loss_w=fclip_loss_w, use_fclip_direction_loss=use_fclip_direction_loss, use_fclip_direction_loss_vision=use_fclip_direction_loss_vision, use_fclip_direction_loss_only=use_fclip_direction_loss_only, ref_semantic_concept=ref_semantic_concept, ref_image_file_path=ref_image_file_path, fclip_direction_loss_w=fclip_direction_loss_w, laplacian_loss_w=laplacian_loss_w, cos_loss_w=cos_loss_w, G1_loss_w=G1_loss_w, L2_loss_w=L2_loss_w, conformal_loss_w=conformal_loss_w, tone_loss_w=tone_loss_w, multiple_attributes=multiple_attributes, multiple_text_encoders=multiple_attributes_preserve_init, target_attributes=target_attributes, target_attributes_weights=target_attributes_weights, use_tone_loss=use_tone_loss, use_conformal_loss=use_conformal_loss, use_laplacian_loss=use_laplacian_loss, use_cos_loss=use_cos_loss, use_G1_loss=use_G1_loss, reduce_cp=reduce_cp, epsilon=epsilon, checkpoint_path=checkpoint_path, visual_optimize=visual_optimize, image_file_path=image_file_path, use_tone_loss_schedular=use_tone_loss_schedular, use_lr_scheduler=use_lr_schedular, use_L2_loss=use_L2_loss, is_counter=is_counter, num_per_curve=num_per_curve, skip_control_points=skip_control_points, skip_corners=skip_corners, skip_corner_threshold=skip_corner_threshold, skip_edge_laplacian=skip_edge_laplacian, skip_edge_cos=skip_edge_cos, only_edge_laplacian=only_edge_laplacian, use_laplacian_between_beziers_loss=use_laplacian_between_beziers_loss, laplacian_between_beziers_loss_w=laplacian_between_beziers_loss_w, laplacian_between_beziers_loss_threshold=laplacian_between_beziers_loss_threshold, use_Xing_loss=use_Xing_loss, Xing_loss_w=Xing_loss_w, use_direction_loss=use_direction_loss, direction_loss_w=direction_loss_w, ) with open(cfg.config, "r") as f: cfg_full = yaml.load(f, Loader=yaml.Loader) print(cfg_full) print(cfg_full[cfg.experiment]["loss"]) # recursively traverse parent_config pointers in the config dicts cfg_key = cfg.experiment # convert cfg to dict cfgs = [vars(cfg)] while cfg_key: # print(cfg_full[cfg_key]['loss']) cfgs.append(cfg_full[cfg_key]) cfg_key = cfgs[-1].get("parent_config", "baseline") tmp_cfg = edict() for options in reversed(cfgs): # print(options['loss']) update(tmp_cfg, options) cfg = tmp_cfg del cfgs if target_image_path is not None: target_img = to_tensor( Image.open(target_image_path).convert("L").convert("RGB") ) display(to_pil_image(target_img)) target_img = change_size_transform(size)(target_img).to(device) else: if image_file_path is not None: target_img = Image.open(image_file_path) target_img = to_tensor(target_img) else: target_img = draw_image_through_svg_from_font_path( visual_font_path, optimized_letter, svg_size=size, char_size=char_size, image_size=size, ) cfg.target_img = target_img display(SVG(cfg.target)) if semantic_concept is None: display(to_pil_image(cfg.target_img)) else: print("================== semantic concept ====================") print(semantic_concept) print("========================================================") cfg.semantic_concept = semantic_concept return cfg ``` -------------------------------- ### Finetune CLIP with train.py Source: https://github.com/yukistavailable/fontclip/blob/main/README.md Runs the processor finetuning for FontCLIP. Requires downloading more than 50 missing font files from the gwfonts download website, as noted in the warning. ```bash python train.py --random_prompt_num_per_font 10000 --sample_num 50 --color_jitter_sample_num 200 --use_lora_text ``` -------------------------------- ### Configuration Dictionary Output Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Displays the configuration dictionary used for the optimization, including baseline settings, loss weights, and diffusion model parameters. The output shows the full configuration structure with nested dictionaries for tone and conformal losses. ```text ascii character {'baseline': {'token': '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', 'parent_config': '', 'save': {'init': True, 'image': True, 'video': True, 'video_frame_freq': 1}, 'trainable': {'point': True}, 'lr_base': {'point': 1}, 'lr': {'lr_init': 0.002, 'lr_final': 0.0008, 'lr_delay_mult': 0.1, 'lr_delay_steps': 100}, 'num_iter': 500, 'render_size': 300, 'cut_size': 300, 'is_custom_svg_path': True, 'custom_svg_path': 'Y_300', 'level_of_cc': 0, 'seed': 0, 'diffusion': {'model': 'runwayml/stable-diffusion-v1-5', 'timesteps': 1000, 'guidance_scale': 100}, 'loss': {'use_sds_loss': True, 'tone': {'use_tone_loss': False}, 'conformal': {'use_conformal_loss': False}}}, 'conformal_1.0_dist_pixel_100_kernel201': {'parent_config': 'baseline', 'level_of_cc': 1, 'loss': {'tone': {'use_tone_loss': True, 'dist_loss_weight': 1.0, 'pixel_dist_kernel_blur': 201, 'pixel_dist_sigma': 100}, 'conformal': {'use_conformal_loss': True, 'angeles_w': 1.0}}}} {'tone': {'use_tone_loss': True, 'dist_loss_weight': 1.0, 'pixel_dist_kernel_blur': 201, 'pixel_dist_sigma': 100}, 'conformal': {'use_conformal_loss': True, 'angeles_w': 1.0}} ``` -------------------------------- ### Configure loss functions and weights Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Toggles and weights for Conformal, Tone, G1, L2, Laplacian, Xing, and Direction losses. Includes assertions for Xing and Direction losses requiring preprocessing. ```python # Conformal Loss use_conformal_loss = True conformal_loss_w = 3.0 conformal_loss_w = 5.0 # conformal_loss_w = 1.0 # conformal_loss_w = 20.0 # conformal_loss_w = 1.0 conformal_loss_ws = [3.0, 5.0, 10.0, 20.0, 50.0, 100.0] # Tone Loss use_tone_loss = True tone_loss_w = 5.0 # tone_loss_w = 1.0 # tone_loss_w = 50.0 tone_loss_ws = [10.0, 15.0, 20.0, 25.0, 30.0, 50.0, 100.0] # G1 loss use_G1_loss = False skip_corners = True G1_loss_w = 5.0 G1_loss_ws = [0.1, 0.5, 1.0, 5.0, 10.0, 20.0, 50.0, 100.0] skip_corner_threshold = -0.85 # skip_corner_thresholds = [-0.95, -0.75, -0.5, -0.25, 0., 0.25, 0.5, 0.75, 0.95] # L2 Loss use_L2_loss = False L2_loss_w = 1.0 # Laplacian Loss use_laplacian_loss = False laplacian_loss_w = 1.0 only_edge_laplacian = True # Laplacian Loss between beziers use_laplacian_between_beziers_loss = False laplacian_between_beziers_loss_w = 1.0 laplacian_between_beziers_loss_ws = [0.000001, 0.00001, 0.0001, 0.0005, 0.001] laplacian_between_beziers_loss_thresholds = [-0.8, -0.5, -0.25] laplacian_between_beziers_loss_threshold = -0.8 # Xing Loss for solving self-intersection use_xing_loss = False xing_loss_w = 1.0 xing_loss_ws = [0.1, 1.0, 5.0, 10.0, 20.0, 50.0, 100.0] if use_xing_loss: assert do_preprocess # Direction Loss use_direction_loss = False direction_loss_w = 1.0 direction_loss_ws = [0.01, 0.1, 1.0, 5.0, 10.0, 20.0, 50.0, 100.0] if use_direction_loss: assert do_preprocess ``` -------------------------------- ### Set checkpoint path and target image path Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Sets the checkpoint path for the model and constructs the target image file path for the visual font. ```python # set your own checkpoint path checkpoint_path = '../model_checkpoints/ViT-B_32_bce_lora_t-qkvo_256-1024.0_91011_batch64_aug50_cj200_lbound_of_scale0.35_max_attr_num_3_random_p_num_10000_geta0.2_use_negative_lr0.0002-0.1_image_file_dir.pt' # checkpoint_path = None font_name = os.path.splitext(os.path.basename(visual_font_path))[0] image_file_path = os.path.join(font_image_dir_path, font_name + ".png") ``` -------------------------------- ### CLIP model loading output Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Shows the output when loading the CLIP ViT-B-32 JIT archive from the cache. This output appears after importing the optimizer modules. ```text Output: loading JIT archive /home/yuki/.cache/clip/ViT-B-32.pt ``` -------------------------------- ### Define get_config function for optimization settings Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Defines the get_config function that builds a configuration dictionary for font optimization. It sets default values for many loss weights and flags, and determines visual_optimize based on whether semantic_concept is provided. The function includes commented-out checkpoint paths and sets target_attributes to ["thin", "formal", "legible"]. ```python def get_config( optimized_letter, font_path, visual_font_path=None, num_iter=3001, use_lr_scheduler=True, target_image_path=None, semantic_concept=None, do_preprocess=True, lr=1.0, use_aug=False, use_single_character_image=False, image_file_path=None, use_L2_loss=False, L2_loss_w=1.0, use_visual_encoder=False, use_fclip=False, fclip_loss_w=1.0, use_fclip_direction_loss=False, use_fclip_direction_loss_vision=False, use_fclip_direction_loss_only=False, ref_semantic_concept=None, ref_image_file_path=None, fclip_direction_loss_w=1.0, checkpoint_path=None, use_conformal_loss=False, conformal_loss_w=1.0, use_tone_loss=False, tone_loss_w=1.0, use_tone_loss_schedular=False, use_G1_loss=False, G1_loss_w=1.0, skip_corner_threshold=-0.95, skip_corners=True, use_laplacian_loss=False, laplacian_loss_w=0.2, is_counter=False, skip_edge_laplacian=False, only_edge_laplacian=False, use_laplacian_between_beziers_loss=False, laplacian_between_beziers_loss_w=1.0, laplacian_between_beziers_loss_threshold=-0.8, use_Xing_loss=False, Xing_loss_w=1.0, use_direction_loss=False, direction_loss_w=1.0, ): visual_optimize = True if semantic_concept is not None: visual_optimize = False char_size = 150 size = 200 num_iter = num_iter lr = lr multiple_attributes = False multiple_attributes_preserve_init = False target_attributes = ["thin", "formal", "legible"] reduce_cp = False epsilon = 50 # checkpoint_path = '../model_checkpoints/new_best_fox_negative_91011_1011_use_weight_image_file_dir_ex.pt' # checkpoint_path = '/home/yuki/learn-computer-vision/CLIPFont/model_checkpoints/new_best_fox_negative_91011_91011_multiple_3_1000_aug_ex.pt' target_attributes_weights = [fclip_loss_w] + [ fclip_loss_w / 10 for _ in range(len(target_attributes) - 1) ] use_conformal_loss = use_conformal_loss use_laplacian_loss = use_laplacian_loss use_laplacian_between_beziers_loss = use_laplacian_between_beziers_loss use_cos_loss = False use_G1_loss = use_G1_loss conformal_loss_w = conformal_loss_w laplacian_loss_w = laplacian_loss_w laplacian_between_beziers_loss_w = laplacian_between_beziers_loss_w cos_loss_w = 0.05 G1_loss_w = G1_loss_w # G1_loss_w = 0.0 is_counter = is_counter num_per_curve = 8 skip_control_points = True skip_corners = skip_corners skip_corner_threshold = skip_corner_threshold laplacian_between_beziers_loss_threshold = laplacian_between_beziers_loss_threshold skip_edge_laplacian = skip_edge_laplacian only_edge_laplacian = only_edge_laplacian skip_edge_cos = True use_lr_schedular = use_lr_scheduler ``` -------------------------------- ### Optimization Loop with show_outline Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Iterates over training iterations, calling show_outline for each iteration to visualize the optimized letter with control points. After the loop, a final show_outline call renders the optimized letter without iteration-specific parameters, using convert_cubic_bezier=True and reverse=False. ```python for iteration in iterations: show_outline( optimized_letter, signature, iteration, show_control_points=True, figsize=(10, 10), num_samples_per_stroke=100, save_image=True, reverse=True, font_path=font_path, convert_cubic_bezier=do_preprocess, ) show_outline( optimized_letter, signature=None, iteration=None, show_control_points=True, convert_cubic_bezier=True, figsize=(10, 10), num_samples_per_stroke=100, save_image=True, reverse=False, font_path=visual_font_path, ) ``` -------------------------------- ### Training Output with Progress Bar Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Shows the training progress with a progress bar, loss value, and a warning about a low contrast image. The output includes the path to the saved SVG file and the model checkpoint used. ```text 100%|██████████| 501/501 [00:22<00:00, 22.75it/s, loss=0.0761] /home/yuki/anaconda3/envs/fontclip/lib/python3.8/site-packages/diffvg-0.0.1-py3.8-linux-x86_64.egg/pydiffvg/image.py:22: UserWarning: /home/yuki/learn-computer-vision/FontCLIP/output/conformal_1.0_dist_pixel_100_kernel201_A/Arimo-Regular/A_Arimo-Regular_aug_italic, formal, legible font_fclip3.5_tone5.0-100-201_conf5.0_cp1_lr1.5use_lr_scheduler/Arimo-Regular_A_A.png is a low contrast image skimage.io.imsave(filename, (img * 255).astype(np.uint8)) ``` -------------------------------- ### Set FontCLIP Optimization Parameters Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Defines paths, optimization settings, and loss weights for a FontCLIP run. Includes assertions that iteration counts are below num_iter, and sets a semantic concept for the optimization. ```python font_image_dir_path = "../gwfonts_images/" target_image_path = None font_path = "../gwfonts/Arimo-Regular.ttf" visual_font_path = "../gwfonts/Amethysta-Regular.ttf" optimized_letters = ["A"] do_preprocess = True lr = 1.5 num_iter = 501 target_iters = [0, 5, 10, 30, 50, 70, 90, 100, 110, 130, 150, 170, 200, 300, 500] iterations = [0, 80, 100, 500] assert all([i < num_iter for i in target_iters]) assert all([i < num_iter for i in iterations]) use_lr_scheduler = True semantic_concept = "italic, formal, legible font" # FontCLIP Loss use_aug = True use_fclip = True use_fclip_direction_loss = False use_fclip_direction_loss_vision = False use_fclip_direction_loss_only = False fclip_loss_w = 3.5 fclip_direction_loss_w = 0.1 fclip_direction_loss_ws = [0.01, 0.025, 0.05, 0.75, 0.1] ``` -------------------------------- ### Import optimizer modules and utilities Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Imports required modules and functions from the optimizer package, including Config, train, and drawing utilities. Adds the parent directory to sys.path to access the optimizer package. ```python import os from IPython.display import SVG, display import yaml from easydict import EasyDict as edict from torchvision.transforms.functional import to_pil_image, to_tensor from PIL import Image, ImageFont import sys sys.path.append('../') from optimizer.optimize import ( Config, draw_image_through_svg_from_font_path, get_signature, train, create_svg_from_font, show_outline, change_size_transform, device, draw_single_character, ) from optimizer.util_tools import update ``` -------------------------------- ### Set reference semantic concept and image paths Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Defines the reference semantic concept and constructs file paths for the reference font image based on the font name. ```python ref_semantic_concept = "formal, legible font" ref_font_name = os.path.splitext(os.path.basename(font_path))[0] ref_image_file_path = os.path.join(font_image_dir_path, ref_font_name + ".png") ``` -------------------------------- ### Optimize letters and generate SVG Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Loops over optimized letters, builds config, trains, and creates SVG output for each letter. ```python for optimized_letter in optimized_letters: if True: if True: cfg = get_config( optimized_letter, font_path, visual_font_path, num_iter, use_lr_scheduler=use_lr_scheduler, lr=lr, use_aug=use_aug, do_preprocess=do_preprocess, semantic_concept=semantic_concept, image_file_path=image_file_path, target_image_path=target_image_path, use_L2_loss=use_L2_loss, L2_loss_w=L2_loss_w, checkpoint_path=checkpoint_path, use_fclip=use_fclip, fclip_loss_w=fclip_loss_w, use_fclip_direction_loss=use_fclip_direction_loss, use_fclip_direction_loss_vision=use_fclip_direction_loss_vision, use_fclip_direction_loss_only=use_fclip_direction_loss_only, ref_semantic_concept=ref_semantic_concept, ref_image_file_path=ref_image_file_path, fclip_direction_loss_w=fclip_direction_loss_w, use_conformal_loss=use_conformal_loss, conformal_loss_w=conformal_loss_w, use_tone_loss=use_tone_loss, tone_loss_w=tone_loss_w, use_G1_loss=use_G1_loss, G1_loss_w=G1_loss_w, skip_corners=skip_corners, skip_corner_threshold=skip_corner_threshold, use_laplacian_loss=use_laplacian_loss, laplacian_loss_w=laplacian_loss_w, only_edge_laplacian=only_edge_laplacian, use_laplacian_between_beziers_loss=use_laplacian_between_beziers_loss, laplacian_between_beziers_loss_w=laplacian_between_beziers_loss_w, laplacian_between_beziers_loss_threshold=laplacian_between_beziers_loss_threshold, use_Xing_loss=use_xing_loss, Xing_loss_w=xing_loss_w, use_direction_loss=use_direction_loss, direction_loss_w=direction_loss_w, ) signature = get_signature( cfg, font_path, visual_font_path, target_image_path ) print(signature) train(cfg, signature) output_path = create_svg_from_font( target_iters, character=optimized_letter, font_path=font_path, output_path=None, signature=signature, target_attribute="", visual_optimize=True, visual_font_path=visual_font_path, char_size=150, ) print(output_path) display(SVG(output_path)) ``` -------------------------------- ### Set WANDB configuration flags Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Sets configuration flags to disable Weights & Biases logging and leave the WANDB user empty. ```python # set config USE_WANDB = 0 WANDB_USER = '' ``` -------------------------------- ### SVG Output Path Source: https://github.com/yukistavailable/fontclip/blob/main/notebooks/optimize.ipynb Displays the file path where the SVG output is saved after the training iteration. ```text saving video /home/yuki/learn-computer-vision/FontCLIP/output/A_Arimo-Regular_aug_italic, formal, legible font_fclip3.5_tone5.0-100-201_conf5.0_cp1_lr1. ``` === COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.