From 68f33b8cdf4fd849f81251c7d6978ff15fa2cd0c Mon Sep 17 00:00:00 2001 From: sa_ddam213 Date: Sat, 4 Jul 2026 13:02:26 +1200 Subject: [PATCH] Pipeline template --- .../Pipelines/Cosmos2Pipeline.py | 445 +++++++++ .../Templates/Cosmos2_T2I/model_index.json | 28 + .../scheduler/scheduler_config.json | 22 + .../Cosmos2_T2I/text_encoder/config.json | 60 ++ .../tokenizer/tokenizer_config.json | 939 ++++++++++++++++++ .../Cosmos2_T2I/transformer/config.json | 29 + .../Templates/Cosmos2_T2I/vae/config.json | 57 ++ TensorStack.Python/TensorStack.Python.csproj | 6 + 8 files changed, 1586 insertions(+) create mode 100644 TensorStack.Python/Pipelines/Cosmos2Pipeline.py create mode 100644 TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/model_index.json create mode 100644 TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/scheduler/scheduler_config.json create mode 100644 TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/text_encoder/config.json create mode 100644 TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/tokenizer/tokenizer_config.json create mode 100644 TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/transformer/config.json create mode 100644 TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/vae/config.json diff --git a/TensorStack.Python/Pipelines/Cosmos2Pipeline.py b/TensorStack.Python/Pipelines/Cosmos2Pipeline.py new file mode 100644 index 0000000..aca4874 --- /dev/null +++ b/TensorStack.Python/Pipelines/Cosmos2Pipeline.py @@ -0,0 +1,445 @@ +import tensorstack.utils as Utils +import tensorstack.data_objects as DataObjects +import tensorstack.quantization as Quantization +from tensorstack.enums import ProcessType, QuantTarget +Utils.redirect_output() +Utils.create_services() + +import torch +import numpy as np +from pathlib import Path +from threading import Event +from functools import partial +from collections.abc import Buffer +from typing import Dict, Sequence, List, Tuple, Optional, Any +from transformers import T5Tokenizer, T5EncoderModel +from diffusers import ( + AutoencoderKLWan, + CosmosTransformer3DModel, + Cosmos2TextToImagePipeline +) + +# Globals +_config = None +_model_config = None +_pipeline = None +_processType = None +_execution_device = None +_device_map = None +_pipeline_device_map = None +_control_net_name = None +_control_net_cache = None +_generator = None +_isMemoryOffload = False +_prompt_cache_key = None +_prompt_cache_value = None +_cancel_event = Event() +_stopwatch = None +_pipelineMap = { + ProcessType.TextToImage: Cosmos2TextToImagePipeline +} + + +#------------------------------------------------ +# Load Pipeline +#------------------------------------------------ +def load(config_args: Dict[str, Any]) -> bool: + global _config, _pipeline, _generator, _processType, _execution_device, _isMemoryOffload + + # Config + _config = DataObjects.PipelineConfig(**config_args) + _execution_device = Utils.get_execution_device(_config) + _generator = torch.Generator(device=_execution_device) + _processType = _config.process_type + + # Initialize Pipeline + _pipeline = initialize(_config) + + # Load Lora + Utils.load_lora_weights(_pipeline, _config) + + # Memory + _isMemoryOffload = Utils.configure_pipeline_memory(_pipeline, _execution_device, _config) + Utils.trim_memory(_isMemoryOffload) + return True + + +#------------------------------------------------ +# Reload Pipeline - ProcessType, LoraAdapters and ControlNet are the only options that can be modified +#------------------------------------------------ +def reload(config_args: Dict[str, Any]) -> bool: + global _config, _pipeline, _processType + + # Config + _config = DataObjects.PipelineConfig(**config_args) + _processType = _config.process_type + + # Rebuild Pipeline + Utils.unload_lora_weights(_pipeline) + _pipeline = create_pipeline(_config) + + # Load Lora + Utils.load_lora_weights(_pipeline, _config) + + # Memory + Utils.configure_pipeline_memory(_pipeline, _execution_device, _config) + Utils.trim_memory(_isMemoryOffload) + return True + + +#------------------------------------------------ +# Switch Pipeline - ProcessType +#------------------------------------------------ +def switch(process_type: ProcessType) -> bool: + global _pipeline, _processType + + # Switch Pipeline + current = _processType + _processType = process_type + _pipeline = create_pipeline(_config) + + print(f"[Generate] Switched pipeline: {current} => {process_type}") + return True + + +#------------------------------------------------ +# Cancel Generation +#------------------------------------------------ +def generateCancel() -> None: + _cancel_event.set() + + +#------------------------------------------------ +# Unload Pipline +#------------------------------------------------ +def unload() -> bool: + global _pipeline, _prompt_cache_key, _prompt_cache_value + _pipeline = None + _prompt_cache_key = None + _prompt_cache_value = None + Utils.trim_memory(_isMemoryOffload) + return True + + +#------------------------------------------------ +# Get the notifications +#------------------------------------------------ +def getNotifications() -> list[(str, Buffer)]: + return Utils.notification_get() + + +#------------------------------------------------ +# Get the log entires +#------------------------------------------------ +def getLogs() -> list[str]: + return Utils.get_output() + + +#------------------------------------------------ +# Initialize Pipeline +#------------------------------------------------ +def initialize(config: DataObjects.PipelineConfig): + global _model_config, _device_map, _pipeline_device_map + + _device_map = Utils.get_device_map(config, _execution_device) + _pipeline_device_map = Utils.get_pipeline_device_map(config, _execution_device) + _model_config = Utils.get_model_config(__file__, config) + return create_pipeline(config) + + +#------------------------------------------------ +# Load T5Tokenizer +#------------------------------------------------ +def load_tokenizer(config: DataObjects.PipelineConfig, pipeline_kwargs: Dict[str, str]): + if _pipeline and _pipeline.tokenizer: + print(f"[Load] Loading Cached Tokenizer") + return _pipeline.tokenizer + + tokenizer_path: Path = _model_config["tokenizer"] + tokenizer_config: Path = _model_config["tokenizer_config"] + + # 1. Load from pretrained folder + print(f"[Load] Loading Pretrained Tokenizer") + tokenizer = T5Tokenizer.from_pretrained( + tokenizer_path, + config=tokenizer_config, + dtype=config.data_type, + **pipeline_kwargs + ) + return tokenizer + + +#------------------------------------------------ +# Load T5EncoderModel +#------------------------------------------------ +def load_text_encoder(config: DataObjects.PipelineConfig, pipeline_kwargs: Dict[str, str]): + if _pipeline and _pipeline.text_encoder: + print(f"[Load] Loading Cached TextEncoder") + return _pipeline.text_encoder + + text_encoder_path: Path = _model_config["text_encoder"] + text_encoder_config: Path = _model_config["text_encoder_config"] + + # 1. Load from pretrained folder + print(f"[Load] Loading Pretrained TextEncoder") + text_encoder = T5EncoderModel.from_pretrained( + text_encoder_path, + config=text_encoder_config, + dtype=config.data_type, + device_map=_device_map if _device_map != "meta" else None, + quantization_config=Quantization.auto_pretrained_config(config, QuantTarget.TEXT_ENCODER), + **pipeline_kwargs + ) + Utils.trim_memory(True) + return text_encoder + + +#------------------------------------------------ +# Load CosmosTransformer3DModel +#------------------------------------------------ +def load_transformer(config: DataObjects.PipelineConfig, pipeline_kwargs: Dict[str, str]): + if _pipeline and _pipeline.transformer: + print(f"[Load] Loading Cached Transformer") + return _pipeline.transformer + + transformer_path: Path = _model_config["transformer"] + transformer_config: Path = _model_config["transformer_config"] + + # 1. Load from single file + if transformer_path.is_file(): + is_gguf = Utils.isGGUF(transformer_path) + print(f"[Load] Loading File Transformer") + transformer = CosmosTransformer3DModel.from_single_file( + str(transformer_path), + config=str(transformer_config), + torch_dtype=config.data_type, + device_map=_device_map, + quantization_config=Quantization.auto_single_file_config(config, QuantTarget.TRANSFORMER, is_gguf), + **pipeline_kwargs + ) + Quantization.quantize_model(config, transformer, is_gguf) + Utils.trim_memory(True) + return transformer + + # 2. Load from pretrained folder + print(f"[Load] Loading Pretrained Transformer") + transformer = CosmosTransformer3DModel.from_pretrained( + str(transformer_path), + torch_dtype=config.data_type, + device_map=_device_map, + quantization_config=Quantization.auto_pretrained_config(config, QuantTarget.TRANSFORMER), + **pipeline_kwargs + ) + Utils.trim_memory(True) + return transformer + + +#------------------------------------------------ +# Load AutoencoderKLWan +#------------------------------------------------ +def load_vae(config: DataObjects.PipelineConfig, pipeline_kwargs: Dict[str, str]): + if _pipeline and _pipeline.vae: + print(f"[Load] Loading Cached Vae") + return _pipeline.vae + + vae_path: Path = _model_config["vae"] + vae_config: Path = _model_config["vae_config"] + single_path: Path = _model_config["single_file"] + template_path: Path = _model_config["template"] + + # 1. Load from single file + if vae_path.is_file(): + print(f"[Load] Loading SingleFile Vae") + auto_encoder = AutoencoderKLWan.from_single_file( + str(vae_path), + config=str(vae_config), + torch_dtype=config.data_type, + device_map=_device_map, + **pipeline_kwargs + ) + Utils.trim_memory(True) + return auto_encoder + + # 2. Load component from single file + if single_path and single_path.is_file(): + print(f"[Load] Loading Component Vae") + auto_encoder = Utils.from_component(Cosmos2TextToImagePipeline, "vae", single_path, template_path, _device_map, config.data_type) + if auto_encoder: + Utils.trim_memory(True) + return auto_encoder + + # 3. Load from pretrained folder + print(f"[Load] Loading Pretrained Vae") + auto_encoder = AutoencoderKLWan.from_pretrained( + str(vae_path), + torch_dtype=config.data_type, + device_map=_device_map, + **pipeline_kwargs + ) + Utils.trim_memory(True) + return auto_encoder + + +#------------------------------------------------ +# Load ControlNetModel +#------------------------------------------------ +def load_control_net(config: DataObjects.PipelineConfig, pipeline_kwargs: Dict[str, str]): + global _control_net_name, _control_net_cache + + if _control_net_cache and _control_net_name == config.control_net.name: + print(f"[Load] Loading Cached ControlNet") + return _control_net_cache + + if config.control_net.name is None: + _control_net_name = None + _control_net_cache = None + return None + + # print(f"[Load] Loading Pretrained ControlNet") + # _control_net_name = config.control_net.name + # _control_net_cache = ControlNetModel.from_pretrained( + # config.control_net.path, + # torch_dtype=config.data_type, + # device_map=_device_map, + # **pipeline_kwargs + # ) + return _control_net_cache + + +#------------------------------------------------ +# Create a new pipeline +#------------------------------------------------ +def create_pipeline(config: DataObjects.PipelineConfig): + template_path: Path = _model_config["template"] + pipeline_kwargs = { + "variant": config.variant, + "use_safetensors":True, + "low_cpu_mem_usage":True, + "local_files_only":True, + } + + # Load Models + tokenizer = load_tokenizer(config, pipeline_kwargs) + text_encoder = load_text_encoder(config, pipeline_kwargs) + transformer = load_transformer(config, pipeline_kwargs) + vae = load_vae(config, pipeline_kwargs) + control_net = load_control_net(config, pipeline_kwargs) + if control_net is not None: + pipeline_kwargs.update({"controlnet": control_net}) + + # Build Pipeline + pipeline = _pipelineMap[_processType] + return pipeline.from_pretrained( + template_path, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + vae=vae, + torch_dtype=config.data_type, + device_map=_pipeline_device_map, + **pipeline_kwargs + ) + + +#------------------------------------------------ +# Generate Image/Video +#------------------------------------------------ +def generate( + inference_args: Dict[str, Any], + input_tensors: Optional[List[Tuple[Sequence[float],Sequence[int]]]] = None, + control_tensors: Optional[List[Tuple[Sequence[float],Sequence[int]]]] = None, + ) -> Sequence[Buffer]: + global _prompt_cache_key, _prompt_cache_value, _stopwatch + _cancel_event.clear() + _pipeline._interrupt = False + _stopwatch = Utils.Stopwatch() + _stopwatch.start() + Utils.notification_push(key="Generate", subkey="Initialize") + + # Input Images + images = Utils.prepare_images(input_tensors) + image_count = Utils.get_len(images) + control_images = Utils.prepare_images(control_tensors) + control_image_count = Utils.get_len(control_images) + print(f"[Generate] Input Received - Tensors: {image_count}, Control Tensors: {control_image_count}") + + # Options + options = DataObjects.PipelineOptions(**inference_args) + + # Scheduler + _pipeline.scheduler = Utils.create_scheduler(options.scheduler_options) + + # AutoEncoder + Utils.configure_vae_memory(_pipeline, options.enable_vae_tiling, options.enable_vae_slicing) + + # Lora Adapters + Utils.set_lora_weights(_pipeline, options) + + # Notify + Utils.notification_push(key="Generate", subkey="TextEncoder", elapsedkey="Initialize", elapsed=_stopwatch.reset()) + + # Prompt Cache + prompt_cache_key = (options.prompt, options.negative_prompt, options.guidance_scale > 1) + if _prompt_cache_key != prompt_cache_key: + print(f"[Generate] Encoding prompt") + with torch.no_grad(): + _prompt_cache_value = _pipeline.encode_prompt( + prompt=options.prompt, + negative_prompt=options.negative_prompt, + do_classifier_free_guidance=options.guidance_scale > 1, + device=_pipeline._execution_device, + max_sequence_length=512 + ) + _prompt_cache_key = prompt_cache_key + + # Notify + Utils.notification_push(key="Generate", subkey="Transformer", elapsedkey="TextEncoder", elapsed=_stopwatch.reset()) + + # Pipeline Options + (prompt_embeds, negative_prompt_embeds) = _prompt_cache_value + pipeline_options = { + "prompt_embeds": prompt_embeds, + "negative_prompt_embeds": negative_prompt_embeds, + "height": options.height, + "width": options.width, + "generator": _generator.manual_seed(options.seed), + "guidance_scale": options.guidance_scale, + "num_inference_steps": options.steps, + "output_type": "np", + "callback_on_step_end": partial(_progress_callback, height=options.height, width=options.width), + "callback_on_step_end_tensor_inputs": ["latents"], + } + + # Run Pipeline + output = _pipeline(**pipeline_options)[0] + + # (Batch, Channel, Height, Width) + output = output.transpose(0, 3, 1, 2).astype(np.float32) + + # Notify + Utils.notification_push(key="Generate", subkey="AutoEncoder", elapsedkey="Transformer", elapsed = _stopwatch.reset()) + Utils.notification_push(key="Generate", subkey="Complete", elapsedkey="AutoEncoder", elapsed = _stopwatch.stop()) + + # Cleanup + Utils.trim_memory(_isMemoryOffload) + return [ np.ascontiguousarray(output) ] + + +#------------------------------------------------ +# Diffusers pipeline callback to capture step artifacts +#------------------------------------------------ +def _progress_callback(pipe, step: int, total_steps: int, info: Dict, height: int, width: int): + if _cancel_event.is_set(): + pipe._interrupt = True + raise Exception("Operation Canceled") + + def preview_latents(latents): + if latents is None: + return [] + return latents.float().cpu() + + steps = pipe._num_timesteps + elapsed = _stopwatch.reset() + step_latents = preview_latents(info.get("latents")) + Utils.notification_push(key="Generate", subkey="Step", elapsedkey="Step", value=step + 1, maximum=steps, elapsed=elapsed, tensor=step_latents) + return info \ No newline at end of file diff --git a/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/model_index.json b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/model_index.json new file mode 100644 index 0000000..40bd3b2 --- /dev/null +++ b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/model_index.json @@ -0,0 +1,28 @@ +{ + "_class_name": "Cosmos2TextToImagePipeline", + "_diffusers_version": "0.34.0.dev0", + "safety_checker": [ + null, + null + ], + "scheduler": [ + "diffusers", + "FlowMatchEulerDiscreteScheduler" + ], + "text_encoder": [ + "transformers", + "T5EncoderModel" + ], + "tokenizer": [ + "transformers", + "T5TokenizerFast" + ], + "transformer": [ + "diffusers", + "CosmosTransformer3DModel" + ], + "vae": [ + "diffusers", + "AutoencoderKLWan" + ] +} diff --git a/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/scheduler/scheduler_config.json b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/scheduler/scheduler_config.json new file mode 100644 index 0000000..07252bf --- /dev/null +++ b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/scheduler/scheduler_config.json @@ -0,0 +1,22 @@ +{ + "_class_name": "FlowMatchEulerDiscreteScheduler", + "_diffusers_version": "0.34.0.dev0", + "base_image_seq_len": 256, + "base_shift": 0.5, + "final_sigmas_type": "sigma_min", + "invert_sigmas": false, + "max_image_seq_len": 4096, + "max_shift": 1.15, + "num_train_timesteps": 1000, + "shift": 1.0, + "shift_terminal": null, + "sigma_data": 1.0, + "sigma_max": 80.0, + "sigma_min": 0.002, + "stochastic_sampling": false, + "time_shift_type": "exponential", + "use_beta_sigmas": false, + "use_dynamic_shifting": false, + "use_exponential_sigmas": false, + "use_karras_sigmas": true +} diff --git a/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/text_encoder/config.json b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/text_encoder/config.json new file mode 100644 index 0000000..6c85dc4 --- /dev/null +++ b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/text_encoder/config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "T5EncoderModel" + ], + "classifier_dropout": 0.0, + "d_ff": 65536, + "d_kv": 128, + "d_model": 1024, + "decoder_start_token_id": 0, + "dense_act_fn": "relu", + "dropout_rate": 0.1, + "eos_token_id": 1, + "feed_forward_proj": "relu", + "initializer_factor": 1.0, + "is_encoder_decoder": true, + "is_gated_act": false, + "layer_norm_epsilon": 1e-06, + "model_type": "t5", + "n_positions": 512, + "num_decoder_layers": 24, + "num_heads": 128, + "num_layers": 24, + "output_past": true, + "pad_token_id": 0, + "relative_attention_max_distance": 128, + "relative_attention_num_buckets": 32, + "task_specific_params": { + "summarization": { + "early_stopping": true, + "length_penalty": 2.0, + "max_length": 200, + "min_length": 30, + "no_repeat_ngram_size": 3, + "num_beams": 4, + "prefix": "summarize: " + }, + "translation_en_to_de": { + "early_stopping": true, + "max_length": 300, + "num_beams": 4, + "prefix": "translate English to German: " + }, + "translation_en_to_fr": { + "early_stopping": true, + "max_length": 300, + "num_beams": 4, + "prefix": "translate English to French: " + }, + "translation_en_to_ro": { + "early_stopping": true, + "max_length": 300, + "num_beams": 4, + "prefix": "translate English to Romanian: " + } + }, + "torch_dtype": "bfloat16", + "transformers_version": "4.52.3", + "use_cache": true, + "vocab_size": 32128 +} diff --git a/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/tokenizer/tokenizer_config.json b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/tokenizer/tokenizer_config.json new file mode 100644 index 0000000..71cc075 --- /dev/null +++ b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/tokenizer/tokenizer_config.json @@ -0,0 +1,939 @@ +{ + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32000": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32001": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32002": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32003": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32004": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32005": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32006": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32007": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32008": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32009": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32010": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32011": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32012": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32013": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32014": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32015": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32016": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32017": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32018": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32019": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32020": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32021": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32022": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32023": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32024": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32025": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32026": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32027": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32028": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32029": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32030": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32031": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32032": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32033": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32034": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32035": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32036": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32037": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32038": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32039": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32040": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32041": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32042": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32043": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32044": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32045": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32046": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32047": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32048": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32049": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32050": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32051": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32052": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32053": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32054": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32055": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32056": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32057": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32058": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32059": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32060": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32061": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32062": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32063": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32064": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32065": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32066": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32067": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32068": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32069": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32070": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32071": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32072": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32073": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32074": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32075": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32076": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32077": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32078": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32079": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32080": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32081": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32082": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32083": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32084": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32085": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32086": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32087": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32088": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32089": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32090": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32091": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32092": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32093": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32094": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32095": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32096": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32097": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32098": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32099": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_ids": 100, + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "tokenizer_class": "T5Tokenizer", + "unk_token": "" +} diff --git a/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/transformer/config.json b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/transformer/config.json new file mode 100644 index 0000000..0cbfb6c --- /dev/null +++ b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/transformer/config.json @@ -0,0 +1,29 @@ +{ + "_class_name": "CosmosTransformer3DModel", + "_diffusers_version": "0.34.0.dev0", + "adaln_lora_dim": 256, + "attention_head_dim": 128, + "concat_padding_mask": true, + "extra_pos_embed_type": null, + "in_channels": 16, + "max_size": [ + 128, + 240, + 240 + ], + "mlp_ratio": 4.0, + "num_attention_heads": 16, + "num_layers": 28, + "out_channels": 16, + "patch_size": [ + 1, + 2, + 2 + ], + "rope_scale": [ + 1.0, + 4.0, + 4.0 + ], + "text_embed_dim": 1024 +} diff --git a/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/vae/config.json b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/vae/config.json new file mode 100644 index 0000000..e95123c --- /dev/null +++ b/TensorStack.Python/Pipelines/Templates/Cosmos2_T2I/vae/config.json @@ -0,0 +1,57 @@ +{ + "_class_name": "AutoencoderKLWan", + "_diffusers_version": "0.34.0.dev0", + "_name_or_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + "attn_scales": [], + "base_dim": 96, + "dim_mult": [ + 1, + 2, + 4, + 4 + ], + "dropout": 0.0, + "latents_mean": [ + -0.7571, + -0.7089, + -0.9113, + 0.1075, + -0.1745, + 0.9653, + -0.1517, + 1.5508, + 0.4134, + -0.0715, + 0.5517, + -0.3632, + -0.1922, + -0.9497, + 0.2503, + -0.2921 + ], + "latents_std": [ + 2.8184, + 1.4541, + 2.3275, + 2.6558, + 1.2196, + 1.7708, + 2.6052, + 2.0743, + 3.2687, + 2.1526, + 2.8652, + 1.5579, + 1.6382, + 1.1253, + 2.8251, + 1.916 + ], + "num_res_blocks": 2, + "temperal_downsample": [ + false, + true, + true + ], + "z_dim": 16 +} diff --git a/TensorStack.Python/TensorStack.Python.csproj b/TensorStack.Python/TensorStack.Python.csproj index 5ab94d6..ccaba6c 100644 --- a/TensorStack.Python/TensorStack.Python.csproj +++ b/TensorStack.Python/TensorStack.Python.csproj @@ -46,6 +46,12 @@ + + contentFiles\any\any\Pipelines\ + PreserveNewest + true + true +