### Install StreamForest Dependencies Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Create a conda environment and install necessary Python packages and download helper files. ```bash conda create --name StreamForest python=3.10 conda activate StreamForest pip install -r requirements.txt python3 download_hf.py ``` -------------------------------- ### Execute Inference with Parameters Source: https://github.com/mcg-nju/streamforest/blob/main/demo/video_demo.ipynb Example of how to call the `run_inference` function with specific video path, question, and time parameters. Ensure `args` is properly initialized before use. ```python # StreamForest video_path = "your video path" question= "your question" question_time=3 #your question time (s) run_inference(args,video_path,question,question_time) ``` -------------------------------- ### Evaluate Predefined Models with lmms-eval Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Run this script to start benchmark evaluation for predefined models. ```bash lmms_eval/scripts/eval_internvl2-8B.sh ``` -------------------------------- ### Argument Parsing for Video Evaluation Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed_multiround.ipynb Parses command-line arguments for video evaluation, including model paths, output directories, and inference settings. This setup is crucial for configuring the evaluation process. ```python import argparse import torch from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN from llava.conversation import conv_templates, SeparatorStyle from llava.model.builder import load_pretrained_model from llava.utils import disable_torch_init from llava.mm_utils import process_anyres_image,tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria, process_anyres_video_nopad import json import os import math from transformers import AutoConfig from llava.video_utils import VIDEO_READER_FUNCS def split_list(lst, n): """Split a list into n (roughly) equal-sized chunks""" chunk_size = math.ceil(len(lst) / n) # integer division return [lst[i : i + chunk_size] for i in range(0, len(lst), chunk_size)] def get_chunk(lst, n, k): chunks = split_list(lst, n) return chunks[k] def parse_args(): """ Parse command-line arguments. """ parser = argparse.ArgumentParser() # Define the command-line arguments # parser.add_argument("--video_path", help="Path to the video files.", default="/mnt/petrelfs/zengxiangyu/OpenSource/Backup/fg-videochat/download/demo_video/legendof1900.mp4") # parser.add_argument("--prompt", default="describe this video in detail.", type=str) parser.add_argument("--output_dir", default="./work_dirs/video_demo/", help="Directory to save the model results JSON.") parser.add_argument("--output_name",default="pred" , help="Name of the file for storing results JSON.") parser.add_argument("--model-path", type=str, default="/your_local_path_to/StreamForest/ckpt/StreamForest-Qwen2-7B_Siglip") parser.add_argument("--inference_device", type=str, default="cuda:0") parser.add_argument("--model-base", type=str, default=None) parser.add_argument("--conv-mode", type=str, default="qwen_2") parser.add_argument("--chunk-idx", type=int, default=0) parser.add_argument("--max_num_frames", type=int, default=4096) parser.add_argument("--load_8bit", type=lambda x: (str(x).lower() == 'true'), default=False) parser.add_argument("--force_sample", type=lambda x: (str(x).lower() == 'true'), default=False) parser.add_argument("--time_msg", type=str, default="") parser.add_argument("--llm_type", type=str, default="") parser.add_argument("--attn_implementation", type=str, default="flash_attention_2") parser.add_argument("--use_hd", type=bool, default=False) args = parser.parse_args(args=[]) return args args = parse_args() ``` -------------------------------- ### Run Video Inference Source: https://github.com/mcg-nju/streamforest/blob/main/demo/video_demo.ipynb This function performs inference on a given video using the VideoChat-Next model. It handles video loading, preprocessing, prompt construction, and model generation. Use this function to get answers to questions about a video. ```python import time def run_inference(args, video_path, question, question_time=0): """ Run inference on a demo video using VideoChat-Next model. Args: args: Command-line arguments. """ if hasattr(model.config, "frame_aspect_ratio"): frame_aspect_ratio = model.config.frame_aspect_ratio else: frame_aspect_ratio = "" # import pdb;pdb.set_trace() print("video_path:", video_path) sample_set = {} sample_set["Q"] = question sample_set["video_name"] = video_path # Check if the video exists # if os.path.exists(video_path) : assert 's3://' in video_path or os.path.exists(video_path), video_path frames, time_msg = load_video(video_path, args, question_time) print("len(frames):", len(frames)) image_sizes = [frames[0].shape[:2]] print("image_sizes:", image_sizes) frames = image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].to(torch.bfloat16).cuda() print("input frames:", frames.shape) video = [frames] # try: # Run inference on the video and add the output to the list qs = question if args.time_msg != "": qs = f'{time_msg.strip()}\n{qs}' if model.config.mm_use_im_start_end: qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs else: qs = DEFAULT_IMAGE_TOKEN + "\n" + qs print(f"Question: {qs}") conv = conv_templates[args.conv_mode].copy() conv.append_message(conv.roles[0], qs) conv.append_message(conv.roles[1], None) prompt = conv.get_prompt() input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda() if tokenizer.pad_token_id is None: if "qwen" in tokenizer.name_or_path.lower(): print("Setting pad token to bos token for qwen model.") tokenizer.pad_token_id = 151643 attention_masks = input_ids.ne(tokenizer.pad_token_id).long().cuda() stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 keywords = [stop_str] stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids) start_time = time.time() with torch.inference_mode(): if "mistral" not in cfg_pretrained._name_or_path.lower(): output_ids = model.generate( inputs=input_ids, images=video, attention_mask=attention_masks, modalities=["video"], image_sizes=image_sizes, do_sample=False, temperature=0.0, max_new_tokens=1024, num_beams=1, use_cache=True, stopping_criteria=[stopping_criteria] ) else: output_ids = model.generate(inputs=input_ids, images=video, attention_mask=attention_masks, modalities="video", do_sample=False, temperature=0.0, max_new_tokens=1024, top_p=0.1, num_beams=1, use_cache=True) outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() end_time = time.time() elapsed_time = end_time - start_time print(f"Inference time: {elapsed_time:.3f} seconds") print(f"Question: {prompt}\n") print(f"Response: {outputs}\n") # import pdb;pdb.set_trace() if "mistral" not in cfg_pretrained._name_or_path.lower(): if outputs.endswith(stop_str): outputs = outputs[: -len(stop_str)] outputs = outputs.strip() sample_set["pred"] = outputs ans_file.write(json.dumps(sample_set, ensure_ascii=False) + "\n") ans_file.flush() ``` -------------------------------- ### Execute Stage 1 Offline Pretraining Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Run the script for the first stage of offline video pretraining. This involves initializing the connector. ```bash bash scripts/train/stage1-init_connector/s1_siglip_tome64_mlp.sh ``` -------------------------------- ### Argument Parsing for Video Demo Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed.ipynb Parses command-line arguments for configuring the video demo, including model paths, inference devices, and output directories. It sets up default values for various parameters. ```python import argparse import torch from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN from llava.conversation import conv_templates, SeparatorStyle from llava.model.builder import load_pretrained_model from llava.utils import disable_torch_init from llava.mm_utils import process_anyres_image,tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria, process_anyres_video_nopad import json import os import math from transformers import AutoConfig from llava.video_utils import VIDEO_READER_FUNCS def split_list(lst, n): """Split a list into n (roughly) equal-sized chunks""" chunk_size = math.ceil(len(lst) / n) # integer division return [lst[i : i + chunk_size] for i in range(0, len(lst), chunk_size)] def get_chunk(lst, n, k): chunks = split_list(lst, n) return chunks[k] def parse_args(): """ Parse command-line arguments. """ parser = argparse.ArgumentParser() parser.add_argument("--output_dir", default="./work_dirs/video_demo/", help="Directory to save the model results JSON.") parser.add_argument("--output_name",default="pred" , help="Name of the file for storing results JSON.") # parser.add_argument("--model-path", type=str, default="/your_local_path_to/StreamForest/ckpt/StreamForest-Qwen2-7B_Siglip") parser.add_argument("--model-path", type=str, default="/your_local_path_to/StreamForest/ckpt/VideoChat-Stream-Drive-Qwen2-7B_Siglip") parser.add_argument("--inference_device", type=str, default="cuda:0") parser.add_argument("--model-base", type=str, default=None) parser.add_argument("--conv-mode", type=str, default="qwen_2") parser.add_argument("--chunk-idx", type=int, default=0) parser.add_argument("--max_num_frames", type=int, default=4096) parser.add_argument("--load_8bit", type=lambda x: (str(x).lower() == 'true'), default=False) parser.add_argument("--force_sample", type=lambda x: (str(x).lower() == 'true'), default=False) parser.add_argument("--time_msg", type=str, default="short") parser.add_argument("--llm_type", type=str, default="") parser.add_argument("--attn_implementation", type=str, default="flash_attention_2") parser.add_argument("--use_hd", type=bool, default=False) args = parser.parse_args(args=[]) return args args = parse_args() ``` -------------------------------- ### Execute Stage 5 Online Drive Fine-tuning Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Run the script for the fifth stage of online video fine-tuning, specifically for drive fine-tuning with tree memory. ```bash bash scripts/train/stage5-drive_ft/s5_siglip_online_tree_memory_drive.sh ``` -------------------------------- ### Parse Command-Line Arguments Source: https://github.com/mcg-nju/streamforest/blob/main/demo/video_demo.ipynb Defines and parses command-line arguments for the video demonstration script. This includes model paths, output directories, and sampling parameters. ```python import argparse import torch from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN from llava.conversation import conv_templates, SeparatorStyle from llava.model.builder import load_pretrained_model from llava.utils import disable_torch_init from llava.mm_utils import process_anyres_image,tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria, process_anyres_video_nopad import json import os import math from transformers import AutoConfig from llava.video_utils import VIDEO_READER_FUNCS def split_list(lst, n): """Split a list into n (roughly) equal-sized chunks""" chunk_size = math.ceil(len(lst) / n) # integer division return [lst[i : i + chunk_size] for i in range(0, len(lst), chunk_size)] def get_chunk(lst, n, k): chunks = split_list(lst, n) return chunks[k] def parse_args(): """ Parse command-line arguments. """ parser = argparse.ArgumentParser() # Define the command-line arguments # parser.add_argument("--video_path", help="Path to the video files.", default="/mnt/petrelfs/zengxiangyu/OpenSource/Backup/fg-videochat/download/demo_video/legendof1900.mp4") # parser.add_argument("--prompt", default="describe this video in detail.", type=str) parser.add_argument("--output_dir", default="./work_dirs/video_demo/", help="Directory to save the model results JSON.") parser.add_argument("--output_name",default="pred" , help="Name of the file for storing results JSON.") # parser.add_argument("--model-path", type=str, default="/your_local_path_to/StreamForest/ckpt/StreamForest-Qwen2-7B_Siglip") parser.add_argument("--model-path", type=str, default="/your_local_path_to/StreamForest/ckpt/VideoChat-Stream-Drive-Qwen2-7B_Siglip") parser.add_argument("--model-base", type=str, default=None) parser.add_argument("--conv-mode", type=str, default="qwen_2") parser.add_argument("--chunk-idx", type=int, default=0) parser.add_argument("--max_num-frames", type=int, default=512) parser.add_argument("--load_8bit", type=lambda x: (str(x).lower() == 'true'), default=False) parser.add_argument("--force_sample", type=lambda x: (str(x).lower() == 'true'), default=False) parser.add_argument("--time_msg", type=str, default="short") parser.add_argument("--llm_type", type=str, default="") parser.add_argument("--attn_implementation", type=str, default="flash_attention_2") parser.add_argument("--use_hd", type=bool, default=False) args = parser.parse_args(args=[]) return args args = parse_args() ``` -------------------------------- ### Initialize LLaVA Model and Configuration Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed.ipynb Loads the pre-trained LLaVA model, tokenizer, and image processor. It configures model parameters like attention implementation and handles potential overrides for sampling and time instructions based on model configuration. ```python # Initialize the model model_name = get_model_name_from_path(args.model_path) # Set model configuration parameters if they exist model_name += args.llm_type cfg_pretrained = AutoConfig.from_pretrained(args.model_path) tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, load_8bit=args.load_8bit, multimodal=True, attn_implementation=args.attn_implementation, **llava_model_args) model.to(torch.bfloat16) # import pdb;pdb.set_trace() if getattr(model.config, "force_sample", None) is not None: args.force_sample = model.config.force_sample else: args.force_sample = False # import pdb;pdb.set_trace() if getattr(model.config, "add_time_instruction", None) is not None: args.add_time_instruction = model.config.add_time_instruction else: args.add_time_instruction = False # Create the output directory if it doesn't exist if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) ``` -------------------------------- ### Load Video and Sample Frames Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed.ipynb Loads a video from a given path, supporting local files or S3. It dynamically samples frames based on arguments and can extract clips based on question time. Use this for general video loading and frame extraction. ```python def load_video(video_path, args, question_time=0): if os.path.isdir(video_path): media_dict = {'video_read_type': 'img'} else: media_dict = {'video_read_type': 'decord'} if type(video_path) != str: assert len(video_path) == 1, video_path video_path = video_path[0] if question_time>0: clip = [0, question_time] else: clip = None if 's3://' in video_path: from petrel_client.client import Client client = Client(conf_path='~/petreloss.conf') else: client = None max_frames_num = args.max_num_frames if 'fps' in media_dict: frames, frame_indices, fps, duration = VIDEO_READER_FUNCS[media_dict['video_read_type']](video_path=video_path, num_frames=max_frames_num, sample='dynamic_fps1', fix_start=None, min_num_frames=4, max_num_frames=max_frames_num, client=client, clip=clip, local_num_frames=1, fps=media_dict['fps']) else: frames, frame_indices, fps, duration = VIDEO_READER_FUNCS[media_dict['video_read_type']](video_path=video_path, num_frames=max_frames_num, sample='dynamic_fps1', fix_start=None, min_num_frames=4, max_num_frames=max_frames_num, client=client, clip=clip, local_num_frames=1) sec = [str(round(f / fps, 1)) for f in frame_indices] if args.time_msg is not None and sec is not None: if args.time_msg == 'short': msg = f"\nThe video lasts for {duration:.2f} seconds, and {len(sec)} frames are uniformly sampled from it. " elif args.time_msg == 'short_online': msg = f"\nThe video segment contains {len(sec)} frames sampled from the past {(float(sec[-1])-float(sec[0])):.1f} seconds ago up to the present moment. " elif args.time_msg == 'short_online_v2': msg = f"\nThe video contains {len(sec)} frames sampled from the past {(float(sec[-1])-float(sec[0])):.1f} seconds ago ({float(sec[0]):.1f}s of the entire video) up to the present moment ({float(sec[-1]):.1f}s of the entire video). " elif args.time_msg == 'short_online_per_frame': msg_overall = f"\nThe video contains {len(sec)} frames sampled from the past {(float(sec[-1])-float(sec[0])):.1f} seconds ago ({float(sec[0]):.1f}s of the entire video) up to the present moment ({float(sec[-1]):.1f}s of the entire video). " msg_per_frame = ''.join([f"[TIME_MSG_PER_FRAME]{sec_time} seconds" for sec_time in sec])+"[TIME_MSG_PER_FRAME]" msg = msg_overall + msg_per_frame else: msg = f"\nThe video lasts for {duration:.2f} seconds, and {len(sec)} frames are uniformly sampled at {', '.join(sec)} seconds. " else: msg = "" return frames, msg ``` -------------------------------- ### Execute Stage 4 Online Video Fine-tuning Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Run the script for the fourth stage of online video fine-tuning. This involves dynamic tree memory. ```bash bash scripts/train/stage4-online_ft/s4_siglip_online_dynamic_tree_memory.sh ``` -------------------------------- ### Execute Stage 2 Offline Pretraining Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Run the script for the second stage of offline video pretraining. This focuses on visual pretraining. ```bash bash scripts/train/stage2-visual_pretraining/s2_siglip_tome64_mlp.sh ``` -------------------------------- ### Execute Stage 3 Offline Video SFT Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Run the script for the third stage of offline video pretraining, which is video supervised fine-tuning (SFT). ```bash bash scripts/train/stage3-video_sft/s3_siglip_tome16_mlp.sh ``` -------------------------------- ### Run StreamForest Evaluation Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Execute this script to evaluate StreamForest on eight benchmark datasets, including ODVBench. ```bash bash scripts/eval/run_eval.sh ``` -------------------------------- ### StreamForest Project Configuration Parameters Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed.ipynb Shows key configuration parameters for the StreamForest project, including embedding window size and weighting factors for similarity, time, and merging. Also indicates the disabling of the depth vision tower. ```text <<>> : 512 <<< self.sim_weight_g: 0.4 >>> <<< self.time_weight_a: 0.2 >>> <<< self.merge_weight_b: 0.4 >>> <<< disable depth vision tower! >>> ``` -------------------------------- ### Load Pre-trained StreamForest Model Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed_multiround.ipynb Initializes the StreamForest model and tokenizer using specified paths and configurations. It sets the model to float16 precision and checks for specific model configuration flags like `force_sample` and `add_time_instruction`. ```python llava_model_args = { } # overwrite_config = {} # mm_projector_type=None # mm_projector_type="tome196_memory_1k" # if mm_projector_type is not None and mm_projector_type!="": # print("<<< warning >>> replace projector with: ", mm_projector_type) # overwrite_config["mm_projector_type"] = mm_projector_type # llava_model_args["overwrite_config"] = overwrite_config # Initialize the model model_name = get_model_name_from_path(args.model_path) # Set model configuration parameters if they exist model_name += args.llm_type cfg_pretrained = AutoConfig.from_pretrained(args.model_path, trust_remote_code=True,) tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, load_8bit=args.load_8bit, multimodal=True, trust_remote_code=True, attn_implementation=args.attn_implementation, **llava_model_args) model.to(torch.float16) print("Model tensor type: ", model.dtype) # import pdb;pdb.set_trace() if getattr(model.config, "force_sample", None) is not None: args.force_sample = model.config.force_sample else: args.force_sample = False # import pdb;pdb.set_trace() if getattr(model.config, "add_time_instruction", None) is not None: args.add_time_instruction = model.config.add_time_instruction else: args.add_time_instruction = False # Create the output directory if it doesn't exist if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) ``` -------------------------------- ### Clone StreamForest Repository Source: https://github.com/mcg-nju/streamforest/blob/main/README.md Use these commands to clone the StreamForest source code locally. ```bash git clone https://github.com/MCG-NJU/StreamForest.git cd StreamForest ``` -------------------------------- ### Model Loading Progress Indicator Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed.ipynb Displays the progress of loading model checkpoint shards. This is useful for monitoring the loading process and estimating the time required. ```text Loading checkpoint shards: 0%| | 0/4 [00:00>> \n") print(f"\n <<< Total average speed: {video[0].shape[0]/elapsed_time:.3f} fps >>> \n") outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() print(f"Question: {prompt}\n") print(f"Response: {outputs}\n") # import pdb;pdb.set_trace() if "mistral" not in cfg_pretrained._name_or_path.lower(): if outputs.endswith(stop_str): outputs = outputs[: -len(stop_str)] outputs = outputs.strip() sample_set["pred"] = outputs ans_file.write(json.dumps(sample_set, ensure_ascii=False) + "\n") ans_file.flush() ``` -------------------------------- ### Run Video Inference with VideoChat-Next Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed.ipynb This function handles the entire inference process for a given video, question, and optional timestamp. It includes video loading, frame preprocessing, prompt formatting, and model generation. ```python output_name = args.output_name answers_file = os.path.join(args.output_dir, f"{output_name}.json") ans_file = open(answers_file, "w") import time import torch.profiler def run_inference(args, video_path, question, question_time=0): """ Run inference on a demo video using VideoChat-Next model. Args: args: Command-line arguments. """ if hasattr(model.config, "frame_aspect_ratio"): frame_aspect_ratio = model.config.frame_aspect_ratio else: frame_aspect_ratio = "" # import pdb;pdb.set_trace() print("video_path:", video_path) sample_set = {} sample_set["Q"] = question sample_set["video_name"] = video_path # Check if the video exists # if os.path.exists(video_path) : assert 's3://' in video_path or os.path.exists(video_path), video_path frames, time_msg = load_video(video_path, args, question_time) print("len(frames):", len(frames)) image_sizes = [frames[0].shape[:2]] print("image_sizes:", image_sizes) frames = image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].to(torch.bfloat16).cuda(args.inference_device) print("input frames:", frames.shape) video = [frames] # try: # Run inference on the video and add the output to the list qs = question if args.time_msg != "": qs = f'{time_msg.strip()}\n{qs}' if model.config.mm_use_im_start_end: qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs else: qs = DEFAULT_IMAGE_TOKEN + "\n" + qs print(f"Question: {qs}") conv = conv_templates[args.conv_mode].copy() conv.append_message(conv.roles[0], qs) conv.append_message(conv.roles[1], None) prompt = conv.get_prompt() input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda(args.inference_device) if tokenizer.pad_token_id is None: if "qwen" in tokenizer.name_or_path.lower(): print("Setting pad token to bos token for qwen model.") tokenizer.pad_token_id = 151643 attention_masks = input_ids.ne(tokenizer.pad_token_id).long().cuda(args.inference_device) stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 keywords = [stop_str] stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids) start_time = time.time() with torch.inference_mode(): if "mistral" not in cfg_pretrained._name_or_path.lower(): output_ids = model.generate( inputs=input_ids, images=video, attention_mask=attention_masks, modalities=["video"], image_sizes=image_sizes, do_sample=False, temperature=0.0, max_new_tokens=32, num_beams=1, use_cache=True, stopping_criteria=[stopping_criteria] ) else: output_ids = model.generate(inputs=input_ids, images=video, attention_mask=attention_masks, modalities="video", do_sample=False, temperature=0.0, max_new_tokens=1024, top_p=0.1, num_beams=1, use_cache=True) end_time = time.time() elapsed_time = end_time - start_time print(f"\n <<< Total inference time: {elapsed_time:.3f} seconds >>> \n") outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() print(f"Question: {prompt}\n") print(f"Response: {outputs}\n") # import pdb;pdb.set_trace() if "mistral" not in cfg_pretrained._name_or_path.lower(): if outputs.endswith(stop_str): outputs = outputs[: -len(stop_str)] outputs = outputs.strip() sample_set["pred"] = outputs ans_file.write(json.dumps(sample_set, ensure_ascii=False) + "\n") ans_file.flush() ``` ```python # StreamForest video_path = "/your_local_path_to/StreamForest/tmp/Forrest_Gump.mp4" question= "Please describe the content of the video in detail." question_time=500 run_inference(args,video_path,question,question_time) ``` -------------------------------- ### LLM Inference Time and Result (Frames 160-187) Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed_multiround.ipynb Logs LLM inference time and the resulting tensor for frames 160 through 187. This data is useful for tracking performance consistency. ```text <<< LLM inference time: 0.086 seconds >>> result at frame 160 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.086 seconds >>> result at frame 161 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 162 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 163 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 164 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.086 seconds >>> result at frame 165 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 166 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 167 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 168 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 169 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 170 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 171 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 172 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 173 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 174 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 175 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 176 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 177 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 178 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 179 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 180 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 181 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 182 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 183 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 184 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 185 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 186 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.090 seconds >>> result at frame 187 : tensor([[785]], device='cuda:0') ``` -------------------------------- ### LLM Inference Time and Result (Frames 139-159) Source: https://github.com/mcg-nju/streamforest/blob/main/demo/eval_speed_multiround.ipynb Logs LLM inference time and the resulting tensor for frames 139 through 159. This data is useful for tracking performance consistency. ```text <<< LLM inference time: 0.088 seconds >>> result at frame 139 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 140 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 141 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 142 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.085 seconds >>> result at frame 143 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.086 seconds >>> result at frame 144 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 145 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 146 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 147 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 148 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 149 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 150 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 151 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.086 seconds >>> result at frame 152 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 153 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 154 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.087 seconds >>> result at frame 155 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 156 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 157 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.093 seconds >>> result at frame 158 : tensor([[785]], device='cuda:0') <<< LLM inference time: 0.088 seconds >>> result at frame 159 : tensor([[785]], device='cuda:0') ```