-
-
Notifications
You must be signed in to change notification settings - Fork 2.7k
feat: WIP: Adjust GPU Layers #3737
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit
Hold shift + click to select a range
790700b
Add GGUF Parser
siddimore a234da5
chore: :arrow_up: Update ggerganov/llama.cpp to `a39ab216aa624308fda7…
localai-bot a3a03a4
chore: :arrow_up: Update ggerganov/whisper.cpp to `ede1718f6d45aa3f7a…
localai-bot 0dc66a6
chore(federated): display a message when nodes are not available (#3721)
mudler 717978e
Update CONTRIBUTING.md (#3723)
jjasghar 78e29f3
models(gallery): add salamandra-7b-instruct (#3726)
mudler af1eb1d
chore: :arrow_up: Update ggerganov/llama.cpp to `d5ed2b929d85bbd7dbee…
localai-bot 16dfee9
chore: :arrow_up: Update ggerganov/whisper.cpp to `ccc2547210e09e3a17…
localai-bot 0ec4dc6
feat(multimodal): allow to template placeholders (#3728)
mudler bb130ff
Update README.md
mudler 5b19cee
feat(vllm): add support for image-to-text and video-to-text (#3729)
mudler 2d11bfc
chore: :arrow_up: Update ggerganov/llama.cpp to `71967c2a6d30da9f6158…
localai-bot fc74bf1
chore: :arrow_up: Update ggerganov/whisper.cpp to `2944cb72d952823780…
localai-bot 4dffc45
feat(shutdown): allow force shutdown of backends (#3733)
mudler 1c0300b
fix(base-grpc): close channel in base grpc server (#3734)
mudler 64ade06
chore: :arrow_up: Update ggerganov/whisper.cpp to `6a94163b913d8e974e…
localai-bot a8f095a
chore: :arrow_up: Update ggerganov/llama.cpp to `8c475b97b8ba7d678d4c…
localai-bot 1ad80c9
fix pr comment
siddimore 6dee2a6
Add tests
siddimore 6d1199d
Save Model Memory Usage
siddimore eaee726
fix merge conflict
siddimore 0a306c8
Merge branch 'master' into adjust_default_gpu_layers
mudler e2fb38f
Merge branch 'master' into adjust_default_gpu_layers
mudler 18bcce2
Add code to query NVIDIA device
siddimore 37f2d65
Merge branch 'master' into adjust_default_gpu_layers
siddimore 7380f80
Add AdjustGPULayers flag
siddimore cd1dc5d
rename file
siddimore File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,233 @@ | ||
| package model | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/url" | ||
| "os" | ||
| "strings" | ||
|
|
||
| ggufparser "github.com/gpustack/gguf-parser-go" | ||
| ) | ||
|
|
||
| // Structs for parsing GGUF data from Parser | ||
| type ModelEstimate struct { | ||
| Estimate ModelEstimateItems `json:"estimate"` | ||
| Architecture Architecture `json:"architecture"` | ||
| Metadata Metadata `json:"metadata"` | ||
| Tokenizer Tokenizer `json:"tokenizer"` | ||
| } | ||
|
|
||
| type ModelEstimateItems struct { | ||
| Items []ModelMemory `json:"items"` | ||
| Type string `json:"type"` | ||
| Architecture string `json:"architecture"` | ||
| ContextSize int `json:"contextSize"` | ||
| FlashAttention bool `json:"flashAttention"` | ||
| NoMMap bool `json:"noMMap"` | ||
| EmbeddingOnly bool `json:"embeddingOnly"` | ||
| Distributable bool `json:"distributable"` | ||
| LogicalBatchSize int32 `json:"logicalBatchSize"` | ||
| PhysicalBatchSize int32 `json:"physicalBatchSize"` | ||
| } | ||
|
|
||
| type ModelMemory struct { | ||
| OffloadLayers uint64 `json:"offloadLayers"` | ||
| FullOffloaded bool `json:"fullOffloaded"` | ||
| RAM EstimateRAM `json:"ram"` | ||
| VRAMs []EstimateVRAM `json:"vrams"` | ||
| } | ||
|
|
||
| type EstimateRAM struct { | ||
| UMA uint64 `json:"uma"` | ||
| NonUMA uint64 `json:"nonuma"` | ||
| } | ||
|
|
||
| type EstimateVRAM struct { | ||
| UMA uint64 `json:"uma"` | ||
| NonUMA uint64 `json:"nonuma"` | ||
| } | ||
|
|
||
| type Architecture struct { | ||
| Type string `json:"type"` | ||
| Architecture string `json:"architecture"` | ||
| MaximumContextLength int `json:"maximumContextLength"` | ||
| EmbeddingLength int `json:"embeddingLength"` | ||
| VocabularyLength int `json:"vocabularyLength"` | ||
| } | ||
|
|
||
| type Metadata struct { | ||
| Type string `json:"type"` | ||
| Architecture string `json:"architecture"` | ||
| QuantizationVersion int `json:"quantizationVersion"` | ||
| Alignment int `json:"alignment"` | ||
| Name string `json:"name"` | ||
| License string `json:"license"` | ||
| FileType int `json:"fileType"` | ||
| LittleEndian bool `json:"littleEndian"` | ||
| FileSize int64 `json:"fileSize"` | ||
| Size int64 `json:"size"` | ||
| Parameters int64 `json:"parameters"` | ||
| } | ||
|
|
||
| type Tokenizer struct { | ||
| Model string `json:"model"` | ||
| TokensLength int `json:"tokensLength"` | ||
| TokensSize int `json:"tokensSize"` | ||
| } | ||
|
|
||
| // Default platform footprint from ggufparser | ||
| const nonUMARamFootprint = uint64(150 * 1024 * 1024) // 150 MiB | ||
| const nonUMAVramFootprint = uint64(250 * 1024 * 1024) // 250 MiB | ||
|
|
||
| func GetModelGGufData(modelPath string) (*ModelEstimate, error) { | ||
| ctx := context.Background() | ||
|
|
||
| // Check if the input is a valid URL | ||
| if isURL(modelPath) { | ||
| fmt.Println("Input is a URL.") | ||
| ggufRemoteData, err := ggufparser.ParseGGUFFileRemote(ctx, modelPath) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error parsing GGUF file from remote URL: %v", err) | ||
| } | ||
| return estimateModelMemoryUsage(ggufRemoteData) | ||
|
|
||
| // Check if the input is an Ollama model | ||
| } else if strings.HasSuffix(modelPath, "ollama") { | ||
| fmt.Println("Input is an Ollama model.") | ||
| ggufOllamaData, err := ggufparser.ParseGGUFFileFromOllama(ctx, modelPath) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error parsing GGUF file from Ollama model: %v", err) | ||
| } | ||
| return estimateModelMemoryUsage(ggufOllamaData) | ||
|
|
||
| // Check if the input is a Hugging Face model reference (format: huggingface.co/<repo>/<file>) | ||
| } else if strings.Contains(modelPath, "huggingface.co") { | ||
| fmt.Println("Input is a Hugging Face model.") | ||
|
|
||
| // Parse the URL to extract the repository and filename | ||
| u, err := url.Parse(modelPath) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid Hugging Face URL: %v", err) | ||
| } | ||
|
|
||
| // Example URL: https://huggingface.co/<repo>/<file>.gguf | ||
| parts := strings.Split(u.Path, "/") | ||
| if len(parts) < 3 { | ||
| return nil, fmt.Errorf("invalid Hugging Face model format. Expected format: huggingface.co/<repo>/<file>") | ||
| } | ||
|
|
||
| repo := parts[1] // Repository name | ||
| file := parts[2] // File name | ||
|
|
||
| ggufHuggingFaceData, err := ggufparser.ParseGGUFFileFromHuggingFace(ctx, repo, file) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error parsing GGUF file from Hugging Face: %v", err) | ||
| } | ||
| return estimateModelMemoryUsage(ggufHuggingFaceData) | ||
|
|
||
| // Otherwise, assume the input is a file path | ||
| } else if fileExists(modelPath) { | ||
| fmt.Println("Input is a file path.") | ||
| ggufData, err := ggufparser.ParseGGUFFile(modelPath) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error parsing GGUF file from file path: %v", err) | ||
| } | ||
| return estimateModelMemoryUsage(ggufData) | ||
| } | ||
|
|
||
| return nil, fmt.Errorf("unsupported input type") | ||
| } | ||
|
|
||
| // Helper function to check if the string is a valid URL | ||
| func isURL(input string) bool { | ||
| _, err := url.ParseRequestURI(input) | ||
| return err == nil | ||
| } | ||
|
|
||
| // Helper function to check if the input is a valid file path | ||
| func fileExists(filename string) bool { | ||
| info, err := os.Stat(filename) | ||
| if os.IsNotExist(err) { | ||
| return false | ||
| } | ||
| return !info.IsDir() | ||
| } | ||
|
|
||
| func estimateModelMemoryUsage(ggufFile *ggufparser.GGUFFile) (*ModelEstimate, error) { | ||
|
|
||
| if ggufFile == nil { | ||
| fmt.Printf("Error Invalid GGUF File \n") | ||
|
|
||
| // Invalid ModelPath return nil and use default values | ||
| return nil, nil | ||
| } | ||
|
|
||
| // | ||
| llamacppRunEstimateOpts := []ggufparser.LLaMACppRunEstimateOption{} | ||
| // | ||
| llamacppRunEstimate := ggufFile.EstimateLLaMACppRun(llamacppRunEstimateOpts...) | ||
|
|
||
| // Summarize the item with mmap and footprint values | ||
| summary := llamacppRunEstimate.SummarizeItem(true, nonUMARamFootprint, nonUMAVramFootprint) | ||
| // Fetch architecture, metadata, and tokenizer from GGUF file | ||
| architecture := ggufFile.Architecture() | ||
| metadata := ggufFile.Metadata() | ||
| tokenizer := ggufFile.Tokenizer() | ||
|
|
||
| // Construct the JSON payload | ||
| payload := ModelEstimate{ | ||
| Estimate: ModelEstimateItems{ | ||
| Items: []ModelMemory{ | ||
| { | ||
| OffloadLayers: summary.OffloadLayers, | ||
| FullOffloaded: summary.FullOffloaded, | ||
| RAM: EstimateRAM{ | ||
| UMA: uint64(summary.RAM.UMA), | ||
| NonUMA: uint64(summary.RAM.NonUMA), | ||
| }, | ||
| VRAMs: []EstimateVRAM{ | ||
| { | ||
| UMA: uint64(summary.VRAMs[0].UMA), | ||
| NonUMA: uint64(summary.VRAMs[0].NonUMA), | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| Type: architecture.Type, | ||
| Architecture: architecture.Architecture, | ||
| ContextSize: int(llamacppRunEstimate.ContextSize), | ||
| FlashAttention: llamacppRunEstimate.FlashAttention, | ||
| NoMMap: llamacppRunEstimate.NoMMap, | ||
| EmbeddingOnly: llamacppRunEstimate.EmbeddingOnly, | ||
| Distributable: llamacppRunEstimate.Distributable, | ||
| LogicalBatchSize: llamacppRunEstimate.LogicalBatchSize, | ||
| PhysicalBatchSize: llamacppRunEstimate.PhysicalBatchSize, | ||
| }, | ||
| Architecture: Architecture{ | ||
| Type: metadata.Type, | ||
| Architecture: architecture.Architecture, | ||
| MaximumContextLength: int(architecture.MaximumContextLength), | ||
| EmbeddingLength: int(architecture.EmbeddingLength), | ||
| VocabularyLength: int(architecture.VocabularyLength), | ||
| }, | ||
| Metadata: Metadata{ | ||
| Type: metadata.Type, | ||
| Architecture: metadata.Architecture, | ||
| QuantizationVersion: int(metadata.QuantizationVersion), | ||
| Name: metadata.Name, | ||
| License: metadata.License, | ||
| FileType: int(metadata.FileType), | ||
| LittleEndian: metadata.LittleEndian, | ||
| FileSize: int64(metadata.FileSize), | ||
| Parameters: int64(metadata.Parameters), | ||
| }, | ||
| Tokenizer: Tokenizer{ | ||
| Model: tokenizer.Model, | ||
| TokensLength: int(tokenizer.TokensLength), | ||
| TokensSize: int(tokenizer.TokensSize), | ||
| }, | ||
| } | ||
|
|
||
| return &payload, nil | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.