pablo/ai-chat-llm-offline
pablo/ai-chat-llm-offline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add preliminary README

Browse Source
This commit is contained in:
Pablo de la Torre Jamardo 2025-06-28 20:43:50 +02:00
parent 60f6f9e55a
commit 3d362d169d
5 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
HELP.md README.md
.mvn .mvn

BIN
demo.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.2 MiB

2
llama.log
View File

@ -1 +1 @@
[1751134161] warming up the model with an empty run [1751135918] warming up the model with an empty run

6
src/main/java/com/pablotj/ia/chat/boot/infraestructure/llm/LlmModelLoader.java
View File

@ -4,18 +4,22 @@ import com.pablotj.ia.chat.boot.domain.exception.BusinessLogicException;
import de.kherud.llama.LlamaModel; import de.kherud.llama.LlamaModel;
import de.kherud.llama.ModelParameters; import de.kherud.llama.ModelParameters;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Component @Component
public class LlmModelLoader implements AutoCloseable { public class LlmModelLoader implements AutoCloseable {
@Value(value = "${model.gguf.name}")
private String modelName;
private LlamaModel model; private LlamaModel model;
@PostConstruct @PostConstruct
public void init() { public void init() {
try { try {
ModelParameters params = new ModelParameters() ModelParameters params = new ModelParameters()
.setModelFilePath("models/openchat-3.5-0106.Q4_K_M.gguf") .setModelFilePath(String.format("models/%s.gguf", modelName))
.setSeed(42) .setSeed(42)
.setNThreads(8) .setNThreads(8)
.setNGpuLayers(0) .setNGpuLayers(0)

3
src/main/resources/application.properties
View File

@ -1,2 +1,5 @@
spring.application.name = ia-chat-boot spring.application.name = ia-chat-boot
server.port = 8080 server.port = 8080
! Model
model.gguf.name = openchat-3.5-0106.Q4_K_M