BAEL-9393 - A Guide to OpenAI’s Moderation Model in Spring AI

spring-ai-4/src/main/java/com/baeldung/springai/moderation/Application.java

@@ -0,0 +1,14 @@
+package com.baeldung.springai.moderation;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class Application {
+
+    public static void main(String[] args) {
+        SpringApplication app = new SpringApplication(Application.class);
+        app.setAdditionalProfiles("moderation");
+        app.run(args);
+    }
+}

spring-ai-4/src/main/java/com/baeldung/springai/moderation/ModerateRequest.java

@@ -0,0 +1,14 @@
+package com.baeldung.springai.moderation;
+
+public class ModerateRequest {
+
+    private String text;
+
+    public String getText() {
+        return text;
+    }
+
+    public void setText(String text) {
+        this.text = text;
+    }
+}

spring-ai-4/src/main/java/com/baeldung/springai/moderation/TextModerationController.java

@@ -0,0 +1,23 @@
+package com.baeldung.springai.moderation;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class TextModerationController {
+
+    private final TextModerationService service;
+
+    @Autowired
+    public TextModerationController(TextModerationService service) {
+        this.service = service;
+    }
+
+    @PostMapping("/moderate")
+    public ResponseEntity<String> moderate(@RequestBody ModerateRequest request) {
+        return ResponseEntity.ok(service.moderate(request.getText()));
+    }
+}

spring-ai-4/src/main/java/com/baeldung/springai/moderation/TextModerationService.java

@@ -0,0 +1,56 @@
+package com.baeldung.springai.moderation;
+
+import org.springframework.ai.moderation.*;
+import org.springframework.ai.openai.OpenAiModerationModel;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Service
+public class TextModerationService {
+
+    private final OpenAiModerationModel openAiModerationModel;
+
+    @Autowired
+    public TextModerationService(OpenAiModerationModel openAiModerationModel) {
+        this.openAiModerationModel = openAiModerationModel;
+    }
+
+    public String moderate(String text) {
+        ModerationPrompt moderationRequest = new ModerationPrompt(text);
+        ModerationResponse response = openAiModerationModel.call(moderationRequest);
+        Moderation output = response.getResult().getOutput();
+
+        return output.getResults().stream()
+          .map(this::buildModerationResult)
+          .collect(Collectors.joining("\n"));
+    }
+
+    private String buildModerationResult(ModerationResult moderationResult) {
+
+        Categories categories = moderationResult.getCategories();
+
+        String violations = Stream.of(
+              Map.entry("Sexual", categories.isSexual()),
+              Map.entry("Hate", categories.isHate()),
+              Map.entry("Harassment", categories.isHarassment()),
+              Map.entry("Self-Harm", categories.isSelfHarm()),
+              Map.entry("Sexual/Minors", categories.isSexualMinors()),
+              Map.entry("Hate/Threatening", categories.isHateThreatening()),
+              Map.entry("Violence/Graphic", categories.isViolenceGraphic()),
+              Map.entry("Self-Harm/Intent", categories.isSelfHarmIntent()),
+              Map.entry("Self-Harm/Instructions", categories.isSelfHarmInstructions()),
+              Map.entry("Harassment/Threatening", categories.isHarassmentThreatening()),
+              Map.entry("Violence", categories.isViolence()))
+          .filter(entry -> Boolean.TRUE.equals(entry.getValue()))
+          .map(Map.Entry::getKey)
+          .collect(Collectors.joining(", "));
+
+        return violations.isEmpty()
+          ? "No category violations detected."
+          : "Violated categories: " + violations;
+    }
+}

spring-ai-4/src/main/resources/application-moderation.yml

@@ -0,0 +1,7 @@
+spring:
+  ai:
+    openai:
+      api-key: ${OPEN_AI_API_KEY}
+      moderation:
+        options:
+          model: omni-moderation-latest

spring-ai-4/src/test/java/com/baeldung/springai/moderation/ModerationApplicationLiveTest.java

@@ -0,0 +1,67 @@
+package com.baeldung.springai.moderation;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.web.servlet.MockMvc;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+
+@AutoConfigureMockMvc
+@ExtendWith(SpringExtension.class)
+@EnableAutoConfiguration
+@SpringBootTest
+@ActiveProfiles("moderation")
+class ModerationApplicationLiveTest {
+
+    @Autowired
+    private MockMvc mockMvc;
+
+    @Test
+    void givenTextWithoutViolation_whenModerating_thenNoCategoryViolationsDetected() throws Exception {
+        String moderationResponse = mockMvc.perform(post("/moderate")
+            .contentType(MediaType.APPLICATION_JSON)
+            .content("{\"text\": \"Please review me\"}"))
+          .andExpect(status().isOk())
+          .andReturn()
+          .getResponse()
+          .getContentAsString();
+
+        assertThat(moderationResponse).contains("No category violations detected");
+    }
+
+    @Test
+    void givenHarassingText_whenModerating_thenHarassmentCategoryShouldBeFlagged() throws Exception {
+        String moderationResponse = mockMvc.perform(post("/moderate")
+            .contentType(MediaType.APPLICATION_JSON)
+            .content("{\"text\": \"You're really Bad Person! I don't like you!\"}"))
+          .andExpect(status().isOk())
+          .andReturn()
+          .getResponse()
+          .getContentAsString();
+
+        assertThat(moderationResponse).contains("Violated categories: Harassment");
+    }
+
+    @Test
+    void givenTextViolatingMultipleCategories_whenModerating_thenAllCategoriesShouldBeFlagged() throws Exception {
+        String moderationResponse = mockMvc.perform(post("/moderate")
+            .contentType(MediaType.APPLICATION_JSON)
+            .content("{\"text\": \"I hate you and I will hurt you!\"}"))
+          .andExpect(status().isOk())
+          .andReturn()
+          .getResponse()
+          .getContentAsString();
+
+        assertThat(moderationResponse).contains("Violated categories: Harassment, Harassment/Threatening, Violence");
+    }
+}