wushenglu/bee-admin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

同步最新代码到简化分支

Browse Source
This commit is contained in:
YunaiV
2024-10-07 16:09:49 +08:00
parent 4faedee4dc
commit d4983752b9
164 changed files with 0 additions and 9824 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java
View File

@@ -1,39 +0,0 @@
package cn.iocoder.yudao.module.ai.enums.knowledge;
import cn.iocoder.yudao.framework.common.core.IntArrayValuable;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.Arrays;
/**
* AI 知识库-文档状态的枚举
*
* @author xiaoxin
*/
@AllArgsConstructor
@Getter
public enum AiKnowledgeDocumentStatusEnum implements IntArrayValuable {
IN_PROGRESS(10, "索引中"),
SUCCESS(20, "可用"),
FAIL(30, "失败");
/**
* 状态
*/
private final Integer status;
/**
* 状态名
*/
private final String name;
public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiKnowledgeDocumentStatusEnum::getStatus).toArray();
@Override
public int[] array() {
return ARRAYS;
}
}

14
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/AiImagePublicPageReqVO.java
View File

@@ -1,14 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 绘画公开的分页 Request VO")
@Data
public class AiImagePublicPageReqVO extends PageParam {
@Schema(description = "提示词")
private String prompt;
}

50
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java
View File

@@ -1,50 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeRespVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId;
@Tag(name = "管理后台 - AI 知识库")
@RestController
@RequestMapping("/ai/knowledge")
@Validated
public class AiKnowledgeController {
@Resource
private AiKnowledgeService knowledgeService;
@GetMapping("/page")
@Operation(summary = "获取知识库分页")
public CommonResult<PageResult<AiKnowledgeRespVO>> getKnowledgePage(@Valid AiKnowledgePageReqVO pageReqVO) {
PageResult<AiKnowledgeDO> pageResult = knowledgeService.getKnowledgePage(getLoginUserId(), pageReqVO);
return success(BeanUtils.toBean(pageResult, AiKnowledgeRespVO.class));
}
@PostMapping("/create")
@Operation(summary = "创建知识库")
public CommonResult<Long> createKnowledge(@RequestBody @Valid AiKnowledgeCreateReqVO createReqVO) {
return success(knowledgeService.createKnowledge(createReqVO, getLoginUserId()));
}
@PutMapping("/update")
@Operation(summary = "更新知识库")
public CommonResult<Boolean> updateKnowledge(@RequestBody @Valid AiKnowledgeUpdateReqVO updateReqVO) {
knowledgeService.updateKnowledge(updateReqVO, getLoginUserId());
return success(true);
}
}

51
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java
View File

@@ -1,51 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentRespVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeDocumentService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
@Tag(name = "管理后台 - AI 知识库文档")
@RestController
@RequestMapping("/ai/knowledge/document")
@Validated
public class AiKnowledgeDocumentController {
@Resource
private AiKnowledgeDocumentService documentService;
@PostMapping("/create")
@Operation(summary = "新建文档")
public CommonResult<Long> createKnowledgeDocument(@Valid AiKnowledgeDocumentCreateReqVO reqVO) {
Long knowledgeDocumentId = documentService.createKnowledgeDocument(reqVO);
return success(knowledgeDocumentId);
}
@GetMapping("/page")
@Operation(summary = "获取文档分页")
public CommonResult<PageResult<AiKnowledgeDocumentRespVO>> getKnowledgeDocumentPage(@Valid AiKnowledgeDocumentPageReqVO pageReqVO) {
PageResult<AiKnowledgeDocumentDO> pageResult = documentService.getKnowledgeDocumentPage(pageReqVO);
return success(BeanUtils.toBean(pageResult, AiKnowledgeDocumentRespVO.class));
}
@PutMapping("/update")
@Operation(summary = "更新文档")
public CommonResult<Boolean> updateKnowledgeDocument(@Valid @RequestBody AiKnowledgeDocumentUpdateReqVO reqVO) {
documentService.updateKnowledgeDocument(reqVO);
return success(true);
}
}

51
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java
View File

@@ -1,51 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentRespVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeSegmentService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
@Tag(name = "管理后台 - AI 知识库段落")
@RestController
@RequestMapping("/ai/knowledge/segment")
@Validated
public class AiKnowledgeSegmentController {
@Resource
private AiKnowledgeSegmentService segmentService;
@GetMapping("/page")
@Operation(summary = "获取段落分页")
public CommonResult<PageResult<AiKnowledgeSegmentRespVO>> getKnowledgeSegmentPage(@Valid AiKnowledgeSegmentPageReqVO pageReqVO) {
PageResult<AiKnowledgeSegmentDO> pageResult = segmentService.getKnowledgeSegmentPage(pageReqVO);
return success(BeanUtils.toBean(pageResult, AiKnowledgeSegmentRespVO.class));
}
@PutMapping("/update")
@Operation(summary = "更新段落内容")
public CommonResult<Boolean> updateKnowledgeSegment(@Valid @RequestBody AiKnowledgeSegmentUpdateReqVO reqVO) {
segmentService.updateKnowledgeSegment(reqVO);
return success(true);
}
@PutMapping("/update-status")
@Operation(summary = "启禁用段落内容")
public CommonResult<Boolean> updateKnowledgeSegmentStatus(@Valid @RequestBody AiKnowledgeSegmentUpdateStatusReqVO reqVO) {
segmentService.updateKnowledgeSegmentStatus(reqVO);
return success(true);
}
}

14
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentPageReqVO.java
View File

@@ -1,14 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库文档的分页 Request VO")
@Data
public class AiKnowledgeDocumentPageReqVO extends PageParam {
@Schema(description = "文档名称", example = "Java 开发手册")
private String name;
}

38
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentRespVO.java
View File

@@ -1,38 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库-文档 Response VO")
@Data
public class AiKnowledgeDocumentRespVO extends PageParam {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long id;
@Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long knowledgeId;
@Schema(description = "名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 开发手册")
private String name;
@Schema(description = "内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 是一门面向对象的语言.....")
private String content;
@Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
private String url;
@Schema(description = "token 数量", requiredMode = Schema.RequiredMode.REQUIRED, example = "1024")
private Integer tokens;
@Schema(description = "字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "1008")
private Integer wordCount;
@Schema(description = "切片状态", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
private Integer sliceStatus;
@Schema(description = "文档状态", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
private Integer status;
}

26
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentUpdateReqVO.java
View File

@@ -1,26 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.validation.InEnum;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
@Schema(description = "管理后台 - AI 更新 知识库-文档 Request VO")
@Data
public class AiKnowledgeDocumentUpdateReqVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "15583")
@NotNull(message = "编号不能为空")
private Long id;
@Schema(description = "是否启用", example = "1")
@InEnum(CommonStatusEnum.class)
private Integer status;
@Schema(description = "名称", example = "Java 开发手册")
private String name;
}

36
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateReqVO.java
View File

@@ -1,36 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Schema(description = "管理后台 - AI 知识库创建 Request VO")
@Data
public class AiKnowledgeCreateReqVO {
@Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南")
@NotBlank(message = "知识库名称不能为空")
private String name;
@Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档")
private String description;
@Schema(description = "可见权限，只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1,2,3]")
private List<Long> visibilityPermissions;
@Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "嵌入模型不能为空")
private Long modelId;
@Schema(description = "相似性阈值", requiredMode = Schema.RequiredMode.REQUIRED, example = "0.5")
@NotNull(message = "相似性阈值不能为空")
private Double similarityThreshold;
@Schema(description = "topK", requiredMode = Schema.RequiredMode.REQUIRED, example = "3")
@NotNull(message = "topK 不能为空")
private Integer topK;
}

46
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java
View File

@@ -1,46 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import org.hibernate.validator.constraints.URL;
@Schema(description = "管理后台 - AI 知识库文档的创建 Request VO")
@Data
public class AiKnowledgeDocumentCreateReqVO {
@Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
@NotNull(message = "知识库编号不能为空")
private Long knowledgeId;
@Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆")
@NotBlank(message = "文档名称不能为空")
private String name;
@Schema(description = "文档 URL", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
@URL(message = "文档 URL 格式不正确")
private String url;
@Schema(description = "每个段落的目标 token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800")
@NotNull(message = "每个段落的目标 token 数不能为空")
private Integer defaultSegmentTokens;
@Schema(description = "每个段落的最小字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "350")
@NotNull(message = "每个段落的最小字符数不能为空")
private Integer minSegmentWordCount;
@Schema(description = "丢弃阈值：低于此阈值的段落会被丢弃", requiredMode = Schema.RequiredMode.REQUIRED, example = "5")
@NotNull(message = "丢弃阈值不能为空")
private Integer minChunkLengthToEmbed;
@Schema(description = "最大段落数", requiredMode = Schema.RequiredMode.REQUIRED, example = "10000")
@NotNull(message = "最大段落数不能为空")
private Integer maxNumSegments;
@Schema(description = "分块是否保留分隔符", requiredMode = Schema.RequiredMode.REQUIRED, example = "true")
@NotNull(message = "分块是否保留分隔符不能为空")
private Boolean keepSeparator;
}

14
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgePageReqVO.java
View File

@@ -1,14 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库的分页 Request VO")
@Data
public class AiKnowledgePageReqVO extends PageParam {
@Schema(description = "知识库名称", example = "Java 开发手册")
private String name;
}

26
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeRespVO.java
View File

@@ -1,26 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库 Response VO")
@Data
public class AiKnowledgeRespVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long id;
@Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南")
private String name;
@Schema(description = "知识库描述", example = "帮助你快速构建系统")
private String description;
@Schema(description = "模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "14")
private Long modelId;
@Schema(description = "模型标识", requiredMode = Schema.RequiredMode.REQUIRED, example = "qwen-72b-chat")
private String model;
}

32
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateReqVO.java
View File

@@ -1,32 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Schema(description = "管理后台 - AI 知识库更新【我的】 Request VO")
@Data
public class AiKnowledgeUpdateReqVO {
@Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
@NotNull(message = "知识库编号不能为空")
private Long id;
@Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
@NotBlank(message = "知识库名称不能为空")
private String name;
@Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
private String description;
@Schema(description = "可见权限，只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "1,2,3")
private List<Long> visibilityPermissions;
@Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "嵌入模型不能为空")
private Long modelId;
}

20
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java
View File

@@ -1,20 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库分段的分页 Request VO")
@Data
public class AiKnowledgeSegmentPageReqVO extends PageParam {
@Schema(description = "分段状态", example = "1")
private Integer status;
@Schema(description = "文档编号", example = "1")
private Integer documentId;
@Schema(description = "分段内容关键字", example = "Java 开发")
private String keyword;
}

34
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentRespVO.java
View File

@@ -1,34 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库-文档 Response VO")
@Data
public class AiKnowledgeSegmentRespVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long id;
@Schema(description = "文档编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long documentId;
@Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long knowledgeId;
@Schema(description = "向量库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1858496a-1dde-4edf-a43e-0aed08f37f8c")
private String vectorId;
@Schema(description = "切片内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 开发手册")
private String content;
@Schema(description = "token 数量", requiredMode = Schema.RequiredMode.REQUIRED, example = "1024")
private Integer tokens;
@Schema(description = "字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "1008")
private Integer wordCount;
@Schema(description = "文档状态", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
private Integer status;
}

17
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentSearchReqVO.java
View File

@@ -1,17 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库段落召回 Request VO")
@Data
public class AiKnowledgeSegmentSearchReqVO {
@Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long knowledgeId;
@Schema(description = "内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 学习路线")
private String content;
}

17
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java
View File

@@ -1,17 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 更新 知识库-段落 request VO")
@Data
public class AiKnowledgeSegmentUpdateReqVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long id;
@Schema(description = "切片内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 开发手册")
private String content;
}

22
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateStatusReqVO.java
View File

@@ -1,22 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.validation.InEnum;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
@Schema(description = "管理后台 - AI 知识库段落的更新状态 Request VO")
@Data
public class AiKnowledgeSegmentUpdateStatusReqVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long id;
@Schema(description = "是否启用", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "是否启用不能为空")
@InEnum(CommonStatusEnum.class)
private Integer status;
}

30
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/mindmap/vo/AiMindMapPageReqVO.java
View File

@@ -1,30 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.mindmap.vo;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import org.springframework.format.annotation.DateTimeFormat;
import java.time.LocalDateTime;
import static cn.iocoder.yudao.framework.common.util.date.DateUtils.FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND;
@Schema(description = "管理后台 - AI 思维导图分页 Request VO")
@Data
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
public class AiMindMapPageReqVO extends PageParam {
@Schema(description = "用户编号", example = "4325")
private Long userId;
@Schema(description = "生成内容提示", example = "Java 学习路线")
private String prompt;
@Schema(description = "创建时间")
@DateTimeFormat(pattern = FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND)
private LocalDateTime[] createTime;
}

36
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/mindmap/vo/AiMindMapRespVO.java
View File

@@ -1,36 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.mindmap.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.time.LocalDateTime;
@Schema(description = "管理后台 - AI 思维导图 Response VO")
@Data
public class AiMindMapRespVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "3373")
private Long id;
@Schema(description = "用户编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "4325")
private Long userId;
@Schema(description = "生成内容提示", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 学习路线")
private String prompt;
@Schema(description = "生成的思维导图内容")
private String generatedContent;
@Schema(description = "平台", requiredMode = Schema.RequiredMode.REQUIRED, example = "OpenAI")
private String platform;
@Schema(description = "模型", requiredMode = Schema.RequiredMode.REQUIRED, example = "gpt-3.5-turbo-0125")
private String model;
@Schema(description = "错误信息")
private String errorMessage;
@Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED)
private LocalDateTime createTime;
}

74
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java
View File

@@ -1,74 +0,0 @@
package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import cn.iocoder.yudao.framework.mybatis.core.type.LongListTypeHandler;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.List;
/**
* AI 知识库 DO
*
* @author xiaoxin
*/
@TableName(value = "ai_knowledge", autoResultMap = true)
@Data
public class AiKnowledgeDO extends BaseDO {
/**
* 编号
*/
@TableId
private Long id;
/**
* 用户编号
* <p>
* 关联 AdminUserDO 的 userId 字段
*/
private Long userId;
/**
* 知识库名称
*/
private String name;
/**
* 知识库描述
*/
private String description;
/**
* 可见权限,选择哪些人可见
* <p>
* -1 所有人可见，其他为各自用户编号
*/
@TableField(typeHandler = LongListTypeHandler.class)
private List<Long> visibilityPermissions;
/**
* 嵌入模型编号
*/
private Long modelId;
/**
* 模型标识
*/
private String model;
/**
* topK
*/
private Integer topK;
/**
* 相似度阈值
*/
private Double similarityThreshold;
/**
* 状态
* <p>
* 枚举 {@link CommonStatusEnum}
*/
private Integer status;
}

90
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
View File

@@ -1,90 +0,0 @@
package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
/**
* AI 知识库-文档 DO
*
* @author xiaoxin
*/
@TableName(value = "ai_knowledge_document")
@Data
public class AiKnowledgeDocumentDO extends BaseDO {
/**
* 编号
*/
@TableId
private Long id;
/**
* 知识库编号
* <p>
* 关联 {@link AiKnowledgeDO#getId()}
*/
private Long knowledgeId;
/**
* 文件名称
*/
private String name;
/**
* 内容
*/
private String content;
/**
* 文件 URL
*/
private String url;
/**
* 文档 token 数量
*/
private Integer tokens;
/**
* 文档字符数
*/
private Integer wordCount;
// ========== 自定义分段所用参数 ==========
// TODO @新：3）defaultChunkSize、defaultChunkSize、minChunkSizeChars、maxNumChunks 这几个字段的命名，可能要微信一起讨论下。尽量命名保持风格统一哈。
/**
* 每个文本块的目标 token 数
*/
private Integer defaultSegmentTokens;
/**
* 每个文本块的最小字符数
*/
private Integer minSegmentWordCount;
/**
* 低于此值的块会被丢弃
*/
private Integer minChunkLengthToEmbed;
/**
* 最大块数
*/
private Integer maxNumSegments;
/**
* 分块是否保留分隔符
*/
private Boolean keepSeparator;
// ===================================
/**
* 切片状态
* <p>
* 枚举 {@link AiKnowledgeDocumentStatusEnum}
*/
private Integer sliceStatus;
/**
* 状态
* <p>
* 枚举 {@link CommonStatusEnum}
*/
private Integer status;
}

60
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java
View File

@@ -1,60 +0,0 @@
package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
/**
* AI 知识库-文档分段 DO
*
* @author xiaoxin
*/
@TableName(value = "ai_knowledge_segment")
@Data
public class AiKnowledgeSegmentDO extends BaseDO {
public static final String FIELD_KNOWLEDGE_ID = "knowledgeId";
/**
* 编号
*/
@TableId
private Long id;
/**
* 向量库的编号
*/
private String vectorId;
/**
* 知识库编号
* <p>
* 关联 {@link AiKnowledgeDO#getId()}
*/
private Long knowledgeId;
/**
* 文档编号
* <p>
* 关联 {@link AiKnowledgeDocumentDO#getId()}
*/
private Long documentId;
/**
* 切片内容
*/
private String content;
/**
* 字符数
*/
private Integer wordCount;
/**
* token 数量
*/
private Integer tokens;
/**
* 状态
* <p>
* 枚举 {@link CommonStatusEnum}
*/
private Integer status;
}

24
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java
View File

@@ -1,24 +0,0 @@
package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import org.apache.ibatis.annotations.Mapper;
/**
* AI 知识库-文档 Mapper
*
* @author xiaoxin
*/
@Mapper
public interface AiKnowledgeDocumentMapper extends BaseMapperX<AiKnowledgeDocumentDO> {
default PageResult<AiKnowledgeDocumentDO> selectPage(AiKnowledgeDocumentPageReqVO reqVO) {
return selectPage(reqVO, new LambdaQueryWrapperX<AiKnowledgeDocumentDO>()
.likeIfPresent(AiKnowledgeDocumentDO::getName, reqVO.getName())
.orderByDesc(AiKnowledgeDocumentDO::getId));
}
}

26
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java
View File

@@ -1,26 +0,0 @@
package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
import org.apache.ibatis.annotations.Mapper;
/**
* AI 知识库基础信息 Mapper
*
* @author xiaoxin
*/
@Mapper
public interface AiKnowledgeMapper extends BaseMapperX<AiKnowledgeDO> {
default PageResult<AiKnowledgeDO> selectPage(Long userId, AiKnowledgePageReqVO pageReqVO) {
return selectPage(pageReqVO, new LambdaQueryWrapperX<AiKnowledgeDO>()
.eq(AiKnowledgeDO::getStatus, CommonStatusEnum.ENABLE.getStatus())
.likeIfPresent(AiKnowledgeDO::getName, pageReqVO.getName())
.and(e -> e.apply("FIND_IN_SET(" + userId + ",visibility_permissions)").or(m -> m.apply("FIND_IN_SET(-1,visibility_permissions)")))
.orderByDesc(AiKnowledgeDO::getId));
}
}

34
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java
View File

@@ -1,34 +0,0 @@
package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
/**
* AI 知识库-分片 Mapper
*
* @author xiaoxin
*/
@Mapper
public interface AiKnowledgeSegmentMapper extends BaseMapperX<AiKnowledgeSegmentDO> {
default PageResult<AiKnowledgeSegmentDO> selectPage(AiKnowledgeSegmentPageReqVO reqVO) {
return selectPage(reqVO, new LambdaQueryWrapperX<AiKnowledgeSegmentDO>()
.eq(AiKnowledgeSegmentDO::getDocumentId, reqVO.getDocumentId())
.eqIfPresent(AiKnowledgeSegmentDO::getStatus, reqVO.getStatus())
.likeIfPresent(AiKnowledgeSegmentDO::getContent, reqVO.getKeyword())
.orderByDesc(AiKnowledgeSegmentDO::getId));
}
default List<AiKnowledgeSegmentDO> selectListByVectorIds(List<String> vectorIdList) {
return selectList(new LambdaQueryWrapperX<AiKnowledgeSegmentDO>()
.in(AiKnowledgeSegmentDO::getVectorId, vectorIdList)
.orderByDesc(AiKnowledgeSegmentDO::getId));
}
}

39
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java
View File

@@ -1,39 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
/**
* AI 知识库-文档 Service 接口
*
* @author xiaoxin
*/
public interface AiKnowledgeDocumentService {
/**
* 创建文档
*
* @param createReqVO 文档创建 Request VO
* @return 文档编号
*/
Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO);
/**
* 获取文档分页
*
* @param pageReqVO 分页参数
* @return 文档分页
*/
PageResult<AiKnowledgeDocumentDO> getKnowledgeDocumentPage(AiKnowledgeDocumentPageReqVO pageReqVO);
/**
* 更新文档
*
* @param reqVO 更新信息
*/
void updateKnowledgeDocument(AiKnowledgeDocumentUpdateReqVO reqVO);
}

130
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
View File

@@ -1,130 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.http.HttpUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.tika.TikaDocumentReader;
import org.springframework.ai.tokenizer.TokenCountEstimator;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_DOCUMENT_NOT_EXISTS;
/**
* AI 知识库文档 Service 实现类
*
* @author xiaoxin
*/
@Service
@Slf4j
public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService {
@Resource
private AiKnowledgeDocumentMapper documentMapper;
@Resource
private AiKnowledgeSegmentMapper segmentMapper;
@Resource
private TokenCountEstimator tokenCountEstimator;
@Resource
private AiKnowledgeService knowledgeService;
@Override
@Transactional(rollbackFor = Exception.class)
public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
// 0. 校验并获取向量存储实例
VectorStore vectorStore = knowledgeService.getVectorStoreById(createReqVO.getKnowledgeId());
// 1.1 下载文档
TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl()));
List<Document> documents = loader.get();
Document document = CollUtil.getFirst(documents);
// 1.2 文档记录入库
String content = document.getContent();
AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class)
.setTokens(tokenCountEstimator.estimate(content)).setWordCount(content.length())
.setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus());
documentMapper.insert(documentDO);
Long documentId = documentDO.getId();
if (CollUtil.isEmpty(documents)) {
return documentId;
}
// 2 构造文本分段器
TokenTextSplitter tokenTextSplitter = new TokenTextSplitter(createReqVO.getDefaultSegmentTokens(), createReqVO.getMinSegmentWordCount(), createReqVO.getMinChunkLengthToEmbed(),
createReqVO.getMaxNumSegments(), createReqVO.getKeepSeparator());
// 2.1 文档分段
List<Document> segments = tokenTextSplitter.apply(documents);
// 2.2 分段内容入库
List<AiKnowledgeSegmentDO> segmentDOList = CollectionUtils.convertList(segments,
segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId)
.setKnowledgeId(createReqVO.getKnowledgeId()).setVectorId(segment.getId())
.setTokens(tokenCountEstimator.estimate(segment.getContent())).setWordCount(segment.getContent().length())
.setStatus(CommonStatusEnum.ENABLE.getStatus()));
segmentMapper.insertBatch(segmentDOList);
// 3. 向量化并存储
segments.forEach(segment -> segment.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, createReqVO.getKnowledgeId()));
vectorStore.add(segments);
return documentId;
}
@Override
public PageResult<AiKnowledgeDocumentDO> getKnowledgeDocumentPage(AiKnowledgeDocumentPageReqVO pageReqVO) {
return documentMapper.selectPage(pageReqVO);
}
@Override
public void updateKnowledgeDocument(AiKnowledgeDocumentUpdateReqVO reqVO) {
// 1. 校验文档是否存在
validateKnowledgeDocumentExists(reqVO.getId());
// 2. 更新文档
AiKnowledgeDocumentDO document = BeanUtils.toBean(reqVO, AiKnowledgeDocumentDO.class);
documentMapper.updateById(document);
}
/**
* 校验文档是否存在
*
* @param id 文档编号
* @return 文档信息
*/
private AiKnowledgeDocumentDO validateKnowledgeDocumentExists(Long id) {
AiKnowledgeDocumentDO knowledgeDocument = documentMapper.selectById(id);
if (knowledgeDocument == null) {
throw exception(KNOWLEDGE_DOCUMENT_NOT_EXISTS);
}
return knowledgeDocument;
}
private org.springframework.core.io.Resource downloadFile(String url) {
try {
byte[] bytes = HttpUtil.downloadBytes(url);
return new ByteArrayResource(bytes);
} catch (Exception e) {
log.error("[downloadFile][url({}) 下载失败]", url, e);
throw new RuntimeException(e);
}
}
}

49
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
View File

@@ -1,49 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import java.util.List;
/**
* AI 知识库段落 Service 接口
*
* @author xiaoxin
*/
public interface AiKnowledgeSegmentService {
/**
* 获取段落分页
*
* @param pageReqVO 分页查询
* @return 文档分页
*/
PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO);
/**
* 更新段落的内容
*
* @param reqVO 更新内容
*/
void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO);
/**
* 更新段落的状态
*
* @param reqVO 更新内容
*/
void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO);
/**
* 召回段落
*
* @param reqVO 召回请求信息
* @return 召回的段落
*/
List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO);
}

134
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
View File

@@ -1,134 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ListUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.document.Document;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Objects;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS;
/**
* AI 知识库分片 Service 实现类
*
* @author xiaoxin
*/
@Service
@Slf4j
public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService {
@Resource
private AiKnowledgeSegmentMapper segmentMapper;
@Resource
private AiKnowledgeService knowledgeService;
@Resource
private AiChatModelService chatModelService;
@Resource
private AiApiKeyService apiKeyService;
@Override
public PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) {
return segmentMapper.selectPage(pageReqVO);
}
@Override
public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) {
// 1. 校验
AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId());
// 2.1 获取知识库向量实例
VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId());
// 2.2 删除原向量
vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId()));
// 2.3 重新向量化
Document document = new Document(reqVO.getContent());
document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId());
vectorStore.add(List.of(document));
// 3. 更新段落内容
AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
knowledgeSegment.setVectorId(document.getId());
segmentMapper.updateById(knowledgeSegment);
}
@Override
public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) {
// 0 校验
AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId());
// 1 获取知识库向量实例
VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId());
AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
if (Objects.equals(reqVO.getStatus(), CommonStatusEnum.ENABLE.getStatus())) {
// 2.1 启用重新向量化
Document document = new Document(oldKnowledgeSegment.getContent());
document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId());
vectorStore.add(List.of(document));
knowledgeSegment.setVectorId(document.getId());
} else {
// 2.2 禁用删除向量
vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId()));
knowledgeSegment.setVectorId("");
}
// 3 更新段落状态
segmentMapper.updateById(knowledgeSegment);
}
@Override
public List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO) {
// 1. 校验
AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(reqVO.getKnowledgeId());
AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
// 2. 获取向量存储实例
VectorStore vectorStore = apiKeyService.getOrCreateVectorStore(model.getKeyId());
// 3.1 向量检索
List<Document> documentList = vectorStore.similaritySearch(SearchRequest.query(reqVO.getContent())
.withTopK(knowledge.getTopK())
.withSimilarityThreshold(knowledge.getSimilarityThreshold())
.withFilterExpression(new FilterExpressionBuilder().eq(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, reqVO.getKnowledgeId()).build()));
if (CollUtil.isEmpty(documentList)) {
return ListUtil.empty();
}
// 3.2 段落召回
return segmentMapper.selectListByVectorIds(CollUtil.getFieldValues(documentList, "id", String.class));
}
/**
* 校验段落是否存在
*
* @param id 文档编号
* @return 段落信息
*/
private AiKnowledgeSegmentDO validateKnowledgeSegmentExists(Long id) {
AiKnowledgeSegmentDO knowledgeSegment = segmentMapper.selectById(id);
if (knowledgeSegment == null) {
throw exception(KNOWLEDGE_SEGMENT_NOT_EXISTS);
}
return knowledgeSegment;
}
}

58
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java
View File

@@ -1,58 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
import org.springframework.ai.vectorstore.VectorStore;
/**
* AI 知识库-基础信息 Service 接口
*
* @author xiaoxin
*/
public interface AiKnowledgeService {
/**
* 创建知识库
*
* @param createReqVO 创建信息
* @param userId 用户编号
* @return 编号
*/
Long createKnowledge(AiKnowledgeCreateReqVO createReqVO, Long userId);
/**
* 更新知识库
*
* @param updateReqVO 更新信息
* @param userId 用户编号
*/
void updateKnowledge(AiKnowledgeUpdateReqVO updateReqVO, Long userId);
/**
* 校验知识库是否存在
*
* @param id 记录编号
*/
AiKnowledgeDO validateKnowledgeExists(Long id);
/**
* 获得知识库分页
*
* @param userId 用户编号
* @param pageReqVO 分页查询
* @return 知识库分页
*/
PageResult<AiKnowledgeDO> getKnowledgePage(Long userId, AiKnowledgePageReqVO pageReqVO);
/**
* 根据知识库编号获取向量存储实例
*
* @param id 知识库编号
* @return 向量存储实例
*/
VectorStore getVectorStoreById(Long id);
}

90
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java
View File

@@ -1,90 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.hutool.core.util.ObjUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper;
import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.stereotype.Service;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS;
/**
* AI 知识库-基础信息 Service 实现类
*
* @author xiaoxin
*/
@Service
@Slf4j
public class AiKnowledgeServiceImpl implements AiKnowledgeService {
@Resource
private AiKnowledgeMapper knowledgeMapper;
@Resource
private AiChatModelService chatModelService;
@Resource
private AiApiKeyService apiKeyService;
@Override
public Long createKnowledge(AiKnowledgeCreateReqVO createReqVO, Long userId) {
// 1. 校验模型配置
AiChatModelDO model = chatModelService.validateChatModel(createReqVO.getModelId());
// 2. 插入知识库
AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class)
.setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus());
knowledgeMapper.insert(knowledgeBase);
return knowledgeBase.getId();
}
@Override
public void updateKnowledge(AiKnowledgeUpdateReqVO updateReqVO, Long userId) {
// 1.1 校验知识库存在
AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId());
if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) {
throw exception(KNOWLEDGE_NOT_EXISTS);
}
// 1.2 校验模型配置
AiChatModelDO model = chatModelService.validateChatModel(updateReqVO.getModelId());
// 2. 更新知识库
AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class);
updateDO.setModel(model.getModel());
knowledgeMapper.updateById(updateDO);
}
@Override
public AiKnowledgeDO validateKnowledgeExists(Long id) {
AiKnowledgeDO knowledgeBase = knowledgeMapper.selectById(id);
if (knowledgeBase == null) {
throw exception(KNOWLEDGE_NOT_EXISTS);
}
return knowledgeBase;
}
@Override
public PageResult<AiKnowledgeDO> getKnowledgePage(Long userId, AiKnowledgePageReqVO pageReqVO) {
return knowledgeMapper.selectPage(userId, pageReqVO);
}
@Override
public VectorStore getVectorStoreById(Long id) {
AiKnowledgeDO knowledge = validateKnowledgeExists(id);
AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
// 创建或获取 VectorStore 对象
return apiKeyService.getOrCreateVectorStore(model.getKeyId());
}
}

61
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java
View File

@@ -1,61 +0,0 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.autoconfigure.vectorstore.redis;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.RedisVectorStore;
import org.springframework.ai.vectorstore.RedisVectorStore.RedisVectorStoreConfig;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
import redis.clients.jedis.JedisPooled;
/**
* TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突
*
* TODO 这个官方，有说啥时候 fix 哇？
* TODO 看着是列在1.0.0-M2版本
*
* @author Christian Tzolov
* @author Eddú Meléndez
*/
@AutoConfiguration(after = RedisAutoConfiguration.class)
@ConditionalOnClass({JedisPooled.class, JedisConnectionFactory.class, RedisVectorStore.class, EmbeddingModel.class})
@ConditionalOnBean(JedisConnectionFactory.class)
@EnableConfigurationProperties(RedisVectorStoreProperties.class)
public class RedisVectorStoreAutoConfiguration {
@Bean
@ConditionalOnMissingBean
public RedisVectorStore vectorStore(EmbeddingModel embeddingModel, RedisVectorStoreProperties properties,
JedisConnectionFactory jedisConnectionFactory) {
var config = RedisVectorStoreConfig.builder()
.withIndexName(properties.getIndex())
.withPrefix(properties.getPrefix())
.build();
return new RedisVectorStore(config, embeddingModel,
new JedisPooled(jedisConnectionFactory.getHostName(), jedisConnectionFactory.getPort()),
properties.isInitializeSchema());
}
}

456
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/vectorstore/RedisVectorStore.java
View File

@@ -1,456 +0,0 @@
/*
* Copyright 2023 - 2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.vectorstore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import redis.clients.jedis.JedisPooled;
import redis.clients.jedis.Pipeline;
import redis.clients.jedis.json.Path2;
import redis.clients.jedis.search.*;
import redis.clients.jedis.search.Schema.FieldType;
import redis.clients.jedis.search.schemafields.*;
import redis.clients.jedis.search.schemafields.VectorField.VectorAlgorithm;
import java.text.MessageFormat;
import java.util.*;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
/**
* The RedisVectorStore is for managing and querying vector data in a Redis database. It
* offers functionalities like adding, deleting, and performing similarity searches on
* documents.
*
* The store utilizes RedisJSON and RedisSearch to handle JSON documents and to index and
* search vector data. It supports various vector algorithms (e.g., FLAT, HSNW) for
* efficient similarity searches. Additionally, it allows for custom metadata fields in
* the documents to be stored alongside the vector and content data.
*
* This class requires a RedisVectorStoreConfig configuration object for initialization,
* which includes settings like Redis URI, index name, field names, and vector algorithms.
* It also requires an EmbeddingModel to convert documents into embeddings before storing
* them.
*
* @author Julien Ruaux
* @author Christian Tzolov
* @author Eddú Meléndez
* @see VectorStore
* @see RedisVectorStoreConfig
* @see EmbeddingModel
*/
public class RedisVectorStore implements VectorStore, InitializingBean {
public enum Algorithm {
FLAT, HSNW
}
public record MetadataField(String name, FieldType fieldType) {
public static MetadataField text(String name) {
return new MetadataField(name, FieldType.TEXT);
}
public static MetadataField numeric(String name) {
return new MetadataField(name, FieldType.NUMERIC);
}
public static MetadataField tag(String name) {
return new MetadataField(name, FieldType.TAG);
}
}
/**
* Configuration for the Redis vector store.
*/
public static final class RedisVectorStoreConfig {
private final String indexName;
private final String prefix;
private final String contentFieldName;
private final String embeddingFieldName;
private final Algorithm vectorAlgorithm;
private final List<MetadataField> metadataFields;
private RedisVectorStoreConfig() {
this(builder());
}
private RedisVectorStoreConfig(Builder builder) {
this.indexName = builder.indexName;
this.prefix = builder.prefix;
this.contentFieldName = builder.contentFieldName;
this.embeddingFieldName = builder.embeddingFieldName;
this.vectorAlgorithm = builder.vectorAlgorithm;
this.metadataFields = builder.metadataFields;
}
/**
* Start building a new configuration.
* @return The entry point for creating a new configuration.
*/
public static Builder builder() {
return new Builder();
}
/**
* {@return the default config}
*/
public static RedisVectorStoreConfig defaultConfig() {
return builder().build();
}
public static class Builder {
private String indexName = DEFAULT_INDEX_NAME;
private String prefix = DEFAULT_PREFIX;
private String contentFieldName = DEFAULT_CONTENT_FIELD_NAME;
private String embeddingFieldName = DEFAULT_EMBEDDING_FIELD_NAME;
private Algorithm vectorAlgorithm = DEFAULT_VECTOR_ALGORITHM;
private List<MetadataField> metadataFields = new ArrayList<>();
private Builder() {
}
/**
* Configures the Redis index name to use.
* @param name the index name to use
* @return this builder
*/
public Builder withIndexName(String name) {
this.indexName = name;
return this;
}
/**
* Configures the Redis key prefix to use (default: "embedding:").
* @param prefix the prefix to use
* @return this builder
*/
public Builder withPrefix(String prefix) {
this.prefix = prefix;
return this;
}
/**
* Configures the Redis content field name to use.
* @param name the content field name to use
* @return this builder
*/
public Builder withContentFieldName(String name) {
this.contentFieldName = name;
return this;
}
/**
* Configures the Redis embedding field name to use.
* @param name the embedding field name to use
* @return this builder
*/
public Builder withEmbeddingFieldName(String name) {
this.embeddingFieldName = name;
return this;
}
/**
* Configures the Redis vector algorithmto use.
* @param algorithm the vector algorithm to use
* @return this builder
*/
public Builder withVectorAlgorithm(Algorithm algorithm) {
this.vectorAlgorithm = algorithm;
return this;
}
public Builder withMetadataFields(MetadataField... fields) {
return withMetadataFields(Arrays.asList(fields));
}
public Builder withMetadataFields(List<MetadataField> fields) {
this.metadataFields = fields;
return this;
}
/**
* {@return the immutable configuration}
*/
public RedisVectorStoreConfig build() {
return new RedisVectorStoreConfig(this);
}
}
}
private final boolean initializeSchema;
public static final String DEFAULT_INDEX_NAME = "spring-ai-index";
public static final String DEFAULT_CONTENT_FIELD_NAME = "content";
public static final String DEFAULT_EMBEDDING_FIELD_NAME = "embedding";
public static final String DEFAULT_PREFIX = "embedding:";
public static final Algorithm DEFAULT_VECTOR_ALGORITHM = Algorithm.HSNW;
private static final String QUERY_FORMAT = "%s=>[KNN %s @%s $%s AS %s]";
private static final Path2 JSON_SET_PATH = Path2.of("$");
private static final String JSON_PATH_PREFIX = "$.";
private static final Logger logger = LoggerFactory.getLogger(RedisVectorStore.class);
private static final Predicate<Object> RESPONSE_OK = Predicate.isEqual("OK");
private static final Predicate<Object> RESPONSE_DEL_OK = Predicate.isEqual(1l);
private static final String VECTOR_TYPE_FLOAT32 = "FLOAT32";
private static final String EMBEDDING_PARAM_NAME = "BLOB";
public static final String DISTANCE_FIELD_NAME = "vector_score";
private static final String DEFAULT_DISTANCE_METRIC = "COSINE";
private final JedisPooled jedis;
private final EmbeddingModel embeddingModel;
private final RedisVectorStoreConfig config;
private FilterExpressionConverter filterExpressionConverter;
public RedisVectorStore(RedisVectorStoreConfig config, EmbeddingModel embeddingModel, JedisPooled jedis,
boolean initializeSchema) {
Assert.notNull(config, "Config must not be null");
Assert.notNull(embeddingModel, "Embedding model must not be null");
this.initializeSchema = initializeSchema;
this.jedis = jedis;
this.embeddingModel = embeddingModel;
this.config = config;
this.filterExpressionConverter = new RedisFilterExpressionConverter(this.config.metadataFields);
}
public JedisPooled getJedis() {
return this.jedis;
}
@Override
public void add(List<Document> documents) {
try (Pipeline pipeline = this.jedis.pipelined()) {
for (Document document : documents) {
var embedding = this.embeddingModel.embed(document);
document.setEmbedding(embedding);
var fields = new HashMap<String, Object>();
fields.put(this.config.embeddingFieldName, embedding);
fields.put(this.config.contentFieldName, document.getContent());
fields.putAll(document.getMetadata());
pipeline.jsonSetWithEscape(key(document.getId()), JSON_SET_PATH, fields);
}
List<Object> responses = pipeline.syncAndReturnAll();
Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_OK)).findAny();
if (errResponse.isPresent()) {
String message = MessageFormat.format("Could not add document: {0}", errResponse.get());
if (logger.isErrorEnabled()) {
logger.error(message);
}
throw new RuntimeException(message);
}
}
}
private String key(String id) {
return this.config.prefix + id;
}
@Override
public Optional<Boolean> delete(List<String> idList) {
try (Pipeline pipeline = this.jedis.pipelined()) {
for (String id : idList) {
pipeline.jsonDel(key(id));
}
List<Object> responses = pipeline.syncAndReturnAll();
Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_DEL_OK)).findAny();
if (errResponse.isPresent()) {
if (logger.isErrorEnabled()) {
logger.error("Could not delete document: {}", errResponse.get());
}
return Optional.of(false);
}
return Optional.of(true);
}
}
@Override
public List<Document> similaritySearch(SearchRequest request) {
Assert.isTrue(request.getTopK() > 0, "The number of documents to returned must be greater than zero");
Assert.isTrue(request.getSimilarityThreshold() >= 0 && request.getSimilarityThreshold() <= 1,
"The similarity score is bounded between 0 and 1; least to most similar respectively.");
String filter = nativeExpressionFilter(request);
String queryString = String.format(QUERY_FORMAT, filter, request.getTopK(), this.config.embeddingFieldName,
EMBEDDING_PARAM_NAME, DISTANCE_FIELD_NAME);
List<String> returnFields = new ArrayList<>();
this.config.metadataFields.stream().map(MetadataField::name).forEach(returnFields::add);
returnFields.add(this.config.embeddingFieldName);
returnFields.add(this.config.contentFieldName);
returnFields.add(DISTANCE_FIELD_NAME);
var embedding = toFloatArray(this.embeddingModel.embed(request.getQuery()));
Query query = new Query(queryString).addParam(EMBEDDING_PARAM_NAME, RediSearchUtil.toByteArray(embedding))
.returnFields(returnFields.toArray(new String[0]))
.setSortBy(DISTANCE_FIELD_NAME, true)
.dialect(2);
SearchResult result = this.jedis.ftSearch(this.config.indexName, query);
return result.getDocuments()
.stream()
.filter(d -> similarityScore(d) >= request.getSimilarityThreshold())
.map(this::toDocument)
.toList();
}
private Document toDocument(redis.clients.jedis.search.Document doc) {
var id = doc.getId().substring(this.config.prefix.length());
var content = doc.hasProperty(this.config.contentFieldName) ? doc.getString(this.config.contentFieldName)
: null;
Map<String, Object> metadata = this.config.metadataFields.stream()
.map(MetadataField::name)
.filter(doc::hasProperty)
.collect(Collectors.toMap(Function.identity(), doc::getString));
metadata.put(DISTANCE_FIELD_NAME, 1 - similarityScore(doc));
return new Document(id, content, metadata);
}
private float similarityScore(redis.clients.jedis.search.Document doc) {
return (2 - Float.parseFloat(doc.getString(DISTANCE_FIELD_NAME))) / 2;
}
private String nativeExpressionFilter(SearchRequest request) {
if (request.getFilterExpression() == null) {
return "*";
}
return "(" + this.filterExpressionConverter.convertExpression(request.getFilterExpression()) + ")";
}
@Override
public void afterPropertiesSet() {
if (!this.initializeSchema) {
return;
}
// If index already exists don't do anything
if (this.jedis.ftList().contains(this.config.indexName)) {
return;
}
String response = this.jedis.ftCreate(this.config.indexName,
FTCreateParams.createParams().on(IndexDataType.JSON).addPrefix(this.config.prefix), schemaFields());
if (!RESPONSE_OK.test(response)) {
String message = MessageFormat.format("Could not create index: {0}", response);
throw new RuntimeException(message);
}
}
private Iterable<SchemaField> schemaFields() {
Map<String, Object> vectorAttrs = new HashMap<>();
vectorAttrs.put("DIM", this.embeddingModel.dimensions());
vectorAttrs.put("DISTANCE_METRIC", DEFAULT_DISTANCE_METRIC);
vectorAttrs.put("TYPE", VECTOR_TYPE_FLOAT32);
List<SchemaField> fields = new ArrayList<>();
fields.add(TextField.of(jsonPath(this.config.contentFieldName)).as(this.config.contentFieldName).weight(1.0));
fields.add(VectorField.builder()
.fieldName(jsonPath(this.config.embeddingFieldName))
.algorithm(vectorAlgorithm())
.attributes(vectorAttrs)
.as(this.config.embeddingFieldName)
.build());
if (!CollectionUtils.isEmpty(this.config.metadataFields)) {
for (MetadataField field : this.config.metadataFields) {
fields.add(schemaField(field));
}
}
return fields;
}
private SchemaField schemaField(MetadataField field) {
String fieldName = jsonPath(field.name);
switch (field.fieldType) {
case NUMERIC:
return NumericField.of(fieldName).as(field.name);
case TAG:
return TagField.of(fieldName).as(field.name);
case TEXT:
return TextField.of(fieldName).as(field.name);
default:
throw new IllegalArgumentException(
MessageFormat.format("Field {0} has unsupported type {1}", field.name, field.fieldType));
}
}
private VectorAlgorithm vectorAlgorithm() {
if (config.vectorAlgorithm == Algorithm.HSNW) {
return VectorAlgorithm.HNSW;
}
return VectorAlgorithm.FLAT;
}
private String jsonPath(String field) {
return JSON_PATH_PREFIX + field;
}
private static float[] toFloatArray(List<Double> embeddingDouble) {
float[] embeddingFloat = new float[embeddingDouble.size()];
int i = 0;
for (Double d : embeddingDouble) {
embeddingFloat[i++] = d.floatValue();
}
return embeddingFloat;
}
}

70
yudao-module-ai/yudao-spring-boot-starter-ai/src/test/java/cn/iocoder/yudao/framework/ai/chat/AzureOpenAIChatModelTests.java
View File

@@ -1,70 +0,0 @@
package cn.iocoder.yudao.framework.ai.chat;
import com.azure.ai.openai.OpenAIClient;
import com.azure.ai.openai.OpenAIClientBuilder;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.core.util.ClientOptions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.ai.azure.openai.AzureOpenAiChatModel;
import org.springframework.ai.azure.openai.AzureOpenAiChatOptions;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import reactor.core.publisher.Flux;
import java.util.ArrayList;
import java.util.List;
import static org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiChatProperties.DEFAULT_DEPLOYMENT_NAME;
/**
* {@link AzureOpenAiChatModel} 集成测试
*
* @author 芋道源码
*/
public class AzureOpenAIChatModelTests {
private final OpenAIClient openAiApi = (new OpenAIClientBuilder())
.endpoint("https://eastusprejade.openai.azure.com")
.credential(new AzureKeyCredential("xxx"))
.clientOptions((new ClientOptions()).setApplicationId("spring-ai"))
.buildClient();
private final AzureOpenAiChatModel chatModel = new AzureOpenAiChatModel(openAiApi,
AzureOpenAiChatOptions.builder().withDeploymentName(DEFAULT_DEPLOYMENT_NAME).build());
@Test
@Disabled
public void testCall() {
// 准备参数
List<Message> messages = new ArrayList<>();
messages.add(new SystemMessage("你是一个优质的文言文作者，用文言文描述着各城市的人文风景。"));
messages.add(new UserMessage("1 + 1 = ？"));
// 调用
ChatResponse response = chatModel.call(new Prompt(messages));
// 打印结果
System.out.println(response);
System.out.println(response.getResult().getOutput());
}
@Test
@Disabled
public void testStream() {
// 准备参数
List<Message> messages = new ArrayList<>();
messages.add(new SystemMessage("你是一个优质的文言文作者，用文言文描述着各城市的人文风景。"));
messages.add(new UserMessage("1 + 1 = ？"));
// 调用
Flux<ChatResponse> flux = chatModel.stream(new Prompt(messages));
// 打印结果
flux.doOnNext(response -> {
// System.out.println(response);
System.out.println(response.getResult().getOutput());
}).then().block();
}
}