api dataset pagination & yuque dataset file folder can enter

docSite/content/zh-cn/docs/guide/knowledge_base/api_dataset.md

@@ -61,6 +61,8 @@ type FileListItem = {
 {{% alert icon=" " context="success" %}}
 - parentId - 父级 id，可选，或者 null。
 - searchKey - 检索词，可选
+- pageSize - 每页显示的数据项的数量
+- offset - 偏移量
 {{% /alert %}}
 
 ```bash
@@ -69,7 +71,9 @@ curl --location --request POST '{{baseURL}}/v1/file/list' \
 --header 'Content-Type: application/json' \
 --data-raw '{
     "parentId": null,
-    "searchKey": ""
+    "searchKey": "",
+    "pageSize": 15,
+    "offset": 0
 }'
 ```
 
@@ -84,16 +88,19 @@ curl --location --request POST '{{baseURL}}/v1/file/list' \
     "code": 200,
     "success": true,
     "message": "",
-    "data": [
-        {
-            "id": "xxxx",
-            "parentId": "xxxx",
-            "type": "file",  // file | folder
-            "name":"test.json",
-            "updateTime":"2024-11-26T03:05:24.759Z",
-            "createTime":"2024-11-26T03:05:24.759Z"
-        }
-   ]
+    "data":{
+        "list": [ 
+            {
+                "id": "xxxx",
+                "parentId": "xxxx",
+                "type": "file",  // file | folder
+                "name":"test.json",
+                "updateTime":"2024-11-26T03:05:24.759Z",
+                "createTime":"2024-11-26T03:05:24.759Z"
+            }
+       ],
+        "total": 1
+    }
 }
 ```
 

packages/global/core/dataset/apiDataset.d.ts

@@ -13,7 +13,10 @@ export type APIFileServer = {
   authorization: string;
 };
 
-export type APIFileListResponse = APIFileItem[];
+export type APIFileListResponse = {
+  list: APIFileItem[];
+  total: number;
+};
 
 export type APIFileContentResponse = {
   content?: string;

packages/service/core/dataset/apiDataset/api.ts

@@ -79,33 +79,51 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
 
   const listFiles = async ({
     searchKey,
-    parentId
+    parentId,
+    offset,
+    pageSize
   }: {
     searchKey?: string;
     parentId?: ParentIdType;
+    offset: number;
+    pageSize: number;
   }) => {
-    const files = await request<APIFileListResponse>(
+    const response = await request<APIFileListResponse>(
       `/v1/file/list`,
       {
         searchKey,
-        parentId
+        parentId,
+        pageSize,
+        offset
       },
       'POST'
     );
 
-    if (!Array.isArray(files)) {
-      return Promise.reject('Invalid file list format');
+    let list: any[] = [];
+    let total = 0;
+
+    // 兼容旧的数据格式
+    if (Array.isArray(response)) {
+      list = response;
+      total = response.length;
+    } else {
+      list = response.list;
+      total = response.total;
     }
-    if (files.some((file) => !file.id || !file.name || typeof file.type === 'undefined')) {
+
+    if (list.some((file) => !file.id || !file.name || typeof file.type === 'undefined')) {
       return Promise.reject('Invalid file data format');
     }
 
-    const formattedFiles = files.map((file) => ({
+    const formattedFiles = list.map((file) => ({
       ...file,
       hasChild: file.type === 'folder'
     }));
 
-    return formattedFiles;
+    return {
+      list: formattedFiles,
+      total
+    };
   };
 
   const getFileContent = async ({ teamId, apiFileId }: { teamId: string; apiFileId: string }) => {

packages/web/common/fetch/type.d.ts

@@ -2,6 +2,7 @@ import { RequireOnlyOne } from '@fastgpt/global/common/type/utils';
 
 type PaginationProps<T = {}> = T & {
   pageSize: number | string;
+  metaData?: Record<string, any>;
 } & RequireOnlyOne<{
     offset: number | string;
     pageNum: number | string;
@@ -10,4 +11,5 @@ type PaginationProps<T = {}> = T & {
 type PaginationResponse<T = {}> = {
   total: number;
   list: T[];
+  metaData?: Record<string, any>;
 };

packages/web/hooks/useScrollPagination.tsx

@@ -89,7 +89,7 @@ export function useVirtualScrollPagination<
         // init or reload
         setData(res.list);
       } else {
-        setData((prev) => [...prev, ...res.list]);
+        setData((prevData) => [...prevData, ...res.list]);
       }
     } catch (error: any) {
       toast({
@@ -205,27 +205,32 @@ export function useScrollPagination<
 
   const [data, setData] = useState<TData['list']>([]);
   const [total, setTotal] = useState(0);
+  const [metaData, setMetaData] = useState<TData['metaData']>({});
   const [isLoading, { setTrue, setFalse }] = useBoolean(false);
-  const isEmpty = total === 0 && !isLoading;
 
-  const noMore = data.length >= total;
+  const isEmpty = total === 0 && !isLoading;
+  const noMore = data.length >= total || !!metaData?.noMore;
 
   const loadData = useLockFn(
     async (init = false, ScrollContainerRef?: RefObject<HTMLDivElement>) => {
       if (noMore && !init) return;
 
-      const offset = init ? 0 : data.length;
-
       setTrue();
 
       try {
+        const offset = init ? 0 : data.length;
+
         const res = await api({
           offset,
           pageSize,
+          ...metaData,
           ...params
         } as TParams);
 
-        setTotal(res.total);
+        const { total: totalCount, list: dataList, metaData: resMetaData } = res;
+
+        setTotal(totalCount);
+        setMetaData(resMetaData);
 
         if (scrollLoadType === 'top') {
           const prevHeight = ScrollContainerRef?.current?.scrollHeight || 0;
@@ -245,10 +250,10 @@ export function useScrollPagination<
             );
           }
 
-          setData((prevData) => (offset === 0 ? res.list : [...res.list, ...prevData]));
+          setData((prevData) => (init ? dataList : [...dataList, ...prevData]));
           adjustScrollPosition();
         } else {
-          setData((prevData) => (offset === 0 ? res.list : [...prevData, ...res.list]));
+          setData((prevData) => (init ? dataList : [...prevData, ...dataList]));
         }
       } catch (error: any) {
         if (showErrorToast) {

projects/app/src/pages/api/core/dataset/apiDataset/list.ts

@@ -12,12 +12,24 @@ export type GetApiDatasetFileListProps = {
   searchKey?: string;
   parentId?: ParentIdType;
   datasetId: string;
+  offset: number;
+  pageSize: number;
 };
 
-export type GetApiDatasetFileListResponse = APIFileItem[];
+export type GetApiDatasetFileListResponse = {
+  list: APIFileItem[];
+  total: number;
+};
 
 async function handler(req: NextApiRequest) {
-  let { searchKey = '', parentId = null, datasetId } = req.body;
+  let {
+    searchKey = '',
+    parentId = null,
+    datasetId,
+    nextPageToken = '',
+    pageSize,
+    offset = 0
+  } = req.body;
 
   const { dataset } = await authDataset({
     req,
@@ -32,13 +44,25 @@ async function handler(req: NextApiRequest) {
   const yuqueServer = dataset.yuqueServer;
 
   if (apiServer) {
-    return useApiDatasetRequest({ apiServer }).listFiles({ searchKey, parentId });
+    const { list, total } = await useApiDatasetRequest({ apiServer }).listFiles({
+      searchKey,
+      parentId,
+      offset,
+      pageSize
+    });
+    return {
+      list,
+      total
+    };
   }
   if (feishuServer || yuqueServer) {
     return getFeishuAndYuqueDatasetFileList({
       feishuServer,
       yuqueServer,
-      parentId
+      parentId,
+      nextPageToken,
+      offset,
+      pageSize
     });
   }