From 5a754e5625f1b7c73872b6fab000043d44c3cb8f Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 15 Jul 2024 15:31:26 -0400 Subject: [PATCH 1/4] cleaning namespace --- pephub/main.py | 2 - pephub/routers/api/v1/base.py | 5 +- pephub/routers/api/v1/namespace.py | 109 ++++++++++++------------- pephub/routers/api/v1/user.py | 124 ++++++++++++++--------------- pephub/routers/models.py | 13 +++ 5 files changed, 127 insertions(+), 126 deletions(-) diff --git a/pephub/main.py b/pephub/main.py index 76c21375..d46b77a2 100644 --- a/pephub/main.py +++ b/pephub/main.py @@ -15,7 +15,6 @@ from .routers.api.v1.project import project as api_project from .routers.api.v1.project import projects as api_projects from .routers.api.v1.search import search as api_search -from .routers.api.v1.user import user as api_user from .routers.auth.base import auth as auth_router from .routers.eido.eido import router as eido_router @@ -78,7 +77,6 @@ # build routes app.include_router(api_base) -app.include_router(api_user) app.include_router(api_namespace) app.include_router(api_namespaces) app.include_router(api_project) diff --git a/pephub/routers/api/v1/base.py b/pephub/routers/api/v1/base.py index 0da308d5..d16bb9c1 100644 --- a/pephub/routers/api/v1/base.py +++ b/pephub/routers/api/v1/base.py @@ -2,13 +2,14 @@ from fastapi import APIRouter from ....const import ALL_VERSIONS +from ...models import BaseEndpointResponseModel, VersionResponseModel load_dotenv() api = APIRouter(prefix="/api/v1", tags=["base"]) -@api.get("/") +@api.get("/", response_model=BaseEndpointResponseModel) async def api_base(): """ Base API endpoint. @@ -19,6 +20,6 @@ async def api_base(): } -@api.get("/_version") +@api.get("/_version", response_model=VersionResponseModel) async def version(): return dict(**ALL_VERSIONS) diff --git a/pephub/routers/api/v1/namespace.py b/pephub/routers/api/v1/namespace.py index 1fe16e2b..33fa5f5a 100644 --- a/pephub/routers/api/v1/namespace.py +++ b/pephub/routers/api/v1/namespace.py @@ -53,10 +53,10 @@ @namespace.get( "/", summary="Fetch details about a particular namespace.", + response_model=Namespace, ) async def get_namespace( - request: Request, - nspace: Namespace = Depends(get_namespace_info), + namespace_info: Namespace = Depends(get_namespace_info), ): """ Fetch namespace. Returns a JSON representation of the namespace. @@ -67,9 +67,7 @@ async def get_namespace( namespace: databio """ - nspace = nspace.model_dump() - nspace["projects_endpoint"] = f"{str(request.url)[:-1]}/projects" - return JSONResponse(content=nspace) + return namespace_info @namespace.get( @@ -83,8 +81,7 @@ async def get_namespace_projects( limit: int = 10, offset: int = 0, query: str = None, - session_info: dict = Depends(read_authorization_header), - namespace_access: List[str] = Depends(get_namespace_access_list), + admin_list: List[str] = Depends(get_namespace_access_list), order_by: str = "update_date", order_desc: bool = False, filter_by: Annotated[ @@ -112,7 +109,7 @@ async def get_namespace_projects( namespace=namespace, limit=limit, offset=offset, - admin=namespace_access, + admin=admin_list, order_by=order_by, order_desc=order_desc, filter_by=filter_by, @@ -125,7 +122,7 @@ async def get_namespace_projects( namespace=namespace, limit=limit, offset=offset, - admin=namespace_access, + admin=admin_list, order_by=order_by, order_desc=order_desc, filter_by=filter_by, @@ -133,17 +130,8 @@ async def get_namespace_projects( filter_end_date=filter_end_date, pep_type=pep_type, ) - results = [p.model_dump() for p in search_result.results] - return JSONResponse( - content={ - "count": search_result.count, - "limit": limit, - "offset": offset, - "items": results, - "session_info": session_info, - } - ) + return search_result # url format based on: @@ -170,8 +158,8 @@ async def create_pep( Create a PEP for a particular namespace you have write access to. Don't know your namespace? Log in to see. - """ + if files is not None: init_file = parse_user_file_upload(files) init_file, other_files = split_upload_files_on_init_file(files, init_file) @@ -212,8 +200,6 @@ async def create_pep( content={ "namespace": namespace, "name": name, - "proj": p.to_dict(), - "init_file": init_file.filename, "tag": tag, "registry_path": f"{namespace}/{name}:{tag}", }, @@ -221,45 +207,48 @@ async def create_pep( ) # create a blank peppy.Project object with fake files else: - # create temp dir that gets deleted when we're done - with tempfile.TemporaryDirectory() as dirpath: - config_file_name = "project_config.yaml" - sample_table_name = BLANK_PEP_CONFIG["sample_table"] - - # create 'empty' config and sample table files - with open(f"{dirpath}/{config_file_name}", "w") as cfg_fh: - cfg_fh.write(json.dumps(BLANK_PEP_CONFIG)) - with open(f"{dirpath}/{sample_table_name}", "w") as cfg_fh: - cfg_fh.write(BLANK_PEP_SAMPLE_TABLE) + raise HTTPException( + detail=f"Project files were not provided", + status_code=400, + ) - # init project - p = Project(f"{dirpath}/{config_file_name}") - p.name = name - p.description = description - p.pep_schema = pep_schema - try: - agent.project.create( - p, - namespace=namespace, - name=name, - tag=tag, - is_private=is_private, - ) - except ProjectUniqueNameError as _: - raise HTTPException( - detail=f"Project '{namespace}/{p.name}:{tag}' already exists in namespace", - status_code=400, - ) - return JSONResponse( - content={ - "namespace": namespace, - "name": name, - "proj": p.to_dict(), - "tag": tag, - "registry_path": f"{namespace}/{name}:{tag}", - }, - status_code=202, - ) + # # create temp dir that gets deleted when we're done + # with tempfile.TemporaryDirectory() as dirpath: + # config_file_name = "project_config.yaml" + # sample_table_name = BLANK_PEP_CONFIG["sample_table"] + # + # # create 'empty' config and sample table files + # with open(f"{dirpath}/{config_file_name}", "w") as cfg_fh: + # cfg_fh.write(json.dumps(BLANK_PEP_CONFIG)) + # with open(f"{dirpath}/{sample_table_name}", "w") as cfg_fh: + # cfg_fh.write(BLANK_PEP_SAMPLE_TABLE) + # + # # init project + # p = Project(f"{dirpath}/{config_file_name}") + # try: + # agent.project.create( + # p, + # namespace=namespace, + # name=name, + # tag=tag, + # description=description, + # is_private=is_private, + # pep_schema=pep_schema, + # ) + # except ProjectUniqueNameError as _: + # raise HTTPException( + # detail=f"Project '{namespace}/{p.name}:{tag}' already exists in namespace", + # status_code=400, + # ) + # return JSONResponse( + # content={ + # "namespace": namespace, + # "name": name, + # "tag": tag, + # "registry_path": f"{namespace}/{name}:{tag}", + # }, + # status_code=202, + # ) @namespace.post( diff --git a/pephub/routers/api/v1/user.py b/pephub/routers/api/v1/user.py index 43ef9b46..d4250006 100644 --- a/pephub/routers/api/v1/user.py +++ b/pephub/routers/api/v1/user.py @@ -1,62 +1,62 @@ -from platform import python_version - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import JSONResponse, RedirectResponse -from fastapi.templating import Jinja2Templates -from pepdbagent import PEPDatabaseAgent - -from ...._version import __version__ as pephub_version -from ....const import BASE_TEMPLATES_PATH -from ....dependencies import get_db, read_authorization_header - -user = APIRouter(prefix="/api/v1/me", tags=["profile"]) - -templates = Jinja2Templates(directory=BASE_TEMPLATES_PATH) - - -# return users data from session_info -@user.get("/") -def profile_data( - session_info=Depends(read_authorization_header), - agent: PEPDatabaseAgent = Depends(get_db), -): - """ - Return the user's profile data. - """ - if session_info is None: - return RedirectResponse(url="/auth/login") - else: - peps = agent.namespace.get( - namespace=session_info["login"], admin=session_info["login"] - ) - return JSONResponse( - content={ - "session_info": session_info, - "peps": [pep.dict() for pep in peps.projects], - } - ) - - -@user.get("/data") -def profile_data2( - request: Request, - session_info=Depends(read_authorization_header), - agent: PEPDatabaseAgent = Depends(get_db), -): - """ - Return the user's profile data. - """ - if session_info is None: - return RedirectResponse(url="/auth/login") - else: - peps = agent.namespace.get( - namespace=session_info["login"], admin=session_info["login"] - ) - return { - "request": request, - "session_info": session_info, - "python_version": python_version(), - "pephub_version": pephub_version, - "logged_in": session_info is not None, - "peps": peps, - } +# from platform import python_version +# +# from fastapi import APIRouter, Depends, Request +# from fastapi.responses import JSONResponse, RedirectResponse +# from fastapi.templating import Jinja2Templates +# from pepdbagent import PEPDatabaseAgent +# +# from ...._version import __version__ as pephub_version +# from ....const import BASE_TEMPLATES_PATH +# from ....dependencies import get_db, read_authorization_header +# +# user = APIRouter(prefix="/api/v1/me", tags=["profile"]) +# +# templates = Jinja2Templates(directory=BASE_TEMPLATES_PATH) +# +# +# # return users data from session_info +# @user.get("/") +# def profile_data( +# session_info=Depends(read_authorization_header), +# agent: PEPDatabaseAgent = Depends(get_db), +# ): +# """ +# Return the user's profile data. +# """ +# if session_info is None: +# return RedirectResponse(url="/auth/login") +# else: +# peps = agent.namespace.get( +# namespace=session_info["login"], admin=session_info["login"] +# ) +# return JSONResponse( +# content={ +# "session_info": session_info, +# "peps": [pep.dict() for pep in peps.projects], +# } +# ) +# +# +# @user.get("/data") +# def profile_data2( +# request: Request, +# session_info=Depends(read_authorization_header), +# agent: PEPDatabaseAgent = Depends(get_db), +# ): +# """ +# Return the user's profile data. +# """ +# if session_info is None: +# return RedirectResponse(url="/auth/login") +# else: +# peps = agent.namespace.get( +# namespace=session_info["login"], admin=session_info["login"] +# ) +# return { +# "request": request, +# "session_info": session_info, +# "python_version": python_version(), +# "pephub_version": pephub_version, +# "logged_in": session_info is not None, +# "peps": peps, +# } diff --git a/pephub/routers/models.py b/pephub/routers/models.py index c88d2b68..7b549f66 100644 --- a/pephub/routers/models.py +++ b/pephub/routers/models.py @@ -106,3 +106,16 @@ class DeveloperKey(BaseModel): key: str created_at: str expires: str + + +class VersionResponseModel(BaseModel): + pephub_version: str + peppy_version: str + python_version: str + fastapi_version: str + pepdbagent_version: str + api_version: int + + +class BaseEndpointResponseModel(VersionResponseModel): + message: str From 60bfe6bc22a72f979cc69cb69b238f036b9b0f6d Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 15 Jul 2024 15:47:37 -0400 Subject: [PATCH 2/4] cleaned project --- pephub/routers/api/v1/project.py | 57 +++++++++++++++----------------- pephub/routers/api/v1/search.py | 19 ++++------- pephub/routers/models.py | 9 +++++ 3 files changed, 42 insertions(+), 43 deletions(-) diff --git a/pephub/routers/api/v1/project.py b/pephub/routers/api/v1/project.py index cd54efbc..7b95b1e2 100644 --- a/pephub/routers/api/v1/project.py +++ b/pephub/routers/api/v1/project.py @@ -9,7 +9,7 @@ from dotenv import load_dotenv from fastapi import APIRouter, Body, Depends, Query from fastapi.exceptions import HTTPException -from fastapi.responses import JSONResponse, PlainTextResponse +from fastapi.responses import JSONResponse, PlainTextResponse, FileResponse from pepdbagent import PEPDatabaseAgent from pepdbagent.exceptions import ( ProjectNotFoundError, @@ -51,6 +51,8 @@ ProjectRawModel, ProjectRawRequest, ProjectHistoryResponse, + SamplesResponseModel, + ConfigResponseModel, ) from .helpers import verify_updated_project @@ -217,7 +219,7 @@ async def delete_a_pep( ) -@project.get("/samples") +@project.get("/samples", response_model=SamplesResponseModel) async def get_pep_samples( proj: dict = Depends(get_project), format: Optional[str] = None, @@ -233,6 +235,10 @@ async def get_pep_samples( project: example namespace: databio """ + + if isinstance(proj, dict): + proj = peppy.Project.from_dict(proj) + if format is not None: conversion_func: Callable = SAMPLE_CONVERSION_FUNCTIONS.get(format, None) if conversion_func is not None: @@ -245,25 +251,20 @@ async def get_pep_samples( else: if raw: df = pd.DataFrame(proj[SAMPLE_RAW_DICT_KEY]) - return JSONResponse( - { - "count": df.shape[0], - "items": df.replace({np.nan: None}).to_dict(orient="records"), - } + return SamplesResponseModel( + count=df.shape[0], + items=df.replace({np.nan: None}).to_dict(orient="records"), ) else: - return JSONResponse( - { - "count": len(proj.samples), - "items": [s.to_dict() for s in proj.samples], - } + return SamplesResponseModel( + count=len(proj.samples), + items=[s.to_dict() for s in proj.samples], ) @project.get("/config", summary="Get project configuration file") async def get_pep_config( config: dict = Depends(get_config), - # format: Optional[Literal["JSON", "String"]] = "JSON", ): """ Get project configuration file from a certain project and namespace @@ -274,10 +275,8 @@ async def get_pep_config( namespace: databio tag: default """ - return JSONResponse( - { - "config": yaml.dump(config, sort_keys=False), - } + return ConfigResponseModel( + config=config, ) @@ -463,7 +462,7 @@ async def delete_sample( ) -@project.get("/subsamples") +@project.get("/subsamples", response_model=SamplesResponseModel) async def get_subsamples_endpoint( subsamples: peppy.Project = Depends(get_subsamples), download: bool = False, @@ -489,19 +488,15 @@ async def get_subsamples_endpoint( if download: return subsamples.to_csv() else: - return JSONResponse( - { - "count": subsamples.shape[0], - "items": subsamples.to_dict(orient="records"), - } + return SamplesResponseModel( + count=subsamples.shape[0], + items=subsamples.to_dict(orient="records"), ) else: - return JSONResponse( - { - "count": 0, - "items": [], - } + return SamplesResponseModel( + count=0, + items=[], ) @@ -552,7 +547,7 @@ async def convert_pep( return resp_obj -@project.get("/zip") +@project.get("/zip", response_class=FileResponse) async def zip_pep_for_download(proj: Dict[str, Any] = Depends(get_project)): """ Zip a pep @@ -618,7 +613,7 @@ async def fork_pep_to_namespace( ) -@project.get("/annotation") +@project.get("/annotation", response_model=AnnotationModel) async def get_project_annotation( proj_annotation: AnnotationModel = Depends(get_project_annotation), ): @@ -752,6 +747,7 @@ async def create_view_of_the_project( "/views/{view}/zip", summary="Zip a view", tags=["views"], + response_class=FileResponse, ) async def zip_view_of_the_view( namespace: str, @@ -1069,6 +1065,7 @@ def restore_project_history_by_id( @project.get( "/history/{history_id}/zip", summary="Zip a project history by id", + response_class=FileResponse, ) def get_zip_snapshot( namespace: str, diff --git a/pephub/routers/api/v1/search.py b/pephub/routers/api/v1/search.py index e004e598..deb2a8ce 100644 --- a/pephub/routers/api/v1/search.py +++ b/pephub/routers/api/v1/search.py @@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse from fastembed.embedding import FlagEmbedding as Embedding from pepdbagent import PEPDatabaseAgent -from pepdbagent.models import ListOfNamespaceInfo +from pepdbagent.models import NamespaceList from qdrant_client import QdrantClient from ....const import DEFAULT_QDRANT_COLLECTION_NAME @@ -22,23 +22,16 @@ search = APIRouter(prefix="/api/v1/search", tags=["search"]) -@search.get("/namespaces", summary="Search for namespaces") +@search.get( + "/namespaces", summary="Search for namespaces", response_model=NamespaceList +) async def search_for_namespaces( limit: Optional[int] = 1_000, query: Optional[str] = "", offset: Optional[int] = 0, agent: PEPDatabaseAgent = Depends(get_db), -) -> ListOfNamespaceInfo: - res = agent.namespace.get(limit=limit, query=query or "", offset=offset) - - return JSONResponse( - content={ - "results": [r.model_dump() for r in res.results], - "count": res.count, - "limit": limit, - "offset": offset, - } - ) +) -> NamespaceList: + return agent.namespace.get(limit=limit, query=query or "", offset=offset) # perform a search diff --git a/pephub/routers/models.py b/pephub/routers/models.py index 7b549f66..95350bb3 100644 --- a/pephub/routers/models.py +++ b/pephub/routers/models.py @@ -119,3 +119,12 @@ class VersionResponseModel(BaseModel): class BaseEndpointResponseModel(VersionResponseModel): message: str + + +class SamplesResponseModel(BaseModel): + count: int + items: list + + +class ConfigResponseModel(BaseModel): + config: dict From 37e578b2fb55d7196c5b29769b093f7471b09171 Mon Sep 17 00:00:00 2001 From: Nathan LeRoy Date: Mon, 15 Jul 2024 15:57:39 -0400 Subject: [PATCH 3/4] fix namespace page --- web/src/api/namespace.ts | 2 +- web/src/components/forms/components/pep-selector.tsx | 6 +++--- web/src/hooks/queries/useNamespaceInfo.ts | 4 ++-- web/src/hooks/queries/useNamespaceProjects.ts | 2 +- web/src/pages/Namespace.tsx | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/web/src/api/namespace.ts b/web/src/api/namespace.ts index 1592f004..2fda1475 100644 --- a/web/src/api/namespace.ts +++ b/web/src/api/namespace.ts @@ -17,7 +17,7 @@ export interface NamespaceProjectsResponse { count: number; offset: number; limit: number; - items: ProjectAnnotation[]; + results: ProjectAnnotation[]; } export interface PaginationParams { diff --git a/web/src/components/forms/components/pep-selector.tsx b/web/src/components/forms/components/pep-selector.tsx index f2d9b962..a6ee5c9c 100644 --- a/web/src/components/forms/components/pep-selector.tsx +++ b/web/src/components/forms/components/pep-selector.tsx @@ -40,9 +40,9 @@ const PepSelector: FC = ({ onChange, value }) => { // projects we've seen before, because it // changes each time we change namespace useEffect(() => { - if (projects?.items) { + if (projects?.results) { // see if any new projects are in the list - const newProjects = projects.items.filter((project) => !cachedOptions.find((p) => p.digest === project.digest)); + const newProjects = projects.results.filter((project) => !cachedOptions.find((p) => p.digest === project.digest)); if (newProjects.length > 0) { setCachedOptions([...cachedOptions, ...newProjects]); } @@ -71,7 +71,7 @@ const PepSelector: FC = ({ onChange, value }) => { annotation: v, }))} onInputChange={(newValue) => setSearch(newValue)} - options={mapOptions(projects?.items || [])} + options={mapOptions(projects?.results || [])} onChange={(newValue: MultiValue<{ label: string; value: string; annotation: ProjectAnnotation }>) => { const mapped = newValue?.map((v) => v.annotation) || []; onChange(mapped); diff --git a/web/src/hooks/queries/useNamespaceInfo.ts b/web/src/hooks/queries/useNamespaceInfo.ts index f9c9ed59..321a9429 100644 --- a/web/src/hooks/queries/useNamespaceInfo.ts +++ b/web/src/hooks/queries/useNamespaceInfo.ts @@ -3,11 +3,11 @@ import { useQuery } from '@tanstack/react-query'; import { getNamespaceInfo } from '../../api/namespace'; import { useSession } from '../useSession'; -export const useNamespaceInfo = (namespace: string | undefined) => { +export const useNamespaceInfo = (namespace: string) => { const session = useSession(); const query = useQuery({ queryKey: [namespace], - queryFn: () => getNamespaceInfo(namespace || '', session.jwt || ''), + queryFn: () => getNamespaceInfo(namespace, session.jwt), enabled: namespace !== undefined && session.jwt !== null, retry: false, }); diff --git a/web/src/hooks/queries/useNamespaceProjects.ts b/web/src/hooks/queries/useNamespaceProjects.ts index 4073b2c4..0706a39b 100644 --- a/web/src/hooks/queries/useNamespaceProjects.ts +++ b/web/src/hooks/queries/useNamespaceProjects.ts @@ -12,7 +12,7 @@ export interface NamespaceProjectsParams extends PaginationParams { export const useNamespaceProjects = (namespace: string | undefined, params: NamespaceProjectsParams) => { const session = useSession(); const query = useQuery({ - queryKey: [namespace, params, session.jwt], + queryKey: [namespace, params], queryFn: () => getNamespaceProjects(namespace || '', session.jwt, params, params.type), enabled: namespace !== undefined && namespace !== '', }); diff --git a/web/src/pages/Namespace.tsx b/web/src/pages/Namespace.tsx index 4e73d713..64ed4aec 100644 --- a/web/src/pages/Namespace.tsx +++ b/web/src/pages/Namespace.tsx @@ -76,7 +76,7 @@ export const NamespacePage = () => { const stars = starsQuery.data; // left over from when we were filtering on sample number - const projectsFiltered = projects?.items.filter((p) => p.number_of_samples) || []; + const projectsFiltered = projects?.results?.filter((p) => p.number_of_samples) || []; if (namespaceInfoIsLoading || starsQuery.isLoading) { return ( From 9b20ecb744029ef33c847adcd256522263520295 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 15 Jul 2024 16:06:47 -0400 Subject: [PATCH 4/4] fixed peppy issue --- pephub/routers/api/v1/project.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pephub/routers/api/v1/project.py b/pephub/routers/api/v1/project.py index 7b95b1e2..7fef3a7e 100644 --- a/pephub/routers/api/v1/project.py +++ b/pephub/routers/api/v1/project.py @@ -235,10 +235,6 @@ async def get_pep_samples( project: example namespace: databio """ - - if isinstance(proj, dict): - proj = peppy.Project.from_dict(proj) - if format is not None: conversion_func: Callable = SAMPLE_CONVERSION_FUNCTIONS.get(format, None) if conversion_func is not None: @@ -256,6 +252,7 @@ async def get_pep_samples( items=df.replace({np.nan: None}).to_dict(orient="records"), ) else: + proj = peppy.Project.from_dict(proj) return SamplesResponseModel( count=len(proj.samples), items=[s.to_dict() for s in proj.samples],