Skip to content

Commit

Permalink
Merge pull request #80 from henilp105/main
Browse files Browse the repository at this point in the history 
add: categories, keywords, markdown support for registry description
  • Loading branch information
@henilp105
henilp105 authored May 25, 2024
2 parents 5a095f3 + 74359c0 commit 2f98784
Show file tree
Hide file tree
Showing 10 changed files with 193 additions and 118 deletions.
59 changes: 59 additions & 0 deletions backend/_unverify.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
from mongo import db
import pymongo

# t = db.packages.update_many({}, {"$set": {"is_verified": False}})
# print(t.modified_count)
# packages = list(db.packages.find())

# # Step 2: Make keywords unique for each package
# for package in packages:
# unique_keywords = list(set(package.get("categories", []))) # Making keywords unique
# package["categories"] = unique_keywords

# # Step 3: Store the updated packages in a list
# updated_packages = [{"_id": package["_id"], "categories": package["categories"]} for package in packages]

# # Step 4: Perform bulk update operation
# bulk_operations = [pymongo.UpdateOne({"_id": package["_id"]}, {"$set": {"categories": package["categories"]}}) for package in updated_packages]
# result = db.packages.bulk_write(bulk_operations)

# print("Number of documents matched:", result.matched_count)
# print("Number of documents modified:", result.modified_count)
# remove key k from all the documents in the collection
t = db.packages.update_many({}, {"$set": {"is_verified": False}})
# result = db.packages.update_many({}, {"$set": {"unable_to_verify": False}})
# t = list(db.packages.find())
# for i in t:
# n = db.namespaces.find_one({"_id": i["namespace"]})['namespace']
# db.packages.update_one({"_id": i["_id"]}, {"$set": {"description": "Package Under Verification","registry_description": "Package Under Verification"}})
# print(i["namespace"], n)

# print("Number of documents matched:", result.matched_count)
# print("Number of documents modified:", result.modified_count)
# t = db.packages.update_many({}, {"$set": {"registry_description": "Package Under Verification"}})
# print(t.modified_count,t.matched_count)
# t = db.packages.update_many({}, {"$set": {"description": "Package Under Verification"}})
# print(t.modified_count,t.matched_count)


# report check if access_token is not undefined.

# port from namespace obj id to name #
# optimise package search, and rendering #
# readme rendering in packages bring clarity on how to support README.md #
# registry-description #
# time and speed benchmarks #
# erase GPF # NEW RELASE NEW DB
# keywords, description, categories ( search and sort by categories and keywords https://github.com/Beliavsky?tab=repositories ) #
# bring clarity on multiple namespaces and hierarchy #

# support catch-22
# test-drive for module naming
# ability to register programs as well as libraries; where plug-ins and programs could be registered and easily installed.
# fpm new folder_name --namespace n --package p --version v
# DOCS!!!!!!!!!!!!!



# tests, docs, PR reviews, CI , Vercel migration.
# fpm release, fpm pr, (meeting time + discourse support).
10 changes: 8 additions & 2 deletions backend/check_digests.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import Tuple
import numpy as np
import json
import re


def hash(input_string):
Expand Down Expand Up @@ -28,6 +29,10 @@ def dilate(instr):
outstr.append(outstr_c)
return outstr

def process(lines):
cleaned = re.sub(r'(?<=\S) +(?=\n)', '', ''.join(lines))
cleaned = re.sub(r'(?<=\n)\t+(?=\n)', '', cleaned)
return re.sub(r'\n\s+\n', '\n\n\n', cleaned)

def check_digests(file_path: str) -> Tuple[int, bool]:
"""
Expand Down Expand Up @@ -65,9 +70,10 @@ def check_digests(file_path: str) -> Tuple[int, bool]:

try:
# Read the content of the file
with open(f"{file_path}{file_name}", 'r',newline='') as file:
file_content: str = file.read()
with open(f'{file_path}{file_name.replace("./", "")}', 'r',newline='') as file:
file_content: str = process(file.readlines())
except:
print(f'Error reading file content: {file_path}{file_name}')
return (-1, "Error reading file content.")

# Compute the digest of the file content
Expand Down
15 changes: 12 additions & 3 deletions backend/models/package.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
class Package:
def __init__(self, name, namespace, description, homepage, repository,
copyright, license, created_at, updated_at, author, maintainers, keywords, categories, is_deprecated, versions=[], id=None,
malicious_report={}, is_verified=False, is_malicious=False, security_status="No security issues found", ratings={"users": {}, "avg_ratings": 0}):
def __init__(self, name, namespace, namespace_name, description, homepage, repository,
copyright, license, created_at, updated_at, author, maintainers, keywords, categories, is_deprecated, versions=[], id=None,unable_to_verify=False,
malicious_report={}, registry_description=None,is_verified=False, is_malicious=False, security_status="No security issues found", ratings={"users": {}, "avg_ratings": 0}):
self.id = id
self.name = name
self.namespace = namespace
self.namespace_name = namespace_name
self.registry_description = registry_description
self.description = description
self.homepage = homepage
self.repository = repository
Expand All @@ -24,6 +26,7 @@ def __init__(self, name, namespace, description, homepage, repository,
self.downloads_stats = {}
self.ratings = ratings
self.categories = categories
self.unable_to_verify = unable_to_verify

# Ensure that versions list only contains instances of Version class
for v in self.versions:
Expand All @@ -43,7 +46,9 @@ def to_json(self):
"id": str(self.id),
"name": self.name,
"namespace": self.namespace,
"namespace_name": self.namespace_name,
"description": self.description,
"registry_description": self.registry_description,
"homepage": self.homepage,
"repository": self.repository,
"copyright": self.copyright,
Expand All @@ -61,6 +66,7 @@ def to_json(self):
"is_malicious": self.is_malicious,
"security_status": self.security_status,
"ratings": self.ratings,
"unable_to_verify": self.unable_to_verify,
}

# Create a from_json method.
Expand All @@ -76,7 +82,9 @@ def from_json(json_data):
id=str(json_data.get("_id")),
name=json_data.get("name"),
namespace=json_data.get("namespace"),
namespace_name=json_data.get("namespace_name"),
description=json_data.get("description"),
registry_description=json_data.get("registry_description"),
homepage=json_data.get("homepage"),
repository=json_data.get("repository"),
copyright=json_data.get("copyright"),
Expand All @@ -94,6 +102,7 @@ def from_json(json_data):
is_malicious=json_data.get("is_malicious"),
security_status=json_data.get("security_status"),
ratings=json_data.get("ratings"),
unable_to_verify=json_data.get("unable_to_verify"),
)

class Version:
Expand Down
70 changes: 14 additions & 56 deletions backend/packages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ def search_packages():
"_id": 0,
"name": 1,
"namespace": 1,
"namespace_name": 1,
"author": 1,
"description": 1,
"keywords": 1,
Expand All @@ -132,24 +133,13 @@ def search_packages():
for i in packages:
package_obj = Package.from_json(i)

namespace = db.namespaces.find_one({"_id": package_obj.namespace})
namespace_obj = Namespace.from_json(namespace)

author = db.users.find_one({"_id": package_obj.author})
author_obj = User.from_json(author)

package_obj.namespace = namespace_obj.namespace
package_obj.author = author_obj.username
search_packages.append({
"name": package_obj.name,
"namespace": package_obj.namespace,
"author": package_obj.author,
"namespace": package_obj.namespace_name,
"description": package_obj.description,
"keywords": package_obj.keywords,
"categories": package_obj.categories,
"keywords": package_obj.keywords+package_obj.categories,
"updated_at": package_obj.updated_at,
})

return (
jsonify(
{"code": 200, "packages": search_packages, "total_pages": total_pages}
Expand Down Expand Up @@ -207,11 +197,8 @@ def upload():
namespace_doc = db.namespaces.find_one(
{"upload_tokens": {"$elemMatch": {"token": upload_token}}}
)
package_doc = db.packages.find_one(
{"upload_tokens": {"$elemMatch": {"token": upload_token}}}
)

if not namespace_doc and not package_doc:
if not namespace_doc:
return jsonify({"code": 401, "message": "Invalid upload token"}), 401

if namespace_doc:
Expand All @@ -225,18 +212,6 @@ def upload():
{"name": package_name, "namespace": namespace_obj.id}
)

elif package_doc:
package_obj = Package.from_json(package_doc)
if package_obj.name != package_name:
return jsonify({"code": 401, "message": "Invalid upload token"}), 401

upload_token_doc = next(
item
for item in package_doc["upload_tokens"]
if item["token"] == upload_token
)
namespace_doc = db.namespaces.find_one({"_id": package_doc["namespace"]})

# Check if the token is expired.
# Expire the token after one week of it's creation.
if check_token_expiry(upload_token_created_at=upload_token_doc["createdAt"]):
Expand Down Expand Up @@ -294,6 +269,7 @@ def upload():
"description": "Package Under Verification",
"copyright": "Package Under Verification",
"homepage": "Package Under Verification",
"registry_description": "Package Under Verification",
}

file_object_id = file_storage.put(
Expand All @@ -306,6 +282,7 @@ def upload():
package_obj = Package(
name=package_name,
namespace=namespace_obj.id,
namespace_name=namespace_obj.namespace,
description=package_data["description"],
homepage=package_data["homepage"],
repository=package_data["repository"],
Expand Down Expand Up @@ -478,20 +455,9 @@ def check_version(current_version, new_version):
@app.route("/packages/<namespace_name>/<package_name>", methods=["GET"])
@swag_from("documentation/get_package.yaml", methods=["GET"])
def get_package(namespace_name, package_name):
# Get namespace from namespace name.
namespace = db.namespaces.find_one({"namespace": namespace_name})

# Check if namespace exists.
if not namespace:
return (
jsonify({"status": "error", "message": "Namespace not found", "code": 404}),
404,
)

namespace_obj = Namespace.from_json(namespace)
# Get package from a package_name and namespace's id.
# Get package from a package_name and namespace's name.
package = db.packages.find_one(
{"name": package_name, "namespace": namespace_obj.id}
{"name": package_name, "namespace_name": namespace_name}
)

# Check if package is not found.
Expand Down Expand Up @@ -549,7 +515,7 @@ def get_package(namespace_name, package_name):
# Only latest version of the package will be sent as a response.
package_response_data = {
"name": package_obj.name,
"namespace": namespace_obj.namespace,
"namespace": package_obj.namespace_name,
"latest_version_data": latest_version_data,
"author": package_author_obj.username,
"keywords": package_obj.keywords if package_obj.keywords else [],
Expand All @@ -559,6 +525,7 @@ def get_package(namespace_name, package_name):
"version_history": version_history,
"updated_at": package_obj.updated_at,
"description": package_obj.description,
"registry_description": package_obj.registry_description,
"ratings": ratings,
"downloads": downloads_stats,
"ratings_count": rating_count
Expand Down Expand Up @@ -586,7 +553,7 @@ def verify_user_role(namespace_name, package_name):
)

package = db.packages.find_one(
{"name": package_name, "namespace": namespace["_id"]}
{"name": package_name, "namespace": namespace_name}
)

if not package:
Expand All @@ -612,20 +579,11 @@ def verify_user_role(namespace_name, package_name):
@app.route("/packages/<namespace_name>/<package_name>/<version>", methods=["GET"])
@swag_from("documentation/get_version.yaml", methods=["GET"])
def get_package_from_version(namespace_name, package_name, version):
# Get namespace from namespace name.
namespace = db.namespaces.find_one({"namespace": namespace_name})

# Check if namespace does not exists.
if not namespace:
return jsonify({"message": "Namespace not found", "code": 404}), 404

namespace_obj = Namespace.from_json(namespace)

# Get package from a package_name, namespace's id and version.
# Get package from a package_name, namespace_name and version.
package = db.packages.find_one(
{
"name": package_name,
"namespace": namespace["_id"],
"namespace_name": namespace_name,
"versions.version": version,
}
)
Expand All @@ -652,7 +610,7 @@ def get_package_from_version(namespace_name, package_name, version):
# Only queried version should be sent as response.
package_response_data = {
"name": package_obj.name,
"namespace": namespace_obj.namespace,
"namespace": package_obj.namespace_name,
"author": package_author_obj.username,
"keywords": package_obj.keywords,
"categories": package_obj.categories,
Expand Down
Loading

0 comments on commit 2f98784

Please sign in to comment.