Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refine keys for app versions and catalog.json #18

Merged
merged 8 commits into from
May 15, 2024
Merged

Refine keys for app versions and catalog.json #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
617a094
Adjust app.yaml json schema to account for normalized values
sonicaj May 14, 2024
b20179d
Merge metadata.yaml into app.yaml
sonicaj May 14, 2024
60a7ee5
Remove chart_metadata usages
sonicaj May 15, 2024
13cc43f
Remove duplication of apps_metadata json schema
sonicaj May 15, 2024
b6ee64e
Add a todo statement
sonicaj May 15, 2024
40a4f2f
Fix readme usage
sonicaj May 15, 2024
d830c7e
Improve error reporting of malformed data
sonicaj May 15, 2024
1e30270
Make version schema more strict
sonicaj May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps_ci/scripts/catalog_update.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def validate_train_data(train_data):
except (json.JSONDecodeError, JsonValidationError) as e:
verrors.add(
'catalog_json',
f'Failed to validate contents of train data: {e!r}'
f'Failed to validate contents of train data ({".".join(list(e.path))}): {e!r}'
sonicaj marked this conversation as resolved.
Show resolved Hide resolved
)
verrors.check()

Expand Down
168 changes: 51 additions & 117 deletions apps_validation/json_schema_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@
'properties': {
'name': {'type': 'string'},
'train': {'type': 'string'},
'description': {'type': 'string'},
'home': {'type': 'string'},
'app_version': {'type': 'string'},
'annotations': {
'type': 'object',
'properties': {
'min_scale_version': {'type': 'string'},
'max_scale_version': {'type': 'string'},
},
},
'title': {'type': 'string'},
'sources': {
'type': 'array',
'items': {'type': 'string'},
Expand All @@ -21,10 +25,15 @@
'properties': {
'name': {'type': 'string'},
'email': {'type': 'string'},
'url': {'type': 'string'},
},
'required': ['name', 'email'],
},
},
'keywords': {
'type': 'array',
'items': {'type': 'string'},
},
'version': {
'type': 'string',
'pattern': '[0-9]+.[0-9]+.[0-9]+',
Expand All @@ -34,9 +43,46 @@
'pattern': '[0-9]+.[0-9]+.[0-9]+',
},
'lib_version_hash': {'type': 'string'},
'run_as_context': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'description': {'type': 'string'},
'gid': {'type': 'integer'},
'groupName': {'type': 'string'},
'userName': {'type': 'string'},
'uid': {'type': 'integer'},
},
'required': ['description'],
},
},
'capabilities': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'description': {'type': 'string'},
'name': {'type': 'string'},
},
'required': ['description', 'name'],
},
},
'host_mounts': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'description': {'type': 'string'},
'hostPath': {'type': 'string'},
},
'required': ['description', 'hostPath'],
},
},
},
'required': [
'name', 'train', 'version',
'name', 'train', 'version', 'app_version', 'title', 'description', 'home',
'sources', 'maintainers', 'run_as_context', 'capabilities', 'host_mounts',
],
'if': {
'properties': {
Expand Down Expand Up @@ -192,47 +238,6 @@
}
}
}
METADATA_JSON_SCHEMA = {
'type': 'object',
'properties': {
'runAsContext': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'description': {'type': 'string'},
'gid': {'type': 'integer'},
'groupName': {'type': 'string'},
'userName': {'type': 'string'},
'uid': {'type': 'integer'},
},
'required': ['description'],
},
},
'capabilities': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'description': {'type': 'string'},
'name': {'type': 'string'},
},
'required': ['description', 'name'],
},
},
'hostMounts': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'description': {'type': 'string'},
'hostPath': {'type': 'string'},
},
'required': ['description', 'hostPath'],
},
},
},
}
RECOMMENDED_APPS_JSON_SCHEMA = {
'type': 'object',
'patternProperties': {
Expand Down Expand Up @@ -279,80 +284,9 @@
'type': 'string',
'pattern': '[0-9]+.[0-9]+.[0-9]+'
},
'chart_metadata': {
'type': 'object',
'properties': {
'name': {
'type': 'string'
},
'description': {
'type': 'string'
},
'annotations': {
'type': 'object'
},
'type': {
'type': 'string'
},
'version': {
'type': 'string',
'pattern': '[0-9]+.[0-9]+.[0-9]+'
},
'apiVersion': {
'type': 'string',
},
'appVersion': {
'type': 'string'
},
'kubeVersion': {
'type': 'string'
},
'app_readme': {'type': 'string'},
'detailed_readme': {'type': 'string'},
'changelog': {'type': ['string', 'null']},
'maintainers': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'url': {'type': ['string', 'null']},
'email': {'type': 'string'},
},
'required': ['name', 'email'],
}
},
'dependencies': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'repository': {'type': 'string'},
'version': {'type': 'string'}
}
}
},
'home': {'type': 'string'},
'icon': {'type': 'string'},
'sources': {
'type': 'array',
'items': {
'type': 'string'
}
},
'keywords': {
'type': 'array',
'items': {
'type': 'string'
}
},
}
},
'app_metadata': {
**METADATA_JSON_SCHEMA,
'type': ['object', 'null'],
},
'app_metadata': APP_METADATA_JSON_SCHEMA,
'readme': {'type': ['string', 'null']},
'changelog': {'type': ['string', 'null']},
'schema': {
'type': 'object',
'properties': {
Expand Down Expand Up @@ -398,7 +332,7 @@
},
'required': [
'healthy', 'supported', 'healthy_error', 'location', 'last_update', 'required_features',
'human_version', 'version', 'chart_metadata', 'app_metadata', 'schema',
'human_version', 'version', 'app_metadata', 'schema', 'readme', 'changelog',
],
},
},
Expand Down
1 change: 1 addition & 0 deletions apps_validation/validate_app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def validate_catalog_item(catalog_item_path: str, schema: str, train_name: str,
with open(os.path.join(catalog_item_path, 'item.yaml'), 'r') as f:
item_config = yaml.safe_load(f.read())

# TODO: Remove validate key value type function and have json schemas for all of this
validate_key_value_types(
item_config, (
('categories', list), ('tags', list, False), ('screenshots', list, False),
Expand Down
25 changes: 1 addition & 24 deletions apps_validation/validate_app_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from .app_version import validate_app_version_file
from .ix_values import validate_ix_values_schema
from .json_schema_utils import METADATA_JSON_SCHEMA, VERSION_VALIDATION_SCHEMA
from .json_schema_utils import VERSION_VALIDATION_SCHEMA
from .validate_questions import validate_questions_yaml
from .validate_templates import validate_templates

Expand Down Expand Up @@ -89,13 +89,6 @@ def validate_catalog_item_version(
except ValidationErrors as v:
verrors.extend(v)

metadata_path = os.path.join(version_path, 'metadata.yaml')
if os.path.exists(metadata_path):
try:
validate_metadata_yaml(metadata_path, f'{schema}.metadata_configuration')
except ValidationErrors as v:
verrors.extend(v)

# validate_app_migrations(verrors, version_path, f'{schema}.app_migrations')
# FIXME: Add validation for app migrations

Expand Down Expand Up @@ -124,19 +117,3 @@ def validate_ix_values_yaml(ix_values_yaml_path: str, schema: str):
verrors.add(schema, 'Must be a dictionary')

verrors.check()


def validate_metadata_yaml(metadata_yaml_path: str, schema: str):
verrors = ValidationErrors()
with open(metadata_yaml_path, 'r') as f:
try:
metadata = yaml.safe_load(f.read())
except yaml.YAMLError:
verrors.add(schema, 'Must be a valid yaml file')
else:
try:
json_schema_validate(metadata, METADATA_JSON_SCHEMA)
except JsonValidationError as e:
verrors.add(schema, f'Invalid format specified for application metadata: {e}')

verrors.check()
52 changes: 25 additions & 27 deletions catalog_reader/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from apps_validation.validate_app import validate_catalog_item
from apps_validation.validate_app_version import validate_catalog_item_version # FIXME: rename this

from .app_utils import get_default_questions_context, get_app_details_base
from .app_utils import get_default_questions_context, get_app_details_base, get_human_version
from .git import get_last_updated_date
from .questions import normalize_questions
from .supported_version import version_supported
Expand Down Expand Up @@ -53,32 +53,30 @@ def get_app_details(
'default_values_callable': options.get('default_values_callable'),
}))
unhealthy_versions = []
desired_keys_mapping = {
'maintainers': 'maintainers',
'description': 'description',
'title': 'title',
'home': 'home',
'sources': 'sources',
}
for k, v in sorted(item_data['versions'].items(), key=lambda v: parse_version(v[0]), reverse=True):
if not v['healthy']:
unhealthy_versions.append(k)
else:
chart_metadata = v['chart_metadata']
if not item_data['app_readme']:
item_data['app_readme'] = v['readme']
if not item_data['maintainers'] and chart_metadata.get('maintainers'):
item_data['maintainers'] = chart_metadata['maintainers']
app_metadata = v['app_metadata']
for desired_key in list(desired_keys_mapping):
if not item_data[desired_key]:
item_data[desired_key] = app_metadata[desired_keys_mapping[desired_key]]
desired_keys_mapping.pop(desired_key)

if not item_data['latest_version']:
item_data['latest_version'] = k
item_data['latest_app_version'] = chart_metadata.get('appVersion')
item_data['latest_human_version'] = ''
if item_data['latest_app_version']:
item_data['latest_human_version'] = f'{item_data["latest_app_version"]}_'
item_data['latest_human_version'] += k
if not item_data['description'] and chart_metadata.get('description'):
item_data['description'] = v['chart_metadata']['description']
if item_data['title'] == item_data['name'].capitalize() and chart_metadata.get(
'annotations', {}
).get('title'):
item_data['title'] = chart_metadata['annotations']['title']
if item_data['home'] is None and chart_metadata.get('home'):
item_data['home'] = chart_metadata['home']
if not item_data['sources'] and chart_metadata.get('sources'):
item_data['sources'] = chart_metadata['sources']
item_data['latest_app_version'] = app_metadata['app_version']
item_data['latest_human_version'] = get_human_version(app_metadata['app_version'], k)

if not item_data['app_readme']:
item_data['app_readme'] = v['readme']

if unhealthy_versions:
item_data['healthy_error'] = f'Errors were found with {", ".join(unhealthy_versions)} version(s)'
Expand Down Expand Up @@ -149,10 +147,9 @@ def get_app_version_details(
) -> dict:
version_data = {'location': version_path, 'required_features': set()}
for key, filename, parser in (
('chart_metadata', 'app.yaml', yaml.safe_load),
('app_metadata', 'metadata.yaml', yaml.safe_load),
('app_metadata', 'app.yaml', yaml.safe_load),
('schema', 'questions.yaml', yaml.safe_load),
('readme', 'README.md', markdown.markdown), # TODO: Has been changed, make sure json schema accounts for it
('readme', 'README.md', markdown.markdown),
('changelog', 'CHANGELOG.md', markdown.markdown),
):
if os.path.exists(os.path.join(version_path, filename)):
Expand All @@ -171,8 +168,9 @@ def get_app_version_details(
})
if options and options.get('default_values_callable'):
version_data['values'] = options['default_values_callable'](version_data)
chart_metadata = version_data['chart_metadata']
if chart_metadata['name'] != 'ix-chart' and chart_metadata.get('appVersion'):
version_data['human_version'] = f'{chart_metadata["appVersion"]}_{chart_metadata["version"]}'

app_metadata = version_data['app_metadata']
# TODO: See if this needs to change for our adaptation of ix-chart
version_data['human_version'] = get_human_version(app_metadata['app_version'], app_metadata['version'])

return version_data
4 changes: 4 additions & 0 deletions catalog_reader/app_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,7 @@ def get_values(values_path: str) -> dict:
return yaml.safe_load(f.read())

return {}


def get_human_version(app_version: str, version: str) -> str:
return f'{app_version}_{version}' if app_version != version else version
Loading