Skip to content

Commit

Permalink
fix summary json merging, add displaytype counts to summary
Browse files Browse the repository at this point in the history
  • Loading branch information
jamesamcl committed Nov 5, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent ad79a42 commit 7e1b528
Showing 2 changed files with 27 additions and 2 deletions.
20 changes: 19 additions & 1 deletion 05_materialise/grebi_materialise/src/main.rs
Original file line number Diff line number Diff line change
@@ -113,6 +113,8 @@ fn main() -> std::io::Result<()> {
id_to_group.clear();
id_to_group.shrink_to(0);

let mut displaytype_to_count:HashMap<Vec<u8>, i64> = HashMap::new();

let node_metadata = load_metadata_mapping_table::load_metadata_mapping_table(&args.in_metadata_jsonl);

let mut types_to_count:HashMap<Vec<u8>,i64> = HashMap::new();
@@ -211,9 +213,20 @@ fn main() -> std::io::Result<()> {

nodes_writer.write_all(&line[0..line.len()-1] /* skip closing bracket */).unwrap();
if rarest_type.is_some() {

let displaytype = rarest_type.unwrap();

nodes_writer.write_all(b",\"grebi:displayType\":\"").unwrap();
nodes_writer.write_all(&rarest_type.unwrap()).unwrap();
nodes_writer.write_all(&displaytype).unwrap();
nodes_writer.write_all(b"\"").unwrap();

let mut w_count = displaytype_to_count.get_mut(&displaytype);
if w_count.is_none() {
displaytype_to_count.insert(displaytype.clone(), 0);
}
w_count = displaytype_to_count.get_mut(&displaytype);
let count:&mut i64 = w_count.unwrap();
*count += 1;
}
nodes_writer.write_all(b",\"_refs\":").unwrap();
nodes_writer.write_all(serde_json::to_string(&_refs).unwrap().as_bytes()).unwrap();
@@ -254,6 +267,11 @@ fn main() -> std::io::Result<()> {
"entity_prop_defs": entity_prop_defs,
"edge_prop_defs": edge_prop_defs,
"types": type_defs,
"displaytypes": displaytype_to_count.iter().map(|(k,v)| {
return (String::from_utf8(k.to_vec()).unwrap(), json!({
"count": v
}))
}).collect::<HashMap<String,serde_json::Value>>(),
"edges": edge_summary
})).unwrap().as_bytes()).unwrap();

9 changes: 8 additions & 1 deletion 05_materialise/merge_summary_jsons.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import numbers
import sys
import json
from collections import defaultdict
@@ -7,8 +8,14 @@ def merge(dict1, dict2):
if key in dict1:
if isinstance(dict1[key], dict) and isinstance(value, dict):
merge(dict1[key], value)
else:
elif isinstance(dict1[key], list) and isinstance(value, list):
for val in value:
if val not in dict1[key]:
dict1[key].append(val)
elif isinstance(dict1[key], numbers.Number) and isinstance(value, numbers.Number):
dict1[key] += value
elif dict1[key] != value:
dict1[key] = [dict1[key], value]
else:
dict1[key] = value
return dict1

0 comments on commit 7e1b528

Please sign in to comment.