From 08e84e44bb9ac227ff6b3022462a3b36669f80e2 Mon Sep 17 00:00:00 2001 From: Paul Warren Date: Mon, 18 Sep 2023 22:22:09 +0100 Subject: [PATCH 01/32] Generation-side support for multi-documents. --- iXBRLViewerPlugin/__init__.py | 9 +- iXBRLViewerPlugin/iXBRLViewer.py | 197 +++++++++++++++++-------------- 2 files changed, 115 insertions(+), 91 deletions(-) diff --git a/iXBRLViewerPlugin/__init__.py b/iXBRLViewerPlugin/__init__.py index 8e9a84490..bde388706 100644 --- a/iXBRLViewerPlugin/__init__.py +++ b/iXBRLViewerPlugin/__init__.py @@ -81,6 +81,7 @@ def iXBRLViewerCommandLineOptionExtender(parser, *args, **kwargs): default=False, dest="zipViewerOutput", help="Converts the viewer output into a self contained zip") + parser.add_option("--keepOpen", dest="keepOpen", default=True, action="store_true", help=argparse.SUPPRESS) featureGroup = OptionGroup(parser, "Viewer Features", "See viewer README for information on enabling/disabling features.") for featureConfig in FEATURE_CONFIGS: @@ -110,7 +111,9 @@ def generateViewer( :param features: List of feature names to enable via generated JSON data. """ # extend XBRL-loaded run processing for this option - if cntlr.modelManager is None or cntlr.modelManager.modelXbrl is None or not cntlr.modelManager.modelXbrl.modelDocument: + if (cntlr.modelManager is None + or len(cntlr.modelManager.loadedModelXbrls) == 0 + or any(not mx.modelDocument for mx in cntlr.modelManager.loadedModelXbrls)): cntlr.addToLog("No taxonomy loaded.") return modelXbrl = cntlr.modelManager.modelXbrl @@ -137,7 +140,7 @@ def generateViewer( try: out = saveViewerDest if out: - viewerBuilder = IXBRLViewerBuilder(modelXbrl) + viewerBuilder = IXBRLViewerBuilder(cntlr.modelManager.loadedModelXbrls) if features: for feature in features: viewerBuilder.enableFeature(feature) @@ -306,7 +309,7 @@ def load_plugin_url(): 'author': 'Paul Warren', 'copyright': 'Copyright :: Workiva Inc. :: 2019', 'CntlrCmdLine.Options': commandLineOptionExtender, - 'CntlrCmdLine.Xbrl.Run': commandLineRun, + 'CntlrCmdLine.Filing.End': commandLineRun, 'CntlrWinMain.Menu.Tools': toolsMenuExtender, 'CntlrWinMain.Xbrl.Loaded': guiRun, } diff --git a/iXBRLViewerPlugin/iXBRLViewer.py b/iXBRLViewerPlugin/iXBRLViewer.py index 3915b8c8f..5f7ed4a4e 100644 --- a/iXBRLViewerPlugin/iXBRLViewer.py +++ b/iXBRLViewerPlugin/iXBRLViewer.py @@ -82,17 +82,15 @@ class IXBRLViewerBuilderError(Exception): class IXBRLViewerBuilder: - def __init__(self, dts: ModelXbrl, basenameSuffix: str = ''): + def __init__(self, reports: List[ModelXbrl], basenameSuffix: str = ''): self.nsmap = NamespaceMap() self.roleMap = NamespaceMap() - self.dts = dts + self.reports = reports self.taxonomyData = { - "concepts": {}, - "languages": {}, - "facts": {}, + "reports": [], "features": [], + "languages": {}, } - self.footnoteRelationshipSet = ModelRelationshipSet(dts, "XBRL-footnotes") self.basenameSuffix = basenameSuffix def enableFeature(self, featureName: str): @@ -146,22 +144,26 @@ def makeLanguageName(self, langCode): def addLanguage(self, langCode): if langCode not in self.taxonomyData["languages"]: self.taxonomyData["languages"][langCode] = self.makeLanguageName(langCode) + + @property + def currentReportData(self): + return self.taxonomyData["reports"][-1] - def addELR(self, elr): + def addELR(self, report: ModelXbrl, elr): prefix = self.roleMap.getPrefix(elr) - if self.taxonomyData.setdefault("roleDefs",{}).get(prefix, None) is None: - rts = self.dts.roleTypes.get(elr, []) + if self.currentReportData.setdefault("roleDefs",{}).get(prefix, None) is None: + rts = report.roleTypes.get(elr, []) label = next((rt.definition for rt in rts if rt.definition is not None), None) if label is not None: - self.taxonomyData["roleDefs"].setdefault(prefix,{})["en"] = label + self.currentReportData["roleDefs"].setdefault(prefix,{})["en"] = label - def addConcept(self, concept, dimensionType = None): + def addConcept(self, report: ModelXbrl, concept, dimensionType = None): if concept is None: return - labelsRelationshipSet = self.dts.relationshipSet(XbrlConst.conceptLabel) + labelsRelationshipSet = report.relationshipSet(XbrlConst.conceptLabel) labels = labelsRelationshipSet.fromModelObject(concept) conceptName = self.nsmap.qname(concept.qname) - if conceptName not in self.taxonomyData["concepts"]: + if conceptName not in self.currentReportData["concepts"]: conceptData = { "labels": { } } @@ -194,24 +196,24 @@ def addConcept(self, concept, dimensionType = None): if typedDomainElement is not None: typedDomainName = self.nsmap.qname(typedDomainElement.qname) conceptData['td'] = typedDomainName - self.addConcept(typedDomainElement) + self.addConcept(report, typedDomainElement) - self.taxonomyData["concepts"][conceptName] = conceptData + self.currentReportData["concepts"][conceptName] = conceptData def treeWalk(self, rels, item, indent = 0): for r in rels.fromModelObject(item): if r.toModelObject is not None: self.treeWalk(rels, r.toModelObject, indent + 1) - def getRelationships(self): + def getRelationships(self, report: ModelXbrl): rels = {} - for baseSetKey, baseSetModelLinks in self.dts.baseSets.items(): + for baseSetKey, baseSetModelLinks in report.baseSets.items(): arcrole, ELR, linkqname, arcqname = baseSetKey if arcrole in (XbrlConst.summationItem, WIDER_NARROWER_ARCROLE, XbrlConst.parentChild, XbrlConst.dimensionDefault) and ELR is not None: - self.addELR(ELR) + self.addELR(report, ELR) rr = dict() - relSet = self.dts.relationshipSet(arcrole, ELR) + relSet = report.relationshipSet(arcrole, ELR) for r in relSet.modelRelationships: if r.fromModelObject is not None and r.toModelObject is not None: fromKey = self.nsmap.qname(r.fromModelObject.qname) @@ -221,8 +223,8 @@ def getRelationships(self): if r.weight is not None: rel['w'] = r.weight rr.setdefault(fromKey, []).append(rel) - self.addConcept(r.toModelObject) - self.addConcept(r.fromModelObject) + self.addConcept(report, r.toModelObject) + self.addConcept(report, r.fromModelObject) rels.setdefault(self.roleMap.getPrefix(arcrole),{})[self.roleMap.getPrefix(ELR)] = rr return rels @@ -245,7 +247,7 @@ def validationErrors(self): return errors - def addFact(self, f): + def addFact(self, report: ModelXbrl, f): if f.id is None: f.set("id","ixv-%d" % (self.idGen)) @@ -270,7 +272,7 @@ def addFact(self, f): qnEnums = (qnEnums,) factData["v"] = " ".join(self.nsmap.qname(qn) for qn in qnEnums) for qn in qnEnums: - self.addConcept(self.dts.qnameConcepts.get(qn)) + self.addConcept(report, report.qnameConcepts.get(qn)) else: factData["v"] = f.value if f.value == INVALIDixVALUE: @@ -295,11 +297,11 @@ def addFact(self, f): for d, v in f.context.qnameDims.items(): if v.memberQname is not None: aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname(v.memberQname) - self.addConcept(v.member) - self.addConcept(v.dimension, dimensionType = "e") + self.addConcept(report, v.member) + self.addConcept(report, v.dimension, dimensionType = "e") elif v.typedMember is not None: aspects[self.nsmap.qname(v.dimensionQname)] = v.typedMember.text - self.addConcept(v.dimension, dimensionType = "t") + self.addConcept(report, v.dimension, dimensionType = "t") if f.context.isForeverPeriod: aspects["p"] = "f" @@ -317,8 +319,8 @@ def addFact(self, f): if frel.toModelObject is not None: factData.setdefault("fn", []).append(frel.toModelObject.id) - self.taxonomyData["facts"][f.id] = factData - self.addConcept(f.concept) + self.currentReportData["facts"][f.id] = factData + self.addConcept(report, f.concept) def oimUnitString(self, unit): """ @@ -384,8 +386,8 @@ def createViewer(self, scriptUrl: str = DEFAULT_VIEWER_PATH, useStubViewer: bool :param showValidations: True if validation errors should be included in output taxonomy data. :return: An iXBRLViewer instance that is ready to be saved. """ - dts = self.dts - iv = iXBRLViewer(dts) + # This "dts" is only used for logging + iv = iXBRLViewer(self.reports[0]) self.idGen = 0 self.roleMap.getPrefix(XbrlConst.standardLabel, "std") self.roleMap.getPrefix(XbrlConst.documentationLabel, "doc") @@ -394,79 +396,98 @@ def createViewer(self, scriptUrl: str = DEFAULT_VIEWER_PATH, useStubViewer: bool self.roleMap.getPrefix(XbrlConst.dimensionDefault, "d-d") self.roleMap.getPrefix(WIDER_NARROWER_ARCROLE, "w-n") - docSetFiles = None + # This is the document that we will embed the JSON taxonomy data in. + # This will be either the first document processed, or the stub document + xmlDocument = None + + for n, report in enumerate(self.reports): + self.footnoteRelationshipSet = ModelRelationshipSet(report, "XBRL-footnotes") + self.taxonomyData["reports"].append({ + "concepts": {}, + "facts": {}, + }) + for f in report.facts: + self.addFact(report, f) + self.currentReportData["rels"] = self.getRelationships(report) + + docSetFiles = None + report.info("viewer:info", "Creating iXBRL viewer (%d of %d)" % (n+1, len(self.reports))) + if report.modelDocument.type == Type.INLINEXBRLDOCUMENTSET: + # Sort by object index to preserve order in which files were specified. + xmlDocsByFilename = { + os.path.basename(self.outputFilename(doc.filepath)): deepcopy(doc.xmlDocument) + for doc in sorted(report.modelDocument.referencesDocument.keys(), key=lambda x: x.objectIndex) + } + docSetFiles = list(xmlDocsByFilename.keys()) + + if xmlDocument is None: + if useStubViewer: + xmlDocument = self.getStubDocument() + iv.addFile(iXBRLViewerFile(DEFAULT_OUTPUT_NAME, xmlDocument)) + else: + xmlDocument = next(iter(xmlDocsByFilename.values())) + + for filename, docSetXMLDoc in xmlDocsByFilename.items(): + iv.addFile(iXBRLViewerFile(filename, docSetXMLDoc)) + + elif useStubViewer: + if xmlDocument is None: + xmlDocument = self.getStubDocument() + iv.addFile(iXBRLViewerFile(DEFAULT_OUTPUT_NAME, xmlDocument)) + filename = self.outputFilename(os.path.basename(report.modelDocument.filepath)) + docSetFiles = [ filename ] + iv.addFile(iXBRLViewerFile(filename, report.modelDocument.xmlDocument)) - for f in dts.facts: - self.addFact(f) + else: + if len(self.reports) == 1: + filename = "xbrlviewer.html" + else: + filename = self.outputFilename(os.path.basename(report.modelDocument.filepath)) + if xmlDocument is None: + xmlDocument = deepcopy(report.modelDocument.xmlDocument) + iv.addFile(iXBRLViewerFile(filename, xmlDocument)) + else: + iv.addFile(iXBRLViewerFile(filename, report.modelDocument.xmlDocument)) + + localDocs = defaultdict(set) + for path, doc in report.urlDocs.items(): + if isHttpUrl(path): + continue + if doc.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET): + localDocs[doc.basename].add('inline') + elif doc.type == Type.SCHEMA: + localDocs[doc.basename].add('schema') + elif doc.type == Type.LINKBASE: + linkbaseIdentifed = False + for child in doc.xmlRootElement.iterchildren(): + linkbaseLocalDocumentsKey = LINK_QNAME_TO_LOCAL_DOCUMENTS_LINKBASE_TYPE.get(child.qname) + if linkbaseLocalDocumentsKey is not None: + localDocs[doc.basename].add(linkbaseLocalDocumentsKey) + linkbaseIdentifed = True + if not linkbaseIdentifed: + localDocs[doc.basename].add(UNRECOGNIZED_LINKBASE_LOCAL_DOCUMENTS_TYPE) + self.currentReportData["localDocs"] = { + localDoc: sorted(docTypes) + for localDoc, docTypes in localDocs.items() + } + + if docSetFiles is not None: + self.currentReportData["docSetFiles"] = list(urllib.parse.quote(f) for f in docSetFiles) self.taxonomyData["prefixes"] = self.nsmap.prefixmap self.taxonomyData["roles"] = self.roleMap.prefixmap - self.taxonomyData["rels"] = self.getRelationships() if showValidations: self.taxonomyData["validation"] = self.validationErrors() - dts.info("viewer:info", "Creating iXBRL viewer") - if dts.modelDocument.type == Type.INLINEXBRLDOCUMENTSET: - # Sort by object index to preserve order in which files were specified. - xmlDocsByFilename = { - os.path.basename(self.outputFilename(doc.filepath)): deepcopy(doc.xmlDocument) - for doc in sorted(dts.modelDocument.referencesDocument.keys(), key=lambda x: x.objectIndex) - } - docSetFiles = list(xmlDocsByFilename.keys()) - - if useStubViewer: - xmlDocument = self.getStubDocument() - iv.addFile(iXBRLViewerFile(DEFAULT_OUTPUT_NAME, xmlDocument)) - else: - xmlDocument = next(iter(xmlDocsByFilename.values())) - - for filename, docSetXMLDoc in xmlDocsByFilename.items(): - iv.addFile(iXBRLViewerFile(filename, docSetXMLDoc)) - - elif useStubViewer: - xmlDocument = self.getStubDocument() - filename = self.outputFilename(os.path.basename(dts.modelDocument.filepath)) - docSetFiles = [ filename ] - iv.addFile(iXBRLViewerFile(DEFAULT_OUTPUT_NAME, xmlDocument)) - iv.addFile(iXBRLViewerFile(filename, dts.modelDocument.xmlDocument)) - - else: - xmlDocument = deepcopy(dts.modelDocument.xmlDocument) - iv.addFile(iXBRLViewerFile('xbrlviewer.html', xmlDocument)) - if packageDownloadURL is not None: self.taxonomyData["filingDocuments"] = packageDownloadURL - elif os.path.dirname(self.dts.modelDocument.filepath).endswith('.zip'): + elif len(self.reports) == 1 and os.path.dirname(self.reports[0].modelDocument.filepath).endswith('.zip'): filingDocZipPath = os.path.dirname(self.dts.modelDocument.filepath) filingDocZipName = os.path.basename(filingDocZipPath) iv.addFilingDoc(filingDocZipPath) self.taxonomyData["filingDocuments"] = filingDocZipName - localDocs = defaultdict(set) - for path, doc in dts.urlDocs.items(): - if isHttpUrl(path): - continue - if doc.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET): - localDocs[doc.basename].add('inline') - elif doc.type == Type.SCHEMA: - localDocs[doc.basename].add('schema') - elif doc.type == Type.LINKBASE: - linkbaseIdentifed = False - for child in doc.xmlRootElement.iterchildren(): - linkbaseLocalDocumentsKey = LINK_QNAME_TO_LOCAL_DOCUMENTS_LINKBASE_TYPE.get(child.qname) - if linkbaseLocalDocumentsKey is not None: - localDocs[doc.basename].add(linkbaseLocalDocumentsKey) - linkbaseIdentifed = True - if not linkbaseIdentifed: - localDocs[doc.basename].add(UNRECOGNIZED_LINKBASE_LOCAL_DOCUMENTS_TYPE) - self.taxonomyData["localDocs"] = { - localDoc: sorted(docTypes) - for localDoc, docTypes in localDocs.items() - } - - if docSetFiles is not None: - self.taxonomyData["docSetFiles"] = list(urllib.parse.quote(f) for f in docSetFiles) if not self.addViewerToXMLDocument(xmlDocument, scriptUrl): return None From 730bdf9fa77244d5b4e7c8a5926af2bf350d73c3 Mon Sep 17 00:00:00 2001 From: Paul Warren Date: Tue, 19 Sep 2023 23:47:19 +0100 Subject: [PATCH 02/32] Multi-report - WIP --- iXBRLViewerPlugin/iXBRLViewer.py | 5 ++- iXBRLViewerPlugin/viewer/src/js/fact.js | 4 +- .../viewer/src/js/ixbrlviewer.js | 6 ++- iXBRLViewerPlugin/viewer/src/js/report.js | 35 +++++++++------- iXBRLViewerPlugin/viewer/src/js/util.js | 7 ++++ iXBRLViewerPlugin/viewer/src/js/viewer.js | 40 ++++++++++--------- 6 files changed, 59 insertions(+), 38 deletions(-) diff --git a/iXBRLViewerPlugin/iXBRLViewer.py b/iXBRLViewerPlugin/iXBRLViewer.py index 5f7ed4a4e..dc4c7a1e8 100644 --- a/iXBRLViewerPlugin/iXBRLViewer.py +++ b/iXBRLViewerPlugin/iXBRLViewer.py @@ -443,6 +443,7 @@ def createViewer(self, scriptUrl: str = DEFAULT_VIEWER_PATH, useStubViewer: bool filename = "xbrlviewer.html" else: filename = self.outputFilename(os.path.basename(report.modelDocument.filepath)) + docSetFiles = [ filename ] if xmlDocument is None: xmlDocument = deepcopy(report.modelDocument.xmlDocument) iv.addFile(iXBRLViewerFile(filename, xmlDocument)) @@ -471,8 +472,8 @@ def createViewer(self, scriptUrl: str = DEFAULT_VIEWER_PATH, useStubViewer: bool for localDoc, docTypes in localDocs.items() } - if docSetFiles is not None: - self.currentReportData["docSetFiles"] = list(urllib.parse.quote(f) for f in docSetFiles) + if docSetFiles is not None: + self.currentReportData["docSetFiles"] = list(urllib.parse.quote(f) for f in docSetFiles) self.taxonomyData["prefixes"] = self.nsmap.prefixmap self.taxonomyData["roles"] = self.roleMap.prefixmap diff --git a/iXBRLViewerPlugin/viewer/src/js/fact.js b/iXBRLViewerPlugin/viewer/src/js/fact.js index 685937ab7..c5d66bcf6 100644 --- a/iXBRLViewerPlugin/viewer/src/js/fact.js +++ b/iXBRLViewerPlugin/viewer/src/js/fact.js @@ -9,8 +9,8 @@ import Decimal from "decimal.js"; export class Fact { - constructor(report, factId) { - this.f = report.data.facts[factId]; + constructor(report, factId, factData) { + this.f = factData; this.ixNode = report.getIXNodeForItemId(factId); this._report = report; this.id = factId; diff --git a/iXBRLViewerPlugin/viewer/src/js/ixbrlviewer.js b/iXBRLViewerPlugin/viewer/src/js/ixbrlviewer.js index e3d709cfe..d92bcb019 100644 --- a/iXBRLViewerPlugin/viewer/src/js/ixbrlviewer.js +++ b/iXBRLViewerPlugin/viewer/src/js/ixbrlviewer.js @@ -137,7 +137,9 @@ iXBRLViewer.prototype._loadInspectorHTML = function () { iXBRLViewer.prototype._reparentDocument = function () { var iframeContainer = $('#ixv #iframe-container'); - var iframe = $('